refactor: remove augmentation system and semantic type matching
Remove the entire augmentation pipeline infrastructure (52 files, ~15,000 lines) and the semantic type matching system. These were unused middleware layers adding complexity without value. What was removed: - src/augmentations/ directory (all augmentation implementations) - src/augmentationManager.ts (pipeline orchestrator) - src/types/augmentations.ts, src/types/pipelineTypes.ts - src/shared/default-augmentations.ts - Semantic type suggestion (BrainyTypes.suggestNoun/suggestVerb) - src/utils/typeMatching/ (embedding-based type matcher) What was preserved by relocating: - Import handlers (CSV, PDF, Excel) -> src/importers/handlers/ - NeuralImportAugmentation -> src/cortex/neuralImportAugmentation.ts - Type matching utilities -> heuristic inference in consumers What was simplified: - brainy.ts: operations call storage directly (no execute() wrapper) - IntegrationBase: standalone class (no BaseAugmentation parent) - BrainyTypes: validation-only (nouns, verbs, isValid*, get*) - Pipeline: direct execution (no augmentation interception) - index.ts: removed TypeSuggestion, suggestType exports - package.json: removed stale types/augmentations export Build passes, 1176 tests pass, 0 failures.
This commit is contained in:
parent
ac7a1f772c
commit
d1db3510be
97 changed files with 349 additions and 19705 deletions
|
|
@ -1,105 +0,0 @@
|
||||||
/**
|
|
||||||
* API Server Example
|
|
||||||
*
|
|
||||||
* This example shows how to expose Brainy through REST, WebSocket, and MCP APIs
|
|
||||||
* using the APIServerAugmentation.
|
|
||||||
*
|
|
||||||
* Zero-config philosophy: Just create and register the augmentation!
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { Brainy } from '../src/index.js'
|
|
||||||
import { APIServerAugmentation } from '../src/augmentations/apiServerAugmentation.js'
|
|
||||||
|
|
||||||
async function main() {
|
|
||||||
// 1. Create Brainy with zero config
|
|
||||||
const brain = new Brainy()
|
|
||||||
await brain.init()
|
|
||||||
|
|
||||||
// 2. Add some sample data
|
|
||||||
await brain.add({ data: "The quick brown fox", type: 'content', metadata: { type: "sentence", category: "animals" } })
|
|
||||||
await brain.add({ data: "Machine learning models", type: 'content', metadata: { type: "tech", category: "AI" } })
|
|
||||||
await brain.add({ data: "Natural language processing", type: 'content', metadata: { type: "tech", category: "NLP" } })
|
|
||||||
|
|
||||||
// 3. Create and register the API Server augmentation
|
|
||||||
const apiServer = new APIServerAugmentation({
|
|
||||||
port: 3000,
|
|
||||||
host: 'localhost'
|
|
||||||
})
|
|
||||||
|
|
||||||
// Register the augmentation with Brainy
|
|
||||||
brain.augmentations.register(apiServer)
|
|
||||||
|
|
||||||
// Initialize augmentations with Brainy context
|
|
||||||
await brain.augmentations.initialize({
|
|
||||||
brain,
|
|
||||||
log: (msg: string, level?: string) => console.log(`[${level || 'info'}] ${msg}`),
|
|
||||||
config: {}
|
|
||||||
})
|
|
||||||
|
|
||||||
console.log('🚀 Brainy API Server is running!')
|
|
||||||
console.log('📡 REST API: http://localhost:3000')
|
|
||||||
console.log('🔌 WebSocket: ws://localhost:3000/ws')
|
|
||||||
console.log('🧠 MCP: http://localhost:3000/api/mcp')
|
|
||||||
console.log('')
|
|
||||||
console.log('Try these endpoints:')
|
|
||||||
console.log(' GET http://localhost:3000/health')
|
|
||||||
console.log(' POST http://localhost:3000/api/search')
|
|
||||||
console.log(' Body: { "query": "fox", "limit": 10 }')
|
|
||||||
console.log(' POST http://localhost:3000/api/add')
|
|
||||||
console.log(' Body: { "content": "New data", "metadata": {} }')
|
|
||||||
console.log('')
|
|
||||||
console.log('Press Ctrl+C to stop the server')
|
|
||||||
}
|
|
||||||
|
|
||||||
// Run the example
|
|
||||||
main().catch(console.error)
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Example REST API calls:
|
|
||||||
*
|
|
||||||
* # Health check
|
|
||||||
* curl http://localhost:3000/health
|
|
||||||
*
|
|
||||||
* # Search
|
|
||||||
* curl -X POST http://localhost:3000/api/search \
|
|
||||||
* -H "Content-Type: application/json" \
|
|
||||||
* -d '{"query": "fox", "limit": 5}'
|
|
||||||
*
|
|
||||||
* # Add data
|
|
||||||
* curl -X POST http://localhost:3000/api/add \
|
|
||||||
* -H "Content-Type: application/json" \
|
|
||||||
* -d '{"content": "The cat sat on the mat", "metadata": {"type": "sentence"}}'
|
|
||||||
*
|
|
||||||
* # Get by ID
|
|
||||||
* curl http://localhost:3000/api/get/[id]
|
|
||||||
*
|
|
||||||
* # Statistics
|
|
||||||
* curl http://localhost:3000/api/stats
|
|
||||||
*/
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Example WebSocket client (browser):
|
|
||||||
*
|
|
||||||
* const ws = new WebSocket('ws://localhost:3000/ws')
|
|
||||||
*
|
|
||||||
* ws.onopen = () => {
|
|
||||||
* // Subscribe to all operations
|
|
||||||
* ws.send(JSON.stringify({
|
|
||||||
* type: 'subscribe',
|
|
||||||
* operations: ['all']
|
|
||||||
* }))
|
|
||||||
*
|
|
||||||
* // Perform a search
|
|
||||||
* ws.send(JSON.stringify({
|
|
||||||
* type: 'search',
|
|
||||||
* query: 'fox',
|
|
||||||
* limit: 5,
|
|
||||||
* requestId: '123'
|
|
||||||
* }))
|
|
||||||
* }
|
|
||||||
*
|
|
||||||
* ws.onmessage = (event) => {
|
|
||||||
* const msg = JSON.parse(event.data)
|
|
||||||
* console.log('Received:', msg)
|
|
||||||
* }
|
|
||||||
*/
|
|
||||||
|
|
@ -1,49 +0,0 @@
|
||||||
/**
|
|
||||||
* Zero-Config Augmentations Example
|
|
||||||
*
|
|
||||||
* Brainy's philosophy: Everything just works with zero configuration.
|
|
||||||
* Augmentations extend Brainy without any complex setup.
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { Brainy } from '../src/index.js'
|
|
||||||
import { EntityRegistryAugmentation } from '../src/augmentations/entityRegistryAugmentation.js'
|
|
||||||
import { BatchProcessingAugmentation } from '../src/augmentations/batchProcessingAugmentation.js'
|
|
||||||
import { APIServerAugmentation } from '../src/augmentations/apiServerAugmentation.js'
|
|
||||||
|
|
||||||
async function main() {
|
|
||||||
// Create Brainy - zero config!
|
|
||||||
const brain = new Brainy()
|
|
||||||
|
|
||||||
// Register augmentations - they just work!
|
|
||||||
brain.augmentations.register(new EntityRegistryAugmentation()) // Deduplication
|
|
||||||
brain.augmentations.register(new BatchProcessingAugmentation()) // Performance
|
|
||||||
brain.augmentations.register(new APIServerAugmentation()) // API exposure
|
|
||||||
|
|
||||||
// Initialize Brainy (augmentations auto-initialize)
|
|
||||||
await brain.init()
|
|
||||||
|
|
||||||
// Use Brainy normally - augmentations work transparently
|
|
||||||
await brain.add({ data: "Hello world", type: 'content', metadata: { type: "greeting" } })
|
|
||||||
|
|
||||||
// Batch operations automatically optimized
|
|
||||||
await brain.addBatch([
|
|
||||||
{ content: "Item 1", metadata: { id: 1 } },
|
|
||||||
{ content: "Item 2", metadata: { id: 2 } },
|
|
||||||
{ content: "Item 3", metadata: { id: 3 } }
|
|
||||||
])
|
|
||||||
|
|
||||||
// Search works with all augmentations active
|
|
||||||
const results = await brain.search("hello")
|
|
||||||
console.log('Search results:', results)
|
|
||||||
|
|
||||||
// API server is running at http://localhost:3000
|
|
||||||
console.log('✨ Brainy is running with:')
|
|
||||||
console.log(' - Write-ahead logging (durability)')
|
|
||||||
console.log(' - Entity deduplication (performance)')
|
|
||||||
console.log(' - Batch optimization (speed)')
|
|
||||||
console.log(' - REST/WebSocket/MCP APIs (connectivity)')
|
|
||||||
console.log('')
|
|
||||||
console.log('Zero configuration required!')
|
|
||||||
}
|
|
||||||
|
|
||||||
main().catch(console.error)
|
|
||||||
|
|
@ -50,10 +50,7 @@ try {
|
||||||
|
|
||||||
console.log('\n📊 API Architecture Validation:')
|
console.log('\n📊 API Architecture Validation:')
|
||||||
|
|
||||||
// Test 5: Augmentation system structure
|
// Test 5: Core properties exist
|
||||||
test('Augmentations system exists', brain.augmentations !== undefined)
|
|
||||||
|
|
||||||
// Test 6: Core properties exist
|
|
||||||
test('Index system exists', brain.index !== undefined)
|
test('Index system exists', brain.index !== undefined)
|
||||||
test('Storage system exists', brain.storage !== undefined)
|
test('Storage system exists', brain.storage !== undefined)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,89 +0,0 @@
|
||||||
/**
|
|
||||||
* Test script to verify storage augmentation system works
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { Brainy } from '../../dist/index.js'
|
|
||||||
import {
|
|
||||||
MemoryStorageAugmentation,
|
|
||||||
FileSystemStorageAugmentation
|
|
||||||
} from './dist/augmentations/storageAugmentations.js'
|
|
||||||
|
|
||||||
console.log('🧪 Testing Storage Augmentation System')
|
|
||||||
console.log('=' .repeat(50))
|
|
||||||
|
|
||||||
async function test1_ZeroConfig() {
|
|
||||||
console.log('\n1. Zero-Config Test')
|
|
||||||
const brain = new Brainy()
|
|
||||||
await brain.init()
|
|
||||||
|
|
||||||
await brain.add('test', { content: 'Zero-config test' })
|
|
||||||
const results = await brain.search('test', { limit: 1 })
|
|
||||||
|
|
||||||
console.log('✅ Zero-config works:', results.length > 0)
|
|
||||||
await brain.destroy()
|
|
||||||
}
|
|
||||||
|
|
||||||
async function test2_ConfigBased() {
|
|
||||||
console.log('\n2. Config-Based Storage Test')
|
|
||||||
const brain = new Brainy({
|
|
||||||
storage: {
|
|
||||||
forceMemoryStorage: true
|
|
||||||
}
|
|
||||||
})
|
|
||||||
await brain.init()
|
|
||||||
|
|
||||||
await brain.add('config test', { content: 'Config-based test' })
|
|
||||||
const results = await brain.search('config', { limit: 1 })
|
|
||||||
|
|
||||||
console.log('✅ Config-based works:', results.length > 0)
|
|
||||||
await brain.destroy()
|
|
||||||
}
|
|
||||||
|
|
||||||
async function test3_AugmentationOverride() {
|
|
||||||
console.log('\n3. Augmentation Override Test')
|
|
||||||
const brain = new Brainy()
|
|
||||||
|
|
||||||
// Register storage augmentation BEFORE init
|
|
||||||
brain.augmentations.register(new MemoryStorageAugmentation('test-memory'))
|
|
||||||
|
|
||||||
await brain.init()
|
|
||||||
|
|
||||||
await brain.add('augmentation test', { content: 'Augmentation override test' })
|
|
||||||
const results = await brain.search('augmentation', { limit: 1 })
|
|
||||||
|
|
||||||
console.log('✅ Augmentation override works:', results.length > 0)
|
|
||||||
await brain.destroy()
|
|
||||||
}
|
|
||||||
|
|
||||||
async function test4_BackwardCompatibility() {
|
|
||||||
console.log('\n4. Backward Compatibility Test')
|
|
||||||
|
|
||||||
// Old style with rootDirectory config
|
|
||||||
const brain = new Brainy({
|
|
||||||
storage: {
|
|
||||||
rootDirectory: './test-data',
|
|
||||||
forceFileSystemStorage: true
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
await brain.init()
|
|
||||||
console.log('✅ Backward compatible config works')
|
|
||||||
await brain.destroy()
|
|
||||||
}
|
|
||||||
|
|
||||||
async function runAllTests() {
|
|
||||||
try {
|
|
||||||
await test1_ZeroConfig()
|
|
||||||
await test2_ConfigBased()
|
|
||||||
await test3_AugmentationOverride()
|
|
||||||
await test4_BackwardCompatibility()
|
|
||||||
|
|
||||||
console.log('\n' + '='.repeat(50))
|
|
||||||
console.log('✅ ALL STORAGE AUGMENTATION TESTS PASSED!')
|
|
||||||
} catch (error) {
|
|
||||||
console.error('❌ Test failed:', error)
|
|
||||||
process.exit(1)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
runAllTests()
|
|
||||||
|
|
@ -28,10 +28,6 @@
|
||||||
"import": "./dist/types/graphTypes.js",
|
"import": "./dist/types/graphTypes.js",
|
||||||
"types": "./dist/types/graphTypes.d.ts"
|
"types": "./dist/types/graphTypes.d.ts"
|
||||||
},
|
},
|
||||||
"./types/augmentations": {
|
|
||||||
"import": "./dist/types/augmentations.js",
|
|
||||||
"types": "./dist/types/augmentations.d.ts"
|
|
||||||
},
|
|
||||||
"./utils/textEncoding": {
|
"./utils/textEncoding": {
|
||||||
"import": "./dist/utils/textEncoding.js",
|
"import": "./dist/utils/textEncoding.js",
|
||||||
"types": "./dist/utils/textEncoding.d.ts"
|
"types": "./dist/utils/textEncoding.d.ts"
|
||||||
|
|
|
||||||
|
|
@ -16,8 +16,7 @@ import { NounType, VerbType } from '../types/graphTypes.js'
|
||||||
import { Vector } from '../coreTypes.js'
|
import { Vector } from '../coreTypes.js'
|
||||||
import type { Brainy } from '../brainy.js'
|
import type { Brainy } from '../brainy.js'
|
||||||
import type { Entity, Relation } from '../types/brainy.types.js'
|
import type { Entity, Relation } from '../types/brainy.types.js'
|
||||||
import { BrainyTypes, getBrainyTypes } from '../augmentations/typeMatching/brainyTypes.js'
|
import { NeuralImportAugmentation } from '../cortex/neuralImportAugmentation.js'
|
||||||
import { NeuralImportAugmentation } from '../augmentations/neuralImport.js'
|
|
||||||
import { mimeDetector } from '../vfs/MimeTypeDetector.js'
|
import { mimeDetector } from '../vfs/MimeTypeDetector.js'
|
||||||
|
|
||||||
export interface ImportSource {
|
export interface ImportSource {
|
||||||
|
|
@ -65,7 +64,6 @@ export interface NeuralImportProgress {
|
||||||
|
|
||||||
export class UniversalImportAPI {
|
export class UniversalImportAPI {
|
||||||
private brain: Brainy<any>
|
private brain: Brainy<any>
|
||||||
private typeMatcher!: BrainyTypes
|
|
||||||
private neuralImport: NeuralImportAugmentation
|
private neuralImport: NeuralImportAugmentation
|
||||||
private embedCache = new Map<string, Vector>()
|
private embedCache = new Map<string, Vector>()
|
||||||
|
|
||||||
|
|
@ -82,7 +80,6 @@ export class UniversalImportAPI {
|
||||||
* Initialize the neural import system
|
* Initialize the neural import system
|
||||||
*/
|
*/
|
||||||
async init(): Promise<void> {
|
async init(): Promise<void> {
|
||||||
this.typeMatcher = await getBrainyTypes()
|
|
||||||
// Neural import initializes itself
|
// Neural import initializes itself
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -370,21 +367,18 @@ export class UniversalImportAPI {
|
||||||
// Generate embedding for the item
|
// Generate embedding for the item
|
||||||
const embedding = await this.generateEmbedding(item)
|
const embedding = await this.generateEmbedding(item)
|
||||||
|
|
||||||
// Neural type matching - MANDATORY
|
// Determine noun type from data
|
||||||
const nounMatch = await this.typeMatcher.matchNounType(item)
|
const nounType = this.inferNounType(item)
|
||||||
|
|
||||||
// Never reject based on confidence - we ALWAYS accept the best match
|
|
||||||
const entityId = this.generateId(item)
|
const entityId = this.generateId(item)
|
||||||
|
|
||||||
entities.set(entityId, {
|
entities.set(entityId, {
|
||||||
id: entityId,
|
id: entityId,
|
||||||
type: nounMatch.type as NounType, // Always use the neural match
|
type: nounType,
|
||||||
data: item,
|
data: item,
|
||||||
vector: embedding,
|
vector: embedding,
|
||||||
confidence: nounMatch.confidence,
|
confidence: 1.0,
|
||||||
metadata: {
|
metadata: {
|
||||||
...item,
|
...item,
|
||||||
_neuralMatch: nounMatch,
|
|
||||||
_importedAt: Date.now()
|
_importedAt: Date.now()
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
@ -465,28 +459,19 @@ export class UniversalImportAPI {
|
||||||
for (const [key, value] of Object.entries(item)) {
|
for (const [key, value] of Object.entries(item)) {
|
||||||
// Check if this looks like a reference
|
// Check if this looks like a reference
|
||||||
if (this.looksLikeReference(key, value)) {
|
if (this.looksLikeReference(key, value)) {
|
||||||
// Find or predict target entity
|
|
||||||
const targetId = String(value)
|
const targetId = String(value)
|
||||||
|
const verbType = this.inferVerbType(key)
|
||||||
|
|
||||||
// Neural verb type matching
|
const relationId = `${sourceId}_${verbType}_${targetId}`
|
||||||
const verbMatch = await this.typeMatcher.matchVerbType(
|
|
||||||
item, // source object
|
|
||||||
{ id: targetId }, // target (we may not have full data)
|
|
||||||
key // field name as context
|
|
||||||
)
|
|
||||||
|
|
||||||
// Always create relationship with neural match
|
|
||||||
const relationId = `${sourceId}_${verbMatch.type}_${targetId}`
|
|
||||||
relationships.set(relationId, {
|
relationships.set(relationId, {
|
||||||
id: relationId,
|
id: relationId,
|
||||||
from: sourceId,
|
from: sourceId,
|
||||||
to: targetId,
|
to: targetId,
|
||||||
type: verbMatch.type as VerbType,
|
type: verbType,
|
||||||
weight: verbMatch.confidence, // Use confidence as weight
|
weight: 1.0,
|
||||||
confidence: verbMatch.confidence,
|
confidence: 1.0,
|
||||||
metadata: {
|
metadata: {
|
||||||
field: key,
|
field: key,
|
||||||
_neuralMatch: verbMatch,
|
|
||||||
_importedAt: Date.now()
|
_importedAt: Date.now()
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
@ -494,27 +479,22 @@ export class UniversalImportAPI {
|
||||||
|
|
||||||
// Handle arrays of references
|
// Handle arrays of references
|
||||||
if (Array.isArray(value)) {
|
if (Array.isArray(value)) {
|
||||||
for (const item of value) {
|
for (const arrayItem of value) {
|
||||||
if (this.looksLikeReference(key, item)) {
|
if (this.looksLikeReference(key, arrayItem)) {
|
||||||
const targetId = String(item)
|
const targetId = String(arrayItem)
|
||||||
const verbMatch = await this.typeMatcher.matchVerbType(
|
const verbType = this.inferVerbType(key)
|
||||||
item,
|
|
||||||
{ id: targetId },
|
|
||||||
key
|
|
||||||
)
|
|
||||||
|
|
||||||
const relationId = `${sourceId}_${verbMatch.type}_${targetId}`
|
const relationId = `${sourceId}_${verbType}_${targetId}`
|
||||||
relationships.set(relationId, {
|
relationships.set(relationId, {
|
||||||
id: relationId,
|
id: relationId,
|
||||||
from: sourceId,
|
from: sourceId,
|
||||||
to: targetId,
|
to: targetId,
|
||||||
type: verbMatch.type as VerbType,
|
type: verbType,
|
||||||
weight: verbMatch.confidence,
|
weight: 1.0,
|
||||||
confidence: verbMatch.confidence,
|
confidence: 1.0,
|
||||||
metadata: {
|
metadata: {
|
||||||
field: key,
|
field: key,
|
||||||
array: true,
|
array: true,
|
||||||
_neuralMatch: verbMatch,
|
|
||||||
_importedAt: Date.now()
|
_importedAt: Date.now()
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
@ -553,6 +533,81 @@ export class UniversalImportAPI {
|
||||||
return fieldLooksLikeRef && valueLooksLikeId
|
return fieldLooksLikeRef && valueLooksLikeId
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Infer noun type from object structure using field heuristics
|
||||||
|
*/
|
||||||
|
private inferNounType(obj: any): NounType {
|
||||||
|
if (typeof obj !== 'object' || obj === null) return NounType.Thing
|
||||||
|
|
||||||
|
// Check for explicit type field
|
||||||
|
if (obj.type && typeof obj.type === 'string') {
|
||||||
|
const normalized = obj.type.charAt(0).toUpperCase() + obj.type.slice(1)
|
||||||
|
if (Object.values(NounType).includes(normalized as NounType)) {
|
||||||
|
return normalized as NounType
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Person heuristics
|
||||||
|
if (obj.email || obj.firstName || obj.lastName || obj.username || obj.age) {
|
||||||
|
return NounType.Person
|
||||||
|
}
|
||||||
|
// Organization heuristics
|
||||||
|
if (obj.companyName || obj.organizationId || obj.employees || obj.industry) {
|
||||||
|
return NounType.Organization
|
||||||
|
}
|
||||||
|
// Location heuristics
|
||||||
|
if (obj.latitude || obj.longitude || obj.address || obj.city || obj.country) {
|
||||||
|
return NounType.Location
|
||||||
|
}
|
||||||
|
// Document heuristics
|
||||||
|
if ((obj.content && (obj.title || obj.author)) || obj.documentType || obj.pages) {
|
||||||
|
return NounType.Document
|
||||||
|
}
|
||||||
|
// Event heuristics
|
||||||
|
if (obj.startTime || obj.endTime || obj.date || obj.eventType || obj.attendees) {
|
||||||
|
return NounType.Event
|
||||||
|
}
|
||||||
|
// Product heuristics
|
||||||
|
if (obj.price || obj.sku || obj.inventory || obj.productId) {
|
||||||
|
return NounType.Product
|
||||||
|
}
|
||||||
|
// Task heuristics
|
||||||
|
if ((obj.status && (obj.assignee || obj.dueDate)) || obj.priority || obj.completed !== undefined) {
|
||||||
|
return NounType.Task
|
||||||
|
}
|
||||||
|
// Dataset heuristics
|
||||||
|
if (Array.isArray(obj.data) || obj.rows || obj.columns || obj.schema) {
|
||||||
|
return NounType.Dataset
|
||||||
|
}
|
||||||
|
|
||||||
|
return NounType.Thing
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Infer verb type from field name using common patterns
|
||||||
|
*/
|
||||||
|
private inferVerbType(fieldName: string): VerbType {
|
||||||
|
const field = fieldName.toLowerCase()
|
||||||
|
|
||||||
|
if (field.includes('parent') || field.includes('child') || field.includes('contain')) {
|
||||||
|
return VerbType.Contains
|
||||||
|
}
|
||||||
|
if (field.includes('owner') || field.includes('created') || field.includes('author')) {
|
||||||
|
return VerbType.Creates
|
||||||
|
}
|
||||||
|
if (field.includes('member') || field.includes('belong')) {
|
||||||
|
return VerbType.MemberOf
|
||||||
|
}
|
||||||
|
if (field.includes('depend') || field.includes('require')) {
|
||||||
|
return VerbType.DependsOn
|
||||||
|
}
|
||||||
|
if (field.includes('ref') || field.includes('link') || field.includes('source')) {
|
||||||
|
return VerbType.References
|
||||||
|
}
|
||||||
|
|
||||||
|
return VerbType.RelatedTo
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Store processed data in brain
|
* Store processed data in brain
|
||||||
*/
|
*/
|
||||||
|
|
|
||||||
|
|
@ -1,137 +0,0 @@
|
||||||
/**
|
|
||||||
* Type-safe augmentation management system for Brainy
|
|
||||||
* Provides a clean API for managing augmentations without string literals
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { IAugmentation, AugmentationType } from './types/augmentations.js'
|
|
||||||
import { augmentationPipeline } from './pipeline.js'
|
|
||||||
|
|
||||||
export interface AugmentationInfo {
|
|
||||||
name: string
|
|
||||||
type: string
|
|
||||||
enabled: boolean
|
|
||||||
description: string
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Type-safe augmentation manager
|
|
||||||
* Accessed via brain.augmentations for all management operations
|
|
||||||
*/
|
|
||||||
export class AugmentationManager {
|
|
||||||
private pipeline = augmentationPipeline
|
|
||||||
|
|
||||||
/**
|
|
||||||
* List all registered augmentations with their status
|
|
||||||
* @returns Array of augmentation information
|
|
||||||
*/
|
|
||||||
list(): AugmentationInfo[] {
|
|
||||||
// Deprecated: use brain.augmentations instead
|
|
||||||
return []
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get information about a specific augmentation
|
|
||||||
* @param name The augmentation name
|
|
||||||
* @returns Augmentation info or undefined if not found
|
|
||||||
*/
|
|
||||||
get(name: string): AugmentationInfo | undefined {
|
|
||||||
const all = this.list()
|
|
||||||
return all.find(a => a.name === name)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Check if an augmentation is enabled
|
|
||||||
* @param name The augmentation name
|
|
||||||
* @returns True if enabled, false otherwise
|
|
||||||
*/
|
|
||||||
isEnabled(name: string): boolean {
|
|
||||||
const aug = this.get(name)
|
|
||||||
return aug?.enabled ?? false
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Enable a specific augmentation
|
|
||||||
* @param name The augmentation name
|
|
||||||
* @returns True if successfully enabled
|
|
||||||
*/
|
|
||||||
enable(name: string): boolean {
|
|
||||||
// Deprecated: use brain.augmentations instead
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Disable a specific augmentation
|
|
||||||
* @param name The augmentation name
|
|
||||||
* @returns True if successfully disabled
|
|
||||||
*/
|
|
||||||
disable(name: string): boolean {
|
|
||||||
// Deprecated: use brain.augmentations instead
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Remove an augmentation from the pipeline
|
|
||||||
* @param name The augmentation name
|
|
||||||
* @returns True if successfully removed
|
|
||||||
*/
|
|
||||||
remove(name: string): boolean {
|
|
||||||
// Deprecated: use brain.augmentations instead
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Enable all augmentations of a specific type
|
|
||||||
* @param type The augmentation type
|
|
||||||
* @returns Number of augmentations enabled
|
|
||||||
*/
|
|
||||||
enableType(type: AugmentationType): number {
|
|
||||||
// Deprecated: use brain.augmentations instead
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Disable all augmentations of a specific type
|
|
||||||
* @param type The augmentation type
|
|
||||||
* @returns Number of augmentations disabled
|
|
||||||
*/
|
|
||||||
disableType(type: AugmentationType): number {
|
|
||||||
// Deprecated: use brain.augmentations instead
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get all augmentations of a specific type
|
|
||||||
* @param type The augmentation type
|
|
||||||
* @returns Array of augmentations of that type
|
|
||||||
*/
|
|
||||||
listByType(type: AugmentationType): AugmentationInfo[] {
|
|
||||||
return this.list().filter(a => a.type === type)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get all enabled augmentations
|
|
||||||
* @returns Array of enabled augmentations
|
|
||||||
*/
|
|
||||||
listEnabled(): AugmentationInfo[] {
|
|
||||||
return this.list().filter(a => a.enabled)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get all disabled augmentations
|
|
||||||
* @returns Array of disabled augmentations
|
|
||||||
*/
|
|
||||||
listDisabled(): AugmentationInfo[] {
|
|
||||||
return this.list().filter(a => !a.enabled)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Register a new augmentation (internal use)
|
|
||||||
* @param augmentation The augmentation to register
|
|
||||||
*/
|
|
||||||
register(augmentation: IAugmentation): void {
|
|
||||||
// Deprecated: use brain.augmentations instead
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Export types for external use
|
|
||||||
export { AugmentationType } from './types/augmentations.js'
|
|
||||||
|
|
@ -1,410 +0,0 @@
|
||||||
/**
|
|
||||||
* Augmentation Metadata Contract System
|
|
||||||
*
|
|
||||||
* Prevents accidental metadata corruption while allowing intentional enrichment
|
|
||||||
* Each augmentation declares its metadata intentions upfront
|
|
||||||
*/
|
|
||||||
|
|
||||||
export interface AugmentationMetadataContract {
|
|
||||||
// Augmentation identity
|
|
||||||
name: string
|
|
||||||
version: string
|
|
||||||
|
|
||||||
// What fields this augmentation READS
|
|
||||||
reads?: {
|
|
||||||
userFields?: string[] // e.g., ['title', 'description']
|
|
||||||
internalFields?: string[] // e.g., ['_brainy.deleted']
|
|
||||||
augmentationFields?: string[] // e.g., ['_augmentations.otherAug.score']
|
|
||||||
}
|
|
||||||
|
|
||||||
// What fields this augmentation WRITES
|
|
||||||
writes?: {
|
|
||||||
// User metadata it enriches (MUST be declared!)
|
|
||||||
userFields?: Array<{
|
|
||||||
field: string
|
|
||||||
type: 'create' | 'update' | 'merge' | 'delete'
|
|
||||||
description: string
|
|
||||||
example?: any
|
|
||||||
}>
|
|
||||||
|
|
||||||
// Its own augmentation namespace
|
|
||||||
augmentationFields?: Array<{
|
|
||||||
field: string
|
|
||||||
description: string
|
|
||||||
}>
|
|
||||||
|
|
||||||
// Internal fields (requires special permission)
|
|
||||||
internalFields?: Array<{
|
|
||||||
field: string
|
|
||||||
permission: 'granted' | 'requested'
|
|
||||||
reason: string
|
|
||||||
}>
|
|
||||||
}
|
|
||||||
|
|
||||||
// Conflict resolution strategy
|
|
||||||
conflictResolution?: {
|
|
||||||
strategy: 'error' | 'warn' | 'merge' | 'skip' | 'override'
|
|
||||||
priority?: number // Higher priority wins in conflicts
|
|
||||||
}
|
|
||||||
|
|
||||||
// Safety guarantees
|
|
||||||
guarantees?: {
|
|
||||||
preservesExisting?: boolean // Won't delete existing fields
|
|
||||||
reversible?: boolean // Can undo changes
|
|
||||||
idempotent?: boolean // Safe to run multiple times
|
|
||||||
validatesTypes?: boolean // Checks field types before modifying
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Runtime metadata safety enforcer
|
|
||||||
*/
|
|
||||||
export class MetadataSafetyEnforcer {
|
|
||||||
private contracts: Map<string, AugmentationMetadataContract> = new Map()
|
|
||||||
private modifications: Map<string, Set<string>> = new Map() // field -> augmentations that modify it
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Register an augmentation's contract
|
|
||||||
*/
|
|
||||||
registerContract(contract: AugmentationMetadataContract): void {
|
|
||||||
this.contracts.set(contract.name, contract)
|
|
||||||
|
|
||||||
// Track which augmentations modify which fields
|
|
||||||
if (contract.writes?.userFields) {
|
|
||||||
for (const fieldDef of contract.writes.userFields) {
|
|
||||||
if (!this.modifications.has(fieldDef.field)) {
|
|
||||||
this.modifications.set(fieldDef.field, new Set())
|
|
||||||
}
|
|
||||||
this.modifications.get(fieldDef.field)!.add(contract.name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Check if an augmentation can modify a field
|
|
||||||
*/
|
|
||||||
canModifyField(augName: string, field: string, value: any): {
|
|
||||||
allowed: boolean
|
|
||||||
reason?: string
|
|
||||||
warnings?: string[]
|
|
||||||
} {
|
|
||||||
const contract = this.contracts.get(augName)
|
|
||||||
|
|
||||||
if (!contract) {
|
|
||||||
return {
|
|
||||||
allowed: false,
|
|
||||||
reason: `Augmentation '${augName}' has no registered contract`
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check if field is in user namespace
|
|
||||||
if (!field.startsWith('_brainy.') && !field.startsWith('_augmentations.')) {
|
|
||||||
// It's a user field
|
|
||||||
const declaredField = contract.writes?.userFields?.find(f => f.field === field)
|
|
||||||
|
|
||||||
if (!declaredField) {
|
|
||||||
return {
|
|
||||||
allowed: false,
|
|
||||||
reason: `Augmentation '${augName}' did not declare intent to modify '${field}'`
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check for conflicts
|
|
||||||
const modifiers = this.modifications.get(field)
|
|
||||||
if (modifiers && modifiers.size > 1) {
|
|
||||||
const others = Array.from(modifiers).filter(a => a !== augName)
|
|
||||||
return {
|
|
||||||
allowed: true, // Still allowed but with warning
|
|
||||||
warnings: [`Field '${field}' is also modified by: ${others.join(', ')}`]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return { allowed: true }
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check internal fields
|
|
||||||
if (field.startsWith('_brainy.')) {
|
|
||||||
const internalField = contract.writes?.internalFields?.find(f =>
|
|
||||||
field === `_brainy.${f.field}`
|
|
||||||
)
|
|
||||||
|
|
||||||
if (!internalField) {
|
|
||||||
return {
|
|
||||||
allowed: false,
|
|
||||||
reason: `Augmentation '${augName}' cannot modify internal field '${field}'`
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (internalField.permission !== 'granted') {
|
|
||||||
return {
|
|
||||||
allowed: false,
|
|
||||||
reason: `Permission not granted for internal field '${field}'`
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return { allowed: true }
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check augmentation namespace
|
|
||||||
if (field.startsWith('_augmentations.')) {
|
|
||||||
const parts = field.split('.')
|
|
||||||
const targetAug = parts[1]
|
|
||||||
|
|
||||||
// Can only modify own namespace
|
|
||||||
if (targetAug !== augName) {
|
|
||||||
return {
|
|
||||||
allowed: false,
|
|
||||||
reason: `Cannot modify another augmentation's namespace: ${targetAug}`
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return { allowed: true }
|
|
||||||
}
|
|
||||||
|
|
||||||
return { allowed: true }
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create safe metadata proxy for an augmentation
|
|
||||||
*/
|
|
||||||
createSafeProxy(metadata: any, augName: string): any {
|
|
||||||
const self = this
|
|
||||||
|
|
||||||
return new Proxy(metadata, {
|
|
||||||
set(target, prop, value) {
|
|
||||||
const field = String(prop)
|
|
||||||
|
|
||||||
// Check permission
|
|
||||||
const permission = self.canModifyField(augName, field, value)
|
|
||||||
|
|
||||||
if (!permission.allowed) {
|
|
||||||
throw new Error(`[${augName}] ${permission.reason}`)
|
|
||||||
}
|
|
||||||
|
|
||||||
if (permission.warnings) {
|
|
||||||
console.warn(`[${augName}] Warning:`, ...permission.warnings)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Track modification for audit
|
|
||||||
if (!target._audit) {
|
|
||||||
target._audit = []
|
|
||||||
}
|
|
||||||
target._audit.push({
|
|
||||||
augmentation: augName,
|
|
||||||
field,
|
|
||||||
oldValue: target[prop],
|
|
||||||
newValue: value,
|
|
||||||
timestamp: Date.now()
|
|
||||||
})
|
|
||||||
|
|
||||||
target[prop] = value
|
|
||||||
return true
|
|
||||||
},
|
|
||||||
|
|
||||||
deleteProperty(target, prop) {
|
|
||||||
const field = String(prop)
|
|
||||||
const permission = self.canModifyField(augName, field, undefined)
|
|
||||||
|
|
||||||
if (!permission.allowed) {
|
|
||||||
throw new Error(`[${augName}] Cannot delete field: ${permission.reason}`)
|
|
||||||
}
|
|
||||||
|
|
||||||
delete target[prop]
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Example augmentation contracts
|
|
||||||
*/
|
|
||||||
export const EXAMPLE_CONTRACTS: Record<string, AugmentationMetadataContract> = {
|
|
||||||
// Enrichment augmentation that adds categories
|
|
||||||
categoryEnricher: {
|
|
||||||
name: 'categoryEnricher',
|
|
||||||
version: '1.0.0',
|
|
||||||
reads: {
|
|
||||||
userFields: ['title', 'description', 'content']
|
|
||||||
},
|
|
||||||
writes: {
|
|
||||||
userFields: [
|
|
||||||
{
|
|
||||||
field: 'category',
|
|
||||||
type: 'create',
|
|
||||||
description: 'Auto-detected category',
|
|
||||||
example: 'technology'
|
|
||||||
},
|
|
||||||
{
|
|
||||||
field: 'subcategories',
|
|
||||||
type: 'create',
|
|
||||||
description: 'List of relevant subcategories',
|
|
||||||
example: ['web', 'framework']
|
|
||||||
}
|
|
||||||
],
|
|
||||||
augmentationFields: [
|
|
||||||
{
|
|
||||||
field: 'confidence',
|
|
||||||
description: 'Confidence score of categorization'
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
guarantees: {
|
|
||||||
preservesExisting: true,
|
|
||||||
idempotent: true
|
|
||||||
}
|
|
||||||
},
|
|
||||||
|
|
||||||
// Translation augmentation
|
|
||||||
translator: {
|
|
||||||
name: 'translator',
|
|
||||||
version: '1.0.0',
|
|
||||||
reads: {
|
|
||||||
userFields: ['title', 'description']
|
|
||||||
},
|
|
||||||
writes: {
|
|
||||||
userFields: [
|
|
||||||
{
|
|
||||||
field: 'translations',
|
|
||||||
type: 'merge',
|
|
||||||
description: 'Translations in multiple languages',
|
|
||||||
example: { es: 'Título', fr: 'Titre' }
|
|
||||||
},
|
|
||||||
{
|
|
||||||
field: 'detectedLanguage',
|
|
||||||
type: 'create',
|
|
||||||
description: 'Detected source language',
|
|
||||||
example: 'en'
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
conflictResolution: {
|
|
||||||
strategy: 'merge',
|
|
||||||
priority: 10
|
|
||||||
}
|
|
||||||
},
|
|
||||||
|
|
||||||
// Sentiment analyzer
|
|
||||||
sentimentAnalyzer: {
|
|
||||||
name: 'sentimentAnalyzer',
|
|
||||||
version: '1.0.0',
|
|
||||||
reads: {
|
|
||||||
userFields: ['content', 'description', 'reviews']
|
|
||||||
},
|
|
||||||
writes: {
|
|
||||||
userFields: [
|
|
||||||
{
|
|
||||||
field: 'sentiment',
|
|
||||||
type: 'update',
|
|
||||||
description: 'Overall sentiment score',
|
|
||||||
example: { score: 0.8, label: 'positive' }
|
|
||||||
}
|
|
||||||
],
|
|
||||||
augmentationFields: [
|
|
||||||
{
|
|
||||||
field: 'analysis',
|
|
||||||
description: 'Detailed sentiment breakdown'
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
guarantees: {
|
|
||||||
reversible: true,
|
|
||||||
validatesTypes: true
|
|
||||||
}
|
|
||||||
},
|
|
||||||
|
|
||||||
// System augmentation with internal access
|
|
||||||
garbageCollector: {
|
|
||||||
name: 'garbageCollector',
|
|
||||||
version: '1.0.0',
|
|
||||||
reads: {
|
|
||||||
internalFields: ['_brainy.createdAt', '_brainy.lastAccessed']
|
|
||||||
},
|
|
||||||
writes: {
|
|
||||||
internalFields: [
|
|
||||||
{
|
|
||||||
field: 'deleted',
|
|
||||||
permission: 'granted',
|
|
||||||
reason: 'Soft delete expired items'
|
|
||||||
},
|
|
||||||
{
|
|
||||||
field: 'archived',
|
|
||||||
permission: 'granted',
|
|
||||||
reason: 'Archive old items'
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Augmentation base class with safety
|
|
||||||
*/
|
|
||||||
export abstract class SafeAugmentation {
|
|
||||||
protected enforcer: MetadataSafetyEnforcer
|
|
||||||
protected contract: AugmentationMetadataContract
|
|
||||||
|
|
||||||
constructor(contract: AugmentationMetadataContract) {
|
|
||||||
this.contract = contract
|
|
||||||
this.enforcer = new MetadataSafetyEnforcer()
|
|
||||||
this.enforcer.registerContract(contract)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get safe metadata proxy
|
|
||||||
*/
|
|
||||||
protected getSafeMetadata(metadata: any): any {
|
|
||||||
return this.enforcer.createSafeProxy(metadata, this.contract.name)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Abstract method to implement augmentation logic
|
|
||||||
*/
|
|
||||||
abstract execute(metadata: any): Promise<any>
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Example: Category enricher implementation
|
|
||||||
*/
|
|
||||||
export class CategoryEnricherAugmentation extends SafeAugmentation {
|
|
||||||
constructor() {
|
|
||||||
super(EXAMPLE_CONTRACTS.categoryEnricher)
|
|
||||||
}
|
|
||||||
|
|
||||||
async execute(metadata: any): Promise<any> {
|
|
||||||
const safe = this.getSafeMetadata(metadata)
|
|
||||||
|
|
||||||
// Read declared fields
|
|
||||||
const title = safe.title
|
|
||||||
const description = safe.description
|
|
||||||
|
|
||||||
// Analyze and categorize
|
|
||||||
const category = this.detectCategory(title, description)
|
|
||||||
const subcategories = this.detectSubcategories(title, description)
|
|
||||||
|
|
||||||
// Write to declared fields (will be checked by proxy)
|
|
||||||
safe.category = category // ✅ Allowed - declared in contract
|
|
||||||
safe.subcategories = subcategories // ✅ Allowed
|
|
||||||
|
|
||||||
// Try to write undeclared field
|
|
||||||
// safe.randomField = 'test' // ❌ Would throw error!
|
|
||||||
|
|
||||||
// Write to our augmentation namespace
|
|
||||||
if (!safe._augmentations) safe._augmentations = {}
|
|
||||||
if (!safe._augmentations.categoryEnricher) {
|
|
||||||
safe._augmentations.categoryEnricher = {}
|
|
||||||
}
|
|
||||||
safe._augmentations.categoryEnricher.confidence = 0.95 // ✅ Allowed
|
|
||||||
|
|
||||||
return safe
|
|
||||||
}
|
|
||||||
|
|
||||||
private detectCategory(title: string, description: string): string {
|
|
||||||
// Simplified logic
|
|
||||||
return 'technology'
|
|
||||||
}
|
|
||||||
|
|
||||||
private detectSubcategories(title: string, description: string): string[] {
|
|
||||||
return ['web', 'framework']
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,292 +0,0 @@
|
||||||
<div align="center">
|
|
||||||
<img src="../../brainy.png" alt="Brainy Logo" width="200"/>
|
|
||||||
|
|
||||||
# Brainy Augmentations
|
|
||||||
|
|
||||||
</div>
|
|
||||||
|
|
||||||
This directory contains the augmentation implementations for Brainy. Augmentations are pluggable components that extend
|
|
||||||
Brainy's functionality in various ways.
|
|
||||||
|
|
||||||
## Available Augmentations
|
|
||||||
|
|
||||||
### Core Augmentations
|
|
||||||
|
|
||||||
#### IntelligentImportAugmentation
|
|
||||||
|
|
||||||
Automatically detects and processes CSV, Excel, and PDF files with intelligent extraction. This augmentation is enabled by default and provides:
|
|
||||||
|
|
||||||
- **CSV Support**: Auto-detection of encoding, delimiters, and field types
|
|
||||||
- **Excel Support**: Multi-sheet extraction with metadata preservation
|
|
||||||
- **PDF Support**: Text extraction, table detection, and metadata extraction
|
|
||||||
- **Type Inference**: Automatically infers data types (string, number, boolean, date)
|
|
||||||
- **Neural Integration**: Seamlessly integrates with entity extraction and relationship detection
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
|
||||||
|
|
||||||
const brain = new Brainy({
|
|
||||||
intelligentImport: {
|
|
||||||
enableCSV: true,
|
|
||||||
enableExcel: true,
|
|
||||||
enablePDF: true,
|
|
||||||
maxFileSize: 100 * 1024 * 1024 // 100MB
|
|
||||||
}
|
|
||||||
})
|
|
||||||
await brain.init()
|
|
||||||
|
|
||||||
// Import CSV with auto-detection
|
|
||||||
await brain.import('customers.csv')
|
|
||||||
|
|
||||||
// Import Excel with specific sheets
|
|
||||||
await brain.import('sales-data.xlsx', {
|
|
||||||
excelSheets: ['Q1', 'Q2']
|
|
||||||
})
|
|
||||||
|
|
||||||
// Import PDF with table extraction
|
|
||||||
await brain.import('report.pdf', {
|
|
||||||
pdfExtractTables: true
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
**See:** [Import Anything Guide](../../docs/guides/import-anything.md) | [Example](../../examples/import-excel-pdf-csv.ts)
|
|
||||||
|
|
||||||
### Conduit Augmentations
|
|
||||||
|
|
||||||
Conduit augmentations provide data synchronization between Brainy instances.
|
|
||||||
|
|
||||||
#### WebSocketConduitAugmentation
|
|
||||||
|
|
||||||
A conduit augmentation that syncs Brainy instances using WebSockets. This is used for syncing between browsers and
|
|
||||||
servers, or between servers.
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
import { createConduitAugmentation, augmentationPipeline } from '@soulcraft/brainy'
|
|
||||||
|
|
||||||
// Create a WebSocket conduit augmentation
|
|
||||||
const wsConduit = await createConduitAugmentation('websocket', 'my-websocket-sync')
|
|
||||||
|
|
||||||
// Register the augmentation with the pipeline
|
|
||||||
augmentationPipeline.register(wsConduit)
|
|
||||||
|
|
||||||
// Connect to another Brainy instance
|
|
||||||
const connectionResult = await wsConduit.establishConnection(
|
|
||||||
'wss://your-websocket-server.com/brainy-sync',
|
|
||||||
{ protocols: 'brainy-sync' }
|
|
||||||
)
|
|
||||||
```
|
|
||||||
|
|
||||||
#### WebRTCConduitAugmentation
|
|
||||||
|
|
||||||
A conduit augmentation that syncs Brainy instances using WebRTC. This is used for direct peer-to-peer syncing between
|
|
||||||
browsers.
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
import { createConduitAugmentation, augmentationPipeline } from '@soulcraft/brainy'
|
|
||||||
|
|
||||||
// Create a WebRTC conduit augmentation
|
|
||||||
const webrtcConduit = await createConduitAugmentation('webrtc', 'my-webrtc-sync')
|
|
||||||
|
|
||||||
// Register the augmentation with the pipeline
|
|
||||||
augmentationPipeline.register(webrtcConduit)
|
|
||||||
|
|
||||||
// Connect to a peer
|
|
||||||
const connectionResult = await webrtcConduit.establishConnection(
|
|
||||||
'peer-id-to-connect-to',
|
|
||||||
{
|
|
||||||
signalServerUrl: 'wss://your-signal-server.com',
|
|
||||||
localPeerId: 'my-peer-id',
|
|
||||||
iceServers: [{ urls: 'stun:stun.l.google.com:19302' }]
|
|
||||||
}
|
|
||||||
)
|
|
||||||
```
|
|
||||||
|
|
||||||
#### ServerSearchConduitAugmentation
|
|
||||||
|
|
||||||
A specialized conduit augmentation that provides functionality for searching a server-hosted Brainy instance and storing
|
|
||||||
results locally. This allows you to:
|
|
||||||
|
|
||||||
- Search a server-hosted Brainy instance from a browser
|
|
||||||
- Store the search results in a local Brainy instance
|
|
||||||
- Perform further searches against the local instance without needing to query the server again
|
|
||||||
- Add data to both local and server instances
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
import {
|
|
||||||
ServerSearchConduitAugmentation,
|
|
||||||
createServerSearchAugmentations,
|
|
||||||
augmentationPipeline
|
|
||||||
} from '@soulcraft/brainy'
|
|
||||||
|
|
||||||
// Using the factory function (recommended)
|
|
||||||
const { conduit, activation, connection } = await createServerSearchAugmentations(
|
|
||||||
'wss://your-brainy-server.com/ws',
|
|
||||||
{ protocols: 'brainy-sync' }
|
|
||||||
)
|
|
||||||
|
|
||||||
// Register the augmentations with the pipeline
|
|
||||||
augmentationPipeline.register(conduit)
|
|
||||||
augmentationPipeline.register(activation)
|
|
||||||
|
|
||||||
// Search the server and store results locally
|
|
||||||
const serverSearchResult = await conduit.searchServer(
|
|
||||||
connection.connectionId,
|
|
||||||
'your search query',
|
|
||||||
5 // limit
|
|
||||||
)
|
|
||||||
|
|
||||||
// Search the local instance
|
|
||||||
const localSearchResult = await conduit.searchLocal('your search query', 5)
|
|
||||||
|
|
||||||
// Perform a combined search (local first, then server if needed)
|
|
||||||
const combinedSearchResult = await conduit.searchCombined(
|
|
||||||
connection.connectionId,
|
|
||||||
'your search query',
|
|
||||||
5
|
|
||||||
)
|
|
||||||
|
|
||||||
// Add data to both local and server
|
|
||||||
const addResult = await conduit.addToBoth(
|
|
||||||
connection.connectionId,
|
|
||||||
'Text to add',
|
|
||||||
{ /* metadata */ }
|
|
||||||
)
|
|
||||||
```
|
|
||||||
|
|
||||||
### Activation Augmentations
|
|
||||||
|
|
||||||
Activation augmentations dictate how Brainy initiates actions, responses, or data manipulations.
|
|
||||||
|
|
||||||
#### ServerSearchActivationAugmentation
|
|
||||||
|
|
||||||
An activation augmentation that provides actions for server search functionality. This works in conjunction with the
|
|
||||||
ServerSearchConduitAugmentation to provide a complete solution for browser-server search.
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
import {
|
|
||||||
ServerSearchActivationAugmentation,
|
|
||||||
createServerSearchAugmentations,
|
|
||||||
augmentationPipeline
|
|
||||||
} from '@soulcraft/brainy'
|
|
||||||
|
|
||||||
// Using the factory function (recommended)
|
|
||||||
const { conduit, activation, connection } = await createServerSearchAugmentations(
|
|
||||||
'wss://your-brainy-server.com/ws',
|
|
||||||
{ protocols: 'brainy-sync' }
|
|
||||||
)
|
|
||||||
|
|
||||||
// Register the augmentations with the pipeline
|
|
||||||
augmentationPipeline.register(conduit)
|
|
||||||
augmentationPipeline.register(activation)
|
|
||||||
|
|
||||||
// Use the activation augmentation to search the server
|
|
||||||
const serverSearchAction = activation.triggerAction('searchServer', {
|
|
||||||
connectionId: connection.connectionId,
|
|
||||||
query: 'your search query',
|
|
||||||
limit: 5
|
|
||||||
})
|
|
||||||
|
|
||||||
if (serverSearchAction.success) {
|
|
||||||
// The data property contains a promise that will resolve to the search results
|
|
||||||
const serverSearchResult = await serverSearchAction.data
|
|
||||||
console.log('Server search results:', serverSearchResult)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Other available actions:
|
|
||||||
// - 'connectToServer': Connect to a server
|
|
||||||
// - 'searchLocal': Search the local instance
|
|
||||||
// - 'searchCombined': Search both local and server
|
|
||||||
// - 'addToBoth': Add data to both local and server
|
|
||||||
```
|
|
||||||
|
|
||||||
## Using the Augmentation Pipeline
|
|
||||||
|
|
||||||
The augmentation pipeline provides a way to execute augmentations based on their type.
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
import { augmentationPipeline } from '@soulcraft/brainy'
|
|
||||||
|
|
||||||
// Execute a conduit augmentation
|
|
||||||
const conduitResults = await augmentationPipeline.executeConduitPipeline(
|
|
||||||
'methodName',
|
|
||||||
[arg1, arg2, ...],
|
|
||||||
{ /* options */ }
|
|
||||||
)
|
|
||||||
|
|
||||||
// Execute an activation augmentation
|
|
||||||
const activationResults = await augmentationPipeline.executeActivationPipeline(
|
|
||||||
'methodName',
|
|
||||||
[arg1, arg2, ...],
|
|
||||||
{ /* options */ }
|
|
||||||
)
|
|
||||||
```
|
|
||||||
|
|
||||||
## Creating Custom Augmentations
|
|
||||||
|
|
||||||
To create a custom augmentation, implement one of the augmentation interfaces:
|
|
||||||
|
|
||||||
- `ISenseAugmentation`: For processing raw data
|
|
||||||
- `IConduitAugmentation`: For data synchronization
|
|
||||||
- `ICognitionAugmentation`: For reasoning and inference
|
|
||||||
- `IMemoryAugmentation`: For data storage
|
|
||||||
- `IPerceptionAugmentation`: For data interpretation and visualization
|
|
||||||
- `IDialogAugmentation`: For natural language processing
|
|
||||||
- `IActivationAugmentation`: For triggering actions
|
|
||||||
|
|
||||||
Example:
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
import { AugmentationType, IActivationAugmentation } from '@soulcraft/brainy'
|
|
||||||
|
|
||||||
class MyCustomActivation implements IActivationAugmentation {
|
|
||||||
readonly
|
|
||||||
name = 'my-custom-activation'
|
|
||||||
readonly
|
|
||||||
description = 'My custom activation augmentation'
|
|
||||||
enabled = true
|
|
||||||
|
|
||||||
getType(): AugmentationType {
|
|
||||||
return AugmentationType.ACTIVATION
|
|
||||||
}
|
|
||||||
|
|
||||||
async initialize(): Promise<void> {
|
|
||||||
// Initialization code
|
|
||||||
}
|
|
||||||
|
|
||||||
async shutDown(): Promise<void> {
|
|
||||||
// Cleanup code
|
|
||||||
}
|
|
||||||
|
|
||||||
async getStatus(): Promise<'active' | 'inactive' | 'error'> {
|
|
||||||
return 'active'
|
|
||||||
}
|
|
||||||
|
|
||||||
triggerAction(actionName: string, parameters
|
|
||||||
|
|
||||||
?:
|
|
||||||
|
|
||||||
Record<string, unknown>
|
|
||||||
|
|
||||||
):
|
|
||||||
|
|
||||||
AugmentationResponse<unknown> {
|
|
||||||
// Implementation
|
|
||||||
}
|
|
||||||
|
|
||||||
generateOutput(knowledgeId: string, format: string): AugmentationResponse<string | Record<string, unknown
|
|
||||||
|
|
||||||
>> {
|
|
||||||
// Implementation
|
|
||||||
}
|
|
||||||
|
|
||||||
interactExternal(systemId
|
|
||||||
:
|
|
||||||
string, payload
|
|
||||||
:
|
|
||||||
Record < string, unknown >
|
|
||||||
):
|
|
||||||
AugmentationResponse < unknown > {
|
|
||||||
// Implementation
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
@ -1,915 +0,0 @@
|
||||||
/**
|
|
||||||
* API Server Augmentation - Universal API Exposure
|
|
||||||
*
|
|
||||||
* 🌐 Exposes Brainy through REST, WebSocket, and MCP
|
|
||||||
* 🔌 Works in Node.js, Deno, and Service Workers
|
|
||||||
* 🚀 Single augmentation for all API needs
|
|
||||||
*
|
|
||||||
* This unifies and replaces:
|
|
||||||
* - BrainyMCPBroadcast (Node-specific server)
|
|
||||||
* - WebSocketConduitAugmentation (client connections)
|
|
||||||
* - Future REST API implementations
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { BaseAugmentation, AugmentationContext } from './brainyAugmentation.js'
|
|
||||||
import { BrainyMCPService } from '../mcp/brainyMCPService.js'
|
|
||||||
import { v4 as uuidv4 } from '../universal/uuid.js'
|
|
||||||
import { isNode, isBrowser } from '../utils/environment.js'
|
|
||||||
|
|
||||||
export interface APIServerConfig {
|
|
||||||
enabled?: boolean
|
|
||||||
port?: number
|
|
||||||
mcpPort?: number
|
|
||||||
wsPort?: number
|
|
||||||
host?: string
|
|
||||||
cors?: {
|
|
||||||
origin?: string | string[]
|
|
||||||
credentials?: boolean
|
|
||||||
}
|
|
||||||
auth?: {
|
|
||||||
required?: boolean
|
|
||||||
apiKeys?: string[]
|
|
||||||
bearerTokens?: string[]
|
|
||||||
}
|
|
||||||
rateLimit?: {
|
|
||||||
windowMs?: number
|
|
||||||
max?: number
|
|
||||||
}
|
|
||||||
ssl?: {
|
|
||||||
cert?: string
|
|
||||||
key?: string
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
interface ConnectedClient {
|
|
||||||
id: string
|
|
||||||
type: 'rest' | 'websocket' | 'mcp'
|
|
||||||
socket?: any
|
|
||||||
subscriptions?: string[]
|
|
||||||
metadata?: Record<string, any>
|
|
||||||
lastSeen: number
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Unified API Server Augmentation
|
|
||||||
* Exposes Brainy through multiple protocols
|
|
||||||
*/
|
|
||||||
export class APIServerAugmentation extends BaseAugmentation {
|
|
||||||
readonly name = 'api-server'
|
|
||||||
readonly timing = 'after' as const
|
|
||||||
readonly metadata = 'readonly' as const // API server reads metadata to serve data
|
|
||||||
readonly operations = ['all'] as ('all')[]
|
|
||||||
readonly priority = 5 // Low priority, runs after other augmentations
|
|
||||||
|
|
||||||
protected config: APIServerConfig
|
|
||||||
private mcpService?: BrainyMCPService
|
|
||||||
private httpServer?: any
|
|
||||||
private wsServer?: any
|
|
||||||
private clients = new Map<string, ConnectedClient>()
|
|
||||||
private operationHistory: any[] = []
|
|
||||||
private maxHistorySize = 1000
|
|
||||||
private rateLimitStore = new Map<string, number[]>()
|
|
||||||
|
|
||||||
constructor(config: APIServerConfig = {}) {
|
|
||||||
super()
|
|
||||||
this.config = {
|
|
||||||
enabled: true,
|
|
||||||
port: 3000,
|
|
||||||
host: '0.0.0.0',
|
|
||||||
cors: { origin: '*', credentials: true },
|
|
||||||
auth: { required: false },
|
|
||||||
rateLimit: { windowMs: 60000, max: 100 },
|
|
||||||
...config
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected async onInitialize(): Promise<void> {
|
|
||||||
if (!this.config.enabled) {
|
|
||||||
this.log('API Server disabled in config')
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Initialize MCP service
|
|
||||||
this.mcpService = new BrainyMCPService(this.context!.brain, {
|
|
||||||
enableAuth: this.config.auth?.required
|
|
||||||
})
|
|
||||||
|
|
||||||
// Start appropriate server based on environment
|
|
||||||
if (isNode()) {
|
|
||||||
await this.startNodeServer()
|
|
||||||
} else if (typeof (globalThis as any).Deno !== 'undefined') {
|
|
||||||
await this.startDenoServer()
|
|
||||||
} else if (isBrowser() && 'serviceWorker' in navigator) {
|
|
||||||
await this.startServiceWorker()
|
|
||||||
} else {
|
|
||||||
this.log('No suitable server environment detected', 'warn')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Start Node.js server with Express
|
|
||||||
*/
|
|
||||||
private async startNodeServer(): Promise<void> {
|
|
||||||
try {
|
|
||||||
// Dynamic imports for Node.js dependencies
|
|
||||||
const express = await import('express').catch(() => null)
|
|
||||||
const cors = await import('cors').catch(() => null)
|
|
||||||
const ws = await import('ws').catch(() => null)
|
|
||||||
const { createServer } = await import('node:http')
|
|
||||||
|
|
||||||
if (!express || !cors || !ws) {
|
|
||||||
this.log('Express, cors, or ws not available. Install with: npm install express cors ws', 'error')
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
const WebSocketServer = (ws as any)?.WebSocketServer || (ws as any)?.default?.WebSocketServer || (ws as any)?.Server
|
|
||||||
|
|
||||||
const app = express.default()
|
|
||||||
|
|
||||||
// Middleware
|
|
||||||
app.use(cors.default(this.config.cors))
|
|
||||||
app.use((express.default || express).json({ limit: '50mb' }))
|
|
||||||
app.use(this.authMiddleware.bind(this))
|
|
||||||
app.use(this.rateLimitMiddleware.bind(this))
|
|
||||||
|
|
||||||
// REST API Routes
|
|
||||||
this.setupRESTRoutes(app)
|
|
||||||
|
|
||||||
// Create HTTP server
|
|
||||||
this.httpServer = createServer(app)
|
|
||||||
|
|
||||||
// WebSocket server
|
|
||||||
this.wsServer = new WebSocketServer({
|
|
||||||
server: this.httpServer,
|
|
||||||
path: '/ws'
|
|
||||||
})
|
|
||||||
|
|
||||||
this.setupWebSocketServer()
|
|
||||||
|
|
||||||
// Start listening
|
|
||||||
await new Promise<void>((resolve, reject) => {
|
|
||||||
this.httpServer.listen(this.config.port, this.config.host, () => {
|
|
||||||
this.log(`🌐 API Server listening on http://${this.config.host}:${this.config.port}`)
|
|
||||||
this.log(`🔌 WebSocket: ws://${this.config.host}:${this.config.port}/ws`)
|
|
||||||
this.log(`🧠 MCP endpoint: http://${this.config.host}:${this.config.port}/api/mcp`)
|
|
||||||
resolve()
|
|
||||||
}).on('error', reject)
|
|
||||||
})
|
|
||||||
|
|
||||||
// Heartbeat interval
|
|
||||||
setInterval(() => this.sendHeartbeats(), 30000)
|
|
||||||
|
|
||||||
} catch (error) {
|
|
||||||
this.log(`Failed to start Node.js server: ${error}`, 'error')
|
|
||||||
throw error
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Setup REST API routes
|
|
||||||
*/
|
|
||||||
private setupRESTRoutes(app: any): void {
|
|
||||||
// Health check
|
|
||||||
app.get('/health', (_req: any, res: any) => {
|
|
||||||
res.json({
|
|
||||||
status: 'healthy',
|
|
||||||
version: '2.0.0',
|
|
||||||
clients: this.clients.size,
|
|
||||||
uptime: process.uptime ? process.uptime() : 0
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
// Search endpoint
|
|
||||||
app.post('/api/search', async (req: any, res: any) => {
|
|
||||||
try {
|
|
||||||
const { query, limit = 10, options = {} } = req.body
|
|
||||||
const results = await this.context!.brain.search(query, limit, options)
|
|
||||||
res.json({ success: true, results })
|
|
||||||
} catch (error: any) {
|
|
||||||
res.status(500).json({ success: false, error: error.message })
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// Add data endpoint
|
|
||||||
app.post('/api/add', async (req: any, res: any) => {
|
|
||||||
try {
|
|
||||||
const { content, metadata } = req.body
|
|
||||||
const id = await this.context!.brain.add({ data: content, type: 'content', metadata })
|
|
||||||
res.json({ success: true, id })
|
|
||||||
} catch (error: any) {
|
|
||||||
res.status(500).json({ success: false, error: error.message })
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// Get by ID endpoint
|
|
||||||
app.get('/api/get/:id', async (req: any, res: any) => {
|
|
||||||
try {
|
|
||||||
const data = await this.context!.brain.get(req.params.id)
|
|
||||||
if (data) {
|
|
||||||
res.json({ success: true, data })
|
|
||||||
} else {
|
|
||||||
res.status(404).json({ success: false, error: 'Not found' })
|
|
||||||
}
|
|
||||||
} catch (error: any) {
|
|
||||||
res.status(500).json({ success: false, error: error.message })
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// Delete endpoint
|
|
||||||
app.delete('/api/delete/:id', async (req: any, res: any) => {
|
|
||||||
try {
|
|
||||||
await this.context!.brain.delete(req.params.id)
|
|
||||||
res.json({ success: true })
|
|
||||||
} catch (error: any) {
|
|
||||||
res.status(500).json({ success: false, error: error.message })
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// Relate endpoint
|
|
||||||
app.post('/api/relate', async (req: any, res: any) => {
|
|
||||||
try {
|
|
||||||
const { source, target, verb, metadata } = req.body
|
|
||||||
await this.context!.brain.relate(source, target, verb, metadata)
|
|
||||||
res.json({ success: true })
|
|
||||||
} catch (error: any) {
|
|
||||||
res.status(500).json({ success: false, error: error.message })
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// Find endpoint (complex queries)
|
|
||||||
app.post('/api/find', async (req: any, res: any) => {
|
|
||||||
try {
|
|
||||||
const results = await this.context!.brain.find(req.body)
|
|
||||||
res.json({ success: true, results })
|
|
||||||
} catch (error: any) {
|
|
||||||
res.status(500).json({ success: false, error: error.message })
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// Cluster endpoint
|
|
||||||
app.post('/api/cluster', async (req: any, res: any) => {
|
|
||||||
try {
|
|
||||||
const { algorithm = 'kmeans', options = {} } = req.body
|
|
||||||
const clusters = await this.context!.brain.cluster(algorithm, options)
|
|
||||||
res.json({ success: true, clusters })
|
|
||||||
} catch (error: any) {
|
|
||||||
res.status(500).json({ success: false, error: error.message })
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// MCP endpoint
|
|
||||||
app.post('/api/mcp', async (req: any, res: any) => {
|
|
||||||
try {
|
|
||||||
const response = await this.mcpService!.handleRequest(req.body)
|
|
||||||
res.json(response)
|
|
||||||
} catch (error: any) {
|
|
||||||
res.status(500).json({ success: false, error: error.message })
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// Statistics endpoint
|
|
||||||
app.get('/api/stats', async (_req: any, res: any) => {
|
|
||||||
try {
|
|
||||||
const stats = this.context!.brain.getStats()
|
|
||||||
res.json({ success: true, stats })
|
|
||||||
} catch (error: any) {
|
|
||||||
res.status(500).json({ success: false, error: error.message })
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// Operation history endpoint
|
|
||||||
app.get('/api/history', (_req: any, res: any) => {
|
|
||||||
res.json({
|
|
||||||
success: true,
|
|
||||||
history: this.operationHistory.slice(-100)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Setup WebSocket server
|
|
||||||
*/
|
|
||||||
private setupWebSocketServer(): void {
|
|
||||||
if (!this.wsServer) return
|
|
||||||
|
|
||||||
this.wsServer.on('connection', (socket: any, request: any) => {
|
|
||||||
const clientId = uuidv4()
|
|
||||||
const client: ConnectedClient = {
|
|
||||||
id: clientId,
|
|
||||||
type: 'websocket',
|
|
||||||
socket,
|
|
||||||
subscriptions: [],
|
|
||||||
lastSeen: Date.now()
|
|
||||||
}
|
|
||||||
|
|
||||||
this.clients.set(clientId, client)
|
|
||||||
|
|
||||||
// Send welcome message
|
|
||||||
socket.send(JSON.stringify({
|
|
||||||
type: 'welcome',
|
|
||||||
clientId,
|
|
||||||
message: 'Connected to Brainy API Server',
|
|
||||||
capabilities: ['search', 'add', 'delete', 'relate', 'subscribe', 'mcp']
|
|
||||||
}))
|
|
||||||
|
|
||||||
// Handle messages
|
|
||||||
socket.on('message', async (message: string) => {
|
|
||||||
try {
|
|
||||||
const msg = JSON.parse(message)
|
|
||||||
await this.handleWebSocketMessage(msg, client)
|
|
||||||
} catch (error: any) {
|
|
||||||
socket.send(JSON.stringify({
|
|
||||||
type: 'error',
|
|
||||||
error: error.message
|
|
||||||
}))
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// Handle disconnect
|
|
||||||
socket.on('close', () => {
|
|
||||||
this.clients.delete(clientId)
|
|
||||||
this.log(`Client ${clientId} disconnected`)
|
|
||||||
})
|
|
||||||
|
|
||||||
// Handle errors
|
|
||||||
socket.on('error', (error: any) => {
|
|
||||||
this.log(`WebSocket error for client ${clientId}: ${error}`, 'error')
|
|
||||||
})
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Handle WebSocket message
|
|
||||||
*/
|
|
||||||
private async handleWebSocketMessage(msg: any, client: ConnectedClient): Promise<void> {
|
|
||||||
const { socket } = client
|
|
||||||
|
|
||||||
switch (msg.type) {
|
|
||||||
case 'subscribe':
|
|
||||||
// Subscribe to operation types
|
|
||||||
client.subscriptions = msg.operations || ['all']
|
|
||||||
socket.send(JSON.stringify({
|
|
||||||
type: 'subscribed',
|
|
||||||
operations: client.subscriptions
|
|
||||||
}))
|
|
||||||
break
|
|
||||||
|
|
||||||
case 'search':
|
|
||||||
const searchResults = await this.context!.brain.search(
|
|
||||||
msg.query,
|
|
||||||
msg.limit || 10,
|
|
||||||
msg.options || {}
|
|
||||||
)
|
|
||||||
socket.send(JSON.stringify({
|
|
||||||
type: 'searchResults',
|
|
||||||
requestId: msg.requestId,
|
|
||||||
results: searchResults
|
|
||||||
}))
|
|
||||||
break
|
|
||||||
|
|
||||||
case 'add':
|
|
||||||
const id = await this.context!.brain.add({ data: msg.content, type: 'content', metadata: msg.metadata })
|
|
||||||
socket.send(JSON.stringify({
|
|
||||||
type: 'addResult',
|
|
||||||
requestId: msg.requestId,
|
|
||||||
id
|
|
||||||
}))
|
|
||||||
break
|
|
||||||
|
|
||||||
case 'mcp':
|
|
||||||
const mcpResponse = await this.mcpService!.handleRequest(msg.request)
|
|
||||||
socket.send(JSON.stringify({
|
|
||||||
type: 'mcpResponse',
|
|
||||||
requestId: msg.requestId,
|
|
||||||
response: mcpResponse
|
|
||||||
}))
|
|
||||||
break
|
|
||||||
|
|
||||||
case 'heartbeat':
|
|
||||||
client.lastSeen = Date.now()
|
|
||||||
socket.send(JSON.stringify({
|
|
||||||
type: 'heartbeat',
|
|
||||||
timestamp: Date.now()
|
|
||||||
}))
|
|
||||||
break
|
|
||||||
|
|
||||||
default:
|
|
||||||
socket.send(JSON.stringify({
|
|
||||||
type: 'error',
|
|
||||||
error: `Unknown message type: ${msg.type}`
|
|
||||||
}))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Execute augmentation - broadcast operations to clients
|
|
||||||
*/
|
|
||||||
async execute<T = any>(
|
|
||||||
operation: string,
|
|
||||||
params: any,
|
|
||||||
next: () => Promise<T>
|
|
||||||
): Promise<T> {
|
|
||||||
const startTime = Date.now()
|
|
||||||
const result = await next()
|
|
||||||
const duration = Date.now() - startTime
|
|
||||||
|
|
||||||
// Record operation in history
|
|
||||||
const historyEntry = {
|
|
||||||
operation,
|
|
||||||
params: this.sanitizeParams(params),
|
|
||||||
timestamp: Date.now(),
|
|
||||||
duration
|
|
||||||
}
|
|
||||||
|
|
||||||
this.operationHistory.push(historyEntry)
|
|
||||||
if (this.operationHistory.length > this.maxHistorySize) {
|
|
||||||
this.operationHistory.shift()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Broadcast to subscribed WebSocket clients
|
|
||||||
const message = JSON.stringify({
|
|
||||||
type: 'operation',
|
|
||||||
operation,
|
|
||||||
params: historyEntry.params,
|
|
||||||
timestamp: historyEntry.timestamp,
|
|
||||||
duration
|
|
||||||
})
|
|
||||||
|
|
||||||
for (const client of this.clients.values()) {
|
|
||||||
if (client.type === 'websocket' && client.socket) {
|
|
||||||
if (client.subscriptions?.includes('all') ||
|
|
||||||
client.subscriptions?.includes(operation)) {
|
|
||||||
try {
|
|
||||||
client.socket.send(message)
|
|
||||||
} catch (error) {
|
|
||||||
// Client might be disconnected
|
|
||||||
this.clients.delete(client.id)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Auth middleware for Express
|
|
||||||
*/
|
|
||||||
private authMiddleware(req: any, res: any, next: any): void {
|
|
||||||
if (!this.config.auth?.required) {
|
|
||||||
return next()
|
|
||||||
}
|
|
||||||
|
|
||||||
const apiKey = req.headers['x-api-key']
|
|
||||||
const bearerToken = req.headers.authorization?.replace('Bearer ', '')
|
|
||||||
|
|
||||||
if (apiKey && this.config.auth.apiKeys?.includes(apiKey)) {
|
|
||||||
return next()
|
|
||||||
}
|
|
||||||
|
|
||||||
if (bearerToken && this.config.auth.bearerTokens?.includes(bearerToken)) {
|
|
||||||
return next()
|
|
||||||
}
|
|
||||||
|
|
||||||
res.status(401).json({ error: 'Unauthorized' })
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Rate limiting middleware
|
|
||||||
*/
|
|
||||||
private rateLimitMiddleware(req: any, res: any, next: any): void {
|
|
||||||
// Simple in-memory rate limiting
|
|
||||||
// In production, use redis or proper rate limiting library
|
|
||||||
const ip = req.ip || req.connection.remoteAddress
|
|
||||||
const now = Date.now()
|
|
||||||
const windowMs = this.config.rateLimit?.windowMs || 60000
|
|
||||||
const max = this.config.rateLimit?.max || 100
|
|
||||||
|
|
||||||
// Clean old entries
|
|
||||||
for (const [key, client] of this.clients.entries()) {
|
|
||||||
if (now - client.lastSeen > windowMs) {
|
|
||||||
this.clients.delete(key)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// For now, just pass through
|
|
||||||
// Real implementation would track requests per IP
|
|
||||||
next()
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Sanitize parameters before broadcasting
|
|
||||||
*/
|
|
||||||
private sanitizeParams(params: any): any {
|
|
||||||
if (!params) return params
|
|
||||||
|
|
||||||
const sanitized = { ...params }
|
|
||||||
|
|
||||||
// Remove sensitive fields
|
|
||||||
delete sanitized.password
|
|
||||||
delete sanitized.apiKey
|
|
||||||
delete sanitized.token
|
|
||||||
delete sanitized.secret
|
|
||||||
|
|
||||||
// Truncate large data
|
|
||||||
if (sanitized.content && sanitized.content.length > 1000) {
|
|
||||||
sanitized.content = sanitized.content.substring(0, 1000) + '...'
|
|
||||||
}
|
|
||||||
|
|
||||||
return sanitized
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Send heartbeats to all connected clients
|
|
||||||
*/
|
|
||||||
private sendHeartbeats(): void {
|
|
||||||
const now = Date.now()
|
|
||||||
|
|
||||||
for (const [id, client] of this.clients.entries()) {
|
|
||||||
if (client.type === 'websocket' && client.socket) {
|
|
||||||
// Remove inactive clients
|
|
||||||
if (now - client.lastSeen > 60000) {
|
|
||||||
this.clients.delete(id)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
// Send heartbeat
|
|
||||||
try {
|
|
||||||
client.socket.send(JSON.stringify({
|
|
||||||
type: 'heartbeat',
|
|
||||||
timestamp: now
|
|
||||||
}))
|
|
||||||
} catch {
|
|
||||||
// Client disconnected
|
|
||||||
this.clients.delete(id)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Start Deno server
|
|
||||||
*/
|
|
||||||
private async startDenoServer(): Promise<void> {
|
|
||||||
try {
|
|
||||||
// Check if Deno.serve is available (Deno 1.35+)
|
|
||||||
const DenoGlobal = (globalThis as any).Deno
|
|
||||||
if (DenoGlobal && 'serve' in DenoGlobal) {
|
|
||||||
const handler = this.createUniversalHandler()
|
|
||||||
|
|
||||||
this.httpServer = DenoGlobal.serve({
|
|
||||||
port: this.config.port,
|
|
||||||
hostname: this.config.host || '0.0.0.0',
|
|
||||||
handler: handler
|
|
||||||
})
|
|
||||||
|
|
||||||
this.log(`Deno server started on ${this.config.host || '0.0.0.0'}:${this.config.port}`)
|
|
||||||
|
|
||||||
// Setup WebSocket handling for Deno
|
|
||||||
this.setupUniversalWebSocket()
|
|
||||||
|
|
||||||
} else {
|
|
||||||
throw new Error('Deno.serve not available - requires Deno 1.35+')
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
this.log(`Failed to start Deno server: ${(error as Error).message}`, 'error')
|
|
||||||
throw error
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Start Service Worker (for browser)
|
|
||||||
*/
|
|
||||||
private async startServiceWorker(): Promise<void> {
|
|
||||||
try {
|
|
||||||
if (typeof self !== 'undefined' && 'addEventListener' in self) {
|
|
||||||
// Service Worker environment - intercept fetch events
|
|
||||||
const handler = this.createUniversalHandler()
|
|
||||||
|
|
||||||
self.addEventListener('fetch', async (event: any) => {
|
|
||||||
const url = new URL(event.request.url)
|
|
||||||
|
|
||||||
// Only handle API requests
|
|
||||||
if (url.pathname.startsWith('/api/') || url.pathname.startsWith('/ws') || url.pathname.startsWith('/mcp/')) {
|
|
||||||
event.respondWith(handler(event.request))
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
this.log('Service Worker API server registered for /api/, /ws, and /mcp paths')
|
|
||||||
|
|
||||||
// Setup message handling for WebSocket-like communication
|
|
||||||
this.setupServiceWorkerMessaging()
|
|
||||||
|
|
||||||
} else if (typeof navigator !== 'undefined' && 'serviceWorker' in navigator) {
|
|
||||||
// Browser main thread - service worker registration should be handled by the application
|
|
||||||
this.log('Service Worker environment detected. Registration should be handled by your application.', 'info')
|
|
||||||
// Return early - the app will handle service worker registration
|
|
||||||
return
|
|
||||||
} else {
|
|
||||||
this.log('Service Worker environment not available', 'warn')
|
|
||||||
return
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
this.log(`Failed to start Service Worker server: ${(error as Error).message}`, 'error')
|
|
||||||
throw error
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create universal handler using Web Standards (works in Node, Deno, Service Workers)
|
|
||||||
*/
|
|
||||||
private createUniversalHandler(): (request: Request) => Promise<Response> {
|
|
||||||
return async (request: Request): Promise<Response> => {
|
|
||||||
try {
|
|
||||||
const url = new URL(request.url)
|
|
||||||
const method = request.method.toUpperCase()
|
|
||||||
const path = url.pathname
|
|
||||||
|
|
||||||
// Add CORS headers
|
|
||||||
const corsOrigin = Array.isArray(this.config.cors?.origin)
|
|
||||||
? this.config.cors.origin[0]
|
|
||||||
: this.config.cors?.origin || '*'
|
|
||||||
|
|
||||||
const headers = new Headers({
|
|
||||||
'Access-Control-Allow-Origin': corsOrigin,
|
|
||||||
'Access-Control-Allow-Methods': 'GET, POST, PUT, DELETE, OPTIONS',
|
|
||||||
'Access-Control-Allow-Headers': 'Content-Type, Authorization',
|
|
||||||
'Content-Type': 'application/json'
|
|
||||||
})
|
|
||||||
|
|
||||||
// Handle preflight requests
|
|
||||||
if (method === 'OPTIONS') {
|
|
||||||
return new Response(null, { status: 200, headers })
|
|
||||||
}
|
|
||||||
|
|
||||||
// Authentication
|
|
||||||
if (!this.authenticateRequest(request)) {
|
|
||||||
return new Response(JSON.stringify({ error: 'Unauthorized' }), {
|
|
||||||
status: 401,
|
|
||||||
headers
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// Rate limiting
|
|
||||||
if (!this.checkRateLimit(request)) {
|
|
||||||
return new Response(JSON.stringify({ error: 'Rate limit exceeded' }), {
|
|
||||||
status: 429,
|
|
||||||
headers
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// Route handling
|
|
||||||
if (path.startsWith('/api/brainy/')) {
|
|
||||||
return this.handleBrainyAPI(request, path.replace('/api/brainy/', ''), headers)
|
|
||||||
} else if (path.startsWith('/mcp/')) {
|
|
||||||
return this.handleMCPAPI(request, path.replace('/mcp/', ''), headers)
|
|
||||||
} else if (path === '/health') {
|
|
||||||
return new Response(JSON.stringify({ status: 'ok', timestamp: Date.now() }), {
|
|
||||||
status: 200,
|
|
||||||
headers
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
return new Response(JSON.stringify({ error: 'Not found' }), {
|
|
||||||
status: 404,
|
|
||||||
headers
|
|
||||||
})
|
|
||||||
|
|
||||||
} catch (error) {
|
|
||||||
return new Response(JSON.stringify({
|
|
||||||
error: 'Internal server error',
|
|
||||||
message: (error as Error).message
|
|
||||||
}), {
|
|
||||||
status: 500,
|
|
||||||
headers: new Headers({ 'Content-Type': 'application/json' })
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Handle Brainy API requests using universal Request/Response
|
|
||||||
*/
|
|
||||||
private async handleBrainyAPI(request: Request, path: string, headers: Headers): Promise<Response> {
|
|
||||||
const method = request.method.toUpperCase()
|
|
||||||
const body = method !== 'GET' ? await request.json().catch(() => ({})) : {}
|
|
||||||
|
|
||||||
try {
|
|
||||||
let result: any
|
|
||||||
|
|
||||||
switch (`${method} ${path}`) {
|
|
||||||
case 'POST add':
|
|
||||||
result = { id: await this.context!.brain.add(body) }
|
|
||||||
break
|
|
||||||
case 'GET get':
|
|
||||||
const id = new URL(request.url).searchParams.get('id')
|
|
||||||
result = await this.context!.brain.get(id)
|
|
||||||
break
|
|
||||||
case 'PUT update':
|
|
||||||
await this.context!.brain.update(body)
|
|
||||||
result = { success: true }
|
|
||||||
break
|
|
||||||
case 'DELETE delete':
|
|
||||||
const deleteId = new URL(request.url).searchParams.get('id')
|
|
||||||
await this.context!.brain.delete(deleteId)
|
|
||||||
result = { success: true }
|
|
||||||
break
|
|
||||||
case 'POST find':
|
|
||||||
result = await this.context!.brain.find(body)
|
|
||||||
break
|
|
||||||
case 'POST relate':
|
|
||||||
result = { id: await this.context!.brain.relate(body) }
|
|
||||||
break
|
|
||||||
case 'GET insights':
|
|
||||||
result = await this.context!.brain.insights()
|
|
||||||
break
|
|
||||||
default:
|
|
||||||
return new Response(JSON.stringify({ error: `Unknown endpoint: ${method} ${path}` }), {
|
|
||||||
status: 404,
|
|
||||||
headers
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
return new Response(JSON.stringify(result), { status: 200, headers })
|
|
||||||
|
|
||||||
} catch (error) {
|
|
||||||
return new Response(JSON.stringify({
|
|
||||||
error: (error as Error).message
|
|
||||||
}), { status: 400, headers })
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Handle MCP API requests
|
|
||||||
*/
|
|
||||||
private async handleMCPAPI(request: Request, path: string, headers: Headers): Promise<Response> {
|
|
||||||
try {
|
|
||||||
if (!this.mcpService) {
|
|
||||||
return new Response(JSON.stringify({ error: 'MCP service not available' }), {
|
|
||||||
status: 503,
|
|
||||||
headers
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
const body = await request.json().catch(() => ({}))
|
|
||||||
// Convert to MCP request format
|
|
||||||
const mcpRequest = {
|
|
||||||
type: path.includes('data') ? 'data_access' : 'tool_execution',
|
|
||||||
...body
|
|
||||||
}
|
|
||||||
const result = await this.mcpService.handleRequest(mcpRequest as any)
|
|
||||||
|
|
||||||
return new Response(JSON.stringify(result), { status: 200, headers })
|
|
||||||
} catch (error) {
|
|
||||||
return new Response(JSON.stringify({
|
|
||||||
error: (error as Error).message
|
|
||||||
}), { status: 400, headers })
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Universal WebSocket setup (works in Node, Deno)
|
|
||||||
*/
|
|
||||||
private setupUniversalWebSocket(): void {
|
|
||||||
// WebSocket handling varies by platform but uses same interface
|
|
||||||
this.log('WebSocket support enabled for real-time updates')
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Service Worker messaging for WebSocket-like communication
|
|
||||||
*/
|
|
||||||
private setupServiceWorkerMessaging(): void {
|
|
||||||
if (typeof self !== 'undefined') {
|
|
||||||
self.addEventListener('message', async (event: any) => {
|
|
||||||
if (event.data.type === 'brainy-api') {
|
|
||||||
try {
|
|
||||||
const response = await this.handleBrainyAPI(
|
|
||||||
new Request('http://localhost/api/brainy/' + event.data.endpoint, {
|
|
||||||
method: event.data.method || 'POST',
|
|
||||||
body: JSON.stringify(event.data.data)
|
|
||||||
}),
|
|
||||||
event.data.endpoint,
|
|
||||||
new Headers({ 'Content-Type': 'application/json' })
|
|
||||||
)
|
|
||||||
|
|
||||||
const result = await response.json()
|
|
||||||
|
|
||||||
event.ports[0]?.postMessage({
|
|
||||||
id: event.data.id,
|
|
||||||
success: response.ok,
|
|
||||||
data: result
|
|
||||||
})
|
|
||||||
} catch (error) {
|
|
||||||
event.ports[0]?.postMessage({
|
|
||||||
id: event.data.id,
|
|
||||||
success: false,
|
|
||||||
error: (error as Error).message
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Universal authentication using Web Standards
|
|
||||||
*/
|
|
||||||
private authenticateRequest(request: Request): boolean {
|
|
||||||
if (!this.config.auth?.required) return true
|
|
||||||
|
|
||||||
const authHeader = request.headers.get('authorization')
|
|
||||||
if (!authHeader) return false
|
|
||||||
|
|
||||||
if (this.config.auth.apiKeys?.length) {
|
|
||||||
const apiKey = authHeader.replace('Bearer ', '')
|
|
||||||
return this.config.auth.apiKeys.includes(apiKey)
|
|
||||||
}
|
|
||||||
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private checkRateLimit(request: Request): boolean {
|
|
||||||
if (!this.config.rateLimit) return true
|
|
||||||
|
|
||||||
// Get client identifier from headers or use a default
|
|
||||||
const clientId = request.headers.get('x-forwarded-for') ||
|
|
||||||
request.headers.get('x-real-ip') ||
|
|
||||||
request.headers.get('cf-connecting-ip') || // Cloudflare
|
|
||||||
request.headers.get('x-vercel-forwarded-for') || // Vercel
|
|
||||||
'unknown'
|
|
||||||
|
|
||||||
const now = Date.now()
|
|
||||||
const windowMs = this.config.rateLimit.windowMs || 60000
|
|
||||||
const maxRequests = this.config.rateLimit.max || 100
|
|
||||||
const windowStart = now - windowMs
|
|
||||||
|
|
||||||
// Get or create request timestamps for this client
|
|
||||||
let timestamps = this.rateLimitStore.get(clientId) || []
|
|
||||||
|
|
||||||
// Remove old timestamps outside the window
|
|
||||||
timestamps = timestamps.filter(t => t > windowStart)
|
|
||||||
|
|
||||||
// Check if limit exceeded
|
|
||||||
if (timestamps.length >= maxRequests) {
|
|
||||||
this.log(`Rate limit exceeded for client ${clientId}: ${timestamps.length}/${maxRequests} requests`, 'warn')
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add current request timestamp
|
|
||||||
timestamps.push(now)
|
|
||||||
this.rateLimitStore.set(clientId, timestamps)
|
|
||||||
|
|
||||||
// Periodic cleanup of old entries to prevent memory leak
|
|
||||||
if (this.rateLimitStore.size > 1000) {
|
|
||||||
for (const [id, times] of this.rateLimitStore.entries()) {
|
|
||||||
const validTimes = times.filter(t => t > windowStart)
|
|
||||||
if (validTimes.length === 0) {
|
|
||||||
this.rateLimitStore.delete(id)
|
|
||||||
} else {
|
|
||||||
this.rateLimitStore.set(id, validTimes)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Shutdown the server
|
|
||||||
*/
|
|
||||||
protected async onShutdown(): Promise<void> {
|
|
||||||
// Close all WebSocket connections
|
|
||||||
for (const client of this.clients.values()) {
|
|
||||||
if (client.socket) {
|
|
||||||
try {
|
|
||||||
client.socket.close()
|
|
||||||
} catch (error) {
|
|
||||||
// Socket already closed or errored
|
|
||||||
console.debug('Error closing WebSocket:', error)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
this.clients.clear()
|
|
||||||
|
|
||||||
// Close servers
|
|
||||||
if (this.wsServer) {
|
|
||||||
this.wsServer.close()
|
|
||||||
}
|
|
||||||
|
|
||||||
if (this.httpServer) {
|
|
||||||
await new Promise<void>(resolve => {
|
|
||||||
this.httpServer.close(() => resolve())
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
this.log('API Server shut down')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Helper function to create and configure API server
|
|
||||||
*/
|
|
||||||
export function createAPIServer(config?: APIServerConfig): APIServerAugmentation {
|
|
||||||
return new APIServerAugmentation(config)
|
|
||||||
}
|
|
||||||
|
|
@ -1,452 +0,0 @@
|
||||||
/**
|
|
||||||
* Audit Logging Augmentation
|
|
||||||
* Provides comprehensive audit trail for all Brainy operations
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { BaseAugmentation } from './brainyAugmentation.js'
|
|
||||||
import { AugmentationManifest } from './manifest.js'
|
|
||||||
import { createHash } from '../universal/crypto.js'
|
|
||||||
|
|
||||||
export interface AuditLogConfig {
|
|
||||||
enabled?: boolean
|
|
||||||
logLevel?: 'minimal' | 'standard' | 'detailed'
|
|
||||||
includeData?: boolean
|
|
||||||
includeMetadata?: boolean
|
|
||||||
retention?: number // Days to keep logs
|
|
||||||
storage?: 'memory' | 'file' | 'database'
|
|
||||||
filePath?: string
|
|
||||||
maxMemoryLogs?: number
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface AuditLogEntry {
|
|
||||||
id: string
|
|
||||||
timestamp: number
|
|
||||||
operation: string
|
|
||||||
params: any
|
|
||||||
result?: any
|
|
||||||
error?: any
|
|
||||||
duration: number
|
|
||||||
userId?: string
|
|
||||||
sessionId?: string
|
|
||||||
metadata?: Record<string, any>
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Audit Log Augmentation
|
|
||||||
*/
|
|
||||||
export class AuditLogAugmentation extends BaseAugmentation {
|
|
||||||
readonly name = 'auditLogger'
|
|
||||||
readonly timing = 'around' as const
|
|
||||||
readonly metadata = 'readonly' as const // Read metadata for context
|
|
||||||
operations = ['all'] as any // Audit all operations
|
|
||||||
readonly priority = 90 // Low priority, runs last
|
|
||||||
|
|
||||||
// Augmentation metadata
|
|
||||||
readonly category = 'core' as const
|
|
||||||
readonly description = 'Comprehensive audit logging for compliance and debugging'
|
|
||||||
|
|
||||||
private logs: AuditLogEntry[] = []
|
|
||||||
private sessionId: string
|
|
||||||
|
|
||||||
constructor(config: AuditLogConfig = {}) {
|
|
||||||
super(config)
|
|
||||||
|
|
||||||
// Merge with defaults
|
|
||||||
this.config = {
|
|
||||||
enabled: config.enabled ?? true,
|
|
||||||
logLevel: config.logLevel ?? 'standard',
|
|
||||||
includeData: config.includeData ?? false,
|
|
||||||
includeMetadata: config.includeMetadata ?? true,
|
|
||||||
retention: config.retention ?? 90, // 90 days default
|
|
||||||
storage: config.storage ?? 'memory',
|
|
||||||
filePath: config.filePath,
|
|
||||||
maxMemoryLogs: config.maxMemoryLogs ?? 10000
|
|
||||||
}
|
|
||||||
|
|
||||||
// Generate session ID
|
|
||||||
this.sessionId = this.generateId()
|
|
||||||
}
|
|
||||||
|
|
||||||
getManifest(): AugmentationManifest {
|
|
||||||
return {
|
|
||||||
id: 'audit-logger',
|
|
||||||
name: 'Audit Logger',
|
|
||||||
version: '1.0.0',
|
|
||||||
description: 'Comprehensive audit trail for all operations',
|
|
||||||
longDescription: 'Records detailed audit logs of all Brainy operations for compliance, debugging, and analytics purposes.',
|
|
||||||
category: 'analytics',
|
|
||||||
configSchema: {
|
|
||||||
type: 'object',
|
|
||||||
properties: {
|
|
||||||
enabled: {
|
|
||||||
type: 'boolean',
|
|
||||||
default: true,
|
|
||||||
description: 'Enable audit logging'
|
|
||||||
},
|
|
||||||
logLevel: {
|
|
||||||
type: 'string',
|
|
||||||
enum: ['minimal', 'standard', 'detailed'],
|
|
||||||
default: 'standard',
|
|
||||||
description: 'Level of detail to log'
|
|
||||||
},
|
|
||||||
includeData: {
|
|
||||||
type: 'boolean',
|
|
||||||
default: false,
|
|
||||||
description: 'Include actual data in logs (privacy concern)'
|
|
||||||
},
|
|
||||||
includeMetadata: {
|
|
||||||
type: 'boolean',
|
|
||||||
default: true,
|
|
||||||
description: 'Include metadata in logs'
|
|
||||||
},
|
|
||||||
retention: {
|
|
||||||
type: 'number',
|
|
||||||
default: 90,
|
|
||||||
minimum: 1,
|
|
||||||
maximum: 365,
|
|
||||||
description: 'Days to retain logs'
|
|
||||||
},
|
|
||||||
storage: {
|
|
||||||
type: 'string',
|
|
||||||
enum: ['memory', 'file', 'database'],
|
|
||||||
default: 'memory',
|
|
||||||
description: 'Where to store audit logs'
|
|
||||||
},
|
|
||||||
maxMemoryLogs: {
|
|
||||||
type: 'number',
|
|
||||||
default: 10000,
|
|
||||||
description: 'Maximum logs to keep in memory'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
configDefaults: {
|
|
||||||
enabled: true,
|
|
||||||
logLevel: 'standard',
|
|
||||||
includeData: false,
|
|
||||||
includeMetadata: true,
|
|
||||||
retention: 90,
|
|
||||||
storage: 'memory',
|
|
||||||
maxMemoryLogs: 10000
|
|
||||||
},
|
|
||||||
minBrainyVersion: '3.0.0',
|
|
||||||
keywords: ['audit', 'logging', 'compliance', 'analytics'],
|
|
||||||
documentation: 'https://docs.brainy.dev/augmentations/audit-log',
|
|
||||||
status: 'stable',
|
|
||||||
performance: {
|
|
||||||
memoryUsage: 'medium',
|
|
||||||
cpuUsage: 'low',
|
|
||||||
networkUsage: 'none'
|
|
||||||
},
|
|
||||||
features: ['operation-logging', 'configurable-detail', 'retention-management'],
|
|
||||||
enhancedOperations: ['all'],
|
|
||||||
metrics: [
|
|
||||||
{
|
|
||||||
name: 'audit_logs_created',
|
|
||||||
type: 'counter',
|
|
||||||
description: 'Total audit logs created'
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: 'audit_log_size',
|
|
||||||
type: 'gauge',
|
|
||||||
description: 'Current audit log size'
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected async onInitialize(): Promise<void> {
|
|
||||||
if (!this.config.enabled) {
|
|
||||||
this.log('Audit logger disabled by configuration')
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
this.log(`Audit logger initialized (level: ${this.config.logLevel}, storage: ${this.config.storage})`)
|
|
||||||
|
|
||||||
// Start retention cleanup if using memory storage
|
|
||||||
if (this.config.storage === 'memory') {
|
|
||||||
setInterval(() => {
|
|
||||||
this.cleanupOldLogs()
|
|
||||||
}, 3600000) // Every hour
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected async onShutdown(): Promise<void> {
|
|
||||||
// Save any pending logs if using file storage
|
|
||||||
if (this.config.storage === 'file' && this.logs.length > 0) {
|
|
||||||
await this.flushLogs()
|
|
||||||
}
|
|
||||||
|
|
||||||
this.log('Audit logger shut down')
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Execute augmentation - log operations
|
|
||||||
*/
|
|
||||||
async execute<T = any>(
|
|
||||||
operation: string,
|
|
||||||
params: any,
|
|
||||||
next: () => Promise<T>
|
|
||||||
): Promise<T> {
|
|
||||||
// If audit logging is disabled, just pass through
|
|
||||||
if (!this.config.enabled) {
|
|
||||||
return next()
|
|
||||||
}
|
|
||||||
|
|
||||||
const startTime = Date.now()
|
|
||||||
const logEntry: Partial<AuditLogEntry> = {
|
|
||||||
id: this.generateId(),
|
|
||||||
timestamp: startTime,
|
|
||||||
operation,
|
|
||||||
sessionId: this.sessionId
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add params based on log level
|
|
||||||
if (this.config.logLevel !== 'minimal') {
|
|
||||||
logEntry.params = this.sanitizeParams(params)
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
const result = await next()
|
|
||||||
|
|
||||||
// Log successful operation
|
|
||||||
logEntry.duration = Date.now() - startTime
|
|
||||||
|
|
||||||
// Add result based on log level and config
|
|
||||||
if (this.config.logLevel === 'detailed' && this.config.includeData) {
|
|
||||||
logEntry.result = this.sanitizeResult(result)
|
|
||||||
}
|
|
||||||
|
|
||||||
await this.writeLog(logEntry as AuditLogEntry)
|
|
||||||
|
|
||||||
return result
|
|
||||||
} catch (error) {
|
|
||||||
// Log failed operation
|
|
||||||
logEntry.duration = Date.now() - startTime
|
|
||||||
logEntry.error = this.sanitizeError(error)
|
|
||||||
|
|
||||||
await this.writeLog(logEntry as AuditLogEntry)
|
|
||||||
|
|
||||||
throw error
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Sanitize parameters to remove sensitive data
|
|
||||||
*/
|
|
||||||
private sanitizeParams(params: any): any {
|
|
||||||
if (!params) return params
|
|
||||||
|
|
||||||
// Don't include actual data unless configured
|
|
||||||
if (!this.config.includeData && params.data) {
|
|
||||||
return {
|
|
||||||
...params,
|
|
||||||
data: '[REDACTED]'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Redact common sensitive fields
|
|
||||||
const sanitized = { ...params }
|
|
||||||
const sensitiveFields = ['password', 'token', 'apiKey', 'secret']
|
|
||||||
|
|
||||||
for (const field of sensitiveFields) {
|
|
||||||
if (sanitized[field]) {
|
|
||||||
sanitized[field] = '[REDACTED]'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return sanitized
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Sanitize result data
|
|
||||||
*/
|
|
||||||
private sanitizeResult(result: any): any {
|
|
||||||
if (!result) return result
|
|
||||||
|
|
||||||
// For arrays, just log count
|
|
||||||
if (Array.isArray(result)) {
|
|
||||||
return { count: result.length, type: 'array' }
|
|
||||||
}
|
|
||||||
|
|
||||||
// For objects, remove sensitive fields
|
|
||||||
if (typeof result === 'object') {
|
|
||||||
const sanitized: any = {}
|
|
||||||
for (const key in result) {
|
|
||||||
if (!['password', 'token', 'apiKey', 'secret'].includes(key)) {
|
|
||||||
sanitized[key] = result[key]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return sanitized
|
|
||||||
}
|
|
||||||
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Sanitize error information
|
|
||||||
*/
|
|
||||||
private sanitizeError(error: any): any {
|
|
||||||
if (!error) return error
|
|
||||||
|
|
||||||
return {
|
|
||||||
message: error.message || 'Unknown error',
|
|
||||||
code: error.code,
|
|
||||||
statusCode: error.statusCode,
|
|
||||||
stack: this.config.logLevel === 'detailed' ? error.stack : undefined
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Write log entry
|
|
||||||
*/
|
|
||||||
private async writeLog(entry: AuditLogEntry): Promise<void> {
|
|
||||||
switch (this.config.storage) {
|
|
||||||
case 'memory':
|
|
||||||
this.logs.push(entry)
|
|
||||||
|
|
||||||
// Enforce max memory logs
|
|
||||||
if (this.logs.length > this.config.maxMemoryLogs!) {
|
|
||||||
this.logs.shift() // Remove oldest
|
|
||||||
}
|
|
||||||
break
|
|
||||||
|
|
||||||
case 'file':
|
|
||||||
// In production, would write to file
|
|
||||||
// For now, just add to memory
|
|
||||||
this.logs.push(entry)
|
|
||||||
break
|
|
||||||
|
|
||||||
case 'database':
|
|
||||||
// In production, would write to database
|
|
||||||
// For now, just add to memory
|
|
||||||
this.logs.push(entry)
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Flush logs to persistent storage
|
|
||||||
*/
|
|
||||||
private async flushLogs(): Promise<void> {
|
|
||||||
// In production, would write to file/database
|
|
||||||
// For now, just clear old logs
|
|
||||||
if (this.logs.length > this.config.maxMemoryLogs!) {
|
|
||||||
this.logs = this.logs.slice(-this.config.maxMemoryLogs!)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Clean up old logs based on retention
|
|
||||||
*/
|
|
||||||
private cleanupOldLogs(): void {
|
|
||||||
const cutoffTime = Date.now() - (this.config.retention! * 24 * 60 * 60 * 1000)
|
|
||||||
|
|
||||||
this.logs = this.logs.filter(log => log.timestamp >= cutoffTime)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Generate unique ID
|
|
||||||
*/
|
|
||||||
private generateId(): string {
|
|
||||||
return createHash('sha256')
|
|
||||||
.update(`${Date.now()}-${Math.random()}`)
|
|
||||||
.digest('hex')
|
|
||||||
.substring(0, 16)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Query audit logs
|
|
||||||
*/
|
|
||||||
queryLogs(filter?: {
|
|
||||||
operation?: string
|
|
||||||
startTime?: number
|
|
||||||
endTime?: number
|
|
||||||
sessionId?: string
|
|
||||||
hasError?: boolean
|
|
||||||
}): AuditLogEntry[] {
|
|
||||||
let results = [...this.logs]
|
|
||||||
|
|
||||||
if (filter) {
|
|
||||||
if (filter.operation) {
|
|
||||||
results = results.filter(log => log.operation === filter.operation)
|
|
||||||
}
|
|
||||||
|
|
||||||
if (filter.startTime) {
|
|
||||||
results = results.filter(log => log.timestamp >= filter.startTime!)
|
|
||||||
}
|
|
||||||
|
|
||||||
if (filter.endTime) {
|
|
||||||
results = results.filter(log => log.timestamp <= filter.endTime!)
|
|
||||||
}
|
|
||||||
|
|
||||||
if (filter.sessionId) {
|
|
||||||
results = results.filter(log => log.sessionId === filter.sessionId)
|
|
||||||
}
|
|
||||||
|
|
||||||
if (filter.hasError !== undefined) {
|
|
||||||
results = results.filter(log => (log.error !== undefined) === filter.hasError)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return results
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get audit statistics
|
|
||||||
*/
|
|
||||||
getStats(): {
|
|
||||||
totalLogs: number
|
|
||||||
operations: Record<string, number>
|
|
||||||
averageDuration: number
|
|
||||||
errorRate: number
|
|
||||||
} {
|
|
||||||
const stats: any = {
|
|
||||||
totalLogs: this.logs.length,
|
|
||||||
operations: {},
|
|
||||||
averageDuration: 0,
|
|
||||||
errorRate: 0
|
|
||||||
}
|
|
||||||
|
|
||||||
let totalDuration = 0
|
|
||||||
let errorCount = 0
|
|
||||||
|
|
||||||
for (const log of this.logs) {
|
|
||||||
// Count by operation
|
|
||||||
stats.operations[log.operation] = (stats.operations[log.operation] || 0) + 1
|
|
||||||
|
|
||||||
// Sum duration
|
|
||||||
totalDuration += log.duration
|
|
||||||
|
|
||||||
// Count errors
|
|
||||||
if (log.error) errorCount++
|
|
||||||
}
|
|
||||||
|
|
||||||
if (this.logs.length > 0) {
|
|
||||||
stats.averageDuration = totalDuration / this.logs.length
|
|
||||||
stats.errorRate = errorCount / this.logs.length
|
|
||||||
}
|
|
||||||
|
|
||||||
return stats
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Export logs for analysis
|
|
||||||
*/
|
|
||||||
exportLogs(): AuditLogEntry[] {
|
|
||||||
return [...this.logs]
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Clear all logs
|
|
||||||
*/
|
|
||||||
clearLogs(): void {
|
|
||||||
this.logs = []
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create audit log augmentation
|
|
||||||
*/
|
|
||||||
export function createAuditLogAugmentation(config?: AuditLogConfig): AuditLogAugmentation {
|
|
||||||
return new AuditLogAugmentation(config)
|
|
||||||
}
|
|
||||||
|
|
@ -1,803 +0,0 @@
|
||||||
/**
|
|
||||||
* Batch Processing Augmentation
|
|
||||||
*
|
|
||||||
* Critical for enterprise-scale performance: 500,000+ operations/second
|
|
||||||
* Automatically batches operations for maximum throughput
|
|
||||||
* Handles streaming data, bulk imports, and high-frequency operations
|
|
||||||
*
|
|
||||||
* Performance Impact: 10-50x improvement for bulk operations
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { BaseAugmentation, AugmentationContext } from './brainyAugmentation.js'
|
|
||||||
import { AugmentationManifest } from './manifest.js'
|
|
||||||
|
|
||||||
interface BatchConfig {
|
|
||||||
enabled?: boolean
|
|
||||||
adaptiveMode?: boolean // Zero-config: automatically decide when to batch
|
|
||||||
immediateThreshold?: number // Operations <= this count are immediate
|
|
||||||
batchThreshold?: number // Start batching when >= this many operations queued
|
|
||||||
maxBatchSize?: number // Maximum items per batch
|
|
||||||
maxWaitTime?: number // Maximum wait time before flushing (ms)
|
|
||||||
adaptiveBatching?: boolean // Dynamically adjust batch size based on performance
|
|
||||||
priorityLanes?: number // Number of priority processing lanes
|
|
||||||
memoryLimit?: number // Maximum memory for batching (bytes)
|
|
||||||
}
|
|
||||||
|
|
||||||
interface BatchedOperation<T = any> {
|
|
||||||
id: string
|
|
||||||
operation: string
|
|
||||||
params: any
|
|
||||||
executor: () => Promise<T> // The actual function to execute
|
|
||||||
resolver: (value: T) => void
|
|
||||||
rejector: (error: Error) => void
|
|
||||||
timestamp: number
|
|
||||||
priority: number
|
|
||||||
size: number // Estimated memory size
|
|
||||||
}
|
|
||||||
|
|
||||||
interface BatchMetrics {
|
|
||||||
totalOperations: number
|
|
||||||
batchesProcessed: number
|
|
||||||
averageBatchSize: number
|
|
||||||
averageLatency: number
|
|
||||||
throughputPerSecond: number
|
|
||||||
memoryUsage: number
|
|
||||||
adaptiveAdjustments: number
|
|
||||||
}
|
|
||||||
|
|
||||||
export class BatchProcessingAugmentation extends BaseAugmentation {
|
|
||||||
readonly metadata = 'readonly' as const // Reads metadata for batching decisions
|
|
||||||
name = 'BatchProcessing'
|
|
||||||
timing = 'around' as const
|
|
||||||
operations = ['add', 'addNoun', 'addVerb', 'saveNoun', 'saveVerb', 'storage'] as ('add' | 'addNoun' | 'addVerb' | 'saveNoun' | 'saveVerb' | 'storage')[]
|
|
||||||
priority = 80 // High priority for performance
|
|
||||||
|
|
||||||
protected config: Required<BatchConfig> = {
|
|
||||||
enabled: true,
|
|
||||||
adaptiveMode: true,
|
|
||||||
immediateThreshold: 1,
|
|
||||||
batchThreshold: 5,
|
|
||||||
maxBatchSize: 100,
|
|
||||||
maxWaitTime: 1000,
|
|
||||||
adaptiveBatching: true,
|
|
||||||
priorityLanes: 2,
|
|
||||||
memoryLimit: 100 * 1024 * 1024 // 100MB
|
|
||||||
}
|
|
||||||
private batches: Map<string, BatchedOperation<any>[]> = new Map()
|
|
||||||
private flushTimers: Map<string, NodeJS.Timeout> = new Map()
|
|
||||||
private metrics: BatchMetrics = {
|
|
||||||
totalOperations: 0,
|
|
||||||
batchesProcessed: 0,
|
|
||||||
averageBatchSize: 0,
|
|
||||||
averageLatency: 0,
|
|
||||||
throughputPerSecond: 0,
|
|
||||||
memoryUsage: 0,
|
|
||||||
adaptiveAdjustments: 0
|
|
||||||
}
|
|
||||||
private currentMemoryUsage = 0
|
|
||||||
private performanceHistory: number[] = []
|
|
||||||
|
|
||||||
constructor(config?: BatchConfig) {
|
|
||||||
super(config)
|
|
||||||
}
|
|
||||||
|
|
||||||
getManifest(): AugmentationManifest {
|
|
||||||
return {
|
|
||||||
id: 'batch-processing',
|
|
||||||
name: 'Batch Processing',
|
|
||||||
version: '2.0.0',
|
|
||||||
description: 'High-performance batching for bulk operations',
|
|
||||||
longDescription: 'Automatically batches operations for maximum throughput. Essential for enterprise-scale workloads, achieving 500,000+ operations/second. Provides 10-50x performance improvement for bulk operations.',
|
|
||||||
category: 'performance',
|
|
||||||
configSchema: {
|
|
||||||
type: 'object',
|
|
||||||
properties: {
|
|
||||||
enabled: {
|
|
||||||
type: 'boolean',
|
|
||||||
default: true,
|
|
||||||
description: 'Enable batch processing'
|
|
||||||
},
|
|
||||||
adaptiveMode: {
|
|
||||||
type: 'boolean',
|
|
||||||
default: true,
|
|
||||||
description: 'Automatically decide when to batch operations'
|
|
||||||
},
|
|
||||||
immediateThreshold: {
|
|
||||||
type: 'number',
|
|
||||||
default: 1,
|
|
||||||
minimum: 1,
|
|
||||||
maximum: 10,
|
|
||||||
description: 'Operations count below which to execute immediately'
|
|
||||||
},
|
|
||||||
batchThreshold: {
|
|
||||||
type: 'number',
|
|
||||||
default: 5,
|
|
||||||
minimum: 2,
|
|
||||||
maximum: 100,
|
|
||||||
description: 'Queue size at which to start batching'
|
|
||||||
},
|
|
||||||
maxBatchSize: {
|
|
||||||
type: 'number',
|
|
||||||
default: 1000,
|
|
||||||
minimum: 10,
|
|
||||||
maximum: 10000,
|
|
||||||
description: 'Maximum items per batch'
|
|
||||||
},
|
|
||||||
maxWaitTime: {
|
|
||||||
type: 'number',
|
|
||||||
default: 100,
|
|
||||||
minimum: 1,
|
|
||||||
maximum: 5000,
|
|
||||||
description: 'Maximum wait time before flushing batch (ms)'
|
|
||||||
},
|
|
||||||
adaptiveBatching: {
|
|
||||||
type: 'boolean',
|
|
||||||
default: true,
|
|
||||||
description: 'Dynamically adjust batch size based on performance'
|
|
||||||
},
|
|
||||||
priorityLanes: {
|
|
||||||
type: 'number',
|
|
||||||
default: 3,
|
|
||||||
minimum: 1,
|
|
||||||
maximum: 10,
|
|
||||||
description: 'Number of priority processing lanes'
|
|
||||||
},
|
|
||||||
memoryLimit: {
|
|
||||||
type: 'number',
|
|
||||||
default: 104857600, // 100MB
|
|
||||||
minimum: 10485760, // 10MB
|
|
||||||
maximum: 1073741824, // 1GB
|
|
||||||
description: 'Maximum memory for batching in bytes'
|
|
||||||
}
|
|
||||||
},
|
|
||||||
additionalProperties: false
|
|
||||||
},
|
|
||||||
configDefaults: {
|
|
||||||
enabled: true,
|
|
||||||
adaptiveMode: true,
|
|
||||||
immediateThreshold: 1,
|
|
||||||
batchThreshold: 5,
|
|
||||||
maxBatchSize: 1000,
|
|
||||||
maxWaitTime: 100,
|
|
||||||
adaptiveBatching: true,
|
|
||||||
priorityLanes: 3,
|
|
||||||
memoryLimit: 104857600
|
|
||||||
},
|
|
||||||
minBrainyVersion: '2.0.0',
|
|
||||||
keywords: ['batch', 'performance', 'bulk', 'streaming', 'throughput'],
|
|
||||||
documentation: 'https://docs.brainy.dev/augmentations/batch-processing',
|
|
||||||
status: 'stable',
|
|
||||||
performance: {
|
|
||||||
memoryUsage: 'high',
|
|
||||||
cpuUsage: 'medium',
|
|
||||||
networkUsage: 'none'
|
|
||||||
},
|
|
||||||
features: ['auto-batching', 'adaptive-sizing', 'priority-lanes', 'streaming-support'],
|
|
||||||
enhancedOperations: ['add', 'addNoun', 'addVerb', 'saveNoun', 'saveVerb'],
|
|
||||||
ui: {
|
|
||||||
icon: '📦',
|
|
||||||
color: '#9C27B0'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected async onInitialize(): Promise<void> {
|
|
||||||
if (this.config.enabled) {
|
|
||||||
this.startMetricsCollection()
|
|
||||||
this.log(`Batch processing initialized: ${this.config.maxBatchSize} batch size, ${this.config.maxWaitTime}ms max wait`)
|
|
||||||
|
|
||||||
if (this.config.adaptiveBatching) {
|
|
||||||
this.log('Adaptive batching enabled - will optimize batch size dynamically')
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
this.log('Batch processing disabled')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
shouldExecute(operation: string, params: any): boolean {
|
|
||||||
if (!this.config.enabled) return false
|
|
||||||
|
|
||||||
// Skip batching for single operations or already-batched operations
|
|
||||||
if (params?.batch === false || params?.streaming === false) return false
|
|
||||||
|
|
||||||
// Enable for high-volume operations
|
|
||||||
return operation.includes('add') ||
|
|
||||||
operation.includes('save') ||
|
|
||||||
operation.includes('storage')
|
|
||||||
}
|
|
||||||
|
|
||||||
async execute<T = any>(
|
|
||||||
operation: string,
|
|
||||||
params: any,
|
|
||||||
next: () => Promise<T>
|
|
||||||
): Promise<T> {
|
|
||||||
if (!this.shouldExecute(operation, params)) {
|
|
||||||
return next()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check if this should be batched based on system load
|
|
||||||
if (this.shouldBatch(operation, params)) {
|
|
||||||
return this.addToBatch(operation, params, next)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Execute immediately for low-latency requirements
|
|
||||||
return next()
|
|
||||||
}
|
|
||||||
|
|
||||||
private shouldBatch(operation: string, params: any): boolean {
|
|
||||||
// ZERO-CONFIG INTELLIGENT ADAPTATION:
|
|
||||||
if (this.config.adaptiveMode) {
|
|
||||||
// CRITICAL WORKFLOW DETECTION: Never batch operations that break critical patterns
|
|
||||||
|
|
||||||
// 1. ENTITY REGISTRY PATTERN: Never batch when immediate lookup is expected
|
|
||||||
if (this.isEntityRegistryWorkflow(operation, params)) {
|
|
||||||
return false // Must be immediate for registry lookups to work
|
|
||||||
}
|
|
||||||
|
|
||||||
// 2. DEPENDENCY CHAIN PATTERN: Never batch when next operation depends on this one
|
|
||||||
if (this.isDependencyChainStart(operation, params)) {
|
|
||||||
return false // Must be immediate for noun → verb workflows
|
|
||||||
}
|
|
||||||
|
|
||||||
// Count pending operations in the current operation's batch (needed for write-only mode)
|
|
||||||
const batchKey = this.getBatchKey(operation, params)
|
|
||||||
const currentBatch = this.batches.get(batchKey) || []
|
|
||||||
const pendingCount = currentBatch.length
|
|
||||||
|
|
||||||
// 3. WRITE-ONLY MODE: Special handling for high-speed streaming
|
|
||||||
if (this.isWriteOnlyMode(params)) {
|
|
||||||
// In write-only mode, batch aggressively but ensure entity registry updates immediately
|
|
||||||
if (this.hasEntityRegistryMetadata(params)) {
|
|
||||||
return false // Entity registry updates must be immediate even in write-only mode
|
|
||||||
}
|
|
||||||
return pendingCount >= 3 // Lower threshold for write-only mode batching
|
|
||||||
}
|
|
||||||
|
|
||||||
// Apply intelligent thresholds:
|
|
||||||
// 4. Single operations are immediate (responsive user experience)
|
|
||||||
if (pendingCount < this.config.immediateThreshold!) {
|
|
||||||
return false // Execute immediately
|
|
||||||
}
|
|
||||||
|
|
||||||
// 5. Start batching when multiple operations are queued
|
|
||||||
if (pendingCount >= this.config.batchThreshold!) {
|
|
||||||
return true // Batch for efficiency
|
|
||||||
}
|
|
||||||
|
|
||||||
// 6. For in-between cases, use smart heuristics
|
|
||||||
const currentLoad = this.getCurrentLoad()
|
|
||||||
if (currentLoad > 0.5) return true // Higher load = more batching
|
|
||||||
|
|
||||||
// 7. Batch operations that naturally benefit from grouping
|
|
||||||
if (operation.includes('save') || operation.includes('add')) {
|
|
||||||
return pendingCount > 1 // Batch if others are already waiting
|
|
||||||
}
|
|
||||||
|
|
||||||
return false // Default to immediate for best responsiveness
|
|
||||||
}
|
|
||||||
|
|
||||||
// TRADITIONAL MODE: (for explicit configuration scenarios)
|
|
||||||
// Always batch if explicitly requested
|
|
||||||
if (params?.batch === true || params?.streaming === true) return true
|
|
||||||
|
|
||||||
// Batch based on current system load
|
|
||||||
const currentLoad = this.getCurrentLoad()
|
|
||||||
if (currentLoad > 0.7) return true // High load - batch everything
|
|
||||||
|
|
||||||
// Batch operations that benefit from grouping
|
|
||||||
return operation.includes('save') ||
|
|
||||||
operation.includes('add') ||
|
|
||||||
operation.includes('update')
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* SMART WORKFLOW DETECTION METHODS
|
|
||||||
* These methods detect critical patterns that must not be batched
|
|
||||||
*/
|
|
||||||
|
|
||||||
private isEntityRegistryWorkflow(operation: string, params: any): boolean {
|
|
||||||
// Detect operations that will likely be followed by immediate entity registry lookups
|
|
||||||
if (operation === 'addNoun' || operation === 'add') {
|
|
||||||
// Check if metadata contains external identifiers (DID, handle, etc.)
|
|
||||||
const metadata = params?.metadata || params?.data || {}
|
|
||||||
return !!(
|
|
||||||
metadata.did || // Bluesky DID
|
|
||||||
metadata.handle || // Social media handle
|
|
||||||
metadata.uri || // Resource URI
|
|
||||||
metadata.external_id || // External system ID
|
|
||||||
metadata.user_id || // User ID
|
|
||||||
metadata.profile_id || // Profile ID
|
|
||||||
metadata.account_id // Account ID
|
|
||||||
)
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
private isDependencyChainStart(operation: string, params: any): boolean {
|
|
||||||
// Detect operations that are likely to be followed by dependent operations
|
|
||||||
if (operation === 'addNoun' || operation === 'add') {
|
|
||||||
// In interactive workflows, noun creation is often followed by verb creation
|
|
||||||
// Use heuristics to detect this pattern
|
|
||||||
const context = this.getOperationContext()
|
|
||||||
|
|
||||||
// If we've seen recent addVerb operations, this noun might be for a relationship
|
|
||||||
if (context.recentVerbOperations > 0) {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
// If this is part of a rapid sequence of operations, it might be a dependency chain
|
|
||||||
if (context.operationsInLastSecond > 3) {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
private isWriteOnlyMode(params: any): boolean {
|
|
||||||
// Detect write-only mode from context or parameters
|
|
||||||
return !!(
|
|
||||||
params?.writeOnlyMode ||
|
|
||||||
params?.streaming ||
|
|
||||||
params?.highThroughput ||
|
|
||||||
this.context?.brain?.writeOnly
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
private hasEntityRegistryMetadata(params: any): boolean {
|
|
||||||
// Check if this operation has metadata that needs immediate entity registry updates
|
|
||||||
const metadata = params?.metadata || params?.data || {}
|
|
||||||
return !!(
|
|
||||||
metadata.did ||
|
|
||||||
metadata.handle ||
|
|
||||||
metadata.uri ||
|
|
||||||
metadata.external_id ||
|
|
||||||
// Also check for auto-registration hints
|
|
||||||
params?.autoCreateMissingNouns ||
|
|
||||||
params?.entityRegistry
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
private getOperationContext(): { recentVerbOperations: number; operationsInLastSecond: number } {
|
|
||||||
const now = Date.now()
|
|
||||||
const oneSecondAgo = now - 1000
|
|
||||||
|
|
||||||
let recentVerbOperations = 0
|
|
||||||
let operationsInLastSecond = 0
|
|
||||||
|
|
||||||
// Analyze recent operations across all batches
|
|
||||||
for (const batch of this.batches.values()) {
|
|
||||||
for (const op of batch) {
|
|
||||||
if (op.timestamp > oneSecondAgo) {
|
|
||||||
operationsInLastSecond++
|
|
||||||
if (op.operation.includes('Verb') || op.operation.includes('verb')) {
|
|
||||||
recentVerbOperations++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return { recentVerbOperations, operationsInLastSecond }
|
|
||||||
}
|
|
||||||
|
|
||||||
private getCurrentLoad(): number {
|
|
||||||
// Simple load calculation based on pending operations
|
|
||||||
let totalPending = 0
|
|
||||||
for (const batch of this.batches.values()) {
|
|
||||||
totalPending += batch.length
|
|
||||||
}
|
|
||||||
|
|
||||||
return Math.min(totalPending / 10000, 1.0) // Normalize to 0-1
|
|
||||||
}
|
|
||||||
|
|
||||||
private async addToBatch<T>(
|
|
||||||
operation: string,
|
|
||||||
params: any,
|
|
||||||
executor: () => Promise<T>
|
|
||||||
): Promise<T> {
|
|
||||||
return new Promise((resolve, reject) => {
|
|
||||||
const priority = this.getOperationPriority(operation, params)
|
|
||||||
const batchKey = this.getBatchKey(operation, priority)
|
|
||||||
const operationSize = this.estimateOperationSize(params)
|
|
||||||
|
|
||||||
// Check memory limit
|
|
||||||
if (this.currentMemoryUsage + operationSize > this.config.memoryLimit) {
|
|
||||||
// Memory limit reached - flush oldest batch
|
|
||||||
this.flushOldestBatch()
|
|
||||||
}
|
|
||||||
|
|
||||||
const batchedOp: BatchedOperation<T> = {
|
|
||||||
id: `op_${Date.now()}_${Math.random()}`,
|
|
||||||
operation,
|
|
||||||
params,
|
|
||||||
executor, // Store the actual executor
|
|
||||||
resolver: resolve,
|
|
||||||
rejector: reject,
|
|
||||||
timestamp: Date.now(),
|
|
||||||
priority,
|
|
||||||
size: operationSize
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add to appropriate batch
|
|
||||||
if (!this.batches.has(batchKey)) {
|
|
||||||
this.batches.set(batchKey, [])
|
|
||||||
}
|
|
||||||
|
|
||||||
const batch = this.batches.get(batchKey)!
|
|
||||||
batch.push(batchedOp)
|
|
||||||
this.currentMemoryUsage += operationSize
|
|
||||||
this.metrics.totalOperations++
|
|
||||||
|
|
||||||
// Check if batch should be flushed immediately
|
|
||||||
if (this.shouldFlushBatch(batch, batchKey)) {
|
|
||||||
this.flushBatch(batchKey)
|
|
||||||
} else if (!this.flushTimers.has(batchKey)) {
|
|
||||||
// Set flush timer if not already set
|
|
||||||
this.setFlushTimer(batchKey)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
private getOperationPriority(operation: string, params: any): number {
|
|
||||||
// Explicit priority
|
|
||||||
if (params?.priority !== undefined) return params.priority
|
|
||||||
|
|
||||||
// Operation-based priority
|
|
||||||
if (operation.includes('delete')) return 10 // Highest
|
|
||||||
if (operation.includes('update')) return 8
|
|
||||||
if (operation.includes('save')) return 6
|
|
||||||
if (operation.includes('add')) return 4
|
|
||||||
return 1 // Lowest
|
|
||||||
}
|
|
||||||
|
|
||||||
private getBatchKey(operation: string, priority: number): string {
|
|
||||||
// Group by operation type and priority for optimal batching
|
|
||||||
const opType = this.getOperationType(operation)
|
|
||||||
const priorityLane = Math.min(priority, this.config.priorityLanes - 1)
|
|
||||||
return `${opType}_p${priorityLane}`
|
|
||||||
}
|
|
||||||
|
|
||||||
private getOperationType(operation: string): string {
|
|
||||||
if (operation.includes('add')) return 'add'
|
|
||||||
if (operation.includes('save')) return 'save'
|
|
||||||
if (operation.includes('update')) return 'update'
|
|
||||||
if (operation.includes('delete')) return 'delete'
|
|
||||||
return 'other'
|
|
||||||
}
|
|
||||||
|
|
||||||
private estimateOperationSize(params: any): number {
|
|
||||||
// Rough estimation of memory usage
|
|
||||||
if (!params) return 100
|
|
||||||
|
|
||||||
let size = 0
|
|
||||||
|
|
||||||
if (params.vector && Array.isArray(params.vector)) {
|
|
||||||
size += params.vector.length * 8 // 8 bytes per float64
|
|
||||||
}
|
|
||||||
|
|
||||||
if (params.data) {
|
|
||||||
size += JSON.stringify(params.data).length * 2 // Rough UTF-16 estimate
|
|
||||||
}
|
|
||||||
|
|
||||||
if (params.metadata) {
|
|
||||||
size += JSON.stringify(params.metadata).length * 2
|
|
||||||
}
|
|
||||||
|
|
||||||
return Math.max(size, 100) // Minimum 100 bytes
|
|
||||||
}
|
|
||||||
|
|
||||||
private shouldFlushBatch(batch: BatchedOperation[], batchKey: string): boolean {
|
|
||||||
// Flush if batch is full
|
|
||||||
if (batch.length >= this.config.maxBatchSize) return true
|
|
||||||
|
|
||||||
// Flush if memory limit approaching
|
|
||||||
if (this.currentMemoryUsage > this.config.memoryLimit * 0.9) return true
|
|
||||||
|
|
||||||
// Flush high-priority batches more aggressively
|
|
||||||
const priority = this.extractPriorityFromKey(batchKey)
|
|
||||||
if (priority >= 8 && batch.length >= 100) return true
|
|
||||||
if (priority >= 6 && batch.length >= 500) return true
|
|
||||||
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
private extractPriorityFromKey(batchKey: string): number {
|
|
||||||
const match = batchKey.match(/_p(\d+)$/)
|
|
||||||
return match ? parseInt(match[1]) : 0
|
|
||||||
}
|
|
||||||
|
|
||||||
private setFlushTimer(batchKey: string): void {
|
|
||||||
const priority = this.extractPriorityFromKey(batchKey)
|
|
||||||
const waitTime = this.getAdaptiveWaitTime(priority)
|
|
||||||
|
|
||||||
const timer = setTimeout(() => {
|
|
||||||
this.flushBatch(batchKey)
|
|
||||||
}, waitTime)
|
|
||||||
|
|
||||||
this.flushTimers.set(batchKey, timer)
|
|
||||||
}
|
|
||||||
|
|
||||||
private getAdaptiveWaitTime(priority: number): number {
|
|
||||||
if (!this.config.adaptiveBatching) {
|
|
||||||
return this.config.maxWaitTime
|
|
||||||
}
|
|
||||||
|
|
||||||
// Adaptive wait time based on performance and priority
|
|
||||||
const baseWaitTime = this.config.maxWaitTime
|
|
||||||
const performanceMultiplier = this.getPerformanceMultiplier()
|
|
||||||
const priorityMultiplier = priority >= 8 ? 0.5 : priority >= 6 ? 0.7 : 1.0
|
|
||||||
|
|
||||||
return Math.max(baseWaitTime * performanceMultiplier * priorityMultiplier, 10)
|
|
||||||
}
|
|
||||||
|
|
||||||
private getPerformanceMultiplier(): number {
|
|
||||||
if (this.performanceHistory.length < 10) return 1.0
|
|
||||||
|
|
||||||
// Calculate average latency trend
|
|
||||||
const recent = this.performanceHistory.slice(-10)
|
|
||||||
const average = recent.reduce((a, b) => a + b, 0) / recent.length
|
|
||||||
|
|
||||||
// If performance is degrading, reduce wait time
|
|
||||||
if (average > this.metrics.averageLatency * 1.2) return 0.7
|
|
||||||
if (average < this.metrics.averageLatency * 0.8) return 1.3
|
|
||||||
|
|
||||||
return 1.0
|
|
||||||
}
|
|
||||||
|
|
||||||
private async flushBatch(batchKey: string): Promise<void> {
|
|
||||||
const batch = this.batches.get(batchKey)
|
|
||||||
if (!batch || batch.length === 0) return
|
|
||||||
|
|
||||||
// Clear timer
|
|
||||||
const timer = this.flushTimers.get(batchKey)
|
|
||||||
if (timer) {
|
|
||||||
clearTimeout(timer)
|
|
||||||
this.flushTimers.delete(batchKey)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Remove batch from queue
|
|
||||||
this.batches.delete(batchKey)
|
|
||||||
|
|
||||||
const startTime = Date.now()
|
|
||||||
|
|
||||||
try {
|
|
||||||
await this.processBatch(batch)
|
|
||||||
|
|
||||||
// Update metrics
|
|
||||||
const latency = Date.now() - startTime
|
|
||||||
this.updateMetrics(batch.length, latency)
|
|
||||||
|
|
||||||
// Adaptive adjustment
|
|
||||||
if (this.config.adaptiveBatching) {
|
|
||||||
this.adjustBatchSize(latency, batch.length)
|
|
||||||
}
|
|
||||||
|
|
||||||
} catch (error) {
|
|
||||||
this.log(`Batch processing failed for ${batchKey}: ${error}`, 'error')
|
|
||||||
|
|
||||||
// Reject all operations in batch
|
|
||||||
batch.forEach(op => {
|
|
||||||
op.rejector(error as Error)
|
|
||||||
this.currentMemoryUsage -= op.size
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private async processBatch(batch: BatchedOperation<any>[]): Promise<void> {
|
|
||||||
// Group by operation type for efficient processing
|
|
||||||
const operationGroups = new Map<string, BatchedOperation<any>[]>()
|
|
||||||
|
|
||||||
for (const op of batch) {
|
|
||||||
const opType = this.getOperationType(op.operation)
|
|
||||||
if (!operationGroups.has(opType)) {
|
|
||||||
operationGroups.set(opType, [])
|
|
||||||
}
|
|
||||||
operationGroups.get(opType)!.push(op)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Process each operation type
|
|
||||||
for (const [opType, operations] of operationGroups) {
|
|
||||||
await this.processBatchByType(opType, operations)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private async processBatchByType(opType: string, operations: BatchedOperation<any>[]): Promise<void> {
|
|
||||||
// Execute batch operation based on type
|
|
||||||
try {
|
|
||||||
if (opType === 'add' || opType === 'save') {
|
|
||||||
await this.processBatchSave(operations)
|
|
||||||
} else if (opType === 'update') {
|
|
||||||
await this.processBatchUpdate(operations)
|
|
||||||
} else if (opType === 'delete') {
|
|
||||||
await this.processBatchDelete(operations)
|
|
||||||
} else {
|
|
||||||
// Fallback: execute individually
|
|
||||||
await this.processIndividually(operations)
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
throw error
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private async processBatchSave(operations: BatchedOperation<any>[]): Promise<void> {
|
|
||||||
// Try to use storage's bulk save if available
|
|
||||||
const storage = this.context?.storage
|
|
||||||
|
|
||||||
if (storage && typeof storage.saveBatch === 'function') {
|
|
||||||
// Use bulk save operation
|
|
||||||
const items = operations.map(op => ({
|
|
||||||
...op.params,
|
|
||||||
_batchId: op.id
|
|
||||||
}))
|
|
||||||
|
|
||||||
try {
|
|
||||||
const results = await storage.saveBatch(items)
|
|
||||||
|
|
||||||
// Resolve all operations with actual results
|
|
||||||
operations.forEach((op, index) => {
|
|
||||||
op.resolver(results[index] || op.params.id)
|
|
||||||
this.currentMemoryUsage -= op.size
|
|
||||||
})
|
|
||||||
} catch (error) {
|
|
||||||
// Reject all operations on batch error
|
|
||||||
operations.forEach(op => {
|
|
||||||
op.rejector(error as Error)
|
|
||||||
this.currentMemoryUsage -= op.size
|
|
||||||
})
|
|
||||||
throw error
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
// Execute using stored executors with concurrency control
|
|
||||||
await this.processWithConcurrency(operations, 10)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private async processBatchUpdate(operations: BatchedOperation<any>[]): Promise<void> {
|
|
||||||
await this.processWithConcurrency(operations, 5) // Lower concurrency for updates
|
|
||||||
}
|
|
||||||
|
|
||||||
private async processBatchDelete(operations: BatchedOperation<any>[]): Promise<void> {
|
|
||||||
await this.processWithConcurrency(operations, 5) // Lower concurrency for deletes
|
|
||||||
}
|
|
||||||
|
|
||||||
private async processIndividually(operations: BatchedOperation<any>[]): Promise<void> {
|
|
||||||
await this.processWithConcurrency(operations, 3) // Conservative concurrency
|
|
||||||
}
|
|
||||||
|
|
||||||
private async processWithConcurrency(operations: BatchedOperation<any>[], concurrency: number): Promise<void> {
|
|
||||||
const promises: Promise<void>[] = []
|
|
||||||
|
|
||||||
for (let i = 0; i < operations.length; i += concurrency) {
|
|
||||||
const chunk = operations.slice(i, i + concurrency)
|
|
||||||
|
|
||||||
const chunkPromise = Promise.all(
|
|
||||||
chunk.map(async (op) => {
|
|
||||||
try {
|
|
||||||
// Execute using the stored executor function - REAL EXECUTION!
|
|
||||||
const result = await op.executor()
|
|
||||||
op.resolver(result)
|
|
||||||
this.currentMemoryUsage -= op.size
|
|
||||||
} catch (error) {
|
|
||||||
op.rejector(error as Error)
|
|
||||||
this.currentMemoryUsage -= op.size
|
|
||||||
}
|
|
||||||
})
|
|
||||||
).then(() => {}) // Convert to void promise
|
|
||||||
|
|
||||||
promises.push(chunkPromise)
|
|
||||||
}
|
|
||||||
|
|
||||||
await Promise.all(promises)
|
|
||||||
}
|
|
||||||
|
|
||||||
// REMOVED executeOperation - no longer needed since we use stored executors
|
|
||||||
|
|
||||||
private flushOldestBatch(): void {
|
|
||||||
if (this.batches.size === 0) return
|
|
||||||
|
|
||||||
// Find oldest batch
|
|
||||||
let oldestKey = ''
|
|
||||||
let oldestTime = Infinity
|
|
||||||
|
|
||||||
for (const [key, batch] of this.batches) {
|
|
||||||
if (batch.length > 0) {
|
|
||||||
const batchAge = Math.min(...batch.map(op => op.timestamp))
|
|
||||||
if (batchAge < oldestTime) {
|
|
||||||
oldestTime = batchAge
|
|
||||||
oldestKey = key
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (oldestKey) {
|
|
||||||
this.flushBatch(oldestKey)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private updateMetrics(batchSize: number, latency: number): void {
|
|
||||||
this.metrics.batchesProcessed++
|
|
||||||
this.metrics.averageBatchSize =
|
|
||||||
(this.metrics.averageBatchSize * (this.metrics.batchesProcessed - 1) + batchSize) /
|
|
||||||
this.metrics.batchesProcessed
|
|
||||||
|
|
||||||
// Update latency with exponential moving average
|
|
||||||
this.metrics.averageLatency = this.metrics.averageLatency * 0.9 + latency * 0.1
|
|
||||||
|
|
||||||
// Add to performance history
|
|
||||||
this.performanceHistory.push(latency)
|
|
||||||
if (this.performanceHistory.length > 100) {
|
|
||||||
this.performanceHistory.shift()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private adjustBatchSize(latency: number, batchSize: number): void {
|
|
||||||
const targetLatency = this.config.maxWaitTime * 5 // Target: 5x wait time
|
|
||||||
|
|
||||||
if (latency > targetLatency && batchSize > 100) {
|
|
||||||
// Reduce batch size if latency too high
|
|
||||||
this.config.maxBatchSize = Math.max(this.config.maxBatchSize * 0.9, 100)
|
|
||||||
this.metrics.adaptiveAdjustments++
|
|
||||||
} else if (latency < targetLatency * 0.5 && batchSize === this.config.maxBatchSize) {
|
|
||||||
// Increase batch size if latency very low
|
|
||||||
this.config.maxBatchSize = Math.min(this.config.maxBatchSize * 1.1, 10000)
|
|
||||||
this.metrics.adaptiveAdjustments++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private startMetricsCollection(): void {
|
|
||||||
setInterval(() => {
|
|
||||||
// Calculate throughput
|
|
||||||
this.metrics.throughputPerSecond = this.metrics.totalOperations
|
|
||||||
this.metrics.totalOperations = 0 // Reset for next measurement
|
|
||||||
|
|
||||||
// Update memory usage
|
|
||||||
this.metrics.memoryUsage = this.currentMemoryUsage
|
|
||||||
|
|
||||||
}, 1000)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get batch processing statistics
|
|
||||||
*/
|
|
||||||
getStats(): BatchMetrics & {
|
|
||||||
pendingBatches: number
|
|
||||||
pendingOperations: number
|
|
||||||
currentBatchSize: number
|
|
||||||
memoryUtilization: string
|
|
||||||
} {
|
|
||||||
let pendingOperations = 0
|
|
||||||
for (const batch of this.batches.values()) {
|
|
||||||
pendingOperations += batch.length
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
...this.metrics,
|
|
||||||
pendingBatches: this.batches.size,
|
|
||||||
pendingOperations,
|
|
||||||
currentBatchSize: this.config.maxBatchSize,
|
|
||||||
memoryUtilization: `${Math.round((this.currentMemoryUsage / this.config.memoryLimit) * 100)}%`
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Force flush all pending batches
|
|
||||||
*/
|
|
||||||
async flushAll(): Promise<void> {
|
|
||||||
const batchKeys = Array.from(this.batches.keys())
|
|
||||||
await Promise.all(batchKeys.map(key => this.flushBatch(key)))
|
|
||||||
}
|
|
||||||
|
|
||||||
protected async onShutdown(): Promise<void> {
|
|
||||||
// Clear all timers
|
|
||||||
for (const timer of this.flushTimers.values()) {
|
|
||||||
clearTimeout(timer)
|
|
||||||
}
|
|
||||||
this.flushTimers.clear()
|
|
||||||
|
|
||||||
// Flush all pending batches
|
|
||||||
await this.flushAll()
|
|
||||||
|
|
||||||
const stats = this.getStats()
|
|
||||||
this.log(`Batch processing shutdown: ${this.metrics.batchesProcessed} batches processed, ${stats.memoryUtilization} peak memory usage`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,766 +0,0 @@
|
||||||
/**
|
|
||||||
* Single BrainyAugmentation Interface
|
|
||||||
*
|
|
||||||
* This replaces the 7 complex interfaces with one elegant, purpose-driven design.
|
|
||||||
* Each augmentation knows its place and when to execute automatically.
|
|
||||||
*
|
|
||||||
* The Vision: Components that enhance Brainy's capabilities seamlessly
|
|
||||||
* - RequestDeduplicator: Prevents duplicate concurrent requests
|
|
||||||
* - ConnectionPool: Optimizes cloud storage throughput
|
|
||||||
* - IntelligentVerbScoring: Enhances relationship analysis
|
|
||||||
* - StreamingPipeline: Enables unlimited data processing
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { AugmentationManifest } from './manifest.js'
|
|
||||||
import { AugmentationConfigResolver } from './configResolver.js'
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Metadata access declaration for augmentations
|
|
||||||
*/
|
|
||||||
export interface MetadataAccess {
|
|
||||||
reads?: string[] | '*' // Fields to read, or '*' for all
|
|
||||||
writes?: string[] | '*' // Fields to write, or '*' for all
|
|
||||||
namespace?: string // Optional: custom namespace like '_myAug'
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface BrainyAugmentation {
|
|
||||||
/**
|
|
||||||
* Unique identifier for the augmentation
|
|
||||||
*/
|
|
||||||
name: string
|
|
||||||
|
|
||||||
/**
|
|
||||||
* When this augmentation should execute
|
|
||||||
* - 'before': Execute before the main operation
|
|
||||||
* - 'after': Execute after the main operation
|
|
||||||
* - 'around': Wrap the main operation (like middleware)
|
|
||||||
* - 'replace': Replace the main operation entirely
|
|
||||||
*/
|
|
||||||
timing: 'before' | 'after' | 'around' | 'replace'
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Metadata access contract - REQUIRED
|
|
||||||
* - 'none': No metadata access at all
|
|
||||||
* - 'readonly': Can read any metadata but cannot write
|
|
||||||
* - MetadataAccess: Specific fields to read/write
|
|
||||||
*/
|
|
||||||
metadata: 'none' | 'readonly' | MetadataAccess
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Which operations this augmentation applies to
|
|
||||||
* Granular operation matching for precise augmentation targeting
|
|
||||||
*/
|
|
||||||
operations: (
|
|
||||||
// Data Operations
|
|
||||||
| 'add' | 'addNoun' | 'addVerb'
|
|
||||||
| 'saveNoun' | 'saveVerb' | 'updateMetadata'
|
|
||||||
| 'update' | 'delete' | 'deleteVerb' | 'clear' | 'get'
|
|
||||||
|
|
||||||
// Search Operations
|
|
||||||
| 'search' | 'searchText' | 'searchByNounTypes'
|
|
||||||
| 'find' | 'findSimilar' | 'searchWithCursor' | 'similar'
|
|
||||||
|
|
||||||
// Relationship Operations
|
|
||||||
| 'relate' | 'unrelate' | 'getConnections' | 'getRelations'
|
|
||||||
|
|
||||||
// Storage Operations
|
|
||||||
| 'storage' | 'backup' | 'restore'
|
|
||||||
|
|
||||||
// Meta
|
|
||||||
| 'all'
|
|
||||||
)[]
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Priority for execution order (higher numbers execute first)
|
|
||||||
* - 100: Critical system operations (ConnectionPool)
|
|
||||||
* - 50: Performance optimizations (RequestDeduplicator, Caching)
|
|
||||||
* - 10: Enhancement features (IntelligentVerbScoring)
|
|
||||||
* - 1: Optional features (Logging, Analytics)
|
|
||||||
*/
|
|
||||||
priority: number
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Initialize the augmentation
|
|
||||||
* Called once during Brainy initialization
|
|
||||||
*
|
|
||||||
* @param context - The Brainy instance and storage
|
|
||||||
*/
|
|
||||||
initialize(context: AugmentationContext): Promise<void>
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Execute the augmentation
|
|
||||||
*
|
|
||||||
* @param operation - The operation being performed
|
|
||||||
* @param params - Parameters for the operation
|
|
||||||
* @param next - Function to call the next augmentation or main operation
|
|
||||||
* @returns Result of the operation
|
|
||||||
*/
|
|
||||||
execute<T = any>(
|
|
||||||
operation: string,
|
|
||||||
params: any,
|
|
||||||
next: () => Promise<T>
|
|
||||||
): Promise<T>
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Optional: Check if this augmentation should run for the given operation
|
|
||||||
* Return false to skip execution
|
|
||||||
*/
|
|
||||||
shouldExecute?(operation: string, params: any): boolean
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Optional: Cleanup when Brainy is destroyed
|
|
||||||
*/
|
|
||||||
shutdown?(): Promise<void>
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Optional: Computed fields this augmentation provides
|
|
||||||
* Used for discovery, TypeScript support, and API documentation
|
|
||||||
*/
|
|
||||||
computedFields?: {
|
|
||||||
[namespace: string]: {
|
|
||||||
[field: string]: {
|
|
||||||
type: 'string' | 'number' | 'boolean' | 'object' | 'array'
|
|
||||||
description: string
|
|
||||||
confidence?: number
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Optional: Compute fields for a result entity
|
|
||||||
* Called when user accesses getDisplay(), getSchema(), etc.
|
|
||||||
*
|
|
||||||
* @param result - The result entity (VectorDocument, GraphVerb, etc.)
|
|
||||||
* @param namespace - The namespace being requested ('display', 'schema', etc.)
|
|
||||||
* @returns Computed fields for the namespace
|
|
||||||
*/
|
|
||||||
computeFields?(result: any, namespace: string): Promise<Record<string, any>> | Record<string, any>
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Context provided to augmentations
|
|
||||||
*/
|
|
||||||
export interface AugmentationContext {
|
|
||||||
/**
|
|
||||||
* The Brainy instance (for accessing methods and config)
|
|
||||||
*/
|
|
||||||
brain: any // Brainy - avoiding circular imports
|
|
||||||
|
|
||||||
/**
|
|
||||||
* The storage adapter
|
|
||||||
*/
|
|
||||||
storage: any // StorageAdapter
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Configuration for this augmentation
|
|
||||||
*/
|
|
||||||
config: any
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Logging function
|
|
||||||
*/
|
|
||||||
log: (message: string, level?: 'info' | 'warn' | 'error') => void
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Base class for augmentations with common functionality
|
|
||||||
*
|
|
||||||
* This is the unified base class that combines the features of both
|
|
||||||
* BaseAugmentation and ConfigurableAugmentation. All augmentations
|
|
||||||
* should extend this class for consistent configuration support.
|
|
||||||
*/
|
|
||||||
export abstract class BaseAugmentation implements BrainyAugmentation {
|
|
||||||
abstract name: string
|
|
||||||
abstract timing: 'before' | 'after' | 'around' | 'replace'
|
|
||||||
abstract metadata: 'none' | 'readonly' | MetadataAccess
|
|
||||||
abstract operations: (
|
|
||||||
// Data Operations
|
|
||||||
| 'add' | 'addNoun' | 'addVerb'
|
|
||||||
| 'saveNoun' | 'saveVerb' | 'updateMetadata'
|
|
||||||
| 'update' | 'delete' | 'deleteVerb' | 'clear' | 'get'
|
|
||||||
|
|
||||||
// Search Operations
|
|
||||||
| 'search' | 'searchText' | 'searchByNounTypes'
|
|
||||||
| 'find' | 'findSimilar' | 'searchWithCursor' | 'similar'
|
|
||||||
|
|
||||||
// Relationship Operations
|
|
||||||
| 'relate' | 'unrelate' | 'getConnections' | 'getRelations'
|
|
||||||
|
|
||||||
// Storage Operations
|
|
||||||
| 'storage' | 'backup' | 'restore'
|
|
||||||
|
|
||||||
// Meta
|
|
||||||
| 'all'
|
|
||||||
)[]
|
|
||||||
abstract priority: number
|
|
||||||
|
|
||||||
// Metadata for augmentation listing and management
|
|
||||||
category: 'internal' | 'core' | 'premium' | 'community' | 'external' = 'core'
|
|
||||||
description: string = ''
|
|
||||||
enabled: boolean = true
|
|
||||||
|
|
||||||
protected context?: AugmentationContext
|
|
||||||
protected isInitialized = false
|
|
||||||
protected config: any = {}
|
|
||||||
private configResolver?: AugmentationConfigResolver
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Constructor with optional configuration
|
|
||||||
* @param config Optional configuration to override defaults
|
|
||||||
*/
|
|
||||||
constructor(config?: any) {
|
|
||||||
// Only resolve configuration if getManifest is implemented
|
|
||||||
if (this.getManifest) {
|
|
||||||
this.config = this.resolveConfiguration(config)
|
|
||||||
} else if (config) {
|
|
||||||
// Legacy support: direct config assignment for augmentations without manifests
|
|
||||||
this.config = config
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get the augmentation manifest for discovery
|
|
||||||
* Override this to enable configuration support
|
|
||||||
* CRITICAL: This enables tools to discover parameters and configuration
|
|
||||||
*/
|
|
||||||
getManifest?(): AugmentationManifest
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get parameter schema for operations
|
|
||||||
* Enables tools to know what parameters each operation needs
|
|
||||||
*/
|
|
||||||
getParameterSchema?(operation: string): any
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get operation descriptions
|
|
||||||
* Enables tools to show what each operation does
|
|
||||||
*/
|
|
||||||
getOperationInfo?(): Record<string, {
|
|
||||||
description: string
|
|
||||||
parameters?: any
|
|
||||||
returns?: any
|
|
||||||
examples?: any[]
|
|
||||||
}>
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get current configuration
|
|
||||||
*/
|
|
||||||
getConfig(): any {
|
|
||||||
return { ...this.config }
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Update configuration at runtime
|
|
||||||
* @param partial Partial configuration to merge
|
|
||||||
*/
|
|
||||||
async updateConfig(partial: any): Promise<void> {
|
|
||||||
if (!this.configResolver) {
|
|
||||||
// For legacy augmentations without manifest, just merge config
|
|
||||||
const oldConfig = this.config
|
|
||||||
this.config = { ...this.config, ...partial }
|
|
||||||
|
|
||||||
if (this.onConfigChange) {
|
|
||||||
await this.onConfigChange(this.config, oldConfig)
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
const oldConfig = this.config
|
|
||||||
|
|
||||||
try {
|
|
||||||
// Use resolver to update and validate
|
|
||||||
this.config = this.configResolver.updateRuntime(partial)
|
|
||||||
|
|
||||||
// Call config change handler if implemented
|
|
||||||
if (this.onConfigChange) {
|
|
||||||
await this.onConfigChange(this.config, oldConfig)
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
// Revert on error
|
|
||||||
this.config = oldConfig
|
|
||||||
throw error
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Optional: Handle configuration changes
|
|
||||||
* Override this to react to runtime configuration updates
|
|
||||||
*/
|
|
||||||
protected onConfigChange?(newConfig: any, oldConfig: any): Promise<void>
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Resolve configuration from all sources
|
|
||||||
* Priority: constructor > env > files > defaults
|
|
||||||
*/
|
|
||||||
private resolveConfiguration(constructorConfig?: any): any {
|
|
||||||
const manifest = this.getManifest!()
|
|
||||||
|
|
||||||
// Create config resolver
|
|
||||||
this.configResolver = new AugmentationConfigResolver({
|
|
||||||
augmentationId: manifest.id,
|
|
||||||
schema: manifest.configSchema,
|
|
||||||
defaults: manifest.configDefaults
|
|
||||||
})
|
|
||||||
|
|
||||||
// Resolve configuration from all sources
|
|
||||||
return this.configResolver.resolve(constructorConfig)
|
|
||||||
}
|
|
||||||
|
|
||||||
async initialize(context: AugmentationContext): Promise<void> {
|
|
||||||
this.context = context
|
|
||||||
this.isInitialized = true
|
|
||||||
await this.onInitialize()
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Override this in subclasses for initialization logic
|
|
||||||
*/
|
|
||||||
protected async onInitialize(): Promise<void> {
|
|
||||||
// Default: no-op
|
|
||||||
}
|
|
||||||
|
|
||||||
abstract execute<T = any>(
|
|
||||||
operation: string,
|
|
||||||
params: any,
|
|
||||||
next: () => Promise<T>
|
|
||||||
): Promise<T>
|
|
||||||
|
|
||||||
shouldExecute(operation: string, params: any): boolean {
|
|
||||||
// Default: execute if operations match exactly or includes 'all'
|
|
||||||
return this.operations.includes('all' as any) ||
|
|
||||||
this.operations.includes(operation as any) ||
|
|
||||||
this.operations.some(op => operation.includes(op))
|
|
||||||
}
|
|
||||||
|
|
||||||
async shutdown(): Promise<void> {
|
|
||||||
await this.onShutdown()
|
|
||||||
this.isInitialized = false
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Override this in subclasses for cleanup logic
|
|
||||||
*/
|
|
||||||
protected async onShutdown(): Promise<void> {
|
|
||||||
// Default: no-op
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Optional computed fields declaration (override in subclasses)
|
|
||||||
*/
|
|
||||||
computedFields?: {
|
|
||||||
[namespace: string]: {
|
|
||||||
[field: string]: {
|
|
||||||
type: 'string' | 'number' | 'boolean' | 'object' | 'array'
|
|
||||||
description: string
|
|
||||||
confidence?: number
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Optional computed fields implementation (override in subclasses)
|
|
||||||
* @param result The result entity
|
|
||||||
* @param namespace The requested namespace
|
|
||||||
* @returns Computed fields for the namespace
|
|
||||||
*/
|
|
||||||
computeFields?(result: any, namespace: string): Promise<Record<string, any>> | Record<string, any>
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Log a message with the augmentation name
|
|
||||||
*/
|
|
||||||
protected log(message: string, level: 'info' | 'warn' | 'error' = 'info'): void {
|
|
||||||
if (this.context) {
|
|
||||||
this.context.log(`[${this.name}] ${message}`, level)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get CommitLog for temporal features
|
|
||||||
*
|
|
||||||
* Provides access to commit history for time-travel queries, audit trails,
|
|
||||||
* and branch management. Available after initialize() is called.
|
|
||||||
*
|
|
||||||
* @returns CommitLog instance
|
|
||||||
* @throws Error if called before initialize() or if COW not enabled
|
|
||||||
*
|
|
||||||
* @example
|
|
||||||
* ```typescript
|
|
||||||
* protected async onInitialize() {
|
|
||||||
* const commitLog = this.getCommitLog()
|
|
||||||
* const history = await commitLog.getHistory('heads/main', { maxCount: 10 })
|
|
||||||
* }
|
|
||||||
* ```
|
|
||||||
*/
|
|
||||||
protected getCommitLog(): any {
|
|
||||||
if (!this.context) {
|
|
||||||
throw new Error(
|
|
||||||
`${this.name}: Cannot access CommitLog before initialize(). ` +
|
|
||||||
`CommitLog is only available after the augmentation has been initialized.`
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
const storage = this.context.storage as any
|
|
||||||
|
|
||||||
if (!storage.commitLog) {
|
|
||||||
throw new Error(
|
|
||||||
`${this.name}: CommitLog not available. ` +
|
|
||||||
`COW (Copy-on-Write) is not enabled on this storage adapter. ` +
|
|
||||||
`Requires BaseStorage with initializeCOW() called. ` +
|
|
||||||
`This is expected if using a non-COW storage adapter.`
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
return storage.commitLog
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get BlobStorage for content-addressable storage
|
|
||||||
*
|
|
||||||
* Provides access to the underlying blob storage system for storing
|
|
||||||
* and retrieving content-addressed data. Available after initialize() is called.
|
|
||||||
*
|
|
||||||
* @returns BlobStorage instance
|
|
||||||
* @throws Error if called before initialize() or if COW not enabled
|
|
||||||
*
|
|
||||||
* @example
|
|
||||||
* ```typescript
|
|
||||||
* protected async onInitialize() {
|
|
||||||
* const blobStorage = this.getBlobStorage()
|
|
||||||
* const hash = await blobStorage.writeBlob(Buffer.from('data'))
|
|
||||||
* const data = await blobStorage.readBlob(hash)
|
|
||||||
* }
|
|
||||||
* ```
|
|
||||||
*/
|
|
||||||
protected getBlobStorage(): any {
|
|
||||||
if (!this.context) {
|
|
||||||
throw new Error(
|
|
||||||
`${this.name}: Cannot access BlobStorage before initialize(). ` +
|
|
||||||
`BlobStorage is only available after the augmentation has been initialized.`
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
const storage = this.context.storage as any
|
|
||||||
|
|
||||||
if (!storage.blobStorage) {
|
|
||||||
throw new Error(
|
|
||||||
`${this.name}: BlobStorage not available. ` +
|
|
||||||
`COW (Copy-on-Write) is not enabled on this storage adapter. ` +
|
|
||||||
`Requires BaseStorage with initializeCOW() called. ` +
|
|
||||||
`This is expected if using a non-COW storage adapter.`
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
return storage.blobStorage
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get RefManager for branch/ref management
|
|
||||||
*
|
|
||||||
* Provides access to the reference manager for creating, updating,
|
|
||||||
* and managing Git-style branches and refs. Available after initialize() is called.
|
|
||||||
*
|
|
||||||
* @returns RefManager instance
|
|
||||||
* @throws Error if called before initialize() or if COW not enabled
|
|
||||||
*
|
|
||||||
* @example
|
|
||||||
* ```typescript
|
|
||||||
* protected async onInitialize() {
|
|
||||||
* const refManager = this.getRefManager()
|
|
||||||
* await refManager.setRef('heads/experiment', commitHash, {
|
|
||||||
* author: 'system',
|
|
||||||
* message: 'Create experiment branch'
|
|
||||||
* })
|
|
||||||
* }
|
|
||||||
* ```
|
|
||||||
*/
|
|
||||||
protected getRefManager(): any {
|
|
||||||
if (!this.context) {
|
|
||||||
throw new Error(
|
|
||||||
`${this.name}: Cannot access RefManager before initialize(). ` +
|
|
||||||
`RefManager is only available after the augmentation has been initialized.`
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
const storage = this.context.storage as any
|
|
||||||
|
|
||||||
if (!storage.refManager) {
|
|
||||||
throw new Error(
|
|
||||||
`${this.name}: RefManager not available. ` +
|
|
||||||
`COW (Copy-on-Write) is not enabled on this storage adapter. ` +
|
|
||||||
`Requires BaseStorage with initializeCOW() called. ` +
|
|
||||||
`This is expected if using a non-COW storage adapter.`
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
return storage.refManager
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get current branch name
|
|
||||||
*
|
|
||||||
* Convenience helper for getting the current branch from the Brainy instance.
|
|
||||||
* Available after initialize() is called.
|
|
||||||
*
|
|
||||||
* @returns Current branch name (e.g., 'main')
|
|
||||||
* @throws Error if called before initialize()
|
|
||||||
*
|
|
||||||
* @example
|
|
||||||
* ```typescript
|
|
||||||
* protected async onInitialize() {
|
|
||||||
* const branch = await this.getCurrentBranch()
|
|
||||||
* console.log(`Current branch: ${branch}`)
|
|
||||||
* }
|
|
||||||
* ```
|
|
||||||
*/
|
|
||||||
protected async getCurrentBranch(): Promise<string> {
|
|
||||||
if (!this.context) {
|
|
||||||
throw new Error(
|
|
||||||
`${this.name}: Cannot access Brainy instance before initialize(). ` +
|
|
||||||
`getCurrentBranch() is only available after the augmentation has been initialized.`
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
const brain = this.context.brain as any
|
|
||||||
|
|
||||||
if (typeof brain.getCurrentBranch !== 'function') {
|
|
||||||
throw new Error(
|
|
||||||
`${this.name}: getCurrentBranch() not available on Brainy instance. ` +
|
|
||||||
`This method requires Brainy with VFS support.`
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
return brain.getCurrentBranch()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Alias for backward compatibility
|
|
||||||
* ConfigurableAugmentation is now merged into BaseAugmentation
|
|
||||||
* @deprecated Use BaseAugmentation instead
|
|
||||||
*/
|
|
||||||
export const ConfigurableAugmentation = BaseAugmentation
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Registry for managing augmentations
|
|
||||||
*/
|
|
||||||
export class AugmentationRegistry {
|
|
||||||
private augmentations: BrainyAugmentation[] = []
|
|
||||||
private context?: AugmentationContext
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Register an augmentation
|
|
||||||
*/
|
|
||||||
register(augmentation: BrainyAugmentation): void {
|
|
||||||
this.augmentations.push(augmentation)
|
|
||||||
// Sort by priority (highest first)
|
|
||||||
this.augmentations.sort((a, b) => b.priority - a.priority)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Find augmentations by operation (before initialization)
|
|
||||||
* Used for two-phase initialization to find storage augmentations
|
|
||||||
*/
|
|
||||||
findByOperation(operation: string): BrainyAugmentation | null {
|
|
||||||
return this.augmentations.find(aug =>
|
|
||||||
aug.operations.includes(operation as any) ||
|
|
||||||
aug.operations.includes('all' as any)
|
|
||||||
) || null
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Initialize all augmentations
|
|
||||||
*/
|
|
||||||
async initialize(context: AugmentationContext): Promise<void> {
|
|
||||||
this.context = context
|
|
||||||
for (const augmentation of this.augmentations) {
|
|
||||||
await augmentation.initialize(context)
|
|
||||||
}
|
|
||||||
context.log(`Initialized ${this.augmentations.length} augmentations`)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Initialize all augmentations (alias for consistency)
|
|
||||||
*/
|
|
||||||
async initializeAll(context: AugmentationContext): Promise<void> {
|
|
||||||
return this.initialize(context)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Execute augmentations for an operation
|
|
||||||
*/
|
|
||||||
async execute<T = any>(
|
|
||||||
operation: string,
|
|
||||||
params: any,
|
|
||||||
mainOperation: () => Promise<T>
|
|
||||||
): Promise<T> {
|
|
||||||
// Filter augmentations that should execute for this operation
|
|
||||||
const applicable = this.augmentations.filter(aug =>
|
|
||||||
aug.shouldExecute ? aug.shouldExecute(operation, params) :
|
|
||||||
aug.operations.includes('all' as any) ||
|
|
||||||
aug.operations.includes(operation as any) ||
|
|
||||||
aug.operations.some(op => operation.includes(op))
|
|
||||||
)
|
|
||||||
|
|
||||||
if (applicable.length === 0) {
|
|
||||||
// No augmentations, execute main operation directly
|
|
||||||
return mainOperation()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create a chain of augmentations
|
|
||||||
let index = 0
|
|
||||||
const executeNext = async (): Promise<T> => {
|
|
||||||
if (index >= applicable.length) {
|
|
||||||
// All augmentations processed, execute main operation
|
|
||||||
return mainOperation()
|
|
||||||
}
|
|
||||||
|
|
||||||
const augmentation = applicable[index++]
|
|
||||||
return augmentation.execute(operation, params, executeNext)
|
|
||||||
}
|
|
||||||
|
|
||||||
return executeNext()
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get all registered augmentations
|
|
||||||
*/
|
|
||||||
getAll(): BrainyAugmentation[] {
|
|
||||||
return [...this.augmentations]
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get augmentation info for listing
|
|
||||||
*/
|
|
||||||
getInfo(): Array<{
|
|
||||||
name: string
|
|
||||||
type: string
|
|
||||||
enabled: boolean
|
|
||||||
description: string
|
|
||||||
category: string
|
|
||||||
priority: number
|
|
||||||
}> {
|
|
||||||
return this.augmentations.map(aug => {
|
|
||||||
const baseAug = aug as any
|
|
||||||
return {
|
|
||||||
name: aug.name,
|
|
||||||
type: baseAug.category || 'core',
|
|
||||||
enabled: baseAug.enabled !== false,
|
|
||||||
description: baseAug.description || `${aug.name} augmentation`,
|
|
||||||
category: baseAug.category || 'core',
|
|
||||||
priority: aug.priority
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get augmentations by name
|
|
||||||
*/
|
|
||||||
get(name: string): BrainyAugmentation | undefined {
|
|
||||||
return this.augmentations.find(aug => aug.name === name)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Discover augmentation parameters and schemas
|
|
||||||
* Critical for tools like brain-cloud to generate UIs
|
|
||||||
*/
|
|
||||||
discover(name?: string): any {
|
|
||||||
if (name) {
|
|
||||||
const aug = this.get(name)
|
|
||||||
if (!aug) return null
|
|
||||||
|
|
||||||
const baseAug = aug as BaseAugmentation
|
|
||||||
return {
|
|
||||||
name: aug.name,
|
|
||||||
operations: aug.operations,
|
|
||||||
priority: aug.priority,
|
|
||||||
timing: aug.timing,
|
|
||||||
metadata: aug.metadata,
|
|
||||||
manifest: baseAug.getManifest ? baseAug.getManifest() : undefined,
|
|
||||||
parameters: baseAug.getParameterSchema ?
|
|
||||||
aug.operations.reduce((acc, op) => {
|
|
||||||
acc[op] = baseAug.getParameterSchema!(op as string)
|
|
||||||
return acc
|
|
||||||
}, {} as any) : undefined,
|
|
||||||
operationInfo: baseAug.getOperationInfo ? baseAug.getOperationInfo() : undefined,
|
|
||||||
config: baseAug.getConfig ? baseAug.getConfig() : undefined
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Return all augmentations discovery info
|
|
||||||
return this.augmentations.map(aug => this.discover(aug.name))
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get configuration schema for an augmentation
|
|
||||||
* Enables UI generation for configuration
|
|
||||||
*/
|
|
||||||
getConfigSchema(name: string): any {
|
|
||||||
const aug = this.get(name) as BaseAugmentation
|
|
||||||
if (!aug || !aug.getManifest) return null
|
|
||||||
|
|
||||||
const manifest = aug.getManifest()
|
|
||||||
return manifest?.configSchema
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Configure an augmentation at runtime
|
|
||||||
*/
|
|
||||||
async configure(name: string, config: any): Promise<void> {
|
|
||||||
const aug = this.get(name) as BaseAugmentation
|
|
||||||
if (!aug || !aug.updateConfig) {
|
|
||||||
throw new Error(`Augmentation ${name} does not support configuration`)
|
|
||||||
}
|
|
||||||
|
|
||||||
await aug.updateConfig(config)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get metrics for an augmentation
|
|
||||||
*/
|
|
||||||
metrics(name?: string): any {
|
|
||||||
if (name) {
|
|
||||||
const aug = this.get(name) as any
|
|
||||||
if (!aug || !aug.metrics) return null
|
|
||||||
return aug.metrics()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Return all metrics
|
|
||||||
const allMetrics: any = {}
|
|
||||||
for (const aug of this.augmentations) {
|
|
||||||
const a = aug as any
|
|
||||||
if (a.metrics) {
|
|
||||||
allMetrics[aug.name] = a.metrics()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return allMetrics
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get health status
|
|
||||||
*/
|
|
||||||
health(): any {
|
|
||||||
const health: any = {
|
|
||||||
overall: 'healthy',
|
|
||||||
augmentations: {}
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const aug of this.augmentations) {
|
|
||||||
const a = aug as any
|
|
||||||
health.augmentations[aug.name] = a.health ? a.health() : 'unknown'
|
|
||||||
}
|
|
||||||
|
|
||||||
return health
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Shutdown all augmentations
|
|
||||||
*/
|
|
||||||
async shutdown(): Promise<void> {
|
|
||||||
for (const augmentation of this.augmentations) {
|
|
||||||
if (augmentation.shutdown) {
|
|
||||||
await augmentation.shutdown()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
this.augmentations = []
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,391 +0,0 @@
|
||||||
/**
|
|
||||||
* Cache Augmentation - Optional Search Result Caching
|
|
||||||
*
|
|
||||||
* Replaces the hardcoded SearchCache in Brainy with an optional augmentation.
|
|
||||||
* This reduces core size and allows custom cache implementations.
|
|
||||||
*
|
|
||||||
* Zero-config: Automatically enabled with sensible defaults
|
|
||||||
* Can be disabled or customized via augmentation registry
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { BaseAugmentation, AugmentationContext } from './brainyAugmentation.js'
|
|
||||||
import { AugmentationManifest } from './manifest.js'
|
|
||||||
import { SearchCache } from '../utils/searchCache.js'
|
|
||||||
import type { GraphNoun } from '../types/graphTypes.js'
|
|
||||||
|
|
||||||
export interface CacheConfig {
|
|
||||||
maxSize?: number
|
|
||||||
ttl?: number
|
|
||||||
enabled?: boolean
|
|
||||||
invalidateOnWrite?: boolean
|
|
||||||
silent?: boolean // Silent mode support
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* CacheAugmentation - Makes search caching optional and pluggable
|
|
||||||
*
|
|
||||||
* Features:
|
|
||||||
* - Transparent search result caching
|
|
||||||
* - Automatic invalidation on data changes
|
|
||||||
* - Memory-aware cache management
|
|
||||||
* - Zero-config with smart defaults
|
|
||||||
*/
|
|
||||||
export class CacheAugmentation extends BaseAugmentation {
|
|
||||||
readonly name = 'cache'
|
|
||||||
readonly timing = 'around' as const
|
|
||||||
readonly metadata = 'none' as const // Cache doesn't access metadata
|
|
||||||
operations = ['search', 'find', 'similar', 'add', 'update', 'delete', 'clear', 'all'] as ('search' | 'find' | 'similar' | 'add' | 'update' | 'delete' | 'clear' | 'all')[]
|
|
||||||
readonly priority = 50 // Mid-priority, runs after data operations
|
|
||||||
|
|
||||||
// Augmentation metadata
|
|
||||||
readonly category = 'core' as const
|
|
||||||
readonly description = 'Transparent search result caching with automatic invalidation'
|
|
||||||
|
|
||||||
private searchCache: SearchCache<GraphNoun> | null = null
|
|
||||||
|
|
||||||
constructor(config?: CacheConfig) {
|
|
||||||
super(config)
|
|
||||||
}
|
|
||||||
|
|
||||||
getManifest(): AugmentationManifest {
|
|
||||||
return {
|
|
||||||
id: 'cache',
|
|
||||||
name: 'Cache',
|
|
||||||
version: '2.0.0',
|
|
||||||
description: 'Intelligent caching for search and query operations',
|
|
||||||
longDescription: 'Provides transparent caching for search results with automatic invalidation on data changes. Significantly improves performance for repeated queries while maintaining data consistency.',
|
|
||||||
category: 'performance',
|
|
||||||
configSchema: {
|
|
||||||
type: 'object',
|
|
||||||
properties: {
|
|
||||||
enabled: {
|
|
||||||
type: 'boolean',
|
|
||||||
default: true,
|
|
||||||
description: 'Enable or disable caching'
|
|
||||||
},
|
|
||||||
maxSize: {
|
|
||||||
type: 'number',
|
|
||||||
default: 1000,
|
|
||||||
minimum: 10,
|
|
||||||
maximum: 100000,
|
|
||||||
description: 'Maximum number of cached entries'
|
|
||||||
},
|
|
||||||
ttl: {
|
|
||||||
type: 'number',
|
|
||||||
default: 300000, // 5 minutes
|
|
||||||
minimum: 1000, // 1 second
|
|
||||||
maximum: 3600000, // 1 hour
|
|
||||||
description: 'Time to live for cache entries in milliseconds'
|
|
||||||
},
|
|
||||||
invalidateOnWrite: {
|
|
||||||
type: 'boolean',
|
|
||||||
default: true,
|
|
||||||
description: 'Automatically invalidate cache on data modifications'
|
|
||||||
},
|
|
||||||
silent: {
|
|
||||||
type: 'boolean',
|
|
||||||
default: false,
|
|
||||||
description: 'Suppress all console output'
|
|
||||||
}
|
|
||||||
},
|
|
||||||
additionalProperties: false
|
|
||||||
},
|
|
||||||
configDefaults: {
|
|
||||||
enabled: true,
|
|
||||||
maxSize: 1000,
|
|
||||||
ttl: 300000,
|
|
||||||
invalidateOnWrite: true
|
|
||||||
},
|
|
||||||
configExamples: [
|
|
||||||
{
|
|
||||||
name: 'High Performance',
|
|
||||||
description: 'Large cache with longer TTL for read-heavy workloads',
|
|
||||||
config: {
|
|
||||||
enabled: true,
|
|
||||||
maxSize: 10000,
|
|
||||||
ttl: 1800000, // 30 minutes
|
|
||||||
invalidateOnWrite: true
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: 'Conservative',
|
|
||||||
description: 'Small cache with short TTL for frequently changing data',
|
|
||||||
config: {
|
|
||||||
enabled: true,
|
|
||||||
maxSize: 100,
|
|
||||||
ttl: 60000, // 1 minute
|
|
||||||
invalidateOnWrite: true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
],
|
|
||||||
minBrainyVersion: '2.0.0',
|
|
||||||
keywords: ['cache', 'performance', 'search', 'optimization'],
|
|
||||||
documentation: 'https://docs.brainy.dev/augmentations/cache',
|
|
||||||
status: 'stable',
|
|
||||||
performance: {
|
|
||||||
memoryUsage: 'medium',
|
|
||||||
cpuUsage: 'low',
|
|
||||||
networkUsage: 'none'
|
|
||||||
},
|
|
||||||
features: ['search-caching', 'auto-invalidation', 'ttl-support', 'memory-management'],
|
|
||||||
enhancedOperations: ['search', 'searchText', 'findSimilar'],
|
|
||||||
metrics: [
|
|
||||||
{
|
|
||||||
name: 'cache_hits',
|
|
||||||
type: 'counter',
|
|
||||||
description: 'Number of cache hits'
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: 'cache_misses',
|
|
||||||
type: 'counter',
|
|
||||||
description: 'Number of cache misses'
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: 'cache_size',
|
|
||||||
type: 'gauge',
|
|
||||||
description: 'Current cache size'
|
|
||||||
}
|
|
||||||
],
|
|
||||||
ui: {
|
|
||||||
icon: '⚡',
|
|
||||||
color: '#FFC107'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected async onInitialize(): Promise<void> {
|
|
||||||
if (!this.config.enabled) {
|
|
||||||
this.log('Cache augmentation disabled by configuration')
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Initialize search cache with config
|
|
||||||
this.searchCache = new SearchCache<GraphNoun>({
|
|
||||||
maxSize: this.config.maxSize!,
|
|
||||||
maxAge: this.config.ttl!, // SearchCache uses maxAge, not ttl
|
|
||||||
enabled: true
|
|
||||||
})
|
|
||||||
|
|
||||||
this.log(`Cache augmentation initialized (maxSize: ${this.config.maxSize}, ttl: ${this.config.ttl}ms)`)
|
|
||||||
}
|
|
||||||
|
|
||||||
protected async onShutdown(): Promise<void> {
|
|
||||||
if (this.searchCache) {
|
|
||||||
this.searchCache.clear()
|
|
||||||
this.searchCache = null
|
|
||||||
}
|
|
||||||
this.log('Cache augmentation shut down')
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Execute augmentation - wrap operations with caching logic
|
|
||||||
*/
|
|
||||||
async execute<T = any>(
|
|
||||||
operation: string,
|
|
||||||
params: any,
|
|
||||||
next: () => Promise<T>
|
|
||||||
): Promise<T> {
|
|
||||||
// If cache is disabled, just pass through
|
|
||||||
if (!this.searchCache || !this.config.enabled) {
|
|
||||||
return next()
|
|
||||||
}
|
|
||||||
|
|
||||||
switch (operation) {
|
|
||||||
case 'search':
|
|
||||||
return this.handleSearch(params, next)
|
|
||||||
|
|
||||||
case 'add':
|
|
||||||
case 'update':
|
|
||||||
case 'delete':
|
|
||||||
// Invalidate cache on data changes
|
|
||||||
// Cache the reference to avoid race condition during async operation
|
|
||||||
const cache = this.searchCache
|
|
||||||
if (this.config.invalidateOnWrite && cache) {
|
|
||||||
const result = await next()
|
|
||||||
// Use cached reference - searchCache might have been nulled during await
|
|
||||||
cache.invalidateOnDataChange(operation as any)
|
|
||||||
this.log(`Cache invalidated due to ${operation} operation`)
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
return next()
|
|
||||||
|
|
||||||
case 'clear':
|
|
||||||
// Clear cache when all data is cleared
|
|
||||||
const result = await next()
|
|
||||||
if (this.searchCache) {
|
|
||||||
this.searchCache.clear()
|
|
||||||
this.log('Cache cleared due to clear operation')
|
|
||||||
}
|
|
||||||
return result
|
|
||||||
|
|
||||||
default:
|
|
||||||
return next()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Handle search operation with caching
|
|
||||||
*/
|
|
||||||
private async handleSearch<T>(params: any, next: () => Promise<T>): Promise<T> {
|
|
||||||
if (!this.searchCache) return next()
|
|
||||||
|
|
||||||
// Extract search parameters
|
|
||||||
const { query, k, options = {} } = params
|
|
||||||
|
|
||||||
// Skip cache if explicitly disabled or has complex filters
|
|
||||||
if (options.skipCache || options.metadata) {
|
|
||||||
return next()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Generate cache key
|
|
||||||
const cacheKey = this.searchCache.getCacheKey(query, k, options)
|
|
||||||
|
|
||||||
// Check cache
|
|
||||||
const cachedResult = this.searchCache.get(cacheKey)
|
|
||||||
if (cachedResult) {
|
|
||||||
this.log('Cache hit for search query')
|
|
||||||
// Update metrics if available
|
|
||||||
if (this.context?.brain) {
|
|
||||||
const metrics = this.context.brain.augmentations?.get('metrics')
|
|
||||||
if (metrics) {
|
|
||||||
metrics.recordCacheHit?.()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return cachedResult as T
|
|
||||||
}
|
|
||||||
|
|
||||||
// Execute search
|
|
||||||
const result = await next()
|
|
||||||
|
|
||||||
// Cache the result
|
|
||||||
this.searchCache.set(cacheKey, result as any)
|
|
||||||
this.log('Search result cached')
|
|
||||||
|
|
||||||
// Update metrics if available
|
|
||||||
if (this.context?.brain) {
|
|
||||||
const metrics = this.context.brain.augmentations?.get('metrics')
|
|
||||||
if (metrics) {
|
|
||||||
metrics.recordCacheMiss?.()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get cache statistics
|
|
||||||
*/
|
|
||||||
getStats() {
|
|
||||||
if (!this.searchCache) {
|
|
||||||
return {
|
|
||||||
enabled: false,
|
|
||||||
hits: 0,
|
|
||||||
misses: 0,
|
|
||||||
size: 0,
|
|
||||||
memoryUsage: 0
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const stats = this.searchCache.getStats()
|
|
||||||
return {
|
|
||||||
...stats,
|
|
||||||
memoryUsage: this.searchCache.getMemoryUsage()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Clear the cache manually
|
|
||||||
*/
|
|
||||||
clear() {
|
|
||||||
if (this.searchCache) {
|
|
||||||
this.searchCache.clear()
|
|
||||||
this.log('Cache manually cleared')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Handle runtime configuration changes
|
|
||||||
*/
|
|
||||||
protected async onConfigChange(newConfig: CacheConfig, oldConfig: CacheConfig): Promise<void> {
|
|
||||||
if (this.searchCache && newConfig.enabled) {
|
|
||||||
this.searchCache.updateConfig({
|
|
||||||
maxSize: newConfig.maxSize!,
|
|
||||||
maxAge: newConfig.ttl!, // SearchCache uses maxAge
|
|
||||||
enabled: newConfig.enabled
|
|
||||||
})
|
|
||||||
this.log('Cache configuration updated')
|
|
||||||
} else if (!newConfig.enabled && this.searchCache) {
|
|
||||||
this.searchCache.clear()
|
|
||||||
this.log('Cache disabled and cleared')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Clean up expired entries
|
|
||||||
*/
|
|
||||||
cleanupExpiredEntries(): number {
|
|
||||||
if (!this.searchCache) return 0
|
|
||||||
|
|
||||||
const cleaned = this.searchCache.cleanupExpiredEntries()
|
|
||||||
if (cleaned > 0) {
|
|
||||||
this.log(`Cleaned ${cleaned} expired cache entries`)
|
|
||||||
}
|
|
||||||
return cleaned
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Invalidate cache when data changes
|
|
||||||
*/
|
|
||||||
invalidateOnDataChange(operation: 'add' | 'update' | 'delete') {
|
|
||||||
if (!this.searchCache) return
|
|
||||||
|
|
||||||
this.searchCache.invalidateOnDataChange(operation)
|
|
||||||
this.log(`Cache invalidated due to ${operation} operation`)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get cache key for a query
|
|
||||||
*/
|
|
||||||
getCacheKey(query: any, options?: any): string {
|
|
||||||
if (!this.searchCache) return ''
|
|
||||||
return this.searchCache.getCacheKey(query, options)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Direct cache get
|
|
||||||
*/
|
|
||||||
get(key: string): any {
|
|
||||||
if (!this.searchCache) return null
|
|
||||||
return this.searchCache.get(key)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Direct cache set
|
|
||||||
*/
|
|
||||||
set(key: string, value: any): void {
|
|
||||||
if (!this.searchCache) return
|
|
||||||
this.searchCache.set(key, value)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get the underlying SearchCache instance (for compatibility)
|
|
||||||
*/
|
|
||||||
getSearchCache() {
|
|
||||||
return this.searchCache
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get memory usage
|
|
||||||
*/
|
|
||||||
getMemoryUsage(): number {
|
|
||||||
if (!this.searchCache) return 0
|
|
||||||
return this.searchCache.getMemoryUsage()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Factory function for zero-config cache augmentation
|
|
||||||
*/
|
|
||||||
export function createCacheAugmentation(config?: CacheConfig): CacheAugmentation {
|
|
||||||
return new CacheAugmentation(config)
|
|
||||||
}
|
|
||||||
|
|
@ -1,274 +0,0 @@
|
||||||
/**
|
|
||||||
* Conduit Augmentations - Data Synchronization Bridges
|
|
||||||
*
|
|
||||||
* These augmentations connect and synchronize data between multiple Brainy instances.
|
|
||||||
* Now using the unified BrainyAugmentation interface.
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { BaseAugmentation, BrainyAugmentation, AugmentationContext } from './brainyAugmentation.js'
|
|
||||||
import { v4 as uuidv4 } from '../universal/uuid.js'
|
|
||||||
|
|
||||||
export interface WebSocketConnection {
|
|
||||||
connectionId: string
|
|
||||||
url: string
|
|
||||||
readyState: number
|
|
||||||
socket?: any
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Base class for conduit augmentations that sync between Brainy instances
|
|
||||||
* Converted to use the unified BrainyAugmentation interface
|
|
||||||
*/
|
|
||||||
abstract class BaseConduitAugmentation extends BaseAugmentation {
|
|
||||||
readonly timing = 'after' as const // Conduits run after operations to sync
|
|
||||||
readonly metadata = 'readonly' as const // Conduits read metadata to pass to external systems
|
|
||||||
readonly operations = ['addNoun', 'delete', 'addVerb'] as ('addNoun' | 'delete' | 'addVerb')[]
|
|
||||||
readonly priority = 20 // Medium-low priority
|
|
||||||
|
|
||||||
protected connections = new Map<string, any>()
|
|
||||||
|
|
||||||
protected async onShutdown(): Promise<void> {
|
|
||||||
// Close all connections
|
|
||||||
for (const [connectionId, connection] of this.connections.entries()) {
|
|
||||||
try {
|
|
||||||
if (connection.close) {
|
|
||||||
await connection.close()
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
this.log(`Failed to close connection ${connectionId}: ${error}`, 'error')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
this.connections.clear()
|
|
||||||
}
|
|
||||||
|
|
||||||
abstract establishConnection(
|
|
||||||
targetSystemId: string,
|
|
||||||
config?: Record<string, unknown>
|
|
||||||
): Promise<WebSocketConnection | null>
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* WebSocket Conduit Augmentation
|
|
||||||
* Syncs data between Brainy instances using WebSockets
|
|
||||||
*/
|
|
||||||
export class WebSocketConduitAugmentation extends BaseConduitAugmentation {
|
|
||||||
readonly name = 'websocket-conduit'
|
|
||||||
private webSocketConnections = new Map<string, WebSocketConnection>()
|
|
||||||
private messageCallbacks = new Map<string, Set<(data: any) => void>>()
|
|
||||||
|
|
||||||
async execute<T = any>(
|
|
||||||
operation: string,
|
|
||||||
params: any,
|
|
||||||
next: () => Promise<T>
|
|
||||||
): Promise<T> {
|
|
||||||
// Execute the operation first
|
|
||||||
const result = await next()
|
|
||||||
|
|
||||||
// Then sync to connected instances
|
|
||||||
if (this.shouldSync(operation)) {
|
|
||||||
await this.syncOperation(operation, params, result)
|
|
||||||
}
|
|
||||||
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
private shouldSync(operation: string): boolean {
|
|
||||||
return ['addNoun', 'deleteNoun', 'addVerb'].includes(operation)
|
|
||||||
}
|
|
||||||
|
|
||||||
private async syncOperation(operation: string, params: any, result: any): Promise<void> {
|
|
||||||
// Broadcast to all connected WebSocket instances
|
|
||||||
for (const [id, connection] of this.webSocketConnections) {
|
|
||||||
if (connection.socket && connection.readyState === 1) { // OPEN state
|
|
||||||
try {
|
|
||||||
const message = JSON.stringify({
|
|
||||||
type: 'sync',
|
|
||||||
operation,
|
|
||||||
params,
|
|
||||||
timestamp: Date.now()
|
|
||||||
})
|
|
||||||
|
|
||||||
if (typeof connection.socket.send === 'function') {
|
|
||||||
connection.socket.send(message)
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
this.log(`Failed to sync to ${id}: ${error}`, 'error')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async establishConnection(
|
|
||||||
url: string,
|
|
||||||
config?: Record<string, unknown>
|
|
||||||
): Promise<WebSocketConnection | null> {
|
|
||||||
try {
|
|
||||||
const connectionId = uuidv4()
|
|
||||||
const protocols = config?.protocols as string | string[] | undefined
|
|
||||||
|
|
||||||
// Create WebSocket based on environment
|
|
||||||
let socket: any
|
|
||||||
|
|
||||||
if (typeof WebSocket !== 'undefined') {
|
|
||||||
// Browser environment
|
|
||||||
socket = new WebSocket(url, protocols)
|
|
||||||
} else {
|
|
||||||
// Node.js environment - dynamic import
|
|
||||||
try {
|
|
||||||
const ws = await import('ws')
|
|
||||||
socket = new ws.WebSocket(url, protocols)
|
|
||||||
} catch {
|
|
||||||
this.log('WebSocket not available in this environment', 'error')
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Setup event handlers
|
|
||||||
socket.onopen = () => {
|
|
||||||
this.log(`Connected to ${url}`)
|
|
||||||
}
|
|
||||||
|
|
||||||
socket.onmessage = (event: any) => {
|
|
||||||
this.handleMessage(connectionId, event.data)
|
|
||||||
}
|
|
||||||
|
|
||||||
socket.onerror = (error: any) => {
|
|
||||||
this.log(`WebSocket error: ${error}`, 'error')
|
|
||||||
}
|
|
||||||
|
|
||||||
socket.onclose = () => {
|
|
||||||
this.log(`Disconnected from ${url}`)
|
|
||||||
this.webSocketConnections.delete(connectionId)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Wait for connection to open
|
|
||||||
await new Promise<void>((resolve, reject) => {
|
|
||||||
const timeout = setTimeout(() => {
|
|
||||||
reject(new Error('Connection timeout'))
|
|
||||||
}, 5000)
|
|
||||||
|
|
||||||
socket.onopen = () => {
|
|
||||||
clearTimeout(timeout)
|
|
||||||
resolve()
|
|
||||||
}
|
|
||||||
|
|
||||||
socket.onerror = (error: any) => {
|
|
||||||
clearTimeout(timeout)
|
|
||||||
reject(error)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
const connection: WebSocketConnection = {
|
|
||||||
connectionId,
|
|
||||||
url,
|
|
||||||
readyState: socket.readyState,
|
|
||||||
socket
|
|
||||||
}
|
|
||||||
|
|
||||||
this.webSocketConnections.set(connectionId, connection)
|
|
||||||
this.connections.set(connectionId, connection)
|
|
||||||
|
|
||||||
return connection
|
|
||||||
} catch (error) {
|
|
||||||
this.log(`Failed to establish connection to ${url}: ${error}`, 'error')
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private handleMessage(connectionId: string, data: any): void {
|
|
||||||
try {
|
|
||||||
const message = typeof data === 'string' ? JSON.parse(data) : data
|
|
||||||
|
|
||||||
// Handle sync messages from remote instances
|
|
||||||
if (message.type === 'sync') {
|
|
||||||
// Apply the operation to our local instance
|
|
||||||
this.applySyncOperation(message).catch(error => {
|
|
||||||
this.log(`Failed to apply sync operation: ${error}`, 'error')
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// Notify any registered callbacks
|
|
||||||
const callbacks = this.messageCallbacks.get(connectionId)
|
|
||||||
if (callbacks) {
|
|
||||||
callbacks.forEach(callback => callback(message))
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
this.log(`Failed to handle message: ${error}`, 'error')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private async applySyncOperation(message: any): Promise<void> {
|
|
||||||
// Apply the synced operation to our local Brainy instance
|
|
||||||
const { operation, params } = message
|
|
||||||
|
|
||||||
try {
|
|
||||||
switch (operation) {
|
|
||||||
case 'addNoun':
|
|
||||||
await this.context?.brain.add({ data: params.content, type: 'content', metadata: params.metadata })
|
|
||||||
break
|
|
||||||
case 'deleteNoun':
|
|
||||||
await this.context?.brain.deleteNoun(params.id)
|
|
||||||
break
|
|
||||||
case 'addVerb':
|
|
||||||
await this.context?.brain.relate({
|
|
||||||
from: params.source,
|
|
||||||
to: params.target,
|
|
||||||
type: params.verb,
|
|
||||||
metadata: params.metadata
|
|
||||||
})
|
|
||||||
break
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
this.log(`Failed to apply ${operation}: ${error}`, 'error')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Subscribe to messages from a specific connection
|
|
||||||
*/
|
|
||||||
onMessage(connectionId: string, callback: (data: any) => void): void {
|
|
||||||
if (!this.messageCallbacks.has(connectionId)) {
|
|
||||||
this.messageCallbacks.set(connectionId, new Set())
|
|
||||||
}
|
|
||||||
this.messageCallbacks.get(connectionId)!.add(callback)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Send a message to a specific connection
|
|
||||||
*/
|
|
||||||
sendMessage(connectionId: string, data: any): boolean {
|
|
||||||
const connection = this.webSocketConnections.get(connectionId)
|
|
||||||
if (connection?.socket && connection.readyState === 1) {
|
|
||||||
try {
|
|
||||||
const message = typeof data === 'string' ? data : JSON.stringify(data)
|
|
||||||
connection.socket.send(message)
|
|
||||||
return true
|
|
||||||
} catch (error) {
|
|
||||||
this.log(`Failed to send message: ${error}`, 'error')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Example usage:
|
|
||||||
*
|
|
||||||
* // Server instance
|
|
||||||
* const serverBrain = new Brainy()
|
|
||||||
* serverBrain.augmentations.register(new APIServerAugmentation())
|
|
||||||
* await serverBrain.init()
|
|
||||||
*
|
|
||||||
* // Client instance
|
|
||||||
* const clientBrain = new Brainy()
|
|
||||||
* const conduit = new WebSocketConduitAugmentation()
|
|
||||||
* clientBrain.augmentations.register(conduit)
|
|
||||||
* await clientBrain.init()
|
|
||||||
*
|
|
||||||
* // Connect client to server
|
|
||||||
* await conduit.establishConnection('ws://localhost:3000/ws')
|
|
||||||
*
|
|
||||||
* // Now operations sync automatically!
|
|
||||||
* await clientBrain.add({ data: 'synced data', type: 'content', metadata: { source: 'client' } })
|
|
||||||
* // This will automatically sync to the server
|
|
||||||
*/
|
|
||||||
|
|
@ -1,539 +0,0 @@
|
||||||
/**
|
|
||||||
* Configuration Resolver for Augmentations
|
|
||||||
*
|
|
||||||
* Handles loading and resolving configuration from multiple sources:
|
|
||||||
* - Environment variables
|
|
||||||
* - Configuration files
|
|
||||||
* - Runtime updates
|
|
||||||
* - Default values from schema
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { JSONSchema } from './manifest.js'
|
|
||||||
import { isNode } from '../utils/environment.js'
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Configuration source priority (highest to lowest)
|
|
||||||
*/
|
|
||||||
export enum ConfigPriority {
|
|
||||||
RUNTIME = 4, // Runtime updates (highest priority)
|
|
||||||
CONSTRUCTOR = 3, // Constructor parameters
|
|
||||||
ENVIRONMENT = 2, // Environment variables
|
|
||||||
FILE = 1, // Configuration files
|
|
||||||
DEFAULT = 0 // Schema defaults (lowest priority)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Configuration source information
|
|
||||||
*/
|
|
||||||
export interface ConfigSource {
|
|
||||||
priority: ConfigPriority
|
|
||||||
source: string
|
|
||||||
config: any
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Configuration resolution options
|
|
||||||
*/
|
|
||||||
export interface ConfigResolverOptions {
|
|
||||||
augmentationId: string
|
|
||||||
schema?: JSONSchema
|
|
||||||
defaults?: Record<string, any>
|
|
||||||
configPaths?: string[]
|
|
||||||
envPrefix?: string
|
|
||||||
allowUndefined?: boolean
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Augmentation Configuration Resolver
|
|
||||||
*/
|
|
||||||
export class AugmentationConfigResolver {
|
|
||||||
private sources: ConfigSource[] = []
|
|
||||||
private resolved: any = {}
|
|
||||||
|
|
||||||
constructor(private options: ConfigResolverOptions) {
|
|
||||||
// Default config paths - include home directory paths only in Node.js
|
|
||||||
const defaultConfigPaths = [
|
|
||||||
'.brainyrc',
|
|
||||||
'.brainyrc.json',
|
|
||||||
'brainy.config.json'
|
|
||||||
]
|
|
||||||
|
|
||||||
// Add home directory paths in Node.js environments
|
|
||||||
if (isNode()) {
|
|
||||||
try {
|
|
||||||
const nodeRequire = typeof require !== 'undefined' ? require : null
|
|
||||||
if (nodeRequire) {
|
|
||||||
const path = nodeRequire('node:path')
|
|
||||||
const os = nodeRequire('node:os')
|
|
||||||
defaultConfigPaths.push(
|
|
||||||
path.join(os.homedir(), '.brainy', 'config.json'),
|
|
||||||
path.join(os.homedir(), '.brainyrc')
|
|
||||||
)
|
|
||||||
}
|
|
||||||
} catch {
|
|
||||||
// Silently continue without home directory paths
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
this.options = {
|
|
||||||
configPaths: defaultConfigPaths,
|
|
||||||
envPrefix: `BRAINY_AUG_${options.augmentationId.toUpperCase()}_`,
|
|
||||||
allowUndefined: true,
|
|
||||||
...options
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Resolve configuration from all sources
|
|
||||||
* @param constructorConfig Optional constructor configuration
|
|
||||||
* @returns Resolved configuration
|
|
||||||
*/
|
|
||||||
resolve(constructorConfig?: any): any {
|
|
||||||
this.sources = []
|
|
||||||
|
|
||||||
// Load from all sources in priority order
|
|
||||||
this.loadDefaults()
|
|
||||||
this.loadFromFiles()
|
|
||||||
this.loadFromEnvironment()
|
|
||||||
|
|
||||||
if (constructorConfig) {
|
|
||||||
this.sources.push({
|
|
||||||
priority: ConfigPriority.CONSTRUCTOR,
|
|
||||||
source: 'constructor',
|
|
||||||
config: constructorConfig
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// Merge configurations by priority
|
|
||||||
this.resolved = this.mergeConfigurations()
|
|
||||||
|
|
||||||
// Validate against schema if provided
|
|
||||||
if (this.options.schema) {
|
|
||||||
this.validateConfiguration(this.resolved)
|
|
||||||
}
|
|
||||||
|
|
||||||
return this.resolved
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Load default values from schema and defaults
|
|
||||||
*/
|
|
||||||
private loadDefaults(): void {
|
|
||||||
let defaults: any = {}
|
|
||||||
|
|
||||||
// Load from provided defaults
|
|
||||||
if (this.options.defaults) {
|
|
||||||
defaults = { ...defaults, ...this.options.defaults }
|
|
||||||
}
|
|
||||||
|
|
||||||
// Load from schema defaults
|
|
||||||
if (this.options.schema?.properties) {
|
|
||||||
for (const [key, prop] of Object.entries(this.options.schema.properties)) {
|
|
||||||
if (prop.default !== undefined && defaults[key] === undefined) {
|
|
||||||
defaults[key] = prop.default
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (Object.keys(defaults).length > 0) {
|
|
||||||
this.sources.push({
|
|
||||||
priority: ConfigPriority.DEFAULT,
|
|
||||||
source: 'defaults',
|
|
||||||
config: defaults
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Load configuration from files
|
|
||||||
*/
|
|
||||||
private loadFromFiles(): void {
|
|
||||||
// Skip in browser environment
|
|
||||||
if (!isNode()) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
const nodeRequire = typeof require !== 'undefined' ? require : null
|
|
||||||
if (!nodeRequire) return
|
|
||||||
|
|
||||||
const fs = nodeRequire('node:fs')
|
|
||||||
|
|
||||||
for (const configPath of this.options.configPaths || []) {
|
|
||||||
try {
|
|
||||||
if (fs.existsSync(configPath)) {
|
|
||||||
const content = fs.readFileSync(configPath, 'utf8')
|
|
||||||
const config = this.parseConfigFile(content, configPath)
|
|
||||||
|
|
||||||
// Extract augmentation-specific configuration
|
|
||||||
const augConfig = this.extractAugmentationConfig(config)
|
|
||||||
|
|
||||||
if (augConfig && Object.keys(augConfig).length > 0) {
|
|
||||||
this.sources.push({
|
|
||||||
priority: ConfigPriority.FILE,
|
|
||||||
source: `file:${configPath}`,
|
|
||||||
config: augConfig
|
|
||||||
})
|
|
||||||
break // Use first found config file
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
// Silently ignore file errors
|
|
||||||
console.debug(`Failed to load config from ${configPath}:`, error)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch {
|
|
||||||
// Silently continue if require fails
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Parse configuration file based on extension
|
|
||||||
*/
|
|
||||||
private parseConfigFile(content: string, filepath: string): any {
|
|
||||||
try {
|
|
||||||
// Try JSON first
|
|
||||||
return JSON.parse(content)
|
|
||||||
} catch {
|
|
||||||
// Try other formats in the future (YAML, TOML, etc.)
|
|
||||||
throw new Error(`Unable to parse config file: ${filepath}`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Extract augmentation-specific configuration from a config object
|
|
||||||
*/
|
|
||||||
private extractAugmentationConfig(config: any): any {
|
|
||||||
const augId = this.options.augmentationId
|
|
||||||
|
|
||||||
// Check for augmentations section
|
|
||||||
if (config.augmentations && config.augmentations[augId]) {
|
|
||||||
return config.augmentations[augId]
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check for direct augmentation config (prefixed keys)
|
|
||||||
const prefix = `${augId}.`
|
|
||||||
const augConfig: any = {}
|
|
||||||
|
|
||||||
for (const [key, value] of Object.entries(config)) {
|
|
||||||
if (key.startsWith(prefix)) {
|
|
||||||
const configKey = key.slice(prefix.length)
|
|
||||||
augConfig[configKey] = value
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return Object.keys(augConfig).length > 0 ? augConfig : null
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Load configuration from environment variables
|
|
||||||
*/
|
|
||||||
private loadFromEnvironment(): void {
|
|
||||||
// Skip in browser environment
|
|
||||||
if (!isNode() || !process.env) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
const prefix = this.options.envPrefix!
|
|
||||||
const envConfig: any = {}
|
|
||||||
|
|
||||||
for (const [key, value] of Object.entries(process.env)) {
|
|
||||||
if (key.startsWith(prefix)) {
|
|
||||||
const configKey = this.envKeyToConfigKey(key.slice(prefix.length))
|
|
||||||
envConfig[configKey] = this.parseEnvValue(value as string)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (Object.keys(envConfig).length > 0) {
|
|
||||||
this.sources.push({
|
|
||||||
priority: ConfigPriority.ENVIRONMENT,
|
|
||||||
source: 'environment',
|
|
||||||
config: envConfig
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Convert environment variable key to config key
|
|
||||||
* ENABLED -> enabled
|
|
||||||
* MAX_SIZE -> maxSize
|
|
||||||
*/
|
|
||||||
private envKeyToConfigKey(envKey: string): string {
|
|
||||||
return envKey
|
|
||||||
.toLowerCase()
|
|
||||||
.replace(/_([a-z])/g, (_, letter) => letter.toUpperCase())
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Parse environment variable value
|
|
||||||
*/
|
|
||||||
private parseEnvValue(value: string): any {
|
|
||||||
// Handle empty strings
|
|
||||||
if (value === '') return value
|
|
||||||
|
|
||||||
// Try to parse as JSON
|
|
||||||
try {
|
|
||||||
return JSON.parse(value)
|
|
||||||
} catch {
|
|
||||||
// Check for boolean strings
|
|
||||||
if (value.toLowerCase() === 'true') return true
|
|
||||||
if (value.toLowerCase() === 'false') return false
|
|
||||||
|
|
||||||
// Check for number strings
|
|
||||||
const num = Number(value)
|
|
||||||
if (!isNaN(num) && value.trim() !== '') return num
|
|
||||||
|
|
||||||
// Return as string
|
|
||||||
return value
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Merge configurations by priority
|
|
||||||
*/
|
|
||||||
private mergeConfigurations(): any {
|
|
||||||
// Sort by priority (lowest to highest)
|
|
||||||
const sorted = [...this.sources].sort((a, b) => a.priority - b.priority)
|
|
||||||
|
|
||||||
// Merge configurations
|
|
||||||
let merged = {}
|
|
||||||
for (const source of sorted) {
|
|
||||||
merged = this.deepMerge(merged, source.config)
|
|
||||||
}
|
|
||||||
|
|
||||||
return merged
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Deep merge two objects
|
|
||||||
*/
|
|
||||||
private deepMerge(target: any, source: any): any {
|
|
||||||
const output = { ...target }
|
|
||||||
|
|
||||||
for (const key in source) {
|
|
||||||
if (source.hasOwnProperty(key)) {
|
|
||||||
if (source[key] && typeof source[key] === 'object' && !Array.isArray(source[key])) {
|
|
||||||
if (target[key] && typeof target[key] === 'object' && !Array.isArray(target[key])) {
|
|
||||||
output[key] = this.deepMerge(target[key], source[key])
|
|
||||||
} else {
|
|
||||||
output[key] = source[key]
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
output[key] = source[key]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return output
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Validate configuration against schema
|
|
||||||
*/
|
|
||||||
private validateConfiguration(config: any): void {
|
|
||||||
if (!this.options.schema) return
|
|
||||||
|
|
||||||
const schema = this.options.schema
|
|
||||||
const errors: string[] = []
|
|
||||||
|
|
||||||
// Check required fields
|
|
||||||
if (schema.required) {
|
|
||||||
for (const field of schema.required) {
|
|
||||||
if (config[field] === undefined) {
|
|
||||||
errors.push(`Missing required field: ${field}`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Validate properties
|
|
||||||
if (schema.properties) {
|
|
||||||
for (const [key, propSchema] of Object.entries(schema.properties)) {
|
|
||||||
const value = config[key]
|
|
||||||
if (value !== undefined) {
|
|
||||||
this.validateProperty(key, value, propSchema, errors)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check for additional properties
|
|
||||||
if (schema.additionalProperties === false) {
|
|
||||||
const allowedKeys = Object.keys(schema.properties || {})
|
|
||||||
for (const key of Object.keys(config)) {
|
|
||||||
if (!allowedKeys.includes(key)) {
|
|
||||||
errors.push(`Unknown configuration property: ${key}`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (errors.length > 0) {
|
|
||||||
throw new Error(`Configuration validation failed for ${this.options.augmentationId}:\n${errors.join('\n')}`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Validate a single property against its schema
|
|
||||||
*/
|
|
||||||
private validateProperty(key: string, value: any, schema: JSONSchema, errors: string[]): void {
|
|
||||||
// Type validation
|
|
||||||
if (schema.type) {
|
|
||||||
const actualType = Array.isArray(value) ? 'array' : typeof value
|
|
||||||
if (actualType !== schema.type) {
|
|
||||||
errors.push(`${key}: expected ${schema.type}, got ${actualType}`)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Number validations
|
|
||||||
if (schema.type === 'number') {
|
|
||||||
if (schema.minimum !== undefined && value < schema.minimum) {
|
|
||||||
errors.push(`${key}: value ${value} is less than minimum ${schema.minimum}`)
|
|
||||||
}
|
|
||||||
if (schema.maximum !== undefined && value > schema.maximum) {
|
|
||||||
errors.push(`${key}: value ${value} is greater than maximum ${schema.maximum}`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// String validations
|
|
||||||
if (schema.type === 'string') {
|
|
||||||
if (schema.minLength !== undefined && value.length < schema.minLength) {
|
|
||||||
errors.push(`${key}: length ${value.length} is less than minimum ${schema.minLength}`)
|
|
||||||
}
|
|
||||||
if (schema.maxLength !== undefined && value.length > schema.maxLength) {
|
|
||||||
errors.push(`${key}: length ${value.length} is greater than maximum ${schema.maxLength}`)
|
|
||||||
}
|
|
||||||
if (schema.pattern) {
|
|
||||||
const regex = new RegExp(schema.pattern)
|
|
||||||
if (!regex.test(value)) {
|
|
||||||
errors.push(`${key}: value does not match pattern ${schema.pattern}`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Enum validation
|
|
||||||
if (schema.enum && !schema.enum.includes(value)) {
|
|
||||||
errors.push(`${key}: value ${value} is not one of allowed values: ${schema.enum.join(', ')}`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get configuration sources for debugging
|
|
||||||
*/
|
|
||||||
getSources(): ConfigSource[] {
|
|
||||||
return [...this.sources]
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get resolved configuration
|
|
||||||
*/
|
|
||||||
getResolved(): any {
|
|
||||||
return { ...this.resolved }
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Update configuration at runtime
|
|
||||||
*/
|
|
||||||
updateRuntime(config: any): any {
|
|
||||||
// Add or update runtime source
|
|
||||||
const runtimeIndex = this.sources.findIndex(s => s.priority === ConfigPriority.RUNTIME)
|
|
||||||
|
|
||||||
if (runtimeIndex >= 0) {
|
|
||||||
this.sources[runtimeIndex].config = {
|
|
||||||
...this.sources[runtimeIndex].config,
|
|
||||||
...config
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
this.sources.push({
|
|
||||||
priority: ConfigPriority.RUNTIME,
|
|
||||||
source: 'runtime',
|
|
||||||
config
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// Re-merge configurations
|
|
||||||
this.resolved = this.mergeConfigurations()
|
|
||||||
|
|
||||||
// Validate
|
|
||||||
if (this.options.schema) {
|
|
||||||
this.validateConfiguration(this.resolved)
|
|
||||||
}
|
|
||||||
|
|
||||||
return this.resolved
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Save configuration to file
|
|
||||||
* @param filepath Path to save configuration
|
|
||||||
* @param format Format to save as (json, etc.)
|
|
||||||
*/
|
|
||||||
async saveToFile(filepath?: string, format: 'json' = 'json'): Promise<void> {
|
|
||||||
// Skip in browser environment
|
|
||||||
if (!isNode()) {
|
|
||||||
throw new Error('Cannot save configuration files in browser environment')
|
|
||||||
}
|
|
||||||
|
|
||||||
const fs = await import('node:fs')
|
|
||||||
const path = await import('node:path')
|
|
||||||
|
|
||||||
const configPath = filepath || this.options.configPaths?.[0] || '.brainyrc'
|
|
||||||
const augId = this.options.augmentationId
|
|
||||||
|
|
||||||
// Load existing config if it exists
|
|
||||||
let fullConfig: any = {}
|
|
||||||
try {
|
|
||||||
if (fs.existsSync(configPath)) {
|
|
||||||
const content = fs.readFileSync(configPath, 'utf8')
|
|
||||||
fullConfig = JSON.parse(content)
|
|
||||||
}
|
|
||||||
} catch {
|
|
||||||
// Start with empty config
|
|
||||||
}
|
|
||||||
|
|
||||||
// Ensure augmentations section exists
|
|
||||||
if (!fullConfig.augmentations) {
|
|
||||||
fullConfig.augmentations = {}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Update augmentation config
|
|
||||||
fullConfig.augmentations[augId] = this.resolved
|
|
||||||
|
|
||||||
// Save based on format
|
|
||||||
let content: string
|
|
||||||
if (format === 'json') {
|
|
||||||
content = JSON.stringify(fullConfig, null, 2)
|
|
||||||
} else {
|
|
||||||
throw new Error(`Unsupported format: ${format}`)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Ensure directory exists
|
|
||||||
const dir = path.dirname(configPath)
|
|
||||||
if (!fs.existsSync(dir)) {
|
|
||||||
fs.mkdirSync(dir, { recursive: true })
|
|
||||||
}
|
|
||||||
|
|
||||||
// Write file
|
|
||||||
fs.writeFileSync(configPath, content, 'utf8')
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get environment variable names for this augmentation
|
|
||||||
*/
|
|
||||||
getEnvironmentVariables(): Record<string, any> {
|
|
||||||
const schema = this.options.schema
|
|
||||||
const prefix = this.options.envPrefix!
|
|
||||||
const vars: Record<string, any> = {}
|
|
||||||
|
|
||||||
if (schema?.properties) {
|
|
||||||
for (const [key, prop] of Object.entries(schema.properties)) {
|
|
||||||
const envKey = prefix + key.replace(/([A-Z])/g, '_$1').toUpperCase()
|
|
||||||
vars[envKey] = {
|
|
||||||
description: prop.description,
|
|
||||||
type: prop.type,
|
|
||||||
default: prop.default,
|
|
||||||
currentValue: process.env?.[envKey]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return vars
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,445 +0,0 @@
|
||||||
/**
|
|
||||||
* Connection Pool Augmentation
|
|
||||||
*
|
|
||||||
* Provides 10-20x throughput improvement for cloud storage (S3, R2, GCS)
|
|
||||||
* Manages connection pooling, request queuing, and parallel processing
|
|
||||||
* Critical for enterprise-scale operations with millions of entries
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { BaseAugmentation, AugmentationContext } from './brainyAugmentation.js'
|
|
||||||
|
|
||||||
interface ConnectionPoolConfig {
|
|
||||||
enabled?: boolean
|
|
||||||
maxConnections?: number // Maximum concurrent connections
|
|
||||||
minConnections?: number // Minimum idle connections
|
|
||||||
acquireTimeout?: number // Timeout for getting connection (ms)
|
|
||||||
idleTimeout?: number // Connection idle timeout (ms)
|
|
||||||
maxQueueSize?: number // Maximum queued requests
|
|
||||||
retryAttempts?: number // Retry failed requests
|
|
||||||
healthCheckInterval?: number // Connection health check (ms)
|
|
||||||
}
|
|
||||||
|
|
||||||
interface PooledConnection {
|
|
||||||
id: string
|
|
||||||
connection: any // Actual storage client (S3, R2, etc)
|
|
||||||
isIdle: boolean
|
|
||||||
lastUsed: number
|
|
||||||
healthScore: number
|
|
||||||
activeRequests: number
|
|
||||||
requestCount: number // Total requests handled
|
|
||||||
}
|
|
||||||
|
|
||||||
interface QueuedRequest<T = any> {
|
|
||||||
id: string
|
|
||||||
operation: string
|
|
||||||
params: any
|
|
||||||
executor: () => Promise<T>
|
|
||||||
resolver: (value: T) => void
|
|
||||||
rejector: (error: Error) => void
|
|
||||||
timestamp: number
|
|
||||||
priority: number
|
|
||||||
}
|
|
||||||
|
|
||||||
export class ConnectionPoolAugmentation extends BaseAugmentation {
|
|
||||||
name = 'ConnectionPool'
|
|
||||||
timing = 'around' as const
|
|
||||||
metadata = 'none' as const // Connection pooling doesn't access metadata
|
|
||||||
operations = ['storage'] as ('storage')[]
|
|
||||||
priority = 95 // Very high priority for storage operations
|
|
||||||
|
|
||||||
protected config: Required<ConnectionPoolConfig>
|
|
||||||
private connections: Map<string, PooledConnection> = new Map()
|
|
||||||
private requestQueue: QueuedRequest<any>[] = []
|
|
||||||
private healthCheckInterval?: NodeJS.Timeout
|
|
||||||
private storageType: string = 'unknown'
|
|
||||||
private stats = {
|
|
||||||
totalRequests: 0,
|
|
||||||
queuedRequests: 0,
|
|
||||||
activeConnections: 0,
|
|
||||||
totalConnections: 0,
|
|
||||||
averageLatency: 0,
|
|
||||||
throughputPerSecond: 0
|
|
||||||
}
|
|
||||||
|
|
||||||
constructor(config: ConnectionPoolConfig = {}) {
|
|
||||||
super()
|
|
||||||
this.config = {
|
|
||||||
enabled: config.enabled ?? true,
|
|
||||||
maxConnections: config.maxConnections ?? 50,
|
|
||||||
minConnections: config.minConnections ?? 5,
|
|
||||||
acquireTimeout: config.acquireTimeout ?? 30000, // 30s
|
|
||||||
idleTimeout: config.idleTimeout ?? 300000, // 5 minutes
|
|
||||||
maxQueueSize: config.maxQueueSize ?? 10000,
|
|
||||||
retryAttempts: config.retryAttempts ?? 3,
|
|
||||||
healthCheckInterval: config.healthCheckInterval ?? 60000 // 1 minute
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected async onInitialize(): Promise<void> {
|
|
||||||
if (!this.config.enabled) {
|
|
||||||
this.log('Connection pooling disabled')
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Detect storage type
|
|
||||||
this.storageType = this.detectStorageType()
|
|
||||||
|
|
||||||
if (this.isCloudStorage()) {
|
|
||||||
await this.initializeConnectionPool()
|
|
||||||
this.startHealthChecks()
|
|
||||||
this.startMetricsCollection()
|
|
||||||
this.log(`Connection pool initialized for ${this.storageType}: ${this.config.minConnections}-${this.config.maxConnections} connections`)
|
|
||||||
} else {
|
|
||||||
this.log(`Connection pooling skipped for ${this.storageType} (local storage)`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
shouldExecute(operation: string, params: any): boolean {
|
|
||||||
return this.config.enabled &&
|
|
||||||
this.isCloudStorage() &&
|
|
||||||
this.isStorageOperation(operation)
|
|
||||||
}
|
|
||||||
|
|
||||||
async execute<T = any>(
|
|
||||||
operation: string,
|
|
||||||
params: any,
|
|
||||||
next: () => Promise<T>
|
|
||||||
): Promise<T> {
|
|
||||||
if (!this.shouldExecute(operation, params)) {
|
|
||||||
return next()
|
|
||||||
}
|
|
||||||
|
|
||||||
const startTime = Date.now()
|
|
||||||
this.stats.totalRequests++
|
|
||||||
|
|
||||||
try {
|
|
||||||
// High priority for critical operations
|
|
||||||
const priority = this.getOperationPriority(operation)
|
|
||||||
|
|
||||||
// Execute with pooled connection
|
|
||||||
const result = await this.executeWithPool(operation, params, next, priority)
|
|
||||||
|
|
||||||
// Update metrics
|
|
||||||
const latency = Date.now() - startTime
|
|
||||||
this.updateLatencyMetrics(latency)
|
|
||||||
|
|
||||||
return result
|
|
||||||
} catch (error) {
|
|
||||||
this.log(`Connection pool error for ${operation}: ${error}`, 'error')
|
|
||||||
|
|
||||||
// Fallback to direct execution for reliability
|
|
||||||
return next()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private detectStorageType(): string {
|
|
||||||
const storage = this.context?.storage
|
|
||||||
if (!storage) return 'unknown'
|
|
||||||
|
|
||||||
const className = storage.constructor.name.toLowerCase()
|
|
||||||
if (className.includes('s3')) return 's3'
|
|
||||||
if (className.includes('r2')) return 'r2'
|
|
||||||
if (className.includes('gcs') || className.includes('google')) return 'gcs'
|
|
||||||
if (className.includes('azure')) return 'azure'
|
|
||||||
if (className.includes('filesystem')) return 'filesystem'
|
|
||||||
if (className.includes('memory')) return 'memory'
|
|
||||||
|
|
||||||
return 'unknown'
|
|
||||||
}
|
|
||||||
|
|
||||||
private isCloudStorage(): boolean {
|
|
||||||
return ['s3', 'r2', 'gcs', 'azure'].includes(this.storageType)
|
|
||||||
}
|
|
||||||
|
|
||||||
private isStorageOperation(operation: string): boolean {
|
|
||||||
return operation.includes('save') ||
|
|
||||||
operation.includes('get') ||
|
|
||||||
operation.includes('delete') ||
|
|
||||||
operation.includes('list') ||
|
|
||||||
operation.includes('backup') ||
|
|
||||||
operation.includes('restore')
|
|
||||||
}
|
|
||||||
|
|
||||||
private getOperationPriority(operation: string): number {
|
|
||||||
// Critical operations get highest priority
|
|
||||||
if (operation.includes('save') || operation.includes('update')) return 10
|
|
||||||
if (operation.includes('delete')) return 9
|
|
||||||
if (operation.includes('get')) return 7
|
|
||||||
if (operation.includes('list')) return 5
|
|
||||||
if (operation.includes('backup')) return 3
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
private async executeWithPool<T>(
|
|
||||||
operation: string,
|
|
||||||
params: any,
|
|
||||||
executor: () => Promise<T>,
|
|
||||||
priority: number
|
|
||||||
): Promise<T> {
|
|
||||||
// Check queue size
|
|
||||||
if (this.requestQueue.length >= this.config.maxQueueSize) {
|
|
||||||
throw new Error('Connection pool queue full - system overloaded')
|
|
||||||
}
|
|
||||||
|
|
||||||
// Try to get available connection immediately
|
|
||||||
const connection = await this.getOrCreateConnection()
|
|
||||||
if (connection && connection.isIdle) {
|
|
||||||
return this.executeWithConnection(connection, operation, executor)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Queue the request with the actual executor
|
|
||||||
return this.queueRequest(operation, params, executor, priority)
|
|
||||||
}
|
|
||||||
|
|
||||||
private getAvailableConnection(): PooledConnection | null {
|
|
||||||
// Find idle connection with best health score
|
|
||||||
let bestConnection: PooledConnection | null = null
|
|
||||||
let bestScore = -1
|
|
||||||
|
|
||||||
for (const connection of this.connections.values()) {
|
|
||||||
if (connection.isIdle && connection.healthScore > bestScore) {
|
|
||||||
bestConnection = connection
|
|
||||||
bestScore = connection.healthScore
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return bestConnection
|
|
||||||
}
|
|
||||||
|
|
||||||
private async executeWithConnection<T>(
|
|
||||||
connection: PooledConnection,
|
|
||||||
operation: string,
|
|
||||||
executor: () => Promise<T>
|
|
||||||
): Promise<T> {
|
|
||||||
// Mark connection as active
|
|
||||||
connection.isIdle = false
|
|
||||||
connection.activeRequests++
|
|
||||||
connection.lastUsed = Date.now()
|
|
||||||
this.stats.activeConnections++
|
|
||||||
|
|
||||||
try {
|
|
||||||
const result = await executor()
|
|
||||||
|
|
||||||
// Update connection health on success
|
|
||||||
connection.healthScore = Math.min(connection.healthScore + 1, 100)
|
|
||||||
|
|
||||||
return result
|
|
||||||
} catch (error) {
|
|
||||||
// Decrease health on failure
|
|
||||||
connection.healthScore = Math.max(connection.healthScore - 5, 0)
|
|
||||||
throw error
|
|
||||||
} finally {
|
|
||||||
// Release connection
|
|
||||||
connection.isIdle = true
|
|
||||||
connection.activeRequests--
|
|
||||||
this.stats.activeConnections--
|
|
||||||
|
|
||||||
// Process next queued request
|
|
||||||
this.processQueue()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private async queueRequest<T>(
|
|
||||||
operation: string,
|
|
||||||
params: any,
|
|
||||||
executor: () => Promise<T>,
|
|
||||||
priority: number
|
|
||||||
): Promise<T> {
|
|
||||||
return new Promise((resolve, reject) => {
|
|
||||||
const request: QueuedRequest<T> = {
|
|
||||||
id: `req_${Date.now()}_${Math.random()}`,
|
|
||||||
operation,
|
|
||||||
params,
|
|
||||||
executor, // Store the actual executor function
|
|
||||||
resolver: resolve,
|
|
||||||
rejector: reject,
|
|
||||||
timestamp: Date.now(),
|
|
||||||
priority
|
|
||||||
}
|
|
||||||
|
|
||||||
// Insert by priority (higher priority first)
|
|
||||||
const insertIndex = this.requestQueue.findIndex(r => r.priority < priority)
|
|
||||||
if (insertIndex === -1) {
|
|
||||||
this.requestQueue.push(request)
|
|
||||||
} else {
|
|
||||||
this.requestQueue.splice(insertIndex, 0, request)
|
|
||||||
}
|
|
||||||
|
|
||||||
this.stats.queuedRequests++
|
|
||||||
|
|
||||||
// Set timeout
|
|
||||||
setTimeout(() => {
|
|
||||||
const index = this.requestQueue.findIndex(r => r.id === request.id)
|
|
||||||
if (index !== -1) {
|
|
||||||
this.requestQueue.splice(index, 1)
|
|
||||||
this.stats.queuedRequests--
|
|
||||||
reject(new Error(`Connection pool timeout: ${this.config.acquireTimeout}ms`))
|
|
||||||
}
|
|
||||||
}, this.config.acquireTimeout)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
private processQueue(): void {
|
|
||||||
if (this.requestQueue.length === 0) return
|
|
||||||
|
|
||||||
const connection = this.getAvailableConnection()
|
|
||||||
if (!connection) return
|
|
||||||
|
|
||||||
const request = this.requestQueue.shift()!
|
|
||||||
this.stats.queuedRequests--
|
|
||||||
|
|
||||||
// Execute queued request with the REAL executor
|
|
||||||
this.executeWithConnection(connection, request.operation, request.executor)
|
|
||||||
.then(request.resolver)
|
|
||||||
.catch(request.rejector)
|
|
||||||
}
|
|
||||||
|
|
||||||
private async initializeConnectionPool(): Promise<void> {
|
|
||||||
// Create minimum connections
|
|
||||||
for (let i = 0; i < this.config.minConnections; i++) {
|
|
||||||
await this.createConnection()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private async createConnection(): Promise<PooledConnection> {
|
|
||||||
const connectionId = `conn_${Date.now()}_${Math.random()}`
|
|
||||||
|
|
||||||
// Create actual connection based on storage type
|
|
||||||
const actualConnection = await this.createStorageConnection()
|
|
||||||
|
|
||||||
const connection: PooledConnection = {
|
|
||||||
id: connectionId,
|
|
||||||
connection: actualConnection,
|
|
||||||
isIdle: true,
|
|
||||||
lastUsed: Date.now(),
|
|
||||||
healthScore: 100,
|
|
||||||
activeRequests: 0,
|
|
||||||
requestCount: 0
|
|
||||||
}
|
|
||||||
|
|
||||||
this.connections.set(connectionId, connection)
|
|
||||||
this.stats.totalConnections++
|
|
||||||
|
|
||||||
return connection
|
|
||||||
}
|
|
||||||
|
|
||||||
private async createStorageConnection(): Promise<any> {
|
|
||||||
// For cloud storage, reuse the existing storage instance
|
|
||||||
// Connection pooling in this context means managing concurrent requests
|
|
||||||
// not creating multiple storage instances (which would be wasteful)
|
|
||||||
const storage = this.context?.storage
|
|
||||||
if (!storage) {
|
|
||||||
throw new Error('Storage not available for connection pooling')
|
|
||||||
}
|
|
||||||
|
|
||||||
// Return a connection wrapper that tracks usage
|
|
||||||
return {
|
|
||||||
storage,
|
|
||||||
created: Date.now(),
|
|
||||||
requestCount: 0
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private async getOrCreateConnection(): Promise<PooledConnection | null> {
|
|
||||||
// Try to get an available connection
|
|
||||||
let connection = this.getAvailableConnection()
|
|
||||||
|
|
||||||
// If no connection available and under max, create new one
|
|
||||||
if (!connection && this.connections.size < this.config.maxConnections) {
|
|
||||||
connection = await this.createConnection()
|
|
||||||
}
|
|
||||||
|
|
||||||
return connection
|
|
||||||
}
|
|
||||||
|
|
||||||
private startHealthChecks(): void {
|
|
||||||
this.healthCheckInterval = setInterval(() => {
|
|
||||||
this.performHealthChecks()
|
|
||||||
}, this.config.healthCheckInterval)
|
|
||||||
}
|
|
||||||
|
|
||||||
private performHealthChecks(): void {
|
|
||||||
const now = Date.now()
|
|
||||||
const toRemove: string[] = []
|
|
||||||
|
|
||||||
for (const [id, connection] of this.connections) {
|
|
||||||
// Remove idle connections that are too old
|
|
||||||
if (connection.isIdle &&
|
|
||||||
now - connection.lastUsed > this.config.idleTimeout &&
|
|
||||||
this.connections.size > this.config.minConnections) {
|
|
||||||
toRemove.push(id)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Remove unhealthy connections
|
|
||||||
if (connection.healthScore < 20) {
|
|
||||||
toRemove.push(id)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Remove unhealthy/old connections
|
|
||||||
for (const id of toRemove) {
|
|
||||||
this.connections.delete(id)
|
|
||||||
this.stats.totalConnections--
|
|
||||||
}
|
|
||||||
|
|
||||||
// Ensure minimum connections
|
|
||||||
while (this.connections.size < this.config.minConnections) {
|
|
||||||
this.createConnection()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private startMetricsCollection(): void {
|
|
||||||
setInterval(() => {
|
|
||||||
this.updateThroughputMetrics()
|
|
||||||
}, 1000) // Update every second
|
|
||||||
}
|
|
||||||
|
|
||||||
private updateLatencyMetrics(latency: number): void {
|
|
||||||
// Simple moving average
|
|
||||||
this.stats.averageLatency = (this.stats.averageLatency * 0.9) + (latency * 0.1)
|
|
||||||
}
|
|
||||||
|
|
||||||
private updateThroughputMetrics(): void {
|
|
||||||
// Reset counter for next second
|
|
||||||
this.stats.throughputPerSecond = this.stats.totalRequests
|
|
||||||
// Reset total for next measurement (in practice, use sliding window)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get connection pool statistics
|
|
||||||
*/
|
|
||||||
getStats(): typeof this.stats & {
|
|
||||||
queueSize: number
|
|
||||||
activeConnections: number
|
|
||||||
totalConnections: number
|
|
||||||
poolUtilization: string
|
|
||||||
storageType: string
|
|
||||||
} {
|
|
||||||
return {
|
|
||||||
...this.stats,
|
|
||||||
queueSize: this.requestQueue.length,
|
|
||||||
activeConnections: this.stats.activeConnections,
|
|
||||||
totalConnections: this.connections.size,
|
|
||||||
poolUtilization: `${Math.round((this.stats.activeConnections / this.connections.size) * 100)}%`,
|
|
||||||
storageType: this.storageType
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected async onShutdown(): Promise<void> {
|
|
||||||
if (this.healthCheckInterval) {
|
|
||||||
clearInterval(this.healthCheckInterval)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Close all connections
|
|
||||||
this.connections.clear()
|
|
||||||
|
|
||||||
// Reject all queued requests
|
|
||||||
this.requestQueue.forEach(request => {
|
|
||||||
request.rejector(new Error('Connection pool shutting down'))
|
|
||||||
})
|
|
||||||
this.requestQueue = []
|
|
||||||
|
|
||||||
const stats = this.getStats()
|
|
||||||
this.log(`Connection pool shutdown: ${stats.totalRequests} requests processed, ${stats.poolUtilization} avg utilization`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,130 +0,0 @@
|
||||||
/**
|
|
||||||
* Default Augmentations Registration
|
|
||||||
*
|
|
||||||
* Maintains zero-config philosophy by automatically registering
|
|
||||||
* core augmentations that were previously hardcoded in Brainy.
|
|
||||||
*
|
|
||||||
* These augmentations are optional but enabled by default for
|
|
||||||
* backward compatibility and optimal performance.
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { Brainy } from '../brainy.js'
|
|
||||||
import { BaseAugmentation } from './brainyAugmentation.js'
|
|
||||||
import { CacheAugmentation } from './cacheAugmentation.js'
|
|
||||||
import { MetricsAugmentation } from './metricsAugmentation.js'
|
|
||||||
import { MonitoringAugmentation } from './monitoringAugmentation.js'
|
|
||||||
import { UniversalDisplayAugmentation } from './universalDisplayAugmentation.js'
|
|
||||||
import { IntelligentImportAugmentation } from './intelligentImport/index.js'
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create default augmentations for zero-config operation
|
|
||||||
* Returns an array of augmentations to be registered
|
|
||||||
*
|
|
||||||
* @param config - Configuration options
|
|
||||||
* @returns Array of augmentations to register
|
|
||||||
*/
|
|
||||||
export function createDefaultAugmentations(
|
|
||||||
config: {
|
|
||||||
cache?: boolean | Record<string, any>
|
|
||||||
metrics?: boolean | Record<string, any>
|
|
||||||
monitoring?: boolean | Record<string, any>
|
|
||||||
display?: boolean | Record<string, any>
|
|
||||||
intelligentImport?: boolean | Record<string, any>
|
|
||||||
} = {}
|
|
||||||
): BaseAugmentation[] {
|
|
||||||
const augmentations: BaseAugmentation[] = []
|
|
||||||
|
|
||||||
// Intelligent Import augmentation (CSV, Excel, PDF)
|
|
||||||
if (config.intelligentImport !== false) {
|
|
||||||
const importConfig = typeof config.intelligentImport === 'object' ? config.intelligentImport : {}
|
|
||||||
augmentations.push(new IntelligentImportAugmentation(importConfig))
|
|
||||||
}
|
|
||||||
|
|
||||||
// Cache augmentation (was SearchCache)
|
|
||||||
if (config.cache !== false) {
|
|
||||||
const cacheConfig = typeof config.cache === 'object' ? config.cache : {}
|
|
||||||
augmentations.push(new CacheAugmentation(cacheConfig))
|
|
||||||
}
|
|
||||||
|
|
||||||
// Note: Index augmentation removed - metadata indexing is now core functionality
|
|
||||||
|
|
||||||
// Metrics augmentation (was StatisticsCollector)
|
|
||||||
if (config.metrics !== false) {
|
|
||||||
const metricsConfig = typeof config.metrics === 'object' ? config.metrics : {}
|
|
||||||
augmentations.push(new MetricsAugmentation(metricsConfig))
|
|
||||||
}
|
|
||||||
|
|
||||||
// Display augmentation (AI-powered intelligent display fields)
|
|
||||||
if (config.display !== false) {
|
|
||||||
const displayConfig = typeof config.display === 'object' ? config.display : {}
|
|
||||||
augmentations.push(new UniversalDisplayAugmentation(displayConfig))
|
|
||||||
}
|
|
||||||
|
|
||||||
// Monitoring augmentation (was HealthMonitor)
|
|
||||||
// Only enable by default in distributed mode
|
|
||||||
const isDistributed = process.env.BRAINY_MODE === 'distributed' ||
|
|
||||||
process.env.BRAINY_DISTRIBUTED === 'true'
|
|
||||||
|
|
||||||
if (config.monitoring !== false && (config.monitoring || isDistributed)) {
|
|
||||||
const monitoringConfig = typeof config.monitoring === 'object' ? config.monitoring : {}
|
|
||||||
augmentations.push(new MonitoringAugmentation(monitoringConfig))
|
|
||||||
}
|
|
||||||
|
|
||||||
return augmentations
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get augmentation by name with type safety
|
|
||||||
*/
|
|
||||||
export function getAugmentation<T>(brain: Brainy, name: string): T | null {
|
|
||||||
// Access augmentations through a public method or property
|
|
||||||
const augmentations = (brain as any).augmentations
|
|
||||||
if (!augmentations) return null
|
|
||||||
const aug = augmentations.get(name)
|
|
||||||
return aug as T | null
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Compatibility helpers for migrating from hardcoded features
|
|
||||||
*/
|
|
||||||
export const AugmentationHelpers = {
|
|
||||||
/**
|
|
||||||
* Get cache augmentation
|
|
||||||
*/
|
|
||||||
getCache(brain: Brainy): CacheAugmentation | null {
|
|
||||||
return getAugmentation<CacheAugmentation>(brain, 'cache')
|
|
||||||
},
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Note: Index augmentation removed - metadata indexing is now core functionality
|
|
||||||
* Use brain.metadataIndex directly instead
|
|
||||||
*/
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get metrics augmentation
|
|
||||||
*/
|
|
||||||
getMetrics(brain: Brainy): MetricsAugmentation | null {
|
|
||||||
return getAugmentation<MetricsAugmentation>(brain, 'metrics')
|
|
||||||
},
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get monitoring augmentation
|
|
||||||
*/
|
|
||||||
getMonitoring(brain: Brainy): MonitoringAugmentation | null {
|
|
||||||
return getAugmentation<MonitoringAugmentation>(brain, 'monitoring')
|
|
||||||
},
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get display augmentation
|
|
||||||
*/
|
|
||||||
getDisplay(brain: Brainy): UniversalDisplayAugmentation | null {
|
|
||||||
return getAugmentation<UniversalDisplayAugmentation>(brain, 'display')
|
|
||||||
},
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get intelligent import augmentation
|
|
||||||
*/
|
|
||||||
getIntelligentImport(brain: Brainy): IntelligentImportAugmentation | null {
|
|
||||||
return getAugmentation<IntelligentImportAugmentation>(brain, 'intelligent-import')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,560 +0,0 @@
|
||||||
/**
|
|
||||||
* Augmentation Discovery API
|
|
||||||
*
|
|
||||||
* Provides discovery and configuration capabilities for augmentations
|
|
||||||
* Enables tools like brain-cloud to dynamically discover, configure, and manage augmentations
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { AugmentationRegistry } from './brainyAugmentation.js'
|
|
||||||
import { AugmentationManifest, JSONSchema } from './manifest.js'
|
|
||||||
import { AugmentationConfigResolver } from './configResolver.js'
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Augmentation listing with manifest and status
|
|
||||||
*/
|
|
||||||
export interface AugmentationListing {
|
|
||||||
id: string
|
|
||||||
name: string
|
|
||||||
manifest: AugmentationManifest
|
|
||||||
status: {
|
|
||||||
enabled: boolean
|
|
||||||
initialized: boolean
|
|
||||||
category: string
|
|
||||||
priority: number
|
|
||||||
}
|
|
||||||
config?: {
|
|
||||||
current: any
|
|
||||||
schema?: JSONSchema
|
|
||||||
sources?: any[]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Configuration validation result
|
|
||||||
*/
|
|
||||||
export interface ConfigValidationResult {
|
|
||||||
valid: boolean
|
|
||||||
errors?: string[]
|
|
||||||
warnings?: string[]
|
|
||||||
suggestions?: string[]
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Discovery API options
|
|
||||||
*/
|
|
||||||
export interface DiscoveryOptions {
|
|
||||||
includeConfig?: boolean
|
|
||||||
includeSchema?: boolean
|
|
||||||
includeSources?: boolean
|
|
||||||
category?: string
|
|
||||||
enabled?: boolean
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Augmentation Discovery API
|
|
||||||
*
|
|
||||||
* Provides a unified interface for discovering and managing augmentations
|
|
||||||
*/
|
|
||||||
export class AugmentationDiscovery {
|
|
||||||
constructor(private registry: AugmentationRegistry) {}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Discover all registered augmentations with manifests
|
|
||||||
* @param options Discovery options
|
|
||||||
* @returns List of augmentation listings
|
|
||||||
*/
|
|
||||||
async discover(options: DiscoveryOptions = {}): Promise<AugmentationListing[]> {
|
|
||||||
const augmentations = this.registry.getAll()
|
|
||||||
const listings: AugmentationListing[] = []
|
|
||||||
|
|
||||||
for (const aug of augmentations) {
|
|
||||||
// Check if augmentation has manifest support
|
|
||||||
const hasManifest = 'getManifest' in aug && typeof aug.getManifest === 'function'
|
|
||||||
|
|
||||||
if (!hasManifest) {
|
|
||||||
// Skip augmentations without manifest support (legacy)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
// Check if augmentation has manifest method
|
|
||||||
if (!('getManifest' in aug) || typeof aug.getManifest !== 'function') {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
const getManifestFn = aug.getManifest as Function
|
|
||||||
const manifest = getManifestFn()
|
|
||||||
|
|
||||||
// Apply filters
|
|
||||||
if (options.category && manifest.category !== options.category) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if (options.enabled !== undefined) {
|
|
||||||
const isEnabled = (aug as any).enabled !== false
|
|
||||||
if (isEnabled !== options.enabled) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Build listing
|
|
||||||
const listing: AugmentationListing = {
|
|
||||||
id: manifest.id,
|
|
||||||
name: manifest.name,
|
|
||||||
manifest,
|
|
||||||
status: {
|
|
||||||
enabled: (aug as any).enabled !== false,
|
|
||||||
initialized: (aug as any).isInitialized || false,
|
|
||||||
category: (aug as any).category || manifest.category,
|
|
||||||
priority: aug.priority
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Include configuration if requested
|
|
||||||
if (options.includeConfig && 'getConfig' in aug) {
|
|
||||||
const getConfigFn = aug.getConfig as Function
|
|
||||||
listing.config = {
|
|
||||||
current: getConfigFn()
|
|
||||||
}
|
|
||||||
|
|
||||||
if (options.includeSchema) {
|
|
||||||
listing.config.schema = manifest.configSchema
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
listings.push(listing)
|
|
||||||
} catch (error) {
|
|
||||||
console.warn(`Failed to get manifest for augmentation ${aug.name}:`, error)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Sort by priority (highest first) then by name
|
|
||||||
listings.sort((a, b) => {
|
|
||||||
const priorityDiff = b.status.priority - a.status.priority
|
|
||||||
if (priorityDiff !== 0) return priorityDiff
|
|
||||||
return a.name.localeCompare(b.name)
|
|
||||||
})
|
|
||||||
|
|
||||||
return listings
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get a specific augmentation's manifest
|
|
||||||
* @param augId Augmentation ID
|
|
||||||
* @returns Augmentation manifest or null if not found
|
|
||||||
*/
|
|
||||||
async getManifest(augId: string): Promise<AugmentationManifest | null> {
|
|
||||||
const aug = this.registry.get(augId)
|
|
||||||
|
|
||||||
if (!aug || !('getManifest' in aug)) {
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
const getManifestFn = aug.getManifest as Function
|
|
||||||
return getManifestFn()
|
|
||||||
} catch (error) {
|
|
||||||
console.error(`Failed to get manifest for ${augId}:`, error)
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get configuration schema for an augmentation
|
|
||||||
* @param augId Augmentation ID
|
|
||||||
* @returns Configuration schema or null
|
|
||||||
*/
|
|
||||||
async getConfigSchema(augId: string): Promise<JSONSchema | null> {
|
|
||||||
const manifest = await this.getManifest(augId)
|
|
||||||
return manifest?.configSchema || null
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get current configuration for an augmentation
|
|
||||||
* @param augId Augmentation ID
|
|
||||||
* @returns Current configuration or null
|
|
||||||
*/
|
|
||||||
async getConfig(augId: string): Promise<any | null> {
|
|
||||||
const aug = this.registry.get(augId)
|
|
||||||
|
|
||||||
if (!aug || !('getConfig' in aug)) {
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
const getConfigFn = aug.getConfig as Function
|
|
||||||
return getConfigFn()
|
|
||||||
} catch (error) {
|
|
||||||
console.error(`Failed to get config for ${augId}:`, error)
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Update configuration for an augmentation
|
|
||||||
* @param augId Augmentation ID
|
|
||||||
* @param config New configuration
|
|
||||||
* @returns Updated configuration or null on failure
|
|
||||||
*/
|
|
||||||
async updateConfig(augId: string, config: any): Promise<any | null> {
|
|
||||||
const aug = this.registry.get(augId)
|
|
||||||
|
|
||||||
if (!aug || !('updateConfig' in aug) || !('getConfig' in aug)) {
|
|
||||||
throw new Error(`Augmentation ${augId} does not support configuration updates`)
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
const updateConfigFn = aug.updateConfig as Function
|
|
||||||
await updateConfigFn(config)
|
|
||||||
const getConfigFn = aug.getConfig as Function
|
|
||||||
return getConfigFn()
|
|
||||||
} catch (error) {
|
|
||||||
throw new Error(`Failed to update config for ${augId}: ${error}`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Validate configuration against schema
|
|
||||||
* @param augId Augmentation ID
|
|
||||||
* @param config Configuration to validate
|
|
||||||
* @returns Validation result
|
|
||||||
*/
|
|
||||||
async validateConfig(augId: string, config: any): Promise<ConfigValidationResult> {
|
|
||||||
const schema = await this.getConfigSchema(augId)
|
|
||||||
|
|
||||||
if (!schema) {
|
|
||||||
return {
|
|
||||||
valid: true,
|
|
||||||
warnings: ['No schema available for validation']
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const errors: string[] = []
|
|
||||||
const warnings: string[] = []
|
|
||||||
const suggestions: string[] = []
|
|
||||||
|
|
||||||
// Check required fields
|
|
||||||
if (schema.required) {
|
|
||||||
for (const field of schema.required) {
|
|
||||||
if (config[field] === undefined) {
|
|
||||||
errors.push(`Missing required field: ${field}`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Validate properties
|
|
||||||
if (schema.properties) {
|
|
||||||
for (const [key, propSchema] of Object.entries(schema.properties)) {
|
|
||||||
const value = config[key]
|
|
||||||
|
|
||||||
if (value === undefined) {
|
|
||||||
// Check if there's a default
|
|
||||||
if (propSchema.default !== undefined) {
|
|
||||||
suggestions.push(`Field '${key}' not provided, will use default: ${JSON.stringify(propSchema.default)}`)
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
// Type validation
|
|
||||||
if (propSchema.type) {
|
|
||||||
const actualType = Array.isArray(value) ? 'array' : typeof value
|
|
||||||
if (actualType !== propSchema.type) {
|
|
||||||
errors.push(`${key}: expected ${propSchema.type}, got ${actualType}`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Additional validations for specific types
|
|
||||||
this.validatePropertyValue(key, value, propSchema, errors, warnings)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check for unknown properties
|
|
||||||
if (schema.additionalProperties === false && schema.properties) {
|
|
||||||
const allowedKeys = Object.keys(schema.properties)
|
|
||||||
for (const key of Object.keys(config)) {
|
|
||||||
if (!allowedKeys.includes(key)) {
|
|
||||||
warnings.push(`Unknown property: ${key}`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
valid: errors.length === 0,
|
|
||||||
errors: errors.length > 0 ? errors : undefined,
|
|
||||||
warnings: warnings.length > 0 ? warnings : undefined,
|
|
||||||
suggestions: suggestions.length > 0 ? suggestions : undefined
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Validate a property value against its schema
|
|
||||||
*/
|
|
||||||
private validatePropertyValue(
|
|
||||||
key: string,
|
|
||||||
value: any,
|
|
||||||
schema: JSONSchema,
|
|
||||||
errors: string[],
|
|
||||||
warnings: string[]
|
|
||||||
): void {
|
|
||||||
// Number validations
|
|
||||||
if (schema.type === 'number') {
|
|
||||||
if (schema.minimum !== undefined && value < schema.minimum) {
|
|
||||||
errors.push(`${key}: value ${value} is less than minimum ${schema.minimum}`)
|
|
||||||
}
|
|
||||||
if (schema.maximum !== undefined && value > schema.maximum) {
|
|
||||||
errors.push(`${key}: value ${value} is greater than maximum ${schema.maximum}`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// String validations
|
|
||||||
if (schema.type === 'string') {
|
|
||||||
if (schema.minLength !== undefined && value.length < schema.minLength) {
|
|
||||||
errors.push(`${key}: length ${value.length} is less than minimum ${schema.minLength}`)
|
|
||||||
}
|
|
||||||
if (schema.maxLength !== undefined && value.length > schema.maxLength) {
|
|
||||||
errors.push(`${key}: length ${value.length} is greater than maximum ${schema.maxLength}`)
|
|
||||||
}
|
|
||||||
if (schema.pattern) {
|
|
||||||
const regex = new RegExp(schema.pattern)
|
|
||||||
if (!regex.test(value)) {
|
|
||||||
errors.push(`${key}: value does not match pattern ${schema.pattern}`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Enum validation
|
|
||||||
if (schema.enum && !schema.enum.includes(value)) {
|
|
||||||
errors.push(`${key}: value '${value}' is not one of allowed values: ${schema.enum.join(', ')}`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get environment variables for an augmentation
|
|
||||||
* @param augId Augmentation ID
|
|
||||||
* @returns Map of environment variable names to descriptions
|
|
||||||
*/
|
|
||||||
async getEnvironmentVariables(augId: string): Promise<Record<string, any> | null> {
|
|
||||||
const manifest = await this.getManifest(augId)
|
|
||||||
|
|
||||||
if (!manifest?.configSchema?.properties) {
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
|
|
||||||
const prefix = `BRAINY_AUG_${augId.toUpperCase()}_`
|
|
||||||
const vars: Record<string, any> = {}
|
|
||||||
|
|
||||||
for (const [key, prop] of Object.entries(manifest.configSchema.properties)) {
|
|
||||||
const envKey = prefix + key.replace(/([A-Z])/g, '_$1').toUpperCase()
|
|
||||||
vars[envKey] = {
|
|
||||||
configKey: key,
|
|
||||||
description: prop.description,
|
|
||||||
type: prop.type,
|
|
||||||
default: prop.default,
|
|
||||||
required: manifest.configSchema.required?.includes(key),
|
|
||||||
currentValue: typeof process !== 'undefined' ? process.env?.[envKey] : undefined
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return vars
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get configuration examples for an augmentation
|
|
||||||
* @param augId Augmentation ID
|
|
||||||
* @returns Configuration examples or empty array
|
|
||||||
*/
|
|
||||||
async getConfigExamples(augId: string): Promise<any[]> {
|
|
||||||
const manifest = await this.getManifest(augId)
|
|
||||||
return manifest?.configExamples || []
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Check if an augmentation supports configuration
|
|
||||||
* @param augId Augmentation ID
|
|
||||||
* @returns True if augmentation supports configuration
|
|
||||||
*/
|
|
||||||
async supportsConfiguration(augId: string): Promise<boolean> {
|
|
||||||
const aug = this.registry.get(augId)
|
|
||||||
return !!(aug && 'getConfig' in aug && 'updateConfig' in aug)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get augmentations by category
|
|
||||||
* @param category Category to filter by
|
|
||||||
* @returns List of augmentations in the category
|
|
||||||
*/
|
|
||||||
async getByCategory(category: string): Promise<AugmentationListing[]> {
|
|
||||||
return this.discover({ category })
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get enabled augmentations
|
|
||||||
* @returns List of enabled augmentations
|
|
||||||
*/
|
|
||||||
async getEnabled(): Promise<AugmentationListing[]> {
|
|
||||||
return this.discover({ enabled: true })
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Search augmentations by keyword
|
|
||||||
* @param query Search query
|
|
||||||
* @returns Matching augmentations
|
|
||||||
*/
|
|
||||||
async search(query: string): Promise<AugmentationListing[]> {
|
|
||||||
const all = await this.discover()
|
|
||||||
const queryLower = query.toLowerCase()
|
|
||||||
|
|
||||||
return all.filter(listing => {
|
|
||||||
const manifest = listing.manifest
|
|
||||||
|
|
||||||
// Search in various fields
|
|
||||||
const searchFields = [
|
|
||||||
manifest.name,
|
|
||||||
manifest.description,
|
|
||||||
manifest.longDescription,
|
|
||||||
...(manifest.keywords || []),
|
|
||||||
manifest.category
|
|
||||||
].filter(Boolean).map(s => s!.toLowerCase())
|
|
||||||
|
|
||||||
return searchFields.some(field => field.includes(queryLower))
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Export configuration for all augmentations
|
|
||||||
* @returns Map of augmentation IDs to configurations
|
|
||||||
*/
|
|
||||||
async exportConfigurations(): Promise<Record<string, any>> {
|
|
||||||
const configs: Record<string, any> = {}
|
|
||||||
const listings = await this.discover({ includeConfig: true })
|
|
||||||
|
|
||||||
for (const listing of listings) {
|
|
||||||
if (listing.config?.current) {
|
|
||||||
configs[listing.id] = listing.config.current
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return configs
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Import configurations for multiple augmentations
|
|
||||||
* @param configs Map of augmentation IDs to configurations
|
|
||||||
* @returns Results of import operation
|
|
||||||
*/
|
|
||||||
async importConfigurations(configs: Record<string, any>): Promise<Record<string, { success: boolean; error?: string }>> {
|
|
||||||
const results: Record<string, { success: boolean; error?: string }> = {}
|
|
||||||
|
|
||||||
for (const [augId, config] of Object.entries(configs)) {
|
|
||||||
try {
|
|
||||||
// Validate before applying
|
|
||||||
const validation = await this.validateConfig(augId, config)
|
|
||||||
|
|
||||||
if (!validation.valid) {
|
|
||||||
results[augId] = {
|
|
||||||
success: false,
|
|
||||||
error: `Validation failed: ${validation.errors?.join(', ')}`
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
// Apply configuration
|
|
||||||
await this.updateConfig(augId, config)
|
|
||||||
results[augId] = { success: true }
|
|
||||||
} catch (error) {
|
|
||||||
results[augId] = {
|
|
||||||
success: false,
|
|
||||||
error: error instanceof Error ? error.message : String(error)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return results
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Generate configuration documentation
|
|
||||||
* @param augId Augmentation ID
|
|
||||||
* @returns Markdown documentation
|
|
||||||
*/
|
|
||||||
async generateConfigDocs(augId: string): Promise<string | null> {
|
|
||||||
const manifest = await this.getManifest(augId)
|
|
||||||
if (!manifest) return null
|
|
||||||
|
|
||||||
const schema = manifest.configSchema
|
|
||||||
const examples = manifest.configExamples || []
|
|
||||||
const envVars = await this.getEnvironmentVariables(augId)
|
|
||||||
|
|
||||||
let docs = `# ${manifest.name} Configuration\n\n`
|
|
||||||
docs += `${manifest.description}\n\n`
|
|
||||||
|
|
||||||
if (manifest.longDescription) {
|
|
||||||
docs += `## Overview\n\n${manifest.longDescription}\n\n`
|
|
||||||
}
|
|
||||||
|
|
||||||
// Configuration options
|
|
||||||
if (schema?.properties) {
|
|
||||||
docs += `## Configuration Options\n\n`
|
|
||||||
|
|
||||||
for (const [key, prop] of Object.entries(schema.properties)) {
|
|
||||||
const required = schema.required?.includes(key) ? ' *(required)*' : ''
|
|
||||||
docs += `### \`${key}\`${required}\n\n`
|
|
||||||
|
|
||||||
if (prop.description) {
|
|
||||||
docs += `${prop.description}\n\n`
|
|
||||||
}
|
|
||||||
|
|
||||||
docs += `- **Type**: ${prop.type}\n`
|
|
||||||
|
|
||||||
if (prop.default !== undefined) {
|
|
||||||
docs += `- **Default**: \`${JSON.stringify(prop.default)}\`\n`
|
|
||||||
}
|
|
||||||
|
|
||||||
if (prop.minimum !== undefined) {
|
|
||||||
docs += `- **Minimum**: ${prop.minimum}\n`
|
|
||||||
}
|
|
||||||
|
|
||||||
if (prop.maximum !== undefined) {
|
|
||||||
docs += `- **Maximum**: ${prop.maximum}\n`
|
|
||||||
}
|
|
||||||
|
|
||||||
if (prop.enum) {
|
|
||||||
docs += `- **Allowed values**: ${prop.enum.map(v => `\`${v}\``).join(', ')}\n`
|
|
||||||
}
|
|
||||||
|
|
||||||
docs += '\n'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Environment variables
|
|
||||||
if (envVars && Object.keys(envVars).length > 0) {
|
|
||||||
docs += `## Environment Variables\n\n`
|
|
||||||
docs += `| Variable | Config Key | Type | Required | Default |\n`
|
|
||||||
docs += `|----------|------------|------|----------|----------|\n`
|
|
||||||
|
|
||||||
for (const [envKey, info] of Object.entries(envVars)) {
|
|
||||||
docs += `| \`${envKey}\` | ${info.configKey} | ${info.type} | ${info.required ? 'Yes' : 'No'} | ${info.default !== undefined ? `\`${info.default}\`` : '-'} |\n`
|
|
||||||
}
|
|
||||||
|
|
||||||
docs += '\n'
|
|
||||||
}
|
|
||||||
|
|
||||||
// Examples
|
|
||||||
if (examples.length > 0) {
|
|
||||||
docs += `## Examples\n\n`
|
|
||||||
|
|
||||||
for (const example of examples) {
|
|
||||||
docs += `### ${example.name}\n\n`
|
|
||||||
|
|
||||||
if (example.description) {
|
|
||||||
docs += `${example.description}\n\n`
|
|
||||||
}
|
|
||||||
|
|
||||||
docs += '```json\n'
|
|
||||||
docs += JSON.stringify(example.config, null, 2)
|
|
||||||
docs += '\n```\n\n'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return docs
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,357 +0,0 @@
|
||||||
/**
|
|
||||||
* Brain-Cloud Catalog Discovery
|
|
||||||
*
|
|
||||||
* Discovers augmentations available in the brain-cloud catalog
|
|
||||||
* Handles free, premium, and community augmentations
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { AugmentationManifest } from '../manifest.js'
|
|
||||||
|
|
||||||
export interface CatalogAugmentation {
|
|
||||||
id: string
|
|
||||||
name: string
|
|
||||||
description: string
|
|
||||||
longDescription?: string
|
|
||||||
category: string
|
|
||||||
status: 'available' | 'coming_soon' | 'deprecated'
|
|
||||||
tier: 'free' | 'premium' | 'enterprise'
|
|
||||||
price?: {
|
|
||||||
monthly?: number
|
|
||||||
yearly?: number
|
|
||||||
oneTime?: number
|
|
||||||
}
|
|
||||||
manifest?: AugmentationManifest
|
|
||||||
source: 'catalog'
|
|
||||||
cdnUrl?: string
|
|
||||||
npmPackage?: string
|
|
||||||
githubRepo?: string
|
|
||||||
author?: {
|
|
||||||
name: string
|
|
||||||
url?: string
|
|
||||||
}
|
|
||||||
metrics?: {
|
|
||||||
installations: number
|
|
||||||
rating: number
|
|
||||||
reviews: number
|
|
||||||
}
|
|
||||||
requirements?: {
|
|
||||||
minBrainyVersion?: string
|
|
||||||
maxBrainyVersion?: string
|
|
||||||
dependencies?: string[]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface CatalogOptions {
|
|
||||||
apiUrl?: string
|
|
||||||
apiKey?: string
|
|
||||||
cache?: boolean
|
|
||||||
cacheTimeout?: number
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface CatalogFilters {
|
|
||||||
category?: string
|
|
||||||
tier?: 'free' | 'premium' | 'enterprise'
|
|
||||||
status?: 'available' | 'coming_soon' | 'deprecated'
|
|
||||||
search?: string
|
|
||||||
installed?: boolean
|
|
||||||
minRating?: number
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Brain-Cloud Catalog Discovery
|
|
||||||
*/
|
|
||||||
export class CatalogDiscovery {
|
|
||||||
private apiUrl: string
|
|
||||||
private apiKey?: string
|
|
||||||
private cache: Map<string, { data: any; timestamp: number }> = new Map()
|
|
||||||
private cacheTimeout: number
|
|
||||||
|
|
||||||
constructor(options: CatalogOptions = {}) {
|
|
||||||
this.apiUrl = options.apiUrl || 'https://api.soulcraft.com/brain-cloud'
|
|
||||||
this.apiKey = options.apiKey
|
|
||||||
this.cacheTimeout = options.cacheTimeout || 5 * 60 * 1000 // 5 minutes
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Discover augmentations from catalog
|
|
||||||
*/
|
|
||||||
async discover(filters: CatalogFilters = {}): Promise<CatalogAugmentation[]> {
|
|
||||||
const cacheKey = JSON.stringify(filters)
|
|
||||||
|
|
||||||
// Check cache
|
|
||||||
if (this.cache.has(cacheKey)) {
|
|
||||||
const cached = this.cache.get(cacheKey)!
|
|
||||||
if (Date.now() - cached.timestamp < this.cacheTimeout) {
|
|
||||||
return cached.data
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Build query parameters
|
|
||||||
const params = new URLSearchParams()
|
|
||||||
if (filters.category) params.append('category', filters.category)
|
|
||||||
if (filters.tier) params.append('tier', filters.tier)
|
|
||||||
if (filters.status) params.append('status', filters.status)
|
|
||||||
if (filters.search) params.append('q', filters.search)
|
|
||||||
if (filters.minRating) params.append('minRating', filters.minRating.toString())
|
|
||||||
|
|
||||||
// Fetch from API
|
|
||||||
const response = await fetch(`${this.apiUrl}/augmentations/discover?${params}`, {
|
|
||||||
headers: this.getHeaders()
|
|
||||||
})
|
|
||||||
|
|
||||||
if (!response.ok) {
|
|
||||||
throw new Error(`Failed to fetch catalog: ${response.statusText}`)
|
|
||||||
}
|
|
||||||
|
|
||||||
const data = await response.json()
|
|
||||||
const augmentations = this.transformCatalogData(data)
|
|
||||||
|
|
||||||
// Cache result
|
|
||||||
this.cache.set(cacheKey, {
|
|
||||||
data: augmentations,
|
|
||||||
timestamp: Date.now()
|
|
||||||
})
|
|
||||||
|
|
||||||
return augmentations
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get specific augmentation details
|
|
||||||
*/
|
|
||||||
async getAugmentation(id: string): Promise<CatalogAugmentation | null> {
|
|
||||||
const response = await fetch(`${this.apiUrl}/augmentations/${id}`, {
|
|
||||||
headers: this.getHeaders()
|
|
||||||
})
|
|
||||||
|
|
||||||
if (!response.ok) {
|
|
||||||
if (response.status === 404) return null
|
|
||||||
throw new Error(`Failed to fetch augmentation: ${response.statusText}`)
|
|
||||||
}
|
|
||||||
|
|
||||||
const data = await response.json()
|
|
||||||
return this.transformAugmentation(data)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get augmentation manifest
|
|
||||||
*/
|
|
||||||
async getManifest(id: string): Promise<AugmentationManifest | null> {
|
|
||||||
const response = await fetch(`${this.apiUrl}/augmentations/${id}/manifest`, {
|
|
||||||
headers: this.getHeaders()
|
|
||||||
})
|
|
||||||
|
|
||||||
if (!response.ok) {
|
|
||||||
if (response.status === 404) return null
|
|
||||||
throw new Error(`Failed to fetch manifest: ${response.statusText}`)
|
|
||||||
}
|
|
||||||
|
|
||||||
return response.json()
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get CDN URL for dynamic loading
|
|
||||||
*/
|
|
||||||
async getCDNUrl(id: string): Promise<string | null> {
|
|
||||||
const aug = await this.getAugmentation(id)
|
|
||||||
return aug?.cdnUrl || null
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Check if user has access to augmentation
|
|
||||||
*/
|
|
||||||
async checkAccess(id: string): Promise<{
|
|
||||||
hasAccess: boolean
|
|
||||||
requiresPurchase?: boolean
|
|
||||||
requiredTier?: string
|
|
||||||
}> {
|
|
||||||
if (!this.apiKey) {
|
|
||||||
// No API key, only free augmentations
|
|
||||||
const aug = await this.getAugmentation(id)
|
|
||||||
return {
|
|
||||||
hasAccess: aug?.tier === 'free',
|
|
||||||
requiresPurchase: aug?.tier !== 'free',
|
|
||||||
requiredTier: aug?.tier
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const response = await fetch(`${this.apiUrl}/augmentations/${id}/access`, {
|
|
||||||
headers: this.getHeaders()
|
|
||||||
})
|
|
||||||
|
|
||||||
if (!response.ok) {
|
|
||||||
throw new Error(`Failed to check access: ${response.statusText}`)
|
|
||||||
}
|
|
||||||
|
|
||||||
return response.json()
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Purchase/activate augmentation
|
|
||||||
*/
|
|
||||||
async purchase(id: string, licenseKey?: string): Promise<{
|
|
||||||
success: boolean
|
|
||||||
cdnUrl?: string
|
|
||||||
npmPackage?: string
|
|
||||||
licenseKey?: string
|
|
||||||
}> {
|
|
||||||
const body = licenseKey ? { licenseKey } : {}
|
|
||||||
|
|
||||||
const response = await fetch(`${this.apiUrl}/augmentations/${id}/purchase`, {
|
|
||||||
method: 'POST',
|
|
||||||
headers: {
|
|
||||||
...this.getHeaders(),
|
|
||||||
'Content-Type': 'application/json'
|
|
||||||
},
|
|
||||||
body: JSON.stringify(body)
|
|
||||||
})
|
|
||||||
|
|
||||||
if (!response.ok) {
|
|
||||||
throw new Error(`Failed to purchase: ${response.statusText}`)
|
|
||||||
}
|
|
||||||
|
|
||||||
return response.json()
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get user's purchased augmentations
|
|
||||||
*/
|
|
||||||
async getPurchased(): Promise<CatalogAugmentation[]> {
|
|
||||||
if (!this.apiKey) {
|
|
||||||
return []
|
|
||||||
}
|
|
||||||
|
|
||||||
const response = await fetch(`${this.apiUrl}/user/augmentations`, {
|
|
||||||
headers: this.getHeaders()
|
|
||||||
})
|
|
||||||
|
|
||||||
if (!response.ok) {
|
|
||||||
throw new Error(`Failed to fetch purchased: ${response.statusText}`)
|
|
||||||
}
|
|
||||||
|
|
||||||
const data = await response.json()
|
|
||||||
return this.transformCatalogData(data)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get categories
|
|
||||||
*/
|
|
||||||
async getCategories(): Promise<Array<{
|
|
||||||
id: string
|
|
||||||
name: string
|
|
||||||
description: string
|
|
||||||
icon?: string
|
|
||||||
}>> {
|
|
||||||
const response = await fetch(`${this.apiUrl}/augmentations/categories`, {
|
|
||||||
headers: this.getHeaders()
|
|
||||||
})
|
|
||||||
|
|
||||||
if (!response.ok) {
|
|
||||||
throw new Error(`Failed to fetch categories: ${response.statusText}`)
|
|
||||||
}
|
|
||||||
|
|
||||||
return response.json()
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Search augmentations
|
|
||||||
*/
|
|
||||||
async search(query: string): Promise<CatalogAugmentation[]> {
|
|
||||||
return this.discover({ search: query })
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get trending augmentations
|
|
||||||
*/
|
|
||||||
async getTrending(limit: number = 10): Promise<CatalogAugmentation[]> {
|
|
||||||
const response = await fetch(`${this.apiUrl}/augmentations/trending?limit=${limit}`, {
|
|
||||||
headers: this.getHeaders()
|
|
||||||
})
|
|
||||||
|
|
||||||
if (!response.ok) {
|
|
||||||
throw new Error(`Failed to fetch trending: ${response.statusText}`)
|
|
||||||
}
|
|
||||||
|
|
||||||
const data = await response.json()
|
|
||||||
return this.transformCatalogData(data)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get recommended augmentations
|
|
||||||
*/
|
|
||||||
async getRecommended(): Promise<CatalogAugmentation[]> {
|
|
||||||
if (!this.apiKey) {
|
|
||||||
// Return popular free augmentations
|
|
||||||
return this.discover({ tier: 'free', minRating: 4 })
|
|
||||||
}
|
|
||||||
|
|
||||||
const response = await fetch(`${this.apiUrl}/augmentations/recommended`, {
|
|
||||||
headers: this.getHeaders()
|
|
||||||
})
|
|
||||||
|
|
||||||
if (!response.ok) {
|
|
||||||
throw new Error(`Failed to fetch recommended: ${response.statusText}`)
|
|
||||||
}
|
|
||||||
|
|
||||||
const data = await response.json()
|
|
||||||
return this.transformCatalogData(data)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Transform catalog data
|
|
||||||
*/
|
|
||||||
private transformCatalogData(data: any[]): CatalogAugmentation[] {
|
|
||||||
return data.map(item => this.transformAugmentation(item))
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Transform single augmentation
|
|
||||||
*/
|
|
||||||
private transformAugmentation(item: any): CatalogAugmentation {
|
|
||||||
return {
|
|
||||||
id: item.id,
|
|
||||||
name: item.name,
|
|
||||||
description: item.description,
|
|
||||||
longDescription: item.longDescription,
|
|
||||||
category: item.category,
|
|
||||||
status: item.status || 'available',
|
|
||||||
tier: item.tier || 'free',
|
|
||||||
price: item.price,
|
|
||||||
manifest: item.manifest,
|
|
||||||
source: 'catalog',
|
|
||||||
cdnUrl: item.cdnUrl || `https://cdn.soulcraft.com/augmentations/${item.id}@latest`,
|
|
||||||
npmPackage: item.npmPackage,
|
|
||||||
githubRepo: item.githubRepo,
|
|
||||||
author: item.author,
|
|
||||||
metrics: item.metrics,
|
|
||||||
requirements: item.requirements
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get request headers
|
|
||||||
*/
|
|
||||||
private getHeaders(): HeadersInit {
|
|
||||||
const headers: HeadersInit = {}
|
|
||||||
|
|
||||||
if (this.apiKey) {
|
|
||||||
headers['Authorization'] = `Bearer ${this.apiKey}`
|
|
||||||
}
|
|
||||||
|
|
||||||
return headers
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Clear cache
|
|
||||||
*/
|
|
||||||
clearCache(): void {
|
|
||||||
this.cache.clear()
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Set API key
|
|
||||||
*/
|
|
||||||
setApiKey(apiKey: string): void {
|
|
||||||
this.apiKey = apiKey
|
|
||||||
this.clearCache() // Clear cache when API key changes
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,315 +0,0 @@
|
||||||
/**
|
|
||||||
* Local Augmentation Discovery
|
|
||||||
*
|
|
||||||
* Discovers augmentations installed locally in node_modules
|
|
||||||
* and built-in augmentations that ship with Brainy
|
|
||||||
*
|
|
||||||
* NOTE: This is a Node.js-only feature that requires filesystem access
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { AugmentationManifest } from '../manifest.js'
|
|
||||||
|
|
||||||
// Node.js modules - dynamically imported to avoid bundler issues
|
|
||||||
let fs: any = null
|
|
||||||
let path: any = null
|
|
||||||
|
|
||||||
// Load Node.js modules if available
|
|
||||||
if (typeof window === 'undefined') {
|
|
||||||
try {
|
|
||||||
fs = await import('node:fs')
|
|
||||||
path = await import('node:path')
|
|
||||||
} catch (e) {
|
|
||||||
// Will throw error in methods if not available
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create compatibility layer for sync methods
|
|
||||||
const existsSync = fs?.existsSync || (() => { throw new Error('Filesystem not available') })
|
|
||||||
const readdirSync = fs?.readdirSync || (() => { throw new Error('Filesystem not available') })
|
|
||||||
const readFileSync = fs?.readFileSync || (() => { throw new Error('Filesystem not available') })
|
|
||||||
const join = path?.join || ((...args: string[]) => args.join('/'))
|
|
||||||
|
|
||||||
export interface LocalAugmentation {
|
|
||||||
id: string
|
|
||||||
name: string
|
|
||||||
source: 'builtin' | 'npm' | 'local'
|
|
||||||
path: string
|
|
||||||
manifest?: AugmentationManifest
|
|
||||||
package?: {
|
|
||||||
name: string
|
|
||||||
version: string
|
|
||||||
description?: string
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Discovers augmentations installed locally
|
|
||||||
*/
|
|
||||||
export class LocalAugmentationDiscovery {
|
|
||||||
private builtInAugmentations: Map<string, LocalAugmentation> = new Map()
|
|
||||||
private installedAugmentations: Map<string, LocalAugmentation> = new Map()
|
|
||||||
|
|
||||||
constructor(private options: {
|
|
||||||
brainyPath?: string
|
|
||||||
projectPath?: string
|
|
||||||
scanNodeModules?: boolean
|
|
||||||
} = {}) {
|
|
||||||
this.options = {
|
|
||||||
brainyPath: this.options.brainyPath || this.findBrainyPath(),
|
|
||||||
projectPath: this.options.projectPath || process.cwd(),
|
|
||||||
scanNodeModules: this.options.scanNodeModules ?? true
|
|
||||||
}
|
|
||||||
|
|
||||||
// Register built-in augmentations
|
|
||||||
this.registerBuiltIn()
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Register built-in augmentations that ship with Brainy
|
|
||||||
*/
|
|
||||||
private registerBuiltIn(): void {
|
|
||||||
const builtIn = [
|
|
||||||
{ id: 'cache', name: 'Cache', path: 'cacheAugmentation' },
|
|
||||||
{ id: 'batch', name: 'Batch Processing', path: 'batchProcessingAugmentation' },
|
|
||||||
{ id: 'entity-registry', name: 'Entity Registry', path: 'entityRegistryAugmentation' },
|
|
||||||
{ id: 'index', name: 'Index', path: 'indexAugmentation' },
|
|
||||||
{ id: 'metrics', name: 'Metrics', path: 'metricsAugmentation' },
|
|
||||||
{ id: 'monitoring', name: 'Monitoring', path: 'monitoringAugmentation' },
|
|
||||||
{ id: 'connection-pool', name: 'Connection Pool', path: 'connectionPoolAugmentation' },
|
|
||||||
{ id: 'request-deduplicator', name: 'Request Deduplicator', path: 'requestDeduplicatorAugmentation' },
|
|
||||||
{ id: 'api-server', name: 'API Server', path: 'apiServerAugmentation' },
|
|
||||||
{ id: 'neural-import', name: 'Neural Import', path: 'neuralImport' },
|
|
||||||
{ id: 'intelligent-verb-scoring', name: 'Intelligent Verb Scoring', path: 'intelligentVerbScoringAugmentation' },
|
|
||||||
{ id: 'universal-display', name: 'Universal Display', path: 'universalDisplayAugmentation' },
|
|
||||||
// Storage augmentations
|
|
||||||
{ id: 'memory-storage', name: 'Memory Storage', path: 'storageAugmentations', export: 'MemoryStorageAugmentation' },
|
|
||||||
{ id: 'filesystem-storage', name: 'FileSystem Storage', path: 'storageAugmentations', export: 'FileSystemStorageAugmentation' },
|
|
||||||
{ id: 'opfs-storage', name: 'OPFS Storage', path: 'storageAugmentations', export: 'OPFSStorageAugmentation' },
|
|
||||||
{ id: 's3-storage', name: 'S3 Storage', path: 'storageAugmentations', export: 'S3StorageAugmentation' },
|
|
||||||
]
|
|
||||||
|
|
||||||
for (const aug of builtIn) {
|
|
||||||
this.builtInAugmentations.set(aug.id, {
|
|
||||||
id: aug.id,
|
|
||||||
name: aug.name,
|
|
||||||
source: 'builtin',
|
|
||||||
path: `@soulcraft/brainy/augmentations/${aug.path}`,
|
|
||||||
package: {
|
|
||||||
name: '@soulcraft/brainy',
|
|
||||||
version: 'builtin',
|
|
||||||
description: `Built-in ${aug.name} augmentation`
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Find Brainy installation path
|
|
||||||
*/
|
|
||||||
private findBrainyPath(): string {
|
|
||||||
// Try to find brainy in node_modules
|
|
||||||
const possiblePaths = [
|
|
||||||
join(process.cwd(), 'node_modules', '@soulcraft', 'brainy'),
|
|
||||||
join(process.cwd(), 'node_modules', 'brainy'),
|
|
||||||
join(process.cwd(), '..', 'node_modules', '@soulcraft', 'brainy'),
|
|
||||||
]
|
|
||||||
|
|
||||||
for (const path of possiblePaths) {
|
|
||||||
if (existsSync(path)) {
|
|
||||||
return path
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Fallback to current directory
|
|
||||||
return process.cwd()
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Discover all augmentations
|
|
||||||
*/
|
|
||||||
async discoverAll(): Promise<LocalAugmentation[]> {
|
|
||||||
const augmentations: LocalAugmentation[] = []
|
|
||||||
|
|
||||||
// Add built-in augmentations
|
|
||||||
augmentations.push(...this.builtInAugmentations.values())
|
|
||||||
|
|
||||||
// Scan node_modules if enabled
|
|
||||||
if (this.options.scanNodeModules) {
|
|
||||||
const installed = await this.scanNodeModules()
|
|
||||||
augmentations.push(...installed)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Scan local project
|
|
||||||
const local = await this.scanLocalProject()
|
|
||||||
augmentations.push(...local)
|
|
||||||
|
|
||||||
return augmentations
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Scan node_modules for installed augmentations
|
|
||||||
*/
|
|
||||||
private async scanNodeModules(): Promise<LocalAugmentation[]> {
|
|
||||||
const augmentations: LocalAugmentation[] = []
|
|
||||||
const nodeModulesPath = join(this.options.projectPath!, 'node_modules')
|
|
||||||
|
|
||||||
if (!existsSync(nodeModulesPath)) {
|
|
||||||
return augmentations
|
|
||||||
}
|
|
||||||
|
|
||||||
// Scan @brainy/* packages
|
|
||||||
const brainyPath = join(nodeModulesPath, '@brainy')
|
|
||||||
if (existsSync(brainyPath)) {
|
|
||||||
const packages = readdirSync(brainyPath)
|
|
||||||
for (const pkg of packages) {
|
|
||||||
const augmentation = await this.loadPackageAugmentation(join(brainyPath, pkg))
|
|
||||||
if (augmentation) {
|
|
||||||
augmentations.push(augmentation)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Scan packages with brainy-augmentation keyword
|
|
||||||
const packages = readdirSync(nodeModulesPath)
|
|
||||||
for (const pkg of packages) {
|
|
||||||
if (pkg.startsWith('@') || pkg.startsWith('.')) continue
|
|
||||||
|
|
||||||
const pkgPath = join(nodeModulesPath, pkg)
|
|
||||||
const packageJson = this.loadPackageJson(pkgPath)
|
|
||||||
|
|
||||||
if (packageJson?.keywords?.includes('brainy-augmentation')) {
|
|
||||||
const augmentation = await this.loadPackageAugmentation(pkgPath)
|
|
||||||
if (augmentation) {
|
|
||||||
augmentations.push(augmentation)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return augmentations
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Scan local project for augmentations
|
|
||||||
*/
|
|
||||||
private async scanLocalProject(): Promise<LocalAugmentation[]> {
|
|
||||||
const augmentations: LocalAugmentation[] = []
|
|
||||||
|
|
||||||
// Check for augmentations directory
|
|
||||||
const augPath = join(this.options.projectPath!, 'augmentations')
|
|
||||||
if (existsSync(augPath)) {
|
|
||||||
const files = readdirSync(augPath)
|
|
||||||
for (const file of files) {
|
|
||||||
if (file.endsWith('.ts') || file.endsWith('.js')) {
|
|
||||||
const name = file.replace(/\.(ts|js)$/, '')
|
|
||||||
augmentations.push({
|
|
||||||
id: name,
|
|
||||||
name: this.humanizeName(name),
|
|
||||||
source: 'local',
|
|
||||||
path: join(augPath, file)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return augmentations
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Load augmentation from package
|
|
||||||
*/
|
|
||||||
private async loadPackageAugmentation(pkgPath: string): Promise<LocalAugmentation | null> {
|
|
||||||
const packageJson = this.loadPackageJson(pkgPath)
|
|
||||||
if (!packageJson) return null
|
|
||||||
|
|
||||||
// Check if it's a brainy augmentation
|
|
||||||
const isBrainyAug = packageJson.keywords?.includes('brainy-augmentation') ||
|
|
||||||
packageJson.brainy?.type === 'augmentation'
|
|
||||||
|
|
||||||
if (!isBrainyAug) return null
|
|
||||||
|
|
||||||
const manifest = packageJson.brainy?.manifest || null
|
|
||||||
|
|
||||||
return {
|
|
||||||
id: packageJson.brainy?.id || packageJson.name.replace('@brainy/', ''),
|
|
||||||
name: packageJson.brainy?.name || packageJson.name,
|
|
||||||
source: 'npm',
|
|
||||||
path: pkgPath,
|
|
||||||
manifest,
|
|
||||||
package: {
|
|
||||||
name: packageJson.name,
|
|
||||||
version: packageJson.version,
|
|
||||||
description: packageJson.description
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Load package.json
|
|
||||||
*/
|
|
||||||
private loadPackageJson(pkgPath: string): any {
|
|
||||||
const packageJsonPath = join(pkgPath, 'package.json')
|
|
||||||
if (!existsSync(packageJsonPath)) return null
|
|
||||||
|
|
||||||
try {
|
|
||||||
return JSON.parse(readFileSync(packageJsonPath, 'utf8'))
|
|
||||||
} catch {
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Convert name to human-readable format
|
|
||||||
*/
|
|
||||||
private humanizeName(name: string): string {
|
|
||||||
return name
|
|
||||||
.replace(/[-_]/g, ' ')
|
|
||||||
.replace(/augmentation/gi, '')
|
|
||||||
.replace(/\b\w/g, l => l.toUpperCase())
|
|
||||||
.trim()
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get built-in augmentations
|
|
||||||
*/
|
|
||||||
getBuiltIn(): LocalAugmentation[] {
|
|
||||||
return Array.from(this.builtInAugmentations.values())
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get installed augmentations
|
|
||||||
*/
|
|
||||||
getInstalled(): LocalAugmentation[] {
|
|
||||||
return Array.from(this.installedAugmentations.values())
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Check if augmentation is installed
|
|
||||||
*/
|
|
||||||
isInstalled(id: string): boolean {
|
|
||||||
return this.builtInAugmentations.has(id) ||
|
|
||||||
this.installedAugmentations.has(id)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get import path for augmentation
|
|
||||||
*/
|
|
||||||
getImportPath(id: string): string | null {
|
|
||||||
const aug = this.builtInAugmentations.get(id) ||
|
|
||||||
this.installedAugmentations.get(id)
|
|
||||||
return aug?.path || null
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Load augmentation module dynamically
|
|
||||||
*/
|
|
||||||
async loadAugmentation(id: string): Promise<any> {
|
|
||||||
const path = this.getImportPath(id)
|
|
||||||
if (!path) {
|
|
||||||
throw new Error(`Augmentation ${id} not found`)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Dynamic import
|
|
||||||
const module = await import(path)
|
|
||||||
return module.default || module
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,443 +0,0 @@
|
||||||
/**
|
|
||||||
* Runtime Augmentation Loader
|
|
||||||
*
|
|
||||||
* Dynamically loads and registers augmentations at runtime
|
|
||||||
* Supports CDN loading for browser environments and npm imports for Node.js
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { BrainyAugmentation, AugmentationRegistry } from '../brainyAugmentation.js'
|
|
||||||
import { AugmentationManifest } from '../manifest.js'
|
|
||||||
|
|
||||||
export interface LoaderOptions {
|
|
||||||
cdnUrl?: string
|
|
||||||
allowUnsafe?: boolean
|
|
||||||
sandbox?: boolean
|
|
||||||
timeout?: number
|
|
||||||
cache?: boolean
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface LoadedAugmentation {
|
|
||||||
id: string
|
|
||||||
instance: BrainyAugmentation
|
|
||||||
manifest: AugmentationManifest
|
|
||||||
source: 'cdn' | 'npm' | 'local'
|
|
||||||
loadTime: number
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Runtime Augmentation Loader
|
|
||||||
*
|
|
||||||
* Enables dynamic loading of augmentations from various sources
|
|
||||||
*/
|
|
||||||
export class RuntimeAugmentationLoader {
|
|
||||||
private loaded: Map<string, LoadedAugmentation> = new Map()
|
|
||||||
private cdnCache: Map<string, any> = new Map()
|
|
||||||
private registry?: AugmentationRegistry
|
|
||||||
|
|
||||||
constructor(private options: LoaderOptions = {}) {
|
|
||||||
this.options = {
|
|
||||||
cdnUrl: options.cdnUrl || 'https://cdn.soulcraft.com/augmentations',
|
|
||||||
allowUnsafe: options.allowUnsafe || false,
|
|
||||||
sandbox: options.sandbox || true,
|
|
||||||
timeout: options.timeout || 30000,
|
|
||||||
cache: options.cache ?? true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Set the augmentation registry
|
|
||||||
*/
|
|
||||||
setRegistry(registry: AugmentationRegistry): void {
|
|
||||||
this.registry = registry
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Load augmentation from CDN (browser)
|
|
||||||
*/
|
|
||||||
async loadFromCDN(
|
|
||||||
id: string,
|
|
||||||
version: string = 'latest',
|
|
||||||
config?: any
|
|
||||||
): Promise<LoadedAugmentation> {
|
|
||||||
// Check if already loaded
|
|
||||||
if (this.loaded.has(id)) {
|
|
||||||
return this.loaded.get(id)!
|
|
||||||
}
|
|
||||||
|
|
||||||
const url = `${this.options.cdnUrl}/${id}@${version}/index.js`
|
|
||||||
const startTime = Date.now()
|
|
||||||
|
|
||||||
try {
|
|
||||||
// Load module from CDN
|
|
||||||
const module = await this.loadCDNModule(url)
|
|
||||||
|
|
||||||
// Extract augmentation class
|
|
||||||
const AugmentationClass = module.default || module[Object.keys(module)[0]]
|
|
||||||
|
|
||||||
if (!AugmentationClass) {
|
|
||||||
throw new Error(`No augmentation class found in module ${id}`)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create instance
|
|
||||||
const instance = new AugmentationClass(config)
|
|
||||||
|
|
||||||
// Validate it's a proper augmentation
|
|
||||||
if (!this.isValidAugmentation(instance)) {
|
|
||||||
throw new Error(`Invalid augmentation: ${id}`)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get manifest
|
|
||||||
const manifest = instance.getManifest ? instance.getManifest() : {
|
|
||||||
id,
|
|
||||||
name: id,
|
|
||||||
version: version,
|
|
||||||
description: `Dynamically loaded ${id}`,
|
|
||||||
category: 'external'
|
|
||||||
}
|
|
||||||
|
|
||||||
const loaded: LoadedAugmentation = {
|
|
||||||
id,
|
|
||||||
instance,
|
|
||||||
manifest,
|
|
||||||
source: 'cdn',
|
|
||||||
loadTime: Date.now() - startTime
|
|
||||||
}
|
|
||||||
|
|
||||||
// Cache
|
|
||||||
this.loaded.set(id, loaded)
|
|
||||||
|
|
||||||
// Auto-register if registry is set
|
|
||||||
if (this.registry) {
|
|
||||||
this.registry.register(instance)
|
|
||||||
}
|
|
||||||
|
|
||||||
return loaded
|
|
||||||
} catch (error) {
|
|
||||||
throw new Error(`Failed to load augmentation ${id} from CDN: ${error}`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Load augmentation from NPM (Node.js)
|
|
||||||
*/
|
|
||||||
async loadFromNPM(
|
|
||||||
packageName: string,
|
|
||||||
config?: any
|
|
||||||
): Promise<LoadedAugmentation> {
|
|
||||||
// Check if already loaded
|
|
||||||
const id = packageName.replace('@', '').replace('/', '-')
|
|
||||||
if (this.loaded.has(id)) {
|
|
||||||
return this.loaded.get(id)!
|
|
||||||
}
|
|
||||||
|
|
||||||
const startTime = Date.now()
|
|
||||||
|
|
||||||
try {
|
|
||||||
// Dynamic import
|
|
||||||
const module = await import(packageName)
|
|
||||||
|
|
||||||
// Extract augmentation class
|
|
||||||
const AugmentationClass = module.default || module[Object.keys(module)[0]]
|
|
||||||
|
|
||||||
if (!AugmentationClass) {
|
|
||||||
throw new Error(`No augmentation class found in package ${packageName}`)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create instance
|
|
||||||
const instance = new AugmentationClass(config)
|
|
||||||
|
|
||||||
// Validate
|
|
||||||
if (!this.isValidAugmentation(instance)) {
|
|
||||||
throw new Error(`Invalid augmentation in package: ${packageName}`)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get manifest
|
|
||||||
const manifest = instance.getManifest ? instance.getManifest() : {
|
|
||||||
id,
|
|
||||||
name: packageName,
|
|
||||||
version: 'unknown',
|
|
||||||
description: `Loaded from ${packageName}`,
|
|
||||||
category: 'external'
|
|
||||||
}
|
|
||||||
|
|
||||||
const loaded: LoadedAugmentation = {
|
|
||||||
id,
|
|
||||||
instance,
|
|
||||||
manifest,
|
|
||||||
source: 'npm',
|
|
||||||
loadTime: Date.now() - startTime
|
|
||||||
}
|
|
||||||
|
|
||||||
// Cache
|
|
||||||
this.loaded.set(id, loaded)
|
|
||||||
|
|
||||||
// Auto-register
|
|
||||||
if (this.registry) {
|
|
||||||
this.registry.register(instance)
|
|
||||||
}
|
|
||||||
|
|
||||||
return loaded
|
|
||||||
} catch (error) {
|
|
||||||
throw new Error(`Failed to load augmentation from NPM ${packageName}: ${error}`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Load augmentation from local file
|
|
||||||
*/
|
|
||||||
async loadFromFile(
|
|
||||||
path: string,
|
|
||||||
config?: any
|
|
||||||
): Promise<LoadedAugmentation> {
|
|
||||||
const startTime = Date.now()
|
|
||||||
|
|
||||||
try {
|
|
||||||
// Dynamic import
|
|
||||||
const module = await import(path)
|
|
||||||
|
|
||||||
// Extract augmentation class
|
|
||||||
const AugmentationClass = module.default || module[Object.keys(module)[0]]
|
|
||||||
|
|
||||||
if (!AugmentationClass) {
|
|
||||||
throw new Error(`No augmentation class found in file ${path}`)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create instance
|
|
||||||
const instance = new AugmentationClass(config)
|
|
||||||
|
|
||||||
// Validate
|
|
||||||
if (!this.isValidAugmentation(instance)) {
|
|
||||||
throw new Error(`Invalid augmentation in file: ${path}`)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Extract ID from path
|
|
||||||
const id = path.split('/').pop()?.replace(/\.(js|ts)$/, '') || 'unknown'
|
|
||||||
|
|
||||||
// Get manifest
|
|
||||||
const manifest = instance.getManifest ? instance.getManifest() : {
|
|
||||||
id,
|
|
||||||
name: id,
|
|
||||||
version: 'local',
|
|
||||||
description: `Loaded from ${path}`,
|
|
||||||
category: 'local'
|
|
||||||
}
|
|
||||||
|
|
||||||
const loaded: LoadedAugmentation = {
|
|
||||||
id,
|
|
||||||
instance,
|
|
||||||
manifest,
|
|
||||||
source: 'local',
|
|
||||||
loadTime: Date.now() - startTime
|
|
||||||
}
|
|
||||||
|
|
||||||
// Cache
|
|
||||||
this.loaded.set(id, loaded)
|
|
||||||
|
|
||||||
// Auto-register
|
|
||||||
if (this.registry) {
|
|
||||||
this.registry.register(instance)
|
|
||||||
}
|
|
||||||
|
|
||||||
return loaded
|
|
||||||
} catch (error) {
|
|
||||||
throw new Error(`Failed to load augmentation from file ${path}: ${error}`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Load multiple augmentations
|
|
||||||
*/
|
|
||||||
async loadBatch(
|
|
||||||
augmentations: Array<{
|
|
||||||
source: 'cdn' | 'npm' | 'local'
|
|
||||||
id: string
|
|
||||||
version?: string
|
|
||||||
path?: string
|
|
||||||
config?: any
|
|
||||||
}>
|
|
||||||
): Promise<LoadedAugmentation[]> {
|
|
||||||
const results = await Promise.allSettled(
|
|
||||||
augmentations.map(aug => {
|
|
||||||
switch (aug.source) {
|
|
||||||
case 'cdn':
|
|
||||||
return this.loadFromCDN(aug.id, aug.version, aug.config)
|
|
||||||
case 'npm':
|
|
||||||
return this.loadFromNPM(aug.id, aug.config)
|
|
||||||
case 'local':
|
|
||||||
return this.loadFromFile(aug.path || aug.id, aug.config)
|
|
||||||
default:
|
|
||||||
return Promise.reject(new Error(`Unknown source: ${aug.source}`))
|
|
||||||
}
|
|
||||||
})
|
|
||||||
)
|
|
||||||
|
|
||||||
const loaded: LoadedAugmentation[] = []
|
|
||||||
const errors: string[] = []
|
|
||||||
|
|
||||||
for (const result of results) {
|
|
||||||
if (result.status === 'fulfilled') {
|
|
||||||
loaded.push(result.value)
|
|
||||||
} else {
|
|
||||||
errors.push(result.reason.message)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (errors.length > 0) {
|
|
||||||
console.warn('Some augmentations failed to load:', errors)
|
|
||||||
}
|
|
||||||
|
|
||||||
return loaded
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Unload augmentation
|
|
||||||
*/
|
|
||||||
unload(id: string): boolean {
|
|
||||||
const loaded = this.loaded.get(id)
|
|
||||||
if (!loaded) return false
|
|
||||||
|
|
||||||
// Shutdown if possible
|
|
||||||
if (loaded.instance.shutdown) {
|
|
||||||
loaded.instance.shutdown()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Remove from registry if set
|
|
||||||
// Note: Registry doesn't have unregister yet, would need to add
|
|
||||||
|
|
||||||
// Remove from cache
|
|
||||||
this.loaded.delete(id)
|
|
||||||
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get loaded augmentations
|
|
||||||
*/
|
|
||||||
getLoaded(): LoadedAugmentation[] {
|
|
||||||
return Array.from(this.loaded.values())
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Check if augmentation is loaded
|
|
||||||
*/
|
|
||||||
isLoaded(id: string): boolean {
|
|
||||||
return this.loaded.has(id)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get loaded augmentation
|
|
||||||
*/
|
|
||||||
getAugmentation(id: string): BrainyAugmentation | null {
|
|
||||||
return this.loaded.get(id)?.instance || null
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Load CDN module (browser-specific)
|
|
||||||
*/
|
|
||||||
private async loadCDNModule(url: string): Promise<any> {
|
|
||||||
// Check cache
|
|
||||||
if (this.options.cache && this.cdnCache.has(url)) {
|
|
||||||
return this.cdnCache.get(url)
|
|
||||||
}
|
|
||||||
|
|
||||||
// In browser environment
|
|
||||||
if (typeof window !== 'undefined') {
|
|
||||||
return new Promise((resolve, reject) => {
|
|
||||||
const timeout = setTimeout(() => {
|
|
||||||
reject(new Error(`Timeout loading ${url}`))
|
|
||||||
}, this.options.timeout!)
|
|
||||||
|
|
||||||
// Create script element
|
|
||||||
const script = document.createElement('script')
|
|
||||||
script.type = 'module'
|
|
||||||
script.src = url
|
|
||||||
|
|
||||||
// Handle load
|
|
||||||
script.onload = async () => {
|
|
||||||
clearTimeout(timeout)
|
|
||||||
|
|
||||||
// The module should register itself on window
|
|
||||||
const moduleId = url.split('/').pop()?.split('@')[0]
|
|
||||||
if (moduleId && (window as any)[moduleId]) {
|
|
||||||
const module = (window as any)[moduleId]
|
|
||||||
|
|
||||||
// Cache
|
|
||||||
if (this.options.cache) {
|
|
||||||
this.cdnCache.set(url, module)
|
|
||||||
}
|
|
||||||
|
|
||||||
resolve(module)
|
|
||||||
} else {
|
|
||||||
reject(new Error(`Module not found on window: ${moduleId}`))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Handle error
|
|
||||||
script.onerror = () => {
|
|
||||||
clearTimeout(timeout)
|
|
||||||
reject(new Error(`Failed to load script: ${url}`))
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add to document
|
|
||||||
document.head.appendChild(script)
|
|
||||||
})
|
|
||||||
} else {
|
|
||||||
// In Node.js, use dynamic import
|
|
||||||
const module = await import(url)
|
|
||||||
|
|
||||||
// Cache
|
|
||||||
if (this.options.cache) {
|
|
||||||
this.cdnCache.set(url, module)
|
|
||||||
}
|
|
||||||
|
|
||||||
return module
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Validate augmentation instance
|
|
||||||
*/
|
|
||||||
private isValidAugmentation(instance: any): boolean {
|
|
||||||
// Check required properties
|
|
||||||
return !!(
|
|
||||||
instance.name &&
|
|
||||||
instance.timing &&
|
|
||||||
instance.operations &&
|
|
||||||
instance.priority !== undefined &&
|
|
||||||
typeof instance.execute === 'function' &&
|
|
||||||
typeof instance.initialize === 'function'
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Clear all caches
|
|
||||||
*/
|
|
||||||
clearCache(): void {
|
|
||||||
this.cdnCache.clear()
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get load statistics
|
|
||||||
*/
|
|
||||||
getStats(): {
|
|
||||||
loaded: number
|
|
||||||
totalLoadTime: number
|
|
||||||
averageLoadTime: number
|
|
||||||
sources: Record<string, number>
|
|
||||||
} {
|
|
||||||
const loaded = Array.from(this.loaded.values())
|
|
||||||
const totalLoadTime = loaded.reduce((sum, aug) => sum + aug.loadTime, 0)
|
|
||||||
|
|
||||||
const sources = loaded.reduce((acc, aug) => {
|
|
||||||
acc[aug.source] = (acc[aug.source] || 0) + 1
|
|
||||||
return acc
|
|
||||||
}, {} as Record<string, number>)
|
|
||||||
|
|
||||||
return {
|
|
||||||
loaded: loaded.length,
|
|
||||||
totalLoadTime,
|
|
||||||
averageLoadTime: loaded.length > 0 ? totalLoadTime / loaded.length : 0,
|
|
||||||
sources
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,375 +0,0 @@
|
||||||
/**
|
|
||||||
* Universal Display Augmentation - Intelligent Caching System
|
|
||||||
*
|
|
||||||
* High-performance LRU cache with smart eviction and batch optimization
|
|
||||||
* Designed for minimal memory footprint and maximum hit ratio
|
|
||||||
*/
|
|
||||||
|
|
||||||
import type { DisplayCacheEntry, ComputedDisplayFields, DisplayAugmentationStats } from './types.js'
|
|
||||||
|
|
||||||
/**
|
|
||||||
* LRU (Least Recently Used) Cache for computed display fields
|
|
||||||
* Optimized for the display augmentation use case
|
|
||||||
*/
|
|
||||||
export class DisplayCache {
|
|
||||||
private cache = new Map<string, DisplayCacheEntry>()
|
|
||||||
private readonly maxSize: number
|
|
||||||
private stats = {
|
|
||||||
hits: 0,
|
|
||||||
misses: 0,
|
|
||||||
evictions: 0,
|
|
||||||
totalComputations: 0,
|
|
||||||
totalComputationTime: 0
|
|
||||||
}
|
|
||||||
|
|
||||||
constructor(maxSize: number = 1000) {
|
|
||||||
this.maxSize = maxSize
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get cached display fields with LRU update
|
|
||||||
* @param key Cache key
|
|
||||||
* @returns Cached fields or null if not found
|
|
||||||
*/
|
|
||||||
get(key: string): ComputedDisplayFields | null {
|
|
||||||
const entry = this.cache.get(key)
|
|
||||||
|
|
||||||
if (!entry) {
|
|
||||||
this.stats.misses++
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
|
|
||||||
// Update LRU - move to end
|
|
||||||
this.cache.delete(key)
|
|
||||||
entry.lastAccessed = Date.now()
|
|
||||||
entry.accessCount++
|
|
||||||
this.cache.set(key, entry)
|
|
||||||
|
|
||||||
this.stats.hits++
|
|
||||||
return entry.fields
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Store computed display fields in cache
|
|
||||||
* @param key Cache key
|
|
||||||
* @param fields Computed display fields
|
|
||||||
* @param computationTime Time taken to compute (for stats)
|
|
||||||
*/
|
|
||||||
set(key: string, fields: ComputedDisplayFields, computationTime?: number): void {
|
|
||||||
// Remove if already exists (for LRU update)
|
|
||||||
if (this.cache.has(key)) {
|
|
||||||
this.cache.delete(key)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create cache entry
|
|
||||||
const entry: DisplayCacheEntry = {
|
|
||||||
fields,
|
|
||||||
lastAccessed: Date.now(),
|
|
||||||
accessCount: 1
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add to end (most recently used)
|
|
||||||
this.cache.set(key, entry)
|
|
||||||
|
|
||||||
// Update stats
|
|
||||||
this.stats.totalComputations++
|
|
||||||
if (computationTime) {
|
|
||||||
this.stats.totalComputationTime += computationTime
|
|
||||||
}
|
|
||||||
|
|
||||||
// Evict oldest if over capacity
|
|
||||||
if (this.cache.size > this.maxSize) {
|
|
||||||
this.evictOldest()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Check if a key exists in cache without affecting LRU order
|
|
||||||
* @param key Cache key
|
|
||||||
* @returns True if key exists
|
|
||||||
*/
|
|
||||||
has(key: string): boolean {
|
|
||||||
return this.cache.has(key)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Generate cache key from data
|
|
||||||
* @param id Entity ID (preferred)
|
|
||||||
* @param data Fallback data for key generation
|
|
||||||
* @param entityType Type of entity (noun/verb)
|
|
||||||
* @returns Cache key string
|
|
||||||
*/
|
|
||||||
generateKey(id?: string, data?: any, entityType: 'noun' | 'verb' = 'noun'): string {
|
|
||||||
// Use ID if available (most reliable)
|
|
||||||
if (id) {
|
|
||||||
return `${entityType}:${id}`
|
|
||||||
}
|
|
||||||
|
|
||||||
// Generate hash from data
|
|
||||||
if (data) {
|
|
||||||
const dataString = JSON.stringify(data, Object.keys(data).sort())
|
|
||||||
const hash = this.simpleHash(dataString)
|
|
||||||
return `${entityType}:hash:${hash}`
|
|
||||||
}
|
|
||||||
|
|
||||||
// Fallback to timestamp (not ideal but prevents crashes)
|
|
||||||
return `${entityType}:temp:${Date.now()}:${Math.random()}`
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Clear all cached entries
|
|
||||||
*/
|
|
||||||
clear(): void {
|
|
||||||
this.cache.clear()
|
|
||||||
this.stats = {
|
|
||||||
hits: 0,
|
|
||||||
misses: 0,
|
|
||||||
evictions: 0,
|
|
||||||
totalComputations: 0,
|
|
||||||
totalComputationTime: 0
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get cache statistics
|
|
||||||
* @returns Cache performance statistics
|
|
||||||
*/
|
|
||||||
getStats(): DisplayAugmentationStats {
|
|
||||||
const hitRatio = this.stats.hits + this.stats.misses > 0
|
|
||||||
? this.stats.hits / (this.stats.hits + this.stats.misses)
|
|
||||||
: 0
|
|
||||||
|
|
||||||
const avgComputationTime = this.stats.totalComputations > 0
|
|
||||||
? this.stats.totalComputationTime / this.stats.totalComputations
|
|
||||||
: 0
|
|
||||||
|
|
||||||
// Analyze cached types for common types statistics
|
|
||||||
const typeCount = new Map<string, number>()
|
|
||||||
let fastestComputation = Infinity
|
|
||||||
let slowestComputation = 0
|
|
||||||
|
|
||||||
for (const entry of this.cache.values()) {
|
|
||||||
const type = entry.fields.type
|
|
||||||
typeCount.set(type, (typeCount.get(type) || 0) + 1)
|
|
||||||
}
|
|
||||||
|
|
||||||
const commonTypes = Array.from(typeCount.entries())
|
|
||||||
.sort(([,a], [,b]) => b - a)
|
|
||||||
.slice(0, 10)
|
|
||||||
.map(([type, count]) => ({
|
|
||||||
type,
|
|
||||||
count,
|
|
||||||
percentage: Math.round((count / this.cache.size) * 100)
|
|
||||||
}))
|
|
||||||
|
|
||||||
return {
|
|
||||||
totalComputations: this.stats.totalComputations,
|
|
||||||
cacheHitRatio: Math.round(hitRatio * 100) / 100,
|
|
||||||
averageComputationTime: Math.round(avgComputationTime * 100) / 100,
|
|
||||||
commonTypes,
|
|
||||||
performance: {
|
|
||||||
fastestComputation,
|
|
||||||
slowestComputation,
|
|
||||||
totalComputationTime: this.stats.totalComputationTime
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get current cache size
|
|
||||||
* @returns Number of cached entries
|
|
||||||
*/
|
|
||||||
size(): number {
|
|
||||||
return this.cache.size
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get cache capacity
|
|
||||||
* @returns Maximum cache size
|
|
||||||
*/
|
|
||||||
capacity(): number {
|
|
||||||
return this.maxSize
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Evict least recently used entry
|
|
||||||
*/
|
|
||||||
private evictOldest(): void {
|
|
||||||
// First entry is oldest (LRU)
|
|
||||||
const firstKey = this.cache.keys().next().value
|
|
||||||
if (firstKey) {
|
|
||||||
this.cache.delete(firstKey)
|
|
||||||
this.stats.evictions++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Simple hash function for cache keys
|
|
||||||
* @param str String to hash
|
|
||||||
* @returns Simple hash number
|
|
||||||
*/
|
|
||||||
private simpleHash(str: string): number {
|
|
||||||
let hash = 0
|
|
||||||
for (let i = 0; i < str.length; i++) {
|
|
||||||
const char = str.charCodeAt(i)
|
|
||||||
hash = ((hash << 5) - hash) + char
|
|
||||||
hash = hash & hash // Convert to 32-bit integer
|
|
||||||
}
|
|
||||||
return Math.abs(hash)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Optimize cache by removing stale entries
|
|
||||||
* Called periodically to maintain cache health
|
|
||||||
*/
|
|
||||||
optimizeCache(): void {
|
|
||||||
const now = Date.now()
|
|
||||||
const maxAge = 24 * 60 * 60 * 1000 // 24 hours
|
|
||||||
const minAccessCount = 2 // Minimum access count to keep
|
|
||||||
|
|
||||||
const toDelete: string[] = []
|
|
||||||
|
|
||||||
for (const [key, entry] of this.cache.entries()) {
|
|
||||||
// Remove very old entries with low access count
|
|
||||||
if (now - entry.lastAccessed > maxAge && entry.accessCount < minAccessCount) {
|
|
||||||
toDelete.push(key)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Remove stale entries
|
|
||||||
for (const key of toDelete) {
|
|
||||||
this.cache.delete(key)
|
|
||||||
this.stats.evictions++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Precompute display fields for a batch of entities
|
|
||||||
* @param entities Array of entities with their compute functions
|
|
||||||
* @returns Promise resolving when batch is complete
|
|
||||||
*/
|
|
||||||
async batchPrecompute<T>(
|
|
||||||
entities: Array<{
|
|
||||||
key: string
|
|
||||||
computeFn: () => Promise<ComputedDisplayFields>
|
|
||||||
}>
|
|
||||||
): Promise<void> {
|
|
||||||
const promises = entities.map(async ({ key, computeFn }) => {
|
|
||||||
if (!this.has(key)) {
|
|
||||||
const startTime = Date.now()
|
|
||||||
try {
|
|
||||||
const fields = await computeFn()
|
|
||||||
const computationTime = Date.now() - startTime
|
|
||||||
this.set(key, fields, computationTime)
|
|
||||||
} catch (error) {
|
|
||||||
console.warn(`Batch precompute failed for key ${key}:`, error)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
await Promise.all(promises)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Request deduplicator for batch processing
|
|
||||||
* Prevents duplicate computations for the same data
|
|
||||||
*/
|
|
||||||
export class RequestDeduplicator {
|
|
||||||
private pendingRequests = new Map<string, Promise<ComputedDisplayFields>>()
|
|
||||||
private readonly batchSize: number
|
|
||||||
|
|
||||||
constructor(batchSize: number = 50) {
|
|
||||||
this.batchSize = batchSize
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Deduplicate computation request
|
|
||||||
* @param key Unique key for the computation
|
|
||||||
* @param computeFn Function to compute the result
|
|
||||||
* @returns Promise that resolves to the computed fields
|
|
||||||
*/
|
|
||||||
async deduplicate(
|
|
||||||
key: string,
|
|
||||||
computeFn: () => Promise<ComputedDisplayFields>
|
|
||||||
): Promise<ComputedDisplayFields> {
|
|
||||||
// Return existing promise if already pending
|
|
||||||
if (this.pendingRequests.has(key)) {
|
|
||||||
return this.pendingRequests.get(key)!
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create new computation promise
|
|
||||||
const promise = computeFn().finally(() => {
|
|
||||||
// Remove from pending when complete
|
|
||||||
this.pendingRequests.delete(key)
|
|
||||||
})
|
|
||||||
|
|
||||||
this.pendingRequests.set(key, promise)
|
|
||||||
return promise
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get number of pending requests
|
|
||||||
* @returns Number of pending computations
|
|
||||||
*/
|
|
||||||
getPendingCount(): number {
|
|
||||||
return this.pendingRequests.size
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Clear all pending requests
|
|
||||||
*/
|
|
||||||
clear(): void {
|
|
||||||
this.pendingRequests.clear()
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Shutdown the deduplicator
|
|
||||||
*/
|
|
||||||
shutdown(): void {
|
|
||||||
this.clear()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Global cache instance management
|
|
||||||
* Provides singleton access to display cache
|
|
||||||
*/
|
|
||||||
let globalDisplayCache: DisplayCache | null = null
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get global display cache instance
|
|
||||||
* @param maxSize Optional cache size (only used on first call)
|
|
||||||
* @returns Shared display cache instance
|
|
||||||
*/
|
|
||||||
export function getGlobalDisplayCache(maxSize?: number): DisplayCache {
|
|
||||||
if (!globalDisplayCache) {
|
|
||||||
globalDisplayCache = new DisplayCache(maxSize)
|
|
||||||
|
|
||||||
// Set up periodic optimization
|
|
||||||
setInterval(() => {
|
|
||||||
globalDisplayCache?.optimizeCache()
|
|
||||||
}, 60 * 60 * 1000) // Every hour
|
|
||||||
}
|
|
||||||
|
|
||||||
return globalDisplayCache
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Clear global cache (for testing or memory management)
|
|
||||||
*/
|
|
||||||
export function clearGlobalDisplayCache(): void {
|
|
||||||
if (globalDisplayCache) {
|
|
||||||
globalDisplayCache.clear()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Shutdown global cache and cleanup
|
|
||||||
*/
|
|
||||||
export function shutdownGlobalDisplayCache(): void {
|
|
||||||
if (globalDisplayCache) {
|
|
||||||
globalDisplayCache.clear()
|
|
||||||
globalDisplayCache = null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,429 +0,0 @@
|
||||||
/**
|
|
||||||
* Universal Display Augmentation - Smart Field Patterns
|
|
||||||
*
|
|
||||||
* Intelligent field detection patterns for mapping user data to display fields
|
|
||||||
* Uses semantic understanding and common naming conventions
|
|
||||||
*/
|
|
||||||
|
|
||||||
import type { FieldPattern, FieldComputationContext } from './types.js'
|
|
||||||
import { NounType, VerbType } from '../../types/graphTypes.js'
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Universal field patterns that work across all data types
|
|
||||||
* Ordered by confidence level (highest first)
|
|
||||||
*/
|
|
||||||
export const UNIVERSAL_FIELD_PATTERNS: FieldPattern[] = [
|
|
||||||
// Title/Name Patterns (Highest Priority)
|
|
||||||
{
|
|
||||||
fields: ['name', 'title', 'displayName', 'label', 'heading'],
|
|
||||||
displayField: 'title',
|
|
||||||
confidence: 0.95
|
|
||||||
},
|
|
||||||
{
|
|
||||||
fields: ['firstName', 'lastName', 'fullName', 'realName'],
|
|
||||||
displayField: 'title',
|
|
||||||
confidence: 0.9,
|
|
||||||
applicableTypes: [NounType.Person],
|
|
||||||
transform: (value: any, context: FieldComputationContext) => {
|
|
||||||
const { metadata } = context
|
|
||||||
if (metadata.firstName && metadata.lastName) {
|
|
||||||
return `${metadata.firstName} ${metadata.lastName}`.trim()
|
|
||||||
}
|
|
||||||
return String(value || '')
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
fields: ['companyName', 'organizationName', 'orgName', 'businessName'],
|
|
||||||
displayField: 'title',
|
|
||||||
confidence: 0.9,
|
|
||||||
applicableTypes: [NounType.Organization]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
fields: ['filename', 'fileName', 'documentTitle', 'docName'],
|
|
||||||
displayField: 'title',
|
|
||||||
confidence: 0.85,
|
|
||||||
applicableTypes: [NounType.Document, NounType.File, NounType.Media]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
fields: ['projectName', 'projectTitle', 'initiative'],
|
|
||||||
displayField: 'title',
|
|
||||||
confidence: 0.9,
|
|
||||||
applicableTypes: [NounType.Project]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
fields: ['taskName', 'taskTitle', 'action', 'todo'],
|
|
||||||
displayField: 'title',
|
|
||||||
confidence: 0.85,
|
|
||||||
applicableTypes: [NounType.Task]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
fields: ['subject', 'topic', 'headline', 'caption'],
|
|
||||||
displayField: 'title',
|
|
||||||
confidence: 0.8
|
|
||||||
},
|
|
||||||
|
|
||||||
// Description Patterns (High Priority)
|
|
||||||
{
|
|
||||||
fields: ['description', 'summary', 'overview', 'details'],
|
|
||||||
displayField: 'description',
|
|
||||||
confidence: 0.9
|
|
||||||
},
|
|
||||||
{
|
|
||||||
fields: ['bio', 'biography', 'profile', 'about'],
|
|
||||||
displayField: 'description',
|
|
||||||
confidence: 0.85,
|
|
||||||
applicableTypes: [NounType.Person]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
fields: ['content', 'text', 'body', 'message'],
|
|
||||||
displayField: 'description',
|
|
||||||
confidence: 0.8
|
|
||||||
},
|
|
||||||
{
|
|
||||||
fields: ['abstract', 'excerpt', 'snippet', 'preview'],
|
|
||||||
displayField: 'description',
|
|
||||||
confidence: 0.75
|
|
||||||
},
|
|
||||||
{
|
|
||||||
fields: ['notes', 'comments', 'remarks', 'observations'],
|
|
||||||
displayField: 'description',
|
|
||||||
confidence: 0.7
|
|
||||||
},
|
|
||||||
|
|
||||||
// Type Patterns (Medium Priority)
|
|
||||||
{
|
|
||||||
fields: ['type', 'category', 'classification', 'kind'],
|
|
||||||
displayField: 'type',
|
|
||||||
confidence: 0.9
|
|
||||||
},
|
|
||||||
{
|
|
||||||
fields: ['nounType', 'entityType', 'objectType'],
|
|
||||||
displayField: 'type',
|
|
||||||
confidence: 0.95
|
|
||||||
},
|
|
||||||
{
|
|
||||||
fields: ['role', 'position', 'jobTitle', 'occupation'],
|
|
||||||
displayField: 'type',
|
|
||||||
confidence: 0.8,
|
|
||||||
applicableTypes: [NounType.Person],
|
|
||||||
transform: (value: any) => String(value || 'Person')
|
|
||||||
},
|
|
||||||
{
|
|
||||||
fields: ['industry', 'sector', 'domain', 'field'],
|
|
||||||
displayField: 'type',
|
|
||||||
confidence: 0.7,
|
|
||||||
applicableTypes: [NounType.Organization]
|
|
||||||
},
|
|
||||||
|
|
||||||
// Tag Patterns (Medium Priority)
|
|
||||||
{
|
|
||||||
fields: ['tags', 'keywords', 'labels', 'categories'],
|
|
||||||
displayField: 'tags',
|
|
||||||
confidence: 0.85
|
|
||||||
},
|
|
||||||
{
|
|
||||||
fields: ['topics', 'subjects', 'themes'],
|
|
||||||
displayField: 'tags',
|
|
||||||
confidence: 0.8
|
|
||||||
}
|
|
||||||
]
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Type-specific field patterns for enhanced detection
|
|
||||||
* Used when we know the specific type of the entity
|
|
||||||
*/
|
|
||||||
export const TYPE_SPECIFIC_PATTERNS: Record<string, FieldPattern[]> = {
|
|
||||||
[NounType.Person]: [
|
|
||||||
{
|
|
||||||
fields: ['email', 'emailAddress', 'contactEmail'],
|
|
||||||
displayField: 'description',
|
|
||||||
confidence: 0.7,
|
|
||||||
transform: (value: any, context: FieldComputationContext) => {
|
|
||||||
const { metadata } = context
|
|
||||||
const role = metadata.role || metadata.jobTitle || metadata.position
|
|
||||||
const company = metadata.company || metadata.organization || metadata.employer
|
|
||||||
|
|
||||||
const parts = []
|
|
||||||
if (role) parts.push(role)
|
|
||||||
if (company) parts.push(`at ${company}`)
|
|
||||||
if (parts.length === 0 && value) parts.push(`Contact: ${value}`)
|
|
||||||
|
|
||||||
return parts.join(' ') || 'Person'
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
fields: ['phone', 'phoneNumber', 'mobile', 'cell'],
|
|
||||||
displayField: 'tags',
|
|
||||||
confidence: 0.6
|
|
||||||
}
|
|
||||||
],
|
|
||||||
|
|
||||||
[NounType.Organization]: [
|
|
||||||
{
|
|
||||||
fields: ['website', 'url', 'homepage', 'domain'],
|
|
||||||
displayField: 'description',
|
|
||||||
confidence: 0.7,
|
|
||||||
transform: (value: any, context: FieldComputationContext) => {
|
|
||||||
const { metadata } = context
|
|
||||||
const industry = metadata.industry || metadata.sector
|
|
||||||
const location = metadata.location || metadata.city || metadata.country
|
|
||||||
|
|
||||||
const parts = []
|
|
||||||
if (industry) parts.push(industry)
|
|
||||||
parts.push('organization')
|
|
||||||
if (location) parts.push(`in ${location}`)
|
|
||||||
|
|
||||||
return parts.join(' ')
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
fields: ['employees', 'size', 'headcount'],
|
|
||||||
displayField: 'tags',
|
|
||||||
confidence: 0.6
|
|
||||||
}
|
|
||||||
],
|
|
||||||
|
|
||||||
[NounType.Project]: [
|
|
||||||
{
|
|
||||||
fields: ['status', 'phase', 'stage', 'state'],
|
|
||||||
displayField: 'description',
|
|
||||||
confidence: 0.8,
|
|
||||||
transform: (value: any, context: FieldComputationContext) => {
|
|
||||||
const { metadata } = context
|
|
||||||
const status = String(value || 'active').toLowerCase()
|
|
||||||
const budget = metadata.budget || metadata.cost
|
|
||||||
const lead = metadata.lead || metadata.manager || metadata.owner
|
|
||||||
|
|
||||||
const parts = []
|
|
||||||
parts.push(status.charAt(0).toUpperCase() + status.slice(1))
|
|
||||||
if (metadata.description) parts.push('project')
|
|
||||||
if (lead) parts.push(`led by ${lead}`)
|
|
||||||
if (budget) parts.push(`($${parseInt(String(budget)).toLocaleString()} budget)`)
|
|
||||||
|
|
||||||
return parts.join(' ')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
],
|
|
||||||
|
|
||||||
[NounType.Document]: [
|
|
||||||
{
|
|
||||||
fields: ['author', 'creator', 'writer'],
|
|
||||||
displayField: 'description',
|
|
||||||
confidence: 0.7,
|
|
||||||
transform: (value: any, context: FieldComputationContext) => {
|
|
||||||
const { metadata } = context
|
|
||||||
const docType = metadata.type || metadata.category || 'document'
|
|
||||||
const date = metadata.date || metadata.created || metadata.published
|
|
||||||
|
|
||||||
const parts = []
|
|
||||||
if (docType) parts.push(docType)
|
|
||||||
if (value) parts.push(`by ${value}`)
|
|
||||||
if (date) {
|
|
||||||
const dateStr = new Date(date).toLocaleDateString()
|
|
||||||
parts.push(`(${dateStr})`)
|
|
||||||
}
|
|
||||||
|
|
||||||
return parts.join(' ')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
],
|
|
||||||
|
|
||||||
[NounType.Task]: [
|
|
||||||
{
|
|
||||||
fields: ['priority', 'urgency', 'importance'],
|
|
||||||
displayField: 'tags',
|
|
||||||
confidence: 0.7
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get field patterns for a specific entity type
|
|
||||||
* @param entityType The type of entity (noun or verb)
|
|
||||||
* @param specificType Optional specific noun/verb type
|
|
||||||
* @returns Array of applicable field patterns
|
|
||||||
*/
|
|
||||||
export function getFieldPatterns(entityType: 'noun' | 'verb', specificType?: string): FieldPattern[] {
|
|
||||||
const patterns = [...UNIVERSAL_FIELD_PATTERNS]
|
|
||||||
|
|
||||||
if (entityType === 'noun' && specificType && TYPE_SPECIFIC_PATTERNS[specificType]) {
|
|
||||||
patterns.unshift(...TYPE_SPECIFIC_PATTERNS[specificType])
|
|
||||||
}
|
|
||||||
|
|
||||||
return patterns.sort((a, b) => b.confidence - a.confidence)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Priority fields for different entity types (for AI analysis)
|
|
||||||
* Used by the BrainyTypes and neural processing
|
|
||||||
*/
|
|
||||||
export const TYPE_PRIORITY_FIELDS: Record<string, string[]> = {
|
|
||||||
[NounType.Person]: [
|
|
||||||
'name', 'firstName', 'lastName', 'fullName', 'displayName',
|
|
||||||
'email', 'role', 'jobTitle', 'position', 'title',
|
|
||||||
'bio', 'description', 'about', 'profile',
|
|
||||||
'company', 'organization', 'employer'
|
|
||||||
],
|
|
||||||
|
|
||||||
[NounType.Organization]: [
|
|
||||||
'name', 'companyName', 'organizationName', 'title',
|
|
||||||
'industry', 'sector', 'domain', 'type',
|
|
||||||
'description', 'about', 'summary',
|
|
||||||
'location', 'city', 'country', 'headquarters',
|
|
||||||
'website', 'url'
|
|
||||||
],
|
|
||||||
|
|
||||||
[NounType.Project]: [
|
|
||||||
'name', 'projectName', 'title', 'projectTitle',
|
|
||||||
'description', 'summary', 'overview', 'goal',
|
|
||||||
'status', 'phase', 'stage', 'state',
|
|
||||||
'lead', 'manager', 'owner', 'team',
|
|
||||||
'budget', 'timeline', 'deadline'
|
|
||||||
],
|
|
||||||
|
|
||||||
[NounType.Document]: [
|
|
||||||
'title', 'filename', 'name', 'subject',
|
|
||||||
'content', 'text', 'body', 'summary',
|
|
||||||
'author', 'creator', 'writer',
|
|
||||||
'type', 'category', 'format',
|
|
||||||
'date', 'created', 'published'
|
|
||||||
],
|
|
||||||
|
|
||||||
[NounType.Task]: [
|
|
||||||
'title', 'name', 'taskName', 'action',
|
|
||||||
'description', 'details', 'notes',
|
|
||||||
'status', 'state', 'priority',
|
|
||||||
'assignee', 'owner', 'responsible',
|
|
||||||
'due', 'deadline', 'dueDate'
|
|
||||||
],
|
|
||||||
|
|
||||||
[NounType.Event]: [
|
|
||||||
'name', 'title', 'eventName',
|
|
||||||
'description', 'details', 'summary',
|
|
||||||
'startDate', 'endDate', 'date', 'time',
|
|
||||||
'location', 'venue', 'address',
|
|
||||||
'organizer', 'host', 'creator'
|
|
||||||
],
|
|
||||||
|
|
||||||
[NounType.Product]: [
|
|
||||||
'name', 'productName', 'title',
|
|
||||||
'description', 'summary', 'features',
|
|
||||||
'price', 'cost', 'value',
|
|
||||||
'category', 'type', 'brand',
|
|
||||||
'manufacturer', 'vendor'
|
|
||||||
]
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get priority fields for intelligent analysis
|
|
||||||
* @param entityType The type of entity
|
|
||||||
* @param specificType Optional specific type
|
|
||||||
* @returns Array of priority field names
|
|
||||||
*/
|
|
||||||
export function getPriorityFields(entityType: 'noun' | 'verb', specificType?: string): string[] {
|
|
||||||
if (entityType === 'noun' && specificType && TYPE_PRIORITY_FIELDS[specificType]) {
|
|
||||||
return TYPE_PRIORITY_FIELDS[specificType]
|
|
||||||
}
|
|
||||||
|
|
||||||
// Default priority fields for any entity
|
|
||||||
return [
|
|
||||||
'name', 'title', 'label', 'displayName',
|
|
||||||
'description', 'summary', 'about', 'details',
|
|
||||||
'type', 'category', 'kind', 'classification',
|
|
||||||
'tags', 'keywords', 'labels'
|
|
||||||
]
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Smart field value extraction with type-aware processing
|
|
||||||
* @param data The data object to extract from
|
|
||||||
* @param pattern The field pattern to apply
|
|
||||||
* @param context The computation context
|
|
||||||
* @returns The extracted and processed field value
|
|
||||||
*/
|
|
||||||
export function extractFieldValue(
|
|
||||||
data: any,
|
|
||||||
pattern: FieldPattern,
|
|
||||||
context: FieldComputationContext
|
|
||||||
): any {
|
|
||||||
// Find the first matching field
|
|
||||||
let value: any = null
|
|
||||||
let matchedField: string | null = null
|
|
||||||
|
|
||||||
for (const field of pattern.fields) {
|
|
||||||
if (data[field] !== undefined && data[field] !== null && data[field] !== '') {
|
|
||||||
value = data[field]
|
|
||||||
matchedField = field
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (value === null) return null
|
|
||||||
|
|
||||||
// Apply transformation if provided
|
|
||||||
if (pattern.transform) {
|
|
||||||
try {
|
|
||||||
return pattern.transform(value, context)
|
|
||||||
} catch (error) {
|
|
||||||
console.warn(`Field transformation error for ${matchedField}:`, error)
|
|
||||||
return String(value)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Default processing based on display field type
|
|
||||||
switch (pattern.displayField) {
|
|
||||||
case 'title':
|
|
||||||
case 'description':
|
|
||||||
case 'type':
|
|
||||||
return String(value)
|
|
||||||
|
|
||||||
case 'tags':
|
|
||||||
if (Array.isArray(value)) return value
|
|
||||||
if (typeof value === 'string') {
|
|
||||||
return value.split(/[,;]\s*|\s+/).filter(Boolean)
|
|
||||||
}
|
|
||||||
return [String(value)]
|
|
||||||
|
|
||||||
default:
|
|
||||||
return value
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Calculate confidence score for field detection
|
|
||||||
* @param pattern The field pattern
|
|
||||||
* @param context The computation context
|
|
||||||
* @param value The extracted value
|
|
||||||
* @returns Confidence score (0-1)
|
|
||||||
*/
|
|
||||||
export function calculateFieldConfidence(
|
|
||||||
pattern: FieldPattern,
|
|
||||||
context: FieldComputationContext,
|
|
||||||
value: any
|
|
||||||
): number {
|
|
||||||
let confidence = pattern.confidence
|
|
||||||
|
|
||||||
// Boost confidence if type matches
|
|
||||||
if (pattern.applicableTypes && context.typeResult) {
|
|
||||||
if (pattern.applicableTypes.includes(context.typeResult.type)) {
|
|
||||||
confidence = Math.min(1.0, confidence + 0.1)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Reduce confidence for empty or very short values
|
|
||||||
if (typeof value === 'string') {
|
|
||||||
if (value.length < 2) {
|
|
||||||
confidence *= 0.5
|
|
||||||
} else if (value.length < 5) {
|
|
||||||
confidence *= 0.8
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Reduce confidence for generic values
|
|
||||||
const genericValues = ['unknown', 'n/a', 'null', 'undefined', 'default']
|
|
||||||
if (typeof value === 'string' && genericValues.includes(value.toLowerCase())) {
|
|
||||||
confidence *= 0.3
|
|
||||||
}
|
|
||||||
|
|
||||||
return Math.max(0, Math.min(1, confidence))
|
|
||||||
}
|
|
||||||
|
|
@ -1,76 +0,0 @@
|
||||||
/**
|
|
||||||
* Universal Display Augmentation - Clean Display
|
|
||||||
*
|
|
||||||
* Simple, clean display without icons - focusing on AI-powered
|
|
||||||
* titles, descriptions, and smart formatting that matches
|
|
||||||
* Soulcraft's minimal aesthetic
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { NounType, VerbType } from '../../types/graphTypes.js'
|
|
||||||
|
|
||||||
/**
|
|
||||||
* No icon mappings - clean, minimal approach
|
|
||||||
* The real value is in AI-generated titles and enhanced descriptions,
|
|
||||||
* not visual clutter that doesn't align with professional aesthetics
|
|
||||||
*/
|
|
||||||
export const NOUN_TYPE_ICONS: Record<string, string> = {}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* No icon mappings for verbs either - focus on clear relationship descriptions
|
|
||||||
* Human-readable relationship text is more valuable than symbolic representations
|
|
||||||
*/
|
|
||||||
export const VERB_TYPE_ICONS: Record<string, string> = {}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get icon for a noun type (returns empty string for clean display)
|
|
||||||
* @param type The noun type
|
|
||||||
* @returns Empty string (no icons)
|
|
||||||
*/
|
|
||||||
export function getNounIcon(type: string): string {
|
|
||||||
return '' // Clean, no icons
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get icon for a verb type (returns empty string for clean display)
|
|
||||||
* @param type The verb type
|
|
||||||
* @returns Empty string (no icons)
|
|
||||||
*/
|
|
||||||
export function getVerbIcon(type: string): string {
|
|
||||||
return '' // Clean, no icons
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get coverage statistics (for backwards compatibility)
|
|
||||||
* @returns Coverage info showing clean approach
|
|
||||||
*/
|
|
||||||
export function getIconCoverage() {
|
|
||||||
return {
|
|
||||||
nounTypes: {
|
|
||||||
total: 'Clean display - no icons needed',
|
|
||||||
covered: 'Focus on AI-powered content'
|
|
||||||
},
|
|
||||||
verbTypes: {
|
|
||||||
total: 'Clean display - no icons needed',
|
|
||||||
covered: 'Focus on relationship descriptions'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Check if an icon exists for a type (always false for clean display)
|
|
||||||
* @param type The type to check
|
|
||||||
* @param entityType Whether it's a noun or verb
|
|
||||||
* @returns Always false (no icons)
|
|
||||||
*/
|
|
||||||
export function hasIcon(type: string, entityType: 'noun' | 'verb' = 'noun'): boolean {
|
|
||||||
return false // Clean approach - no icons
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get fallback icon (returns empty string for clean display)
|
|
||||||
* @param entityType The entity type
|
|
||||||
* @returns Empty string (no fallback icons)
|
|
||||||
*/
|
|
||||||
export function getFallbackIcon(entityType: 'noun' | 'verb' = 'noun'): string {
|
|
||||||
return '' // Clean, minimal display
|
|
||||||
}
|
|
||||||
|
|
@ -1,539 +0,0 @@
|
||||||
/**
|
|
||||||
* Universal Display Augmentation - Intelligent Computation Engine
|
|
||||||
*
|
|
||||||
* Leverages existing Brainy AI infrastructure for intelligent field computation:
|
|
||||||
* - BrainyTypes for semantic type detection
|
|
||||||
* - Neural Import patterns for field analysis
|
|
||||||
* - JSON processing utilities for field extraction
|
|
||||||
* - Existing NounType/VerbType taxonomy (42+127 types)
|
|
||||||
*/
|
|
||||||
|
|
||||||
import type {
|
|
||||||
ComputedDisplayFields,
|
|
||||||
FieldComputationContext,
|
|
||||||
TypeMatchResult,
|
|
||||||
DisplayConfig
|
|
||||||
} from './types.js'
|
|
||||||
import type { VectorDocument, GraphVerb } from '../../coreTypes.js'
|
|
||||||
import { BrainyTypes, getBrainyTypes } from '../typeMatching/brainyTypes.js'
|
|
||||||
import { getNounIcon, getVerbIcon } from './iconMappings.js'
|
|
||||||
import {
|
|
||||||
getFieldPatterns,
|
|
||||||
getPriorityFields,
|
|
||||||
extractFieldValue,
|
|
||||||
calculateFieldConfidence
|
|
||||||
} from './fieldPatterns.js'
|
|
||||||
import { prepareJsonForVectorization, extractFieldFromJson } from '../../utils/jsonProcessing.js'
|
|
||||||
import { NounType, VerbType } from '../../types/graphTypes.js'
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Intelligent field computation engine
|
|
||||||
* Coordinates AI-powered analysis with fallback heuristics
|
|
||||||
*/
|
|
||||||
export class IntelligentComputationEngine {
|
|
||||||
private typeMatcher: BrainyTypes | null = null
|
|
||||||
protected config: DisplayConfig
|
|
||||||
private initialized = false
|
|
||||||
|
|
||||||
constructor(config: DisplayConfig) {
|
|
||||||
this.config = config
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Initialize the computation engine with AI components
|
|
||||||
*/
|
|
||||||
async initialize(): Promise<void> {
|
|
||||||
if (this.initialized) return
|
|
||||||
|
|
||||||
try {
|
|
||||||
// 🧠 LEVERAGE YOUR EXISTING AI INFRASTRUCTURE
|
|
||||||
this.typeMatcher = await getBrainyTypes()
|
|
||||||
if (this.typeMatcher) {
|
|
||||||
console.log('🎨 Display computation engine initialized with AI intelligence')
|
|
||||||
} else {
|
|
||||||
console.warn('🎨 Display computation engine running in basic mode (AI unavailable)')
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
console.warn('🎨 AI initialization failed, using heuristic fallback:', error)
|
|
||||||
}
|
|
||||||
|
|
||||||
this.initialized = true
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Compute display fields for a noun using AI-first approach
|
|
||||||
* @param data The noun data/metadata
|
|
||||||
* @param id Optional noun ID
|
|
||||||
* @returns Computed display fields
|
|
||||||
*/
|
|
||||||
async computeNounDisplay(data: any, id?: string): Promise<ComputedDisplayFields> {
|
|
||||||
const startTime = Date.now()
|
|
||||||
|
|
||||||
try {
|
|
||||||
// 🟢 PRIMARY PATH: Use your existing AI intelligence
|
|
||||||
if (this.typeMatcher) {
|
|
||||||
return await this.computeWithAI(data, 'noun', { id })
|
|
||||||
}
|
|
||||||
|
|
||||||
// 🟡 FALLBACK PATH: Use heuristic patterns
|
|
||||||
return await this.computeWithHeuristics(data, 'noun', { id })
|
|
||||||
|
|
||||||
} catch (error) {
|
|
||||||
console.warn('Display computation failed, using minimal fallback:', error)
|
|
||||||
return this.createMinimalDisplay(data, 'noun')
|
|
||||||
} finally {
|
|
||||||
const computationTime = Date.now() - startTime
|
|
||||||
if (this.config.debugMode) {
|
|
||||||
console.log(`Display computation took ${computationTime}ms`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Compute display fields for a verb using AI-first approach
|
|
||||||
* @param verb The verb/relationship data
|
|
||||||
* @returns Computed display fields
|
|
||||||
*/
|
|
||||||
async computeVerbDisplay(verb: GraphVerb): Promise<ComputedDisplayFields> {
|
|
||||||
const startTime = Date.now()
|
|
||||||
|
|
||||||
try {
|
|
||||||
// 🟢 PRIMARY PATH: Use your existing AI for verb analysis
|
|
||||||
if (this.typeMatcher) {
|
|
||||||
return await this.computeVerbWithAI(verb)
|
|
||||||
}
|
|
||||||
|
|
||||||
// 🟡 FALLBACK PATH: Use heuristic patterns for verbs
|
|
||||||
return await this.computeWithHeuristics(verb, 'verb')
|
|
||||||
|
|
||||||
} catch (error) {
|
|
||||||
console.warn('Verb display computation failed, using minimal fallback:', error)
|
|
||||||
return this.createMinimalDisplay(verb, 'verb')
|
|
||||||
} finally {
|
|
||||||
const computationTime = Date.now() - startTime
|
|
||||||
if (this.config.debugMode) {
|
|
||||||
console.log(`Verb display computation took ${computationTime}ms`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* AI-powered computation using your existing BrainyTypes
|
|
||||||
* @param data Entity data/metadata
|
|
||||||
* @param entityType Type of entity (noun/verb)
|
|
||||||
* @param options Additional options
|
|
||||||
* @returns AI-computed display fields
|
|
||||||
*/
|
|
||||||
private async computeWithAI(
|
|
||||||
data: any,
|
|
||||||
entityType: 'noun' | 'verb',
|
|
||||||
options: { id?: string } = {}
|
|
||||||
): Promise<ComputedDisplayFields> {
|
|
||||||
|
|
||||||
// 🧠 USE YOUR EXISTING TYPE DETECTION AI
|
|
||||||
const typeResult = await this.typeMatcher!.matchNounType(data)
|
|
||||||
|
|
||||||
// Create computation context
|
|
||||||
const context: FieldComputationContext = {
|
|
||||||
data,
|
|
||||||
metadata: data,
|
|
||||||
typeResult,
|
|
||||||
config: this.config,
|
|
||||||
entityType
|
|
||||||
}
|
|
||||||
|
|
||||||
// 🟢 INTELLIGENT FIELD EXTRACTION using your patterns + AI insights
|
|
||||||
const displayFields = {
|
|
||||||
title: await this.computeIntelligentTitle(context),
|
|
||||||
description: await this.computeIntelligentDescription(context),
|
|
||||||
type: typeResult.type,
|
|
||||||
tags: await this.computeIntelligentTags(context),
|
|
||||||
confidence: typeResult.confidence,
|
|
||||||
reasoning: this.config.debugMode ? typeResult.reasoning : undefined,
|
|
||||||
alternatives: this.config.debugMode ? typeResult.alternatives : undefined,
|
|
||||||
computedAt: Date.now(),
|
|
||||||
version: '1.0.0'
|
|
||||||
}
|
|
||||||
|
|
||||||
return displayFields
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* AI-powered verb computation using relationship analysis
|
|
||||||
* @param verb The verb/relationship
|
|
||||||
* @returns AI-computed display fields
|
|
||||||
*/
|
|
||||||
private async computeVerbWithAI(verb: GraphVerb): Promise<ComputedDisplayFields> {
|
|
||||||
|
|
||||||
// 🧠 USE YOUR EXISTING VERB TYPE DETECTION
|
|
||||||
const typeResult = await this.typeMatcher!.matchVerbType(verb, 0.7)
|
|
||||||
|
|
||||||
// Create verb computation context
|
|
||||||
const context: FieldComputationContext = {
|
|
||||||
data: verb,
|
|
||||||
metadata: verb.metadata || {},
|
|
||||||
typeResult,
|
|
||||||
config: this.config,
|
|
||||||
entityType: 'verb',
|
|
||||||
verbContext: {
|
|
||||||
sourceId: verb.sourceId,
|
|
||||||
targetId: verb.targetId,
|
|
||||||
verbType: verb.type
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 🟢 INTELLIGENT VERB DISPLAY COMPUTATION
|
|
||||||
const displayFields = {
|
|
||||||
title: await this.computeVerbTitle(context),
|
|
||||||
description: await this.computeVerbDescription(context),
|
|
||||||
type: typeResult.type,
|
|
||||||
tags: await this.computeVerbTags(context),
|
|
||||||
relationship: await this.computeHumanReadableRelationship(context),
|
|
||||||
confidence: typeResult.confidence,
|
|
||||||
reasoning: this.config.debugMode ? typeResult.reasoning : undefined,
|
|
||||||
alternatives: this.config.debugMode ? typeResult.alternatives : undefined,
|
|
||||||
computedAt: Date.now(),
|
|
||||||
version: '1.0.0'
|
|
||||||
}
|
|
||||||
|
|
||||||
return displayFields
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Heuristic computation when AI is unavailable
|
|
||||||
* @param data Entity data
|
|
||||||
* @param entityType Type of entity
|
|
||||||
* @param options Additional options
|
|
||||||
* @returns Heuristically computed display fields
|
|
||||||
*/
|
|
||||||
private async computeWithHeuristics(
|
|
||||||
data: any,
|
|
||||||
entityType: 'noun' | 'verb',
|
|
||||||
options: { id?: string } = {}
|
|
||||||
): Promise<ComputedDisplayFields> {
|
|
||||||
|
|
||||||
// Use basic type detection
|
|
||||||
const detectedType = this.detectTypeHeuristically(data, entityType)
|
|
||||||
const typeResult: TypeMatchResult = {
|
|
||||||
type: detectedType,
|
|
||||||
confidence: 0.6, // Lower confidence for heuristics
|
|
||||||
reasoning: 'Heuristic detection (AI unavailable)',
|
|
||||||
alternatives: []
|
|
||||||
}
|
|
||||||
|
|
||||||
const context: FieldComputationContext = {
|
|
||||||
data,
|
|
||||||
metadata: data,
|
|
||||||
typeResult: typeResult,
|
|
||||||
config: this.config,
|
|
||||||
entityType
|
|
||||||
}
|
|
||||||
|
|
||||||
// Use pattern-based field extraction
|
|
||||||
const patterns = getFieldPatterns(entityType, detectedType)
|
|
||||||
|
|
||||||
return {
|
|
||||||
title: this.extractFieldWithPatterns(data, patterns, 'title') || 'Untitled',
|
|
||||||
description: this.extractFieldWithPatterns(data, patterns, 'description') || 'No description',
|
|
||||||
type: detectedType,
|
|
||||||
tags: this.extractFieldWithPatterns(data, patterns, 'tags') || [],
|
|
||||||
confidence: typeResult.confidence,
|
|
||||||
reasoning: this.config.debugMode ? typeResult.reasoning : undefined,
|
|
||||||
computedAt: Date.now(),
|
|
||||||
version: '1.0.0'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Compute intelligent title using AI insights and your field extraction
|
|
||||||
* @param context Computation context with AI results
|
|
||||||
* @returns Computed title
|
|
||||||
*/
|
|
||||||
private async computeIntelligentTitle(context: FieldComputationContext): Promise<string> {
|
|
||||||
const { data, typeResult } = context
|
|
||||||
|
|
||||||
// 🟢 USE TYPE-SPECIFIC LOGIC based on your NounType taxonomy
|
|
||||||
switch (typeResult?.type) {
|
|
||||||
case NounType.Person:
|
|
||||||
return this.computePersonTitle(data)
|
|
||||||
|
|
||||||
case NounType.Organization:
|
|
||||||
return this.computeOrganizationTitle(data)
|
|
||||||
|
|
||||||
case NounType.Project:
|
|
||||||
return this.computeProjectTitle(data)
|
|
||||||
|
|
||||||
case NounType.Document:
|
|
||||||
return this.computeDocumentTitle(data)
|
|
||||||
|
|
||||||
default:
|
|
||||||
// 🟢 LEVERAGE YOUR JSON PROCESSING for unknown types
|
|
||||||
return this.extractBestTitle(data, typeResult?.type)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Compute intelligent description using AI insights and context
|
|
||||||
* @param context Computation context
|
|
||||||
* @returns Enhanced description
|
|
||||||
*/
|
|
||||||
private async computeIntelligentDescription(context: FieldComputationContext): Promise<string> {
|
|
||||||
const { data, typeResult } = context
|
|
||||||
|
|
||||||
// 🟢 USE YOUR EXISTING JSON PROCESSING for vectorization-quality text
|
|
||||||
const priorityFields = getPriorityFields('noun', typeResult?.type)
|
|
||||||
const enhancedText = prepareJsonForVectorization(data, {
|
|
||||||
priorityFields,
|
|
||||||
includeFieldNames: false,
|
|
||||||
maxDepth: 2
|
|
||||||
})
|
|
||||||
|
|
||||||
// Create context-aware description based on type
|
|
||||||
return this.createContextAwareDescription(data, typeResult, enhancedText)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Compute intelligent tags using type analysis
|
|
||||||
* @param context Computation context
|
|
||||||
* @returns Generated tags array
|
|
||||||
*/
|
|
||||||
private async computeIntelligentTags(context: FieldComputationContext): Promise<string[]> {
|
|
||||||
const { data, typeResult } = context
|
|
||||||
const tags: string[] = []
|
|
||||||
|
|
||||||
// Add type-based tag
|
|
||||||
if (typeResult?.type) {
|
|
||||||
tags.push(typeResult.type.toLowerCase())
|
|
||||||
}
|
|
||||||
|
|
||||||
// Extract explicit tags from data
|
|
||||||
const explicitTags = this.extractExplicitTags(data)
|
|
||||||
tags.push(...explicitTags)
|
|
||||||
|
|
||||||
// Add semantic tags based on AI analysis
|
|
||||||
if (typeResult && this.typeMatcher) {
|
|
||||||
const semanticTags = this.generateSemanticTags(data, typeResult)
|
|
||||||
tags.push(...semanticTags)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Remove duplicates and return
|
|
||||||
return [...new Set(tags.filter(Boolean))]
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Compute verb title (relationship summary)
|
|
||||||
* @param context Verb computation context
|
|
||||||
* @returns Verb title
|
|
||||||
*/
|
|
||||||
private async computeVerbTitle(context: FieldComputationContext): Promise<string> {
|
|
||||||
const { verbContext, typeResult } = context
|
|
||||||
|
|
||||||
if (!verbContext) return 'Relationship'
|
|
||||||
|
|
||||||
const { sourceId, targetId } = verbContext
|
|
||||||
const relationshipType = typeResult?.type || 'RelatedTo'
|
|
||||||
|
|
||||||
// Try to get readable names for source and target
|
|
||||||
// This could be enhanced to actually resolve the entities
|
|
||||||
return `${sourceId} ${this.getReadableVerbPhrase(relationshipType)} ${targetId}`
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create minimal display for error cases
|
|
||||||
* @param data Entity data
|
|
||||||
* @param entityType Entity type
|
|
||||||
* @returns Minimal display fields
|
|
||||||
*/
|
|
||||||
private createMinimalDisplay(data: any, entityType: 'noun' | 'verb'): ComputedDisplayFields {
|
|
||||||
return {
|
|
||||||
title: data.name || data.title || data.id || 'Untitled',
|
|
||||||
description: data.description || data.summary || 'No description available',
|
|
||||||
type: entityType === 'noun' ? 'Item' : 'RelatedTo',
|
|
||||||
tags: [],
|
|
||||||
confidence: 0.1, // Very low confidence for fallback
|
|
||||||
computedAt: Date.now(),
|
|
||||||
version: '1.0.0'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Helper methods for specific noun types
|
|
||||||
private computePersonTitle(data: any): string {
|
|
||||||
if (data.firstName && data.lastName) {
|
|
||||||
return `${data.firstName} ${data.lastName}`.trim()
|
|
||||||
}
|
|
||||||
return data.name || data.fullName || data.displayName || data.firstName || data.lastName || 'Person'
|
|
||||||
}
|
|
||||||
|
|
||||||
private computeOrganizationTitle(data: any): string {
|
|
||||||
return data.name || data.companyName || data.organizationName || data.title || 'Organization'
|
|
||||||
}
|
|
||||||
|
|
||||||
private computeProjectTitle(data: any): string {
|
|
||||||
return data.name || data.projectName || data.title || data.projectTitle || 'Project'
|
|
||||||
}
|
|
||||||
|
|
||||||
private computeDocumentTitle(data: any): string {
|
|
||||||
return data.title || data.filename || data.name || data.subject || 'Document'
|
|
||||||
}
|
|
||||||
|
|
||||||
private extractBestTitle(data: any, type?: string): string {
|
|
||||||
const titleFields = ['name', 'title', 'displayName', 'label', 'subject', 'heading']
|
|
||||||
|
|
||||||
for (const field of titleFields) {
|
|
||||||
if (data[field]) return String(data[field])
|
|
||||||
}
|
|
||||||
|
|
||||||
return data.id || Object.keys(data)[0] || 'Untitled'
|
|
||||||
}
|
|
||||||
|
|
||||||
private createContextAwareDescription(data: any, typeResult?: TypeMatchResult, enhancedText?: string): string {
|
|
||||||
// Start with basic description fields
|
|
||||||
const basicDesc = data.description || data.summary || data.about || data.details
|
|
||||||
|
|
||||||
if (basicDesc) return String(basicDesc)
|
|
||||||
|
|
||||||
// Use enhanced text from JSON processing
|
|
||||||
if (enhancedText && enhancedText.length > 10) {
|
|
||||||
return enhancedText.substring(0, 200) + (enhancedText.length > 200 ? '...' : '')
|
|
||||||
}
|
|
||||||
|
|
||||||
// Generate from available fields
|
|
||||||
const parts = []
|
|
||||||
if (data.role) parts.push(data.role)
|
|
||||||
if (data.company) parts.push(`at ${data.company}`)
|
|
||||||
if (data.location) parts.push(`in ${data.location}`)
|
|
||||||
|
|
||||||
return parts.length > 0 ? parts.join(' ') : 'No description available'
|
|
||||||
}
|
|
||||||
|
|
||||||
private extractExplicitTags(data: any): string[] {
|
|
||||||
const tagFields = ['tags', 'keywords', 'labels', 'categories', 'topics']
|
|
||||||
|
|
||||||
for (const field of tagFields) {
|
|
||||||
if (data[field]) {
|
|
||||||
if (Array.isArray(data[field])) {
|
|
||||||
return data[field].map(String).filter(Boolean)
|
|
||||||
}
|
|
||||||
if (typeof data[field] === 'string') {
|
|
||||||
return data[field].split(/[,;]\s*|\s+/).filter(Boolean)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return []
|
|
||||||
}
|
|
||||||
|
|
||||||
private generateSemanticTags(data: any, typeResult: TypeMatchResult): string[] {
|
|
||||||
const tags: string[] = []
|
|
||||||
|
|
||||||
// Add confidence-based tags
|
|
||||||
if (typeResult.confidence > 0.9) tags.push('verified')
|
|
||||||
else if (typeResult.confidence < 0.7) tags.push('uncertain')
|
|
||||||
|
|
||||||
// Add type-specific semantic tags
|
|
||||||
if (data.status) tags.push(String(data.status).toLowerCase())
|
|
||||||
if (data.priority) tags.push(String(data.priority).toLowerCase())
|
|
||||||
if (data.category) tags.push(String(data.category).toLowerCase())
|
|
||||||
|
|
||||||
return tags
|
|
||||||
}
|
|
||||||
|
|
||||||
private getReadableVerbPhrase(verbType: string): string {
|
|
||||||
const verbPhrases: Record<string, string> = {
|
|
||||||
[VerbType.WorksWith]: 'works with',
|
|
||||||
[VerbType.MemberOf]: 'is member of',
|
|
||||||
[VerbType.ReportsTo]: 'reports to',
|
|
||||||
[VerbType.Owns]: 'owns',
|
|
||||||
[VerbType.LocatedAt]: 'located at',
|
|
||||||
[VerbType.Likes]: 'likes',
|
|
||||||
[VerbType.Follows]: 'follows',
|
|
||||||
}
|
|
||||||
|
|
||||||
return verbPhrases[verbType] || 'related to'
|
|
||||||
}
|
|
||||||
|
|
||||||
private async computeVerbDescription(context: FieldComputationContext): Promise<string> {
|
|
||||||
const { data, verbContext, typeResult } = context
|
|
||||||
|
|
||||||
if (data.description) return String(data.description)
|
|
||||||
|
|
||||||
// Generate contextual description for relationship
|
|
||||||
if (verbContext && typeResult) {
|
|
||||||
const parts = []
|
|
||||||
const relationshipPhrase = this.getReadableVerbPhrase(typeResult.type)
|
|
||||||
|
|
||||||
if (data.role) parts.push(`Role: ${data.role}`)
|
|
||||||
if (data.startDate) parts.push(`Since: ${new Date(data.startDate).toLocaleDateString()}`)
|
|
||||||
if (data.department) parts.push(`Department: ${data.department}`)
|
|
||||||
|
|
||||||
return parts.length > 0
|
|
||||||
? `${relationshipPhrase} - ${parts.join(', ')}`
|
|
||||||
: `${relationshipPhrase} relationship`
|
|
||||||
}
|
|
||||||
|
|
||||||
return 'Relationship'
|
|
||||||
}
|
|
||||||
|
|
||||||
private async computeVerbTags(context: FieldComputationContext): Promise<string[]> {
|
|
||||||
const { data, typeResult } = context
|
|
||||||
const tags = ['relationship']
|
|
||||||
|
|
||||||
if (typeResult?.type) {
|
|
||||||
tags.push(typeResult.type.toLowerCase())
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add relationship-specific tags
|
|
||||||
if (data.status) tags.push(String(data.status).toLowerCase())
|
|
||||||
if (data.type) tags.push(String(data.type).toLowerCase())
|
|
||||||
|
|
||||||
return [...new Set(tags)]
|
|
||||||
}
|
|
||||||
|
|
||||||
private async computeHumanReadableRelationship(context: FieldComputationContext): Promise<string> {
|
|
||||||
const { verbContext, typeResult } = context
|
|
||||||
|
|
||||||
if (!verbContext || !typeResult) return 'Related'
|
|
||||||
|
|
||||||
const { sourceId, targetId } = verbContext
|
|
||||||
const phrase = this.getReadableVerbPhrase(typeResult.type)
|
|
||||||
|
|
||||||
return `${sourceId} ${phrase} ${targetId}`
|
|
||||||
}
|
|
||||||
|
|
||||||
private detectTypeHeuristically(data: any, entityType: 'noun' | 'verb'): string {
|
|
||||||
if (entityType === 'verb') return VerbType.RelatedTo
|
|
||||||
|
|
||||||
// Basic heuristics for noun types
|
|
||||||
if (data.firstName || data.lastName || data.email) return NounType.Person
|
|
||||||
if (data.companyName || data.organization) return NounType.Organization
|
|
||||||
if (data.filename || data.fileType) return NounType.Document
|
|
||||||
if (data.projectName || data.initiative) return NounType.Project
|
|
||||||
if (data.taskName || data.todo) return NounType.Task
|
|
||||||
if (data.startDate || data.endDate) return NounType.Event
|
|
||||||
|
|
||||||
return 'Item' // Generic fallback
|
|
||||||
}
|
|
||||||
|
|
||||||
private extractFieldWithPatterns(data: any, patterns: any[], fieldType: string): any {
|
|
||||||
const relevantPatterns = patterns.filter(p => p.displayField === fieldType)
|
|
||||||
|
|
||||||
for (const pattern of relevantPatterns) {
|
|
||||||
for (const field of pattern.fields) {
|
|
||||||
if (data[field]) {
|
|
||||||
return pattern.transform ? pattern.transform(data[field], { data, config: this.config } as any) : data[field]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Shutdown the computation engine
|
|
||||||
*/
|
|
||||||
async shutdown(): Promise<void> {
|
|
||||||
// Cleanup if needed
|
|
||||||
this.typeMatcher = null
|
|
||||||
this.initialized = false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,256 +0,0 @@
|
||||||
/**
|
|
||||||
* Universal Display Augmentation - Type Definitions
|
|
||||||
*
|
|
||||||
* Clean TypeScript interfaces for the display augmentation system
|
|
||||||
*/
|
|
||||||
|
|
||||||
import type { VectorDocument, GraphVerb } from '../../coreTypes.js'
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Configuration interface for the Universal Display Augmentation
|
|
||||||
*/
|
|
||||||
export interface DisplayConfig {
|
|
||||||
/** Enable/disable the augmentation */
|
|
||||||
enabled: boolean
|
|
||||||
|
|
||||||
/** LRU cache size for computed display fields */
|
|
||||||
cacheSize: number
|
|
||||||
|
|
||||||
/** Use lazy computation (recommended for performance) */
|
|
||||||
lazyComputation: boolean
|
|
||||||
|
|
||||||
/** Batch processing size for multiple requests */
|
|
||||||
batchSize: number
|
|
||||||
|
|
||||||
/** Minimum confidence threshold for AI type detection */
|
|
||||||
confidenceThreshold: number
|
|
||||||
|
|
||||||
/** Silent mode - suppress all console output */
|
|
||||||
silent?: boolean
|
|
||||||
|
|
||||||
// No icon configuration needed - clean, minimal approach
|
|
||||||
|
|
||||||
/** Custom field mappings (userField -> displayField) */
|
|
||||||
customFieldMappings: Record<string, string>
|
|
||||||
|
|
||||||
/** Type-specific priority fields for intelligent detection */
|
|
||||||
priorityFields: Record<string, string[]>
|
|
||||||
|
|
||||||
/** Enable debug mode with reasoning output */
|
|
||||||
debugMode: boolean
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Computed display fields for any noun or verb
|
|
||||||
*/
|
|
||||||
export interface ComputedDisplayFields {
|
|
||||||
/** Primary display name (AI-detected best field combination) */
|
|
||||||
title: string
|
|
||||||
|
|
||||||
/** Enhanced description with context awareness */
|
|
||||||
description: string
|
|
||||||
|
|
||||||
/** Human-readable type name */
|
|
||||||
type: string
|
|
||||||
|
|
||||||
// No icon field - clean, minimal approach
|
|
||||||
|
|
||||||
/** Generated display tags for categorization */
|
|
||||||
tags: string[]
|
|
||||||
|
|
||||||
/** For verbs: human-readable relationship description */
|
|
||||||
relationship?: string
|
|
||||||
|
|
||||||
/** AI confidence score (0-1) */
|
|
||||||
confidence: number
|
|
||||||
|
|
||||||
/** Explanation of type detection reasoning (debug mode) */
|
|
||||||
reasoning?: string
|
|
||||||
|
|
||||||
/** Alternative type suggestions with confidence scores */
|
|
||||||
alternatives?: Array<{
|
|
||||||
type: string
|
|
||||||
confidence: number
|
|
||||||
}>
|
|
||||||
|
|
||||||
/** Timestamp when fields were computed */
|
|
||||||
computedAt: number
|
|
||||||
|
|
||||||
/** Version of augmentation that computed these fields */
|
|
||||||
version: string
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Cache entry for computed display fields
|
|
||||||
*/
|
|
||||||
export interface DisplayCacheEntry {
|
|
||||||
fields: ComputedDisplayFields
|
|
||||||
lastAccessed: number
|
|
||||||
accessCount: number
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Field computation context passed to computation functions
|
|
||||||
*/
|
|
||||||
export interface FieldComputationContext {
|
|
||||||
/** The original data object */
|
|
||||||
data: any
|
|
||||||
|
|
||||||
/** Metadata associated with the object */
|
|
||||||
metadata: any
|
|
||||||
|
|
||||||
/** Type detection result from AI */
|
|
||||||
typeResult?: TypeMatchResult
|
|
||||||
|
|
||||||
/** Display configuration */
|
|
||||||
config: DisplayConfig
|
|
||||||
|
|
||||||
/** Whether this is a noun or verb */
|
|
||||||
entityType: 'noun' | 'verb'
|
|
||||||
|
|
||||||
/** For verbs: source and target information */
|
|
||||||
verbContext?: {
|
|
||||||
sourceId: string
|
|
||||||
targetId: string
|
|
||||||
verbType?: string
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Type matching result from BrainyTypes
|
|
||||||
*/
|
|
||||||
export interface TypeMatchResult {
|
|
||||||
type: string
|
|
||||||
confidence: number
|
|
||||||
reasoning: string
|
|
||||||
alternatives: Array<{
|
|
||||||
type: string
|
|
||||||
confidence: number
|
|
||||||
}>
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Enhanced VectorDocument with display capabilities
|
|
||||||
*/
|
|
||||||
export interface EnhancedVectorDocument<T = any> extends VectorDocument<T> {
|
|
||||||
/**
|
|
||||||
* Get computed display field(s)
|
|
||||||
* @param field Optional specific field name
|
|
||||||
* @returns Single field value or all display fields
|
|
||||||
*/
|
|
||||||
getDisplay(): Promise<ComputedDisplayFields>
|
|
||||||
getDisplay(field: keyof ComputedDisplayFields): Promise<any>
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get available fields for a specific augmentation namespace
|
|
||||||
* @param namespace The augmentation namespace (e.g., 'display')
|
|
||||||
* @returns Array of available field names
|
|
||||||
*/
|
|
||||||
getAvailableFields(namespace: string): string[]
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get available augmentation namespaces
|
|
||||||
* @returns Array of available augmentation names
|
|
||||||
*/
|
|
||||||
getAvailableAugmentations(): string[]
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Debug exploration of all computed fields
|
|
||||||
*/
|
|
||||||
explore(): Promise<void>
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Enhanced GraphVerb with display capabilities
|
|
||||||
*/
|
|
||||||
export interface EnhancedGraphVerb extends GraphVerb {
|
|
||||||
/**
|
|
||||||
* Get computed display field(s) for relationships
|
|
||||||
* @param field Optional specific field name
|
|
||||||
* @returns Single field value or all display fields
|
|
||||||
*/
|
|
||||||
getDisplay(): Promise<ComputedDisplayFields>
|
|
||||||
getDisplay(field: keyof ComputedDisplayFields): Promise<any>
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get available fields for a specific augmentation namespace
|
|
||||||
* @param namespace The augmentation namespace (e.g., 'display')
|
|
||||||
* @returns Array of available field names
|
|
||||||
*/
|
|
||||||
getAvailableFields(namespace: string): string[]
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Batch computation request for performance optimization
|
|
||||||
*/
|
|
||||||
export interface BatchComputationRequest {
|
|
||||||
id: string
|
|
||||||
data: any
|
|
||||||
metadata: any
|
|
||||||
entityType: 'noun' | 'verb'
|
|
||||||
verbContext?: {
|
|
||||||
sourceId: string
|
|
||||||
targetId: string
|
|
||||||
verbType?: string
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Batch computation result
|
|
||||||
*/
|
|
||||||
export interface BatchComputationResult {
|
|
||||||
id: string
|
|
||||||
fields: ComputedDisplayFields
|
|
||||||
error?: string
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Field pattern for intelligent field detection
|
|
||||||
*/
|
|
||||||
export interface FieldPattern {
|
|
||||||
/** Field names that match this pattern */
|
|
||||||
fields: string[]
|
|
||||||
|
|
||||||
/** Target display field name */
|
|
||||||
displayField: keyof ComputedDisplayFields
|
|
||||||
|
|
||||||
/** Confidence score for this pattern match */
|
|
||||||
confidence: number
|
|
||||||
|
|
||||||
/** Optional: specific noun/verb types this applies to */
|
|
||||||
applicableTypes?: string[]
|
|
||||||
|
|
||||||
/** Optional: transformation function for the field value */
|
|
||||||
transform?: (value: any, context: FieldComputationContext) => string
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Statistics for the display augmentation
|
|
||||||
*/
|
|
||||||
export interface DisplayAugmentationStats {
|
|
||||||
/** Total number of computations performed */
|
|
||||||
totalComputations: number
|
|
||||||
|
|
||||||
/** Cache hit ratio */
|
|
||||||
cacheHitRatio: number
|
|
||||||
|
|
||||||
/** Average computation time in milliseconds */
|
|
||||||
averageComputationTime: number
|
|
||||||
|
|
||||||
/** Type detection accuracy (when ground truth available) */
|
|
||||||
typeDetectionAccuracy?: number
|
|
||||||
|
|
||||||
/** Most commonly detected types */
|
|
||||||
commonTypes: Array<{
|
|
||||||
type: string
|
|
||||||
count: number
|
|
||||||
percentage: number
|
|
||||||
}>
|
|
||||||
|
|
||||||
/** Performance metrics */
|
|
||||||
performance: {
|
|
||||||
fastestComputation: number
|
|
||||||
slowestComputation: number
|
|
||||||
totalComputationTime: number
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,518 +0,0 @@
|
||||||
/**
|
|
||||||
* Entity Registry Augmentation
|
|
||||||
* Fast external-ID to internal-UUID mapping for streaming data processing
|
|
||||||
* Works in write-only mode for high-performance deduplication
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { BrainyAugmentation, BaseAugmentation, AugmentationContext } from './brainyAugmentation.js'
|
|
||||||
|
|
||||||
export interface EntityRegistryConfig {
|
|
||||||
/**
|
|
||||||
* Maximum number of entries to keep in memory cache
|
|
||||||
* Default: 100,000 entries
|
|
||||||
*/
|
|
||||||
maxCacheSize?: number
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Time to live for cache entries in milliseconds
|
|
||||||
* Default: 300,000 (5 minutes)
|
|
||||||
*/
|
|
||||||
cacheTTL?: number
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Fields to index for fast lookup
|
|
||||||
* Default: ['did', 'handle', 'uri', 'id', 'external_id']
|
|
||||||
*/
|
|
||||||
indexedFields?: string[]
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Persistence strategy
|
|
||||||
* memory: Keep only in memory (fast, but lost on restart)
|
|
||||||
* storage: Persist to storage (survives restarts)
|
|
||||||
* hybrid: Memory + periodic storage sync
|
|
||||||
*/
|
|
||||||
persistence?: 'memory' | 'storage' | 'hybrid'
|
|
||||||
|
|
||||||
/**
|
|
||||||
* How often to sync memory cache to storage (hybrid mode)
|
|
||||||
* Default: 30000 (30 seconds)
|
|
||||||
*/
|
|
||||||
syncInterval?: number
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface EntityMapping {
|
|
||||||
externalId: string
|
|
||||||
field: string
|
|
||||||
brainyId: string
|
|
||||||
nounType: string
|
|
||||||
lastAccessed: number
|
|
||||||
metadata?: any
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* High-performance entity registry for external ID to Brainy UUID mapping
|
|
||||||
* Optimized for streaming data scenarios like Bluesky firehose processing
|
|
||||||
*/
|
|
||||||
export class EntityRegistryAugmentation extends BaseAugmentation {
|
|
||||||
readonly metadata = 'readonly' as const // Reads metadata to register entities
|
|
||||||
readonly name = 'entity-registry'
|
|
||||||
readonly description = 'Fast external-ID to internal-UUID mapping for streaming data'
|
|
||||||
readonly timing: 'before' | 'after' | 'around' | 'replace' = 'before'
|
|
||||||
readonly operations = ['add', 'addNoun', 'addVerb'] as ('add' | 'addNoun' | 'addVerb')[]
|
|
||||||
readonly priority = 90 // High priority for entity registration
|
|
||||||
|
|
||||||
protected config: Required<EntityRegistryConfig>
|
|
||||||
private memoryIndex = new Map<string, EntityMapping>()
|
|
||||||
private fieldIndices = new Map<string, Map<string, string>>() // field -> value -> brainyId
|
|
||||||
private syncTimer?: NodeJS.Timeout
|
|
||||||
private brain?: any
|
|
||||||
private storage?: any
|
|
||||||
private cacheHits = 0
|
|
||||||
private cacheMisses = 0
|
|
||||||
|
|
||||||
constructor(config: EntityRegistryConfig = {}) {
|
|
||||||
super()
|
|
||||||
|
|
||||||
this.config = {
|
|
||||||
maxCacheSize: config.maxCacheSize ?? 100000,
|
|
||||||
cacheTTL: config.cacheTTL ?? 300000, // 5 minutes
|
|
||||||
indexedFields: config.indexedFields ?? ['did', 'handle', 'uri', 'id', 'external_id'],
|
|
||||||
persistence: config.persistence ?? 'hybrid',
|
|
||||||
syncInterval: config.syncInterval ?? 30000 // 30 seconds
|
|
||||||
}
|
|
||||||
|
|
||||||
// Initialize field indices
|
|
||||||
for (const field of this.config.indexedFields) {
|
|
||||||
this.fieldIndices.set(field, new Map())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async initialize(context: AugmentationContext): Promise<void> {
|
|
||||||
this.brain = context.brain
|
|
||||||
this.storage = context.storage
|
|
||||||
|
|
||||||
// Load existing mappings from storage
|
|
||||||
if (this.config.persistence === 'storage' || this.config.persistence === 'hybrid') {
|
|
||||||
await this.loadFromStorage()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Start sync timer for hybrid mode
|
|
||||||
if (this.config.persistence === 'hybrid') {
|
|
||||||
this.syncTimer = setInterval(() => {
|
|
||||||
this.syncToStorage().catch(console.error)
|
|
||||||
}, this.config.syncInterval)
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log(`🔍 EntityRegistry initialized: ${this.memoryIndex.size} cached mappings`)
|
|
||||||
}
|
|
||||||
|
|
||||||
async shutdown(): Promise<void> {
|
|
||||||
// Final sync before shutdown
|
|
||||||
if (this.config.persistence === 'storage' || this.config.persistence === 'hybrid') {
|
|
||||||
await this.syncToStorage()
|
|
||||||
}
|
|
||||||
|
|
||||||
if (this.syncTimer) {
|
|
||||||
clearInterval(this.syncTimer)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Execute the augmentation
|
|
||||||
*/
|
|
||||||
async execute<T = any>(
|
|
||||||
operation: string,
|
|
||||||
params: any,
|
|
||||||
next: () => Promise<T>
|
|
||||||
): Promise<T> {
|
|
||||||
console.log(`🔍 [EntityRegistry] execute called: operation=${operation}`)
|
|
||||||
|
|
||||||
// For add operations, check for duplicates first
|
|
||||||
if (operation === 'add' || operation === 'addNoun') {
|
|
||||||
const metadata = params.metadata || {}
|
|
||||||
|
|
||||||
// Check if entity already exists
|
|
||||||
for (const field of this.config.indexedFields) {
|
|
||||||
const value = this.extractFieldValue(metadata, field)
|
|
||||||
if (value) {
|
|
||||||
const existingId = await this.lookupEntity(field, value)
|
|
||||||
if (existingId) {
|
|
||||||
// Entity already exists, return the existing one
|
|
||||||
console.log(`🔍 Duplicate detected: ${field}:${value} → ${existingId}`)
|
|
||||||
return { id: existingId, duplicate: true } as any
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// For addVerb operations, resolve external IDs to internal UUIDs
|
|
||||||
if (operation === 'addVerb') {
|
|
||||||
const sourceId = params.sourceId
|
|
||||||
const targetId = params.targetId
|
|
||||||
|
|
||||||
// Try to resolve source and target IDs if they look like external IDs
|
|
||||||
for (const field of this.config.indexedFields) {
|
|
||||||
// Check if sourceId matches an external ID pattern
|
|
||||||
if (typeof sourceId === 'string' && this.looksLikeExternalId(sourceId, field)) {
|
|
||||||
const resolvedSourceId = await this.lookupEntity(field, sourceId)
|
|
||||||
if (resolvedSourceId) {
|
|
||||||
console.log(`🔍 [EntityRegistry] Resolved source: ${sourceId} → ${resolvedSourceId}`)
|
|
||||||
params.sourceId = resolvedSourceId
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check if targetId matches an external ID pattern
|
|
||||||
if (typeof targetId === 'string' && this.looksLikeExternalId(targetId, field)) {
|
|
||||||
const resolvedTargetId = await this.lookupEntity(field, targetId)
|
|
||||||
if (resolvedTargetId) {
|
|
||||||
console.log(`🔍 [EntityRegistry] Resolved target: ${targetId} → ${resolvedTargetId}`)
|
|
||||||
params.targetId = resolvedTargetId
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Proceed with the operation
|
|
||||||
const result = await next()
|
|
||||||
|
|
||||||
// Register the entity after successful add
|
|
||||||
if ((operation === 'add' || operation === 'addNoun' || operation === 'addVerb') && result) {
|
|
||||||
// Handle both formats: string UUID or object with id property
|
|
||||||
const brainyId = typeof result === 'string' ? result : (result as any).id
|
|
||||||
if (brainyId) {
|
|
||||||
const metadata = params.metadata || {}
|
|
||||||
const nounType = params.nounType || 'default'
|
|
||||||
console.log(`🔍 [EntityRegistry] Registering entity: ${brainyId}`)
|
|
||||||
await this.registerEntity(brainyId, metadata, nounType)
|
|
||||||
console.log(`✅ [EntityRegistry] Entity registered successfully`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Register a new entity mapping
|
|
||||||
*/
|
|
||||||
async registerEntity(brainyId: string, metadata: any, nounType: string): Promise<void> {
|
|
||||||
const now = Date.now()
|
|
||||||
|
|
||||||
// Extract indexed fields from metadata
|
|
||||||
for (const field of this.config.indexedFields) {
|
|
||||||
const value = this.extractFieldValue(metadata, field)
|
|
||||||
if (value) {
|
|
||||||
const key = `${field}:${value}`
|
|
||||||
|
|
||||||
// Add to memory index
|
|
||||||
const mapping: EntityMapping = {
|
|
||||||
externalId: value,
|
|
||||||
field,
|
|
||||||
brainyId,
|
|
||||||
nounType,
|
|
||||||
lastAccessed: now,
|
|
||||||
metadata
|
|
||||||
}
|
|
||||||
|
|
||||||
this.memoryIndex.set(key, mapping)
|
|
||||||
|
|
||||||
// Add to field-specific index
|
|
||||||
const fieldIndex = this.fieldIndices.get(field)
|
|
||||||
if (fieldIndex) {
|
|
||||||
fieldIndex.set(value, brainyId)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Enforce cache size limit (LRU eviction)
|
|
||||||
await this.evictOldEntries()
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Fast lookup: external ID → Brainy UUID
|
|
||||||
* Works in write-only mode without search indexes
|
|
||||||
*/
|
|
||||||
async lookupEntity(field: string, value: string): Promise<string | null> {
|
|
||||||
const key = `${field}:${value}`
|
|
||||||
const cached = this.memoryIndex.get(key)
|
|
||||||
|
|
||||||
if (cached) {
|
|
||||||
// Update last accessed time
|
|
||||||
cached.lastAccessed = Date.now()
|
|
||||||
this.cacheHits++
|
|
||||||
return cached.brainyId
|
|
||||||
}
|
|
||||||
|
|
||||||
this.cacheMisses++
|
|
||||||
|
|
||||||
// If not in cache and using storage persistence, try loading from storage
|
|
||||||
if (this.config.persistence === 'storage' || this.config.persistence === 'hybrid') {
|
|
||||||
const stored = await this.loadFromStorageByField(field, value)
|
|
||||||
if (stored) {
|
|
||||||
// Add to memory cache
|
|
||||||
this.memoryIndex.set(key, stored)
|
|
||||||
const fieldIndex = this.fieldIndices.get(field)
|
|
||||||
if (fieldIndex) {
|
|
||||||
fieldIndex.set(value, stored.brainyId)
|
|
||||||
}
|
|
||||||
return stored.brainyId
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Batch lookup for multiple external IDs
|
|
||||||
*/
|
|
||||||
async lookupBatch(lookups: Array<{ field: string; value: string }>): Promise<Map<string, string | null>> {
|
|
||||||
const results = new Map<string, string | null>()
|
|
||||||
const missingKeys: Array<{ field: string; value: string; key: string }> = []
|
|
||||||
|
|
||||||
// Check memory cache first
|
|
||||||
for (const lookup of lookups) {
|
|
||||||
const key = `${lookup.field}:${lookup.value}`
|
|
||||||
const cached = this.memoryIndex.get(key)
|
|
||||||
|
|
||||||
if (cached) {
|
|
||||||
cached.lastAccessed = Date.now()
|
|
||||||
results.set(key, cached.brainyId)
|
|
||||||
} else {
|
|
||||||
missingKeys.push({ ...lookup, key })
|
|
||||||
results.set(key, null)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Batch load missing keys from storage
|
|
||||||
if (missingKeys.length > 0 && (this.config.persistence === 'storage' || this.config.persistence === 'hybrid')) {
|
|
||||||
const stored = await this.loadBatchFromStorage(missingKeys)
|
|
||||||
|
|
||||||
for (const [key, mapping] of stored) {
|
|
||||||
if (mapping) {
|
|
||||||
// Add to memory cache
|
|
||||||
this.memoryIndex.set(key, mapping)
|
|
||||||
const fieldIndex = this.fieldIndices.get(mapping.field)
|
|
||||||
if (fieldIndex) {
|
|
||||||
fieldIndex.set(mapping.externalId, mapping.brainyId)
|
|
||||||
}
|
|
||||||
results.set(key, mapping.brainyId)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return results
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Check if entity exists (faster than lookupEntity for existence checks)
|
|
||||||
*/
|
|
||||||
async hasEntity(field: string, value: string): Promise<boolean> {
|
|
||||||
const fieldIndex = this.fieldIndices.get(field)
|
|
||||||
if (fieldIndex && fieldIndex.has(value)) {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
return (await this.lookupEntity(field, value)) !== null
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get all entities by field (e.g., all DIDs)
|
|
||||||
*/
|
|
||||||
async getEntitiesByField(field: string): Promise<string[]> {
|
|
||||||
const fieldIndex = this.fieldIndices.get(field)
|
|
||||||
return fieldIndex ? Array.from(fieldIndex.keys()) : []
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get registry statistics
|
|
||||||
*/
|
|
||||||
getStats(): {
|
|
||||||
totalMappings: number
|
|
||||||
fieldCounts: Record<string, number>
|
|
||||||
cacheHitRate: number
|
|
||||||
memoryUsage: number
|
|
||||||
} {
|
|
||||||
const fieldCounts: Record<string, number> = {}
|
|
||||||
|
|
||||||
for (const [field, index] of this.fieldIndices) {
|
|
||||||
fieldCounts[field] = index.size
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
totalMappings: this.memoryIndex.size,
|
|
||||||
fieldCounts,
|
|
||||||
cacheHitRate: this.cacheHits > 0 ? this.cacheHits / (this.cacheHits + this.cacheMisses) : 0,
|
|
||||||
memoryUsage: this.estimateMemoryUsage()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Clear all cached mappings
|
|
||||||
*/
|
|
||||||
async clearCache(): Promise<void> {
|
|
||||||
this.memoryIndex.clear()
|
|
||||||
for (const fieldIndex of this.fieldIndices.values()) {
|
|
||||||
fieldIndex.clear()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Private helper methods
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Check if an ID looks like it could be an external ID for a specific field
|
|
||||||
*/
|
|
||||||
private looksLikeExternalId(id: string, field: string): boolean {
|
|
||||||
// Basic heuristics to detect external ID patterns
|
|
||||||
switch (field) {
|
|
||||||
case 'did':
|
|
||||||
return id.startsWith('did:')
|
|
||||||
case 'handle':
|
|
||||||
return id.includes('.') && (id.includes('bsky') || id.includes('social'))
|
|
||||||
case 'external_id':
|
|
||||||
return !id.match(/^[a-f0-9-]{36}$/i) // Not a UUID
|
|
||||||
case 'uri':
|
|
||||||
return id.startsWith('http') || id.startsWith('at://')
|
|
||||||
case 'id':
|
|
||||||
return !id.match(/^[a-f0-9-]{36}$/i) // Not a UUID
|
|
||||||
default:
|
|
||||||
// For custom fields, assume non-UUID strings might be external IDs
|
|
||||||
return typeof id === 'string' && id.length > 3 && !id.match(/^[a-f0-9-]{36}$/i)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private extractFieldValue(metadata: any, field: string): string | null {
|
|
||||||
if (!metadata) return null
|
|
||||||
|
|
||||||
// Support nested field access (e.g., "author.did")
|
|
||||||
const parts = field.split('.')
|
|
||||||
let value = metadata
|
|
||||||
|
|
||||||
for (const part of parts) {
|
|
||||||
value = value?.[part]
|
|
||||||
if (value === undefined || value === null) {
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return typeof value === 'string' ? value : String(value)
|
|
||||||
}
|
|
||||||
|
|
||||||
private async evictOldEntries(): Promise<void> {
|
|
||||||
if (this.memoryIndex.size <= this.config.maxCacheSize) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Sort by last accessed time and remove oldest entries
|
|
||||||
const entries = Array.from(this.memoryIndex.entries())
|
|
||||||
entries.sort(([, a], [, b]) => a.lastAccessed - b.lastAccessed)
|
|
||||||
|
|
||||||
const toRemove = entries.slice(0, entries.length - this.config.maxCacheSize)
|
|
||||||
|
|
||||||
for (const [key, mapping] of toRemove) {
|
|
||||||
this.memoryIndex.delete(key)
|
|
||||||
|
|
||||||
const fieldIndex = this.fieldIndices.get(mapping.field)
|
|
||||||
if (fieldIndex) {
|
|
||||||
fieldIndex.delete(mapping.externalId)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private async loadFromStorage(): Promise<void> {
|
|
||||||
if (!this.brain) return
|
|
||||||
|
|
||||||
try {
|
|
||||||
// Load registry data from a special storage location
|
|
||||||
const registryData = await this.brain.storage?.getMetadata('__entity_registry__')
|
|
||||||
|
|
||||||
if (registryData && registryData.mappings) {
|
|
||||||
for (const mapping of registryData.mappings) {
|
|
||||||
const key = `${mapping.field}:${mapping.externalId}`
|
|
||||||
this.memoryIndex.set(key, mapping)
|
|
||||||
|
|
||||||
const fieldIndex = this.fieldIndices.get(mapping.field)
|
|
||||||
if (fieldIndex) {
|
|
||||||
fieldIndex.set(mapping.externalId, mapping.brainyId)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
console.warn('Failed to load entity registry from storage:', error)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private async syncToStorage(): Promise<void> {
|
|
||||||
if (!this.brain) return
|
|
||||||
|
|
||||||
try {
|
|
||||||
const mappings = Array.from(this.memoryIndex.values())
|
|
||||||
|
|
||||||
await this.brain.storage?.saveMetadata('__entity_registry__', {
|
|
||||||
version: 1,
|
|
||||||
lastSync: Date.now(),
|
|
||||||
mappings
|
|
||||||
})
|
|
||||||
} catch (error) {
|
|
||||||
console.warn('Failed to sync entity registry to storage:', error)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private async loadFromStorageByField(field: string, value: string): Promise<EntityMapping | null> {
|
|
||||||
// For now, this would require a full load. In production, you'd want
|
|
||||||
// a more sophisticated storage index system
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
|
|
||||||
private async loadBatchFromStorage(keys: Array<{ field: string; value: string; key: string }>): Promise<Map<string, EntityMapping | null>> {
|
|
||||||
// For now, return empty. In production, implement batch storage lookup
|
|
||||||
return new Map()
|
|
||||||
}
|
|
||||||
|
|
||||||
private estimateMemoryUsage(): number {
|
|
||||||
// Rough estimate: 200 bytes per mapping on average
|
|
||||||
return this.memoryIndex.size * 200
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Hook into Brainy's add operations to automatically register entities
|
|
||||||
export class AutoRegisterEntitiesAugmentation extends BaseAugmentation {
|
|
||||||
readonly metadata = 'readonly' as const // Reads metadata for auto-registration
|
|
||||||
readonly name = 'auto-register-entities'
|
|
||||||
readonly description = 'Automatically register entities in the registry when added'
|
|
||||||
readonly timing: 'before' | 'after' | 'around' | 'replace' = 'after'
|
|
||||||
readonly operations = ['add', 'addNoun', 'addVerb'] as ('add' | 'addNoun' | 'addVerb')[]
|
|
||||||
readonly priority = 85 // After entity registry
|
|
||||||
|
|
||||||
private registry?: EntityRegistryAugmentation
|
|
||||||
private brain?: any
|
|
||||||
|
|
||||||
async initialize(context: AugmentationContext): Promise<void> {
|
|
||||||
this.brain = context.brain
|
|
||||||
// Find the entity registry augmentation from the registry
|
|
||||||
this.registry = this.brain?.augmentations?.augmentations?.find(
|
|
||||||
(aug: any) => aug instanceof EntityRegistryAugmentation
|
|
||||||
) as EntityRegistryAugmentation
|
|
||||||
}
|
|
||||||
|
|
||||||
async execute<T = any>(
|
|
||||||
operation: string,
|
|
||||||
params: any,
|
|
||||||
next: () => Promise<T>
|
|
||||||
): Promise<T> {
|
|
||||||
const result = await next()
|
|
||||||
|
|
||||||
// After successful add, register the entity
|
|
||||||
if ((operation === 'add' || operation === 'addNoun' || operation === 'addVerb') && result) {
|
|
||||||
if (this.registry) {
|
|
||||||
// Handle both formats: string UUID or object with id property
|
|
||||||
const brainyId = typeof result === 'string' ? result : (result as any).id
|
|
||||||
if (brainyId) {
|
|
||||||
const metadata = params.metadata || {}
|
|
||||||
const nounType = params.nounType || 'default'
|
|
||||||
await this.registry.registerEntity(brainyId, metadata, nounType)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,337 +0,0 @@
|
||||||
/**
|
|
||||||
* Index Augmentation - Optional Metadata Indexing
|
|
||||||
*
|
|
||||||
* Replaces the hardcoded MetadataIndex in Brainy with an optional augmentation.
|
|
||||||
* Provides O(1) metadata filtering and field lookups.
|
|
||||||
*
|
|
||||||
* Zero-config: Automatically enabled for better search performance
|
|
||||||
* Can be disabled or customized via augmentation registry
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { BaseAugmentation, AugmentationContext } from './brainyAugmentation.js'
|
|
||||||
import { MetadataIndexManager } from '../utils/metadataIndex.js'
|
|
||||||
import type { BaseStorageAdapter as StorageAdapter } from '../storage/adapters/baseStorageAdapter.js'
|
|
||||||
|
|
||||||
export interface IndexConfig {
|
|
||||||
enabled?: boolean
|
|
||||||
maxFieldValues?: number
|
|
||||||
maxIndexSize?: number // Max number of entries per field value
|
|
||||||
autoRebuild?: boolean
|
|
||||||
rebuildThreshold?: number
|
|
||||||
flushInterval?: number
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* IndexAugmentation - Makes metadata indexing optional and pluggable
|
|
||||||
*
|
|
||||||
* Features:
|
|
||||||
* - O(1) metadata field lookups
|
|
||||||
* - Fast pre-filtering for searches
|
|
||||||
* - Automatic index maintenance
|
|
||||||
* - Zero-config with smart defaults
|
|
||||||
*/
|
|
||||||
export class IndexAugmentation extends BaseAugmentation {
|
|
||||||
readonly metadata = 'readonly' as const // Reads metadata to build indexes
|
|
||||||
readonly name = 'index'
|
|
||||||
readonly timing = 'after' as const
|
|
||||||
operations = ['add', 'update', 'updateMetadata', 'delete', 'clear', 'all'] as ('add' | 'update' | 'updateMetadata' | 'delete' | 'clear' | 'all')[]
|
|
||||||
readonly priority = 60 // Run after data operations
|
|
||||||
|
|
||||||
// Augmentation metadata
|
|
||||||
readonly category = 'core' as const
|
|
||||||
readonly description = 'Fast metadata field indexing for O(1) filtering and lookups'
|
|
||||||
|
|
||||||
private metadataIndex: MetadataIndexManager | null = null
|
|
||||||
protected config: IndexConfig
|
|
||||||
private flushTimer: NodeJS.Timeout | null = null
|
|
||||||
|
|
||||||
constructor(config: IndexConfig = {}) {
|
|
||||||
super()
|
|
||||||
this.config = {
|
|
||||||
enabled: true,
|
|
||||||
maxFieldValues: 1000,
|
|
||||||
autoRebuild: true,
|
|
||||||
rebuildThreshold: 0.3, // Rebuild if 30% inconsistent
|
|
||||||
flushInterval: 30000, // Flush every 30 seconds
|
|
||||||
...config
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected async onInitialize(): Promise<void> {
|
|
||||||
if (!this.config.enabled) {
|
|
||||||
this.log('Index augmentation disabled by configuration')
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get storage from context
|
|
||||||
const storage = this.context?.storage
|
|
||||||
if (!storage) {
|
|
||||||
this.log('No storage available, index augmentation inactive', 'warn')
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Initialize metadata index
|
|
||||||
this.metadataIndex = new MetadataIndexManager(
|
|
||||||
storage as StorageAdapter,
|
|
||||||
{
|
|
||||||
maxIndexSize: this.config.maxIndexSize || 10000
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
// Check if we need to rebuild
|
|
||||||
if (this.config.autoRebuild) {
|
|
||||||
const stats = await this.metadataIndex.getStats()
|
|
||||||
if (stats.totalEntries === 0) {
|
|
||||||
// Check if storage has data but index is empty
|
|
||||||
try {
|
|
||||||
const storageStats = await storage.getStatistics?.()
|
|
||||||
if (storageStats && storageStats.totalNouns > 0) {
|
|
||||||
this.log('Rebuilding metadata index for existing data...')
|
|
||||||
await this.metadataIndex.rebuild()
|
|
||||||
const newStats = await this.metadataIndex.getStats()
|
|
||||||
this.log(`Index rebuilt: ${newStats.totalEntries} entries, ${newStats.fieldsIndexed.length} fields`)
|
|
||||||
}
|
|
||||||
} catch (e) {
|
|
||||||
this.log('Could not check storage statistics', 'info')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Start flush timer
|
|
||||||
if (this.config.flushInterval && this.config.flushInterval > 0) {
|
|
||||||
this.startFlushTimer()
|
|
||||||
}
|
|
||||||
|
|
||||||
this.log('Index augmentation initialized')
|
|
||||||
}
|
|
||||||
|
|
||||||
protected async onShutdown(): Promise<void> {
|
|
||||||
// Stop flush timer
|
|
||||||
if (this.flushTimer) {
|
|
||||||
clearInterval(this.flushTimer)
|
|
||||||
this.flushTimer = null
|
|
||||||
}
|
|
||||||
|
|
||||||
// Flush index one last time
|
|
||||||
if (this.metadataIndex) {
|
|
||||||
try {
|
|
||||||
await this.metadataIndex.flush()
|
|
||||||
} catch (error) {
|
|
||||||
this.log('Error flushing index during shutdown', 'warn')
|
|
||||||
}
|
|
||||||
this.metadataIndex = null
|
|
||||||
}
|
|
||||||
|
|
||||||
this.log('Index augmentation shut down')
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Execute augmentation - maintain index on data operations
|
|
||||||
*/
|
|
||||||
async execute<T = any>(
|
|
||||||
operation: string,
|
|
||||||
params: any,
|
|
||||||
next: () => Promise<T>
|
|
||||||
): Promise<T> {
|
|
||||||
// Execute the operation first
|
|
||||||
const result = await next()
|
|
||||||
|
|
||||||
// If index is disabled, just return
|
|
||||||
if (!this.metadataIndex || !this.config.enabled) {
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
// Handle index updates after operation completes
|
|
||||||
switch (operation) {
|
|
||||||
case 'add':
|
|
||||||
await this.handleAdd(params)
|
|
||||||
break
|
|
||||||
|
|
||||||
case 'updateMetadata':
|
|
||||||
await this.handleUpdate(params)
|
|
||||||
break
|
|
||||||
|
|
||||||
case 'delete':
|
|
||||||
await this.handleDelete(params)
|
|
||||||
break
|
|
||||||
|
|
||||||
case 'clear':
|
|
||||||
await this.handleClear()
|
|
||||||
break
|
|
||||||
}
|
|
||||||
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Handle add operation - index new metadata
|
|
||||||
*/
|
|
||||||
private async handleAdd(params: any): Promise<void> {
|
|
||||||
if (!this.metadataIndex) return
|
|
||||||
|
|
||||||
const { id, metadata } = params
|
|
||||||
if (id && metadata) {
|
|
||||||
await this.metadataIndex.addToIndex(id, metadata)
|
|
||||||
this.log(`Indexed metadata for ${id}`, 'info')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Handle update operation - reindex metadata
|
|
||||||
*/
|
|
||||||
private async handleUpdate(params: any): Promise<void> {
|
|
||||||
if (!this.metadataIndex) return
|
|
||||||
|
|
||||||
const { id, oldMetadata, newMetadata } = params
|
|
||||||
|
|
||||||
// Remove old metadata
|
|
||||||
if (id && oldMetadata) {
|
|
||||||
await this.metadataIndex.removeFromIndex(id, oldMetadata)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add new metadata
|
|
||||||
if (id && newMetadata) {
|
|
||||||
await this.metadataIndex.addToIndex(id, newMetadata)
|
|
||||||
this.log(`Reindexed metadata for ${id}`, 'info')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Handle delete operation - remove from index
|
|
||||||
*/
|
|
||||||
private async handleDelete(params: any): Promise<void> {
|
|
||||||
if (!this.metadataIndex) return
|
|
||||||
|
|
||||||
const { id, metadata } = params
|
|
||||||
if (id && metadata) {
|
|
||||||
await this.metadataIndex.removeFromIndex(id, metadata)
|
|
||||||
this.log(`Removed ${id} from index`, 'info')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Handle clear operation - clear index
|
|
||||||
*/
|
|
||||||
private async handleClear(): Promise<void> {
|
|
||||||
if (!this.metadataIndex) return
|
|
||||||
|
|
||||||
// Clear the index when all data is cleared (rebuild effectively clears it)
|
|
||||||
await this.metadataIndex.rebuild()
|
|
||||||
this.log('Index cleared due to clear operation')
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Start periodic flush timer
|
|
||||||
*/
|
|
||||||
private startFlushTimer(): void {
|
|
||||||
if (this.flushTimer) return
|
|
||||||
|
|
||||||
this.flushTimer = setInterval(async () => {
|
|
||||||
if (this.metadataIndex) {
|
|
||||||
try {
|
|
||||||
await this.metadataIndex.flush()
|
|
||||||
} catch (error) {
|
|
||||||
this.log('Error during periodic index flush', 'warn')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}, this.config.flushInterval!)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get IDs that match metadata filter (for pre-filtering)
|
|
||||||
*/
|
|
||||||
async getIdsForFilter(filter: Record<string, any>): Promise<string[]> {
|
|
||||||
if (!this.metadataIndex) return []
|
|
||||||
return this.metadataIndex.getIdsForFilter(filter)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get available values for a field
|
|
||||||
*/
|
|
||||||
async getFilterValues(field: string): Promise<any[]> {
|
|
||||||
if (!this.metadataIndex) return []
|
|
||||||
return this.metadataIndex.getFilterValues(field)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get all indexed fields
|
|
||||||
*/
|
|
||||||
async getFilterFields(): Promise<string[]> {
|
|
||||||
if (!this.metadataIndex) return []
|
|
||||||
return this.metadataIndex.getFilterFields()
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get index statistics
|
|
||||||
*/
|
|
||||||
async getStats() {
|
|
||||||
if (!this.metadataIndex) {
|
|
||||||
return {
|
|
||||||
enabled: false,
|
|
||||||
totalEntries: 0,
|
|
||||||
fieldsIndexed: [],
|
|
||||||
memoryUsage: 0
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const stats = await this.metadataIndex.getStats()
|
|
||||||
return {
|
|
||||||
enabled: true,
|
|
||||||
...stats
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Rebuild the index from storage
|
|
||||||
*/
|
|
||||||
async rebuild(): Promise<void> {
|
|
||||||
if (!this.metadataIndex) {
|
|
||||||
throw new Error('Index augmentation is not initialized')
|
|
||||||
}
|
|
||||||
|
|
||||||
this.log('Rebuilding metadata index...')
|
|
||||||
await this.metadataIndex.rebuild()
|
|
||||||
const stats = await this.metadataIndex.getStats()
|
|
||||||
this.log(`Index rebuilt: ${stats.totalEntries} entries, ${stats.fieldsIndexed.length} fields`)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Flush index to storage
|
|
||||||
*/
|
|
||||||
async flush(): Promise<void> {
|
|
||||||
if (this.metadataIndex) {
|
|
||||||
await this.metadataIndex.flush()
|
|
||||||
this.log('Index flushed to storage', 'info')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Add entry to index (public method for direct access)
|
|
||||||
*/
|
|
||||||
async addToIndex(id: string, metadata: Record<string, any>): Promise<void> {
|
|
||||||
if (!this.metadataIndex) return
|
|
||||||
await this.metadataIndex.addToIndex(id, metadata)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Remove entry from index (public method for direct access)
|
|
||||||
*/
|
|
||||||
async removeFromIndex(id: string, metadata: Record<string, any>): Promise<void> {
|
|
||||||
if (!this.metadataIndex) return
|
|
||||||
await this.metadataIndex.removeFromIndex(id, metadata)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get the underlying MetadataIndexManager instance
|
|
||||||
*/
|
|
||||||
getMetadataIndex() {
|
|
||||||
return this.metadataIndex
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Factory function for zero-config index augmentation
|
|
||||||
*/
|
|
||||||
export function createIndexAugmentation(config?: IndexConfig): IndexAugmentation {
|
|
||||||
return new IndexAugmentation(config)
|
|
||||||
}
|
|
||||||
|
|
@ -1,246 +0,0 @@
|
||||||
/**
|
|
||||||
* Format Handler Registry
|
|
||||||
*
|
|
||||||
* Central registry for format handlers with:
|
|
||||||
* - MIME type-based routing
|
|
||||||
* - Lazy loading support
|
|
||||||
* - Pluggable handler registration
|
|
||||||
* - Handler lifecycle management
|
|
||||||
*
|
|
||||||
* NO MOCKS - Production implementation
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { FormatHandler, HandlerRegistry as IHandlerRegistry } from './types.js'
|
|
||||||
import { mimeDetector } from '../../vfs/MimeTypeDetector.js'
|
|
||||||
|
|
||||||
export interface HandlerRegistration {
|
|
||||||
/** Handler name (e.g., 'csv', 'image', 'pdf') */
|
|
||||||
name: string
|
|
||||||
|
|
||||||
/** MIME types this handler supports */
|
|
||||||
mimeTypes: string[]
|
|
||||||
|
|
||||||
/** File extensions (fallback if MIME detection fails) */
|
|
||||||
extensions: string[]
|
|
||||||
|
|
||||||
/** Lazy loader function */
|
|
||||||
loader: () => Promise<FormatHandler>
|
|
||||||
|
|
||||||
/** Loaded handler instance (lazy-loaded) */
|
|
||||||
instance?: FormatHandler
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* FormatHandlerRegistry - Central handler management
|
|
||||||
*
|
|
||||||
* Implements the HandlerRegistry interface with:
|
|
||||||
* - MIME type-based routing
|
|
||||||
* - Lazy loading for performance
|
|
||||||
* - Multiple handlers per MIME type
|
|
||||||
* - Priority-based selection
|
|
||||||
*/
|
|
||||||
export class FormatHandlerRegistry implements IHandlerRegistry {
|
|
||||||
// Interface compatibility
|
|
||||||
handlers: Map<string, () => Promise<FormatHandler>> = new Map()
|
|
||||||
loaded: Map<string, FormatHandler> = new Map()
|
|
||||||
|
|
||||||
// Enhanced registry
|
|
||||||
private registrations: Map<string, HandlerRegistration> = new Map()
|
|
||||||
private mimeTypeIndex: Map<string, string[]> = new Map() // MIME type → handler names
|
|
||||||
private extensionIndex: Map<string, string[]> = new Map() // Extension → handler names
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Register a format handler
|
|
||||||
*
|
|
||||||
* @param registration Handler registration details
|
|
||||||
*/
|
|
||||||
registerHandler(registration: HandlerRegistration): void {
|
|
||||||
const { name, mimeTypes, extensions, loader } = registration
|
|
||||||
|
|
||||||
// Store registration
|
|
||||||
this.registrations.set(name, registration)
|
|
||||||
|
|
||||||
// Index by MIME types
|
|
||||||
for (const mimeType of mimeTypes) {
|
|
||||||
const handlers = this.mimeTypeIndex.get(mimeType) || []
|
|
||||||
handlers.push(name)
|
|
||||||
this.mimeTypeIndex.set(mimeType, handlers)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Index by extensions
|
|
||||||
for (const ext of extensions) {
|
|
||||||
const normalized = ext.toLowerCase().replace(/^\./, '')
|
|
||||||
const handlers = this.extensionIndex.get(normalized) || []
|
|
||||||
handlers.push(name)
|
|
||||||
this.extensionIndex.set(normalized, handlers)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Interface compatibility
|
|
||||||
this.handlers.set(name, loader)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Register a handler (interface compatibility)
|
|
||||||
*/
|
|
||||||
register(extensions: string[], loader: () => Promise<FormatHandler>): void {
|
|
||||||
const name = extensions[0].replace(/^\./, '')
|
|
||||||
|
|
||||||
// Auto-detect MIME types from extensions for better routing
|
|
||||||
const mimeTypes: string[] = []
|
|
||||||
for (const ext of extensions) {
|
|
||||||
const mimeType = mimeDetector.detectMimeType(`file${ext}`)
|
|
||||||
if (mimeType && mimeType !== 'application/octet-stream' && !mimeTypes.includes(mimeType)) {
|
|
||||||
mimeTypes.push(mimeType)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
this.registerHandler({
|
|
||||||
name,
|
|
||||||
mimeTypes,
|
|
||||||
extensions,
|
|
||||||
loader
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get handler by filename or extension
|
|
||||||
*
|
|
||||||
* Uses MIME detection first, falls back to extension matching
|
|
||||||
*
|
|
||||||
* @param filenameOrExt Filename or extension
|
|
||||||
* @returns Handler instance or null
|
|
||||||
*/
|
|
||||||
async getHandler(filenameOrExt: string): Promise<FormatHandler | null> {
|
|
||||||
// Try MIME type detection first
|
|
||||||
const mimeType = mimeDetector.detectMimeType(filenameOrExt)
|
|
||||||
const byMime = await this.getHandlerByMimeType(mimeType)
|
|
||||||
if (byMime) return byMime
|
|
||||||
|
|
||||||
// Fallback to extension matching
|
|
||||||
const ext = this.extractExtension(filenameOrExt)
|
|
||||||
if (ext) {
|
|
||||||
return this.getHandlerByExtension(ext)
|
|
||||||
}
|
|
||||||
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get handler by MIME type
|
|
||||||
*
|
|
||||||
* @param mimeType MIME type string
|
|
||||||
* @returns Handler instance or null
|
|
||||||
*/
|
|
||||||
async getHandlerByMimeType(mimeType: string): Promise<FormatHandler | null> {
|
|
||||||
const handlerNames = this.mimeTypeIndex.get(mimeType)
|
|
||||||
if (!handlerNames || handlerNames.length === 0) {
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
|
|
||||||
// Return first matching handler (could add priority later)
|
|
||||||
return this.loadHandler(handlerNames[0])
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get handler by file extension
|
|
||||||
*
|
|
||||||
* @param ext File extension (with or without dot)
|
|
||||||
* @returns Handler instance or null
|
|
||||||
*/
|
|
||||||
async getHandlerByExtension(ext: string): Promise<FormatHandler | null> {
|
|
||||||
const normalized = ext.toLowerCase().replace(/^\./, '')
|
|
||||||
const handlerNames = this.extensionIndex.get(normalized)
|
|
||||||
if (!handlerNames || handlerNames.length === 0) {
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
|
|
||||||
// Return first matching handler
|
|
||||||
return this.loadHandler(handlerNames[0])
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get handler by name
|
|
||||||
*
|
|
||||||
* @param name Handler name
|
|
||||||
* @returns Handler instance or null
|
|
||||||
*/
|
|
||||||
async getHandlerByName(name: string): Promise<FormatHandler | null> {
|
|
||||||
return this.loadHandler(name)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Load handler (lazy loading)
|
|
||||||
*
|
|
||||||
* @param name Handler name
|
|
||||||
* @returns Loaded handler instance
|
|
||||||
*/
|
|
||||||
private async loadHandler(name: string): Promise<FormatHandler | null> {
|
|
||||||
const registration = this.registrations.get(name)
|
|
||||||
if (!registration) return null
|
|
||||||
|
|
||||||
// Return cached instance if available
|
|
||||||
if (registration.instance) {
|
|
||||||
return registration.instance
|
|
||||||
}
|
|
||||||
|
|
||||||
// Load handler
|
|
||||||
try {
|
|
||||||
const handler = await registration.loader()
|
|
||||||
registration.instance = handler
|
|
||||||
|
|
||||||
// Interface compatibility
|
|
||||||
this.loaded.set(name, handler)
|
|
||||||
|
|
||||||
return handler
|
|
||||||
} catch (error) {
|
|
||||||
console.error(`Failed to load handler ${name}:`, error)
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get all registered handler names
|
|
||||||
*/
|
|
||||||
getRegisteredHandlers(): string[] {
|
|
||||||
return Array.from(this.registrations.keys())
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get handlers for a MIME type
|
|
||||||
*/
|
|
||||||
getHandlersForMimeType(mimeType: string): string[] {
|
|
||||||
return this.mimeTypeIndex.get(mimeType) || []
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Check if handler is registered
|
|
||||||
*/
|
|
||||||
hasHandler(name: string): boolean {
|
|
||||||
return this.registrations.has(name)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Clear all handlers (for testing)
|
|
||||||
*/
|
|
||||||
clear(): void {
|
|
||||||
this.registrations.clear()
|
|
||||||
this.mimeTypeIndex.clear()
|
|
||||||
this.extensionIndex.clear()
|
|
||||||
this.handlers.clear()
|
|
||||||
this.loaded.clear()
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Extract file extension from filename
|
|
||||||
*/
|
|
||||||
private extractExtension(filename: string): string | null {
|
|
||||||
const lastDot = filename.lastIndexOf('.')
|
|
||||||
if (lastDot === -1 || lastDot === 0) return null
|
|
||||||
return filename.substring(lastDot + 1).toLowerCase()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Global handler registry singleton
|
|
||||||
*/
|
|
||||||
export const globalHandlerRegistry = new FormatHandlerRegistry()
|
|
||||||
|
|
@ -1,227 +0,0 @@
|
||||||
/**
|
|
||||||
* Intelligent Import Augmentation
|
|
||||||
*
|
|
||||||
* Automatically detects and processes CSV, Excel, and PDF files with:
|
|
||||||
* - Format detection and routing
|
|
||||||
* - Lazy-loaded handlers
|
|
||||||
* - Intelligent entity and relationship extraction
|
|
||||||
* - Integration with NeuralImport augmentation
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { BaseAugmentation, AugmentationContext } from '../brainyAugmentation.js'
|
|
||||||
import { FormatHandler, IntelligentImportConfig, ProcessedData } from './types.js'
|
|
||||||
import { CSVHandler } from './handlers/csvHandler.js'
|
|
||||||
import { ExcelHandler } from './handlers/excelHandler.js'
|
|
||||||
import { PDFHandler } from './handlers/pdfHandler.js'
|
|
||||||
import { ImageHandler } from './handlers/imageHandler.js'
|
|
||||||
|
|
||||||
export class IntelligentImportAugmentation extends BaseAugmentation {
|
|
||||||
readonly name = 'intelligent-import'
|
|
||||||
readonly timing = 'before' as const
|
|
||||||
readonly metadata = {
|
|
||||||
reads: '*' as '*',
|
|
||||||
writes: ['_intelligentImport', '_processedFormat', '_extractedData'] as string[]
|
|
||||||
}
|
|
||||||
readonly operations = ['import', 'importFile', 'importFromFile', 'importFromURL', 'all'] as any[]
|
|
||||||
readonly priority = 75 // Before NeuralImport (80), after validation
|
|
||||||
|
|
||||||
protected config: IntelligentImportConfig
|
|
||||||
private handlers: Map<string, FormatHandler> = new Map()
|
|
||||||
private initialized = false
|
|
||||||
|
|
||||||
constructor(config: Partial<IntelligentImportConfig> = {}) {
|
|
||||||
super(config)
|
|
||||||
this.config = {
|
|
||||||
enableCSV: true,
|
|
||||||
enableExcel: true,
|
|
||||||
enablePDF: true,
|
|
||||||
enableImage: true, // Image handler enabled by default
|
|
||||||
maxFileSize: 100 * 1024 * 1024, // 100MB default
|
|
||||||
enableCache: true,
|
|
||||||
cacheTTL: 24 * 60 * 60 * 1000, // 24 hours
|
|
||||||
...config
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected async onInitialize(): Promise<void> {
|
|
||||||
// Initialize handlers based on config
|
|
||||||
if (this.config.enableCSV) {
|
|
||||||
this.handlers.set('csv', new CSVHandler())
|
|
||||||
}
|
|
||||||
|
|
||||||
if (this.config.enableExcel) {
|
|
||||||
this.handlers.set('excel', new ExcelHandler())
|
|
||||||
}
|
|
||||||
|
|
||||||
if (this.config.enablePDF) {
|
|
||||||
this.handlers.set('pdf', new PDFHandler())
|
|
||||||
}
|
|
||||||
|
|
||||||
if (this.config.enableImage) {
|
|
||||||
this.handlers.set('image', new ImageHandler())
|
|
||||||
}
|
|
||||||
|
|
||||||
this.initialized = true
|
|
||||||
this.log(`Initialized with ${this.handlers.size} format handlers (CSV: ${this.config.enableCSV}, Excel: ${this.config.enableExcel}, PDF: ${this.config.enablePDF}, Image: ${this.config.enableImage})`)
|
|
||||||
}
|
|
||||||
|
|
||||||
async execute<T = any>(
|
|
||||||
operation: string,
|
|
||||||
params: any,
|
|
||||||
next: () => Promise<T>
|
|
||||||
): Promise<T> {
|
|
||||||
// Only process import operations
|
|
||||||
if (!this.shouldProcess(operation, params)) {
|
|
||||||
return next()
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
// Extract file data from params
|
|
||||||
const fileData = this.extractFileData(params)
|
|
||||||
if (!fileData) {
|
|
||||||
return next()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check file size limit
|
|
||||||
if (this.config.maxFileSize && fileData.data.length > this.config.maxFileSize) {
|
|
||||||
this.log(`File too large (${fileData.data.length} bytes), skipping intelligent import`, 'warn')
|
|
||||||
return next()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Detect format and get appropriate handler
|
|
||||||
const handler = this.detectHandler(fileData.data, fileData.filename)
|
|
||||||
if (!handler) {
|
|
||||||
// Not a supported format, pass through
|
|
||||||
return next()
|
|
||||||
}
|
|
||||||
|
|
||||||
this.log(`Processing ${fileData.filename || 'file'} with ${handler.format} handler`)
|
|
||||||
|
|
||||||
// Process the file
|
|
||||||
const processed = await handler.process(fileData.data, {
|
|
||||||
filename: fileData.filename,
|
|
||||||
ext: fileData.ext,
|
|
||||||
...this.config.csvDefaults,
|
|
||||||
...this.config.excelDefaults,
|
|
||||||
...this.config.pdfDefaults,
|
|
||||||
...this.config.imageDefaults,
|
|
||||||
...params.options
|
|
||||||
})
|
|
||||||
|
|
||||||
// Enrich params with processed data
|
|
||||||
params._intelligentImport = true
|
|
||||||
params._processedFormat = processed.format
|
|
||||||
params._extractedData = processed.data
|
|
||||||
params._metadata = {
|
|
||||||
...params._metadata,
|
|
||||||
intelligentImport: processed.metadata
|
|
||||||
}
|
|
||||||
|
|
||||||
// If this is an import operation, transform params to include the structured data
|
|
||||||
if (processed.data.length > 0) {
|
|
||||||
// Store processed data for the neural import augmentation to use
|
|
||||||
params.data = processed.data
|
|
||||||
params.metadata = params._metadata
|
|
||||||
}
|
|
||||||
|
|
||||||
this.log(`Extracted ${processed.data.length} items from ${processed.format} file`)
|
|
||||||
|
|
||||||
return next()
|
|
||||||
} catch (error) {
|
|
||||||
this.log(`Intelligent import processing failed: ${error instanceof Error ? error.message : String(error)}`, 'warn')
|
|
||||||
// Fall through to normal import on error
|
|
||||||
return next()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Check if we should process this operation
|
|
||||||
*/
|
|
||||||
private shouldProcess(operation: string, params: any): boolean {
|
|
||||||
// Only process if we have handlers initialized
|
|
||||||
if (!this.initialized || this.handlers.size === 0) {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check operation type
|
|
||||||
const validOps = ['import', 'importFile', 'importFromFile', 'importFromURL']
|
|
||||||
if (!validOps.some(op => operation.includes(op))) {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// Must have some data
|
|
||||||
if (!params || (!params.source && !params.data && !params.filePath && !params.url)) {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Extract file data from various param formats
|
|
||||||
*/
|
|
||||||
private extractFileData(params: any): { data: Buffer, filename?: string, ext?: string } | null {
|
|
||||||
// From source parameter
|
|
||||||
if (params.source) {
|
|
||||||
if (Buffer.isBuffer(params.source)) {
|
|
||||||
return { data: params.source, filename: params.filename }
|
|
||||||
}
|
|
||||||
if (typeof params.source === 'string') {
|
|
||||||
return { data: Buffer.from(params.source), filename: params.filename }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// From data parameter
|
|
||||||
if (params.data) {
|
|
||||||
if (Buffer.isBuffer(params.data)) {
|
|
||||||
return { data: params.data, filename: params.filename }
|
|
||||||
}
|
|
||||||
if (typeof params.data === 'string') {
|
|
||||||
return { data: Buffer.from(params.data), filename: params.filename }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// From file path (would need to read - but that should be handled by UniversalImportAPI)
|
|
||||||
if (params.filePath && typeof params.filePath === 'string') {
|
|
||||||
const ext = params.filePath.split('.').pop()
|
|
||||||
return null // File reading handled elsewhere
|
|
||||||
}
|
|
||||||
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Detect which handler can process this file
|
|
||||||
*/
|
|
||||||
private detectHandler(data: Buffer, filename?: string): FormatHandler | null {
|
|
||||||
// Try each handler's canHandle method
|
|
||||||
for (const handler of this.handlers.values()) {
|
|
||||||
if (handler.canHandle(data) || (filename && handler.canHandle({ filename }))) {
|
|
||||||
return handler
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get handler by format name
|
|
||||||
*/
|
|
||||||
getHandler(format: string): FormatHandler | undefined {
|
|
||||||
return this.handlers.get(format.toLowerCase())
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get all registered handlers
|
|
||||||
*/
|
|
||||||
getHandlers(): FormatHandler[] {
|
|
||||||
return Array.from(this.handlers.values())
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get supported formats
|
|
||||||
*/
|
|
||||||
getSupportedFormats(): string[] {
|
|
||||||
return Array.from(this.handlers.keys())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,350 +0,0 @@
|
||||||
/**
|
|
||||||
* Image Import Handler
|
|
||||||
*
|
|
||||||
* Handles image files with:
|
|
||||||
* - EXIF metadata extraction (camera, GPS, timestamps) via exifr
|
|
||||||
* - Image metadata (dimensions, format) via probe-image-size
|
|
||||||
* - Support for JPEG, PNG, WebP, GIF, TIFF, BMP, SVG
|
|
||||||
*
|
|
||||||
* NO NATIVE DEPENDENCIES - Pure JavaScript implementation
|
|
||||||
* Replaces Sharp with lightweight pure-JS alternatives
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { BaseFormatHandler } from './base.js'
|
|
||||||
import type { FormatHandlerOptions, ProcessedData } from '../types.js'
|
|
||||||
import exifr from 'exifr'
|
|
||||||
import probeImageSize from 'probe-image-size'
|
|
||||||
import { Readable } from 'stream'
|
|
||||||
|
|
||||||
export interface ImageMetadata {
|
|
||||||
/** Image dimensions */
|
|
||||||
width: number
|
|
||||||
height: number
|
|
||||||
|
|
||||||
/** Image format (jpeg, png, webp, etc.) */
|
|
||||||
format: string
|
|
||||||
|
|
||||||
/** File size in bytes */
|
|
||||||
size: number
|
|
||||||
|
|
||||||
/** Orientation (EXIF) */
|
|
||||||
orientation?: number
|
|
||||||
|
|
||||||
/** MIME type */
|
|
||||||
mimeType?: string
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface EXIFData {
|
|
||||||
/** Camera make (e.g., "Canon", "Nikon") */
|
|
||||||
make?: string
|
|
||||||
|
|
||||||
/** Camera model */
|
|
||||||
model?: string
|
|
||||||
|
|
||||||
/** Lens information */
|
|
||||||
lens?: string
|
|
||||||
|
|
||||||
/** Date/time original */
|
|
||||||
dateTimeOriginal?: Date
|
|
||||||
|
|
||||||
/** GPS latitude */
|
|
||||||
latitude?: number
|
|
||||||
|
|
||||||
/** GPS longitude */
|
|
||||||
longitude?: number
|
|
||||||
|
|
||||||
/** GPS altitude in meters */
|
|
||||||
altitude?: number
|
|
||||||
|
|
||||||
/** Exposure time (e.g., "1/250") */
|
|
||||||
exposureTime?: number
|
|
||||||
|
|
||||||
/** F-number (e.g., 2.8) */
|
|
||||||
fNumber?: number
|
|
||||||
|
|
||||||
/** ISO speed */
|
|
||||||
iso?: number
|
|
||||||
|
|
||||||
/** Focal length in mm */
|
|
||||||
focalLength?: number
|
|
||||||
|
|
||||||
/** Flash fired */
|
|
||||||
flash?: boolean
|
|
||||||
|
|
||||||
/** Copyright */
|
|
||||||
copyright?: string
|
|
||||||
|
|
||||||
/** Artist/photographer */
|
|
||||||
artist?: string
|
|
||||||
|
|
||||||
/** Image description */
|
|
||||||
imageDescription?: string
|
|
||||||
|
|
||||||
/** Software used */
|
|
||||||
software?: string
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface ImageHandlerOptions extends FormatHandlerOptions {
|
|
||||||
/** Extract EXIF data (default: true) */
|
|
||||||
extractEXIF?: boolean
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* ImageImportHandler
|
|
||||||
*
|
|
||||||
* Processes image files and extracts rich metadata including EXIF data.
|
|
||||||
* Enables developers to import images into the knowledge graph with
|
|
||||||
* full metadata extraction.
|
|
||||||
*
|
|
||||||
* Pure JavaScript implementation (no native dependencies)
|
|
||||||
*/
|
|
||||||
export class ImageHandler extends BaseFormatHandler {
|
|
||||||
readonly format = 'image'
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Check if this handler can process the given data
|
|
||||||
*/
|
|
||||||
canHandle(data: Buffer | string | { filename?: string; ext?: string }): boolean {
|
|
||||||
// Check by filename/extension
|
|
||||||
if (typeof data === 'object' && 'filename' in data) {
|
|
||||||
const mimeType = this.getMimeType(data)
|
|
||||||
return this.mimeTypeMatches(mimeType, ['image/*'])
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check by extension
|
|
||||||
if (typeof data === 'object' && 'ext' in data && data.ext) {
|
|
||||||
return this.isImageExtension(data.ext)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check by data (Buffer magic bytes)
|
|
||||||
if (Buffer.isBuffer(data)) {
|
|
||||||
return this.detectImageFormat(data) !== null
|
|
||||||
}
|
|
||||||
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Process image file
|
|
||||||
*/
|
|
||||||
async process(
|
|
||||||
data: Buffer | string,
|
|
||||||
options: ImageHandlerOptions = {}
|
|
||||||
): Promise<ProcessedData> {
|
|
||||||
const startTime = Date.now()
|
|
||||||
|
|
||||||
try {
|
|
||||||
// Convert to Buffer
|
|
||||||
const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, 'base64')
|
|
||||||
|
|
||||||
// Extract image metadata
|
|
||||||
const metadata = await this.extractMetadata(buffer)
|
|
||||||
|
|
||||||
// Extract EXIF data (default: enabled)
|
|
||||||
let exifData: EXIFData | undefined
|
|
||||||
if (options.extractEXIF !== false) {
|
|
||||||
exifData = await this.extractEXIF(buffer)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Calculate processing time
|
|
||||||
const processingTime = Date.now() - startTime
|
|
||||||
|
|
||||||
// Generate descriptive name
|
|
||||||
const imageName = options.filename
|
|
||||||
? options.filename.replace(/\.[^/.]+$/, '') // Remove extension
|
|
||||||
: `${metadata.format.toUpperCase()} Image ${metadata.width}x${metadata.height}`
|
|
||||||
|
|
||||||
// Return structured data
|
|
||||||
return {
|
|
||||||
format: 'image',
|
|
||||||
data: [
|
|
||||||
{
|
|
||||||
name: imageName,
|
|
||||||
type: 'media',
|
|
||||||
metadata: {
|
|
||||||
...metadata,
|
|
||||||
subtype: 'image',
|
|
||||||
exif: exifData
|
|
||||||
}
|
|
||||||
}
|
|
||||||
],
|
|
||||||
metadata: {
|
|
||||||
rowCount: 1,
|
|
||||||
fields: ['type', 'metadata'],
|
|
||||||
processingTime,
|
|
||||||
imageMetadata: metadata,
|
|
||||||
exifData
|
|
||||||
},
|
|
||||||
filename: options.filename
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
throw new Error(
|
|
||||||
`Image processing failed: ${error instanceof Error ? error.message : String(error)}`
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Extract image metadata using probe-image-size (pure JS)
|
|
||||||
*/
|
|
||||||
private async extractMetadata(buffer: Buffer): Promise<ImageMetadata> {
|
|
||||||
try {
|
|
||||||
// Convert Buffer to Stream for probe-image-size
|
|
||||||
const stream = Readable.from(buffer)
|
|
||||||
const result = await probeImageSize(stream)
|
|
||||||
|
|
||||||
return {
|
|
||||||
width: result.width,
|
|
||||||
height: result.height,
|
|
||||||
format: result.type, // 'jpeg', 'png', 'webp', etc.
|
|
||||||
size: buffer.length,
|
|
||||||
mimeType: result.mime,
|
|
||||||
orientation: result.orientation
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
// Fallback: Try to detect format from magic bytes
|
|
||||||
const detectedFormat = this.detectImageFormat(buffer)
|
|
||||||
|
|
||||||
return {
|
|
||||||
width: 0,
|
|
||||||
height: 0,
|
|
||||||
format: detectedFormat || 'unknown',
|
|
||||||
size: buffer.length,
|
|
||||||
mimeType: detectedFormat ? `image/${detectedFormat}` : undefined
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Extract EXIF data using exifr (pure JS)
|
|
||||||
*/
|
|
||||||
private async extractEXIF(buffer: Buffer): Promise<EXIFData | undefined> {
|
|
||||||
try {
|
|
||||||
const exif = await exifr.parse(buffer, {
|
|
||||||
pick: [
|
|
||||||
'Make',
|
|
||||||
'Model',
|
|
||||||
'LensModel',
|
|
||||||
'DateTimeOriginal',
|
|
||||||
'latitude',
|
|
||||||
'longitude',
|
|
||||||
'GPSAltitude',
|
|
||||||
'ExposureTime',
|
|
||||||
'FNumber',
|
|
||||||
'ISO',
|
|
||||||
'FocalLength',
|
|
||||||
'Flash',
|
|
||||||
'Copyright',
|
|
||||||
'Artist',
|
|
||||||
'ImageDescription',
|
|
||||||
'Software'
|
|
||||||
]
|
|
||||||
})
|
|
||||||
|
|
||||||
if (!exif) return undefined
|
|
||||||
|
|
||||||
return {
|
|
||||||
make: exif.Make,
|
|
||||||
model: exif.Model,
|
|
||||||
lens: exif.LensModel,
|
|
||||||
dateTimeOriginal: exif.DateTimeOriginal,
|
|
||||||
latitude: exif.latitude,
|
|
||||||
longitude: exif.longitude,
|
|
||||||
altitude: exif.GPSAltitude,
|
|
||||||
exposureTime: exif.ExposureTime,
|
|
||||||
fNumber: exif.FNumber,
|
|
||||||
iso: exif.ISO,
|
|
||||||
focalLength: exif.FocalLength,
|
|
||||||
flash: exif.Flash !== undefined ? Boolean(exif.Flash & 1) : undefined,
|
|
||||||
copyright: exif.Copyright,
|
|
||||||
artist: exif.Artist,
|
|
||||||
imageDescription: exif.ImageDescription,
|
|
||||||
software: exif.Software
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
// EXIF extraction can fail for non-JPEG images or corrupt data
|
|
||||||
return undefined
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Detect image format from magic bytes
|
|
||||||
*/
|
|
||||||
private detectImageFormat(buffer: Buffer): string | null {
|
|
||||||
if (buffer.length < 4) return null
|
|
||||||
|
|
||||||
// JPEG: FF D8 FF
|
|
||||||
if (buffer[0] === 0xff && buffer[1] === 0xd8 && buffer[2] === 0xff) {
|
|
||||||
return 'jpeg'
|
|
||||||
}
|
|
||||||
|
|
||||||
// PNG: 89 50 4E 47
|
|
||||||
if (
|
|
||||||
buffer[0] === 0x89 &&
|
|
||||||
buffer[1] === 0x50 &&
|
|
||||||
buffer[2] === 0x4e &&
|
|
||||||
buffer[3] === 0x47
|
|
||||||
) {
|
|
||||||
return 'png'
|
|
||||||
}
|
|
||||||
|
|
||||||
// GIF: 47 49 46
|
|
||||||
if (buffer[0] === 0x47 && buffer[1] === 0x49 && buffer[2] === 0x46) {
|
|
||||||
return 'gif'
|
|
||||||
}
|
|
||||||
|
|
||||||
// WebP: 52 49 46 46 ... 57 45 42 50
|
|
||||||
if (
|
|
||||||
buffer[0] === 0x52 &&
|
|
||||||
buffer[1] === 0x49 &&
|
|
||||||
buffer[2] === 0x46 &&
|
|
||||||
buffer[3] === 0x46 &&
|
|
||||||
buffer.length >= 12 &&
|
|
||||||
buffer[8] === 0x57 &&
|
|
||||||
buffer[9] === 0x45 &&
|
|
||||||
buffer[10] === 0x42 &&
|
|
||||||
buffer[11] === 0x50
|
|
||||||
) {
|
|
||||||
return 'webp'
|
|
||||||
}
|
|
||||||
|
|
||||||
// TIFF: 49 49 2A 00 (little-endian) or 4D 4D 00 2A (big-endian)
|
|
||||||
if (
|
|
||||||
(buffer[0] === 0x49 && buffer[1] === 0x49 && buffer[2] === 0x2a && buffer[3] === 0x00) ||
|
|
||||||
(buffer[0] === 0x4d && buffer[1] === 0x4d && buffer[2] === 0x00 && buffer[3] === 0x2a)
|
|
||||||
) {
|
|
||||||
return 'tiff'
|
|
||||||
}
|
|
||||||
|
|
||||||
// BMP: 42 4D
|
|
||||||
if (buffer[0] === 0x42 && buffer[1] === 0x4d) {
|
|
||||||
return 'bmp'
|
|
||||||
}
|
|
||||||
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Detect if extension is an image format
|
|
||||||
*/
|
|
||||||
private isImageExtension(ext: string): boolean {
|
|
||||||
const imageExts = [
|
|
||||||
'.jpg',
|
|
||||||
'.jpeg',
|
|
||||||
'.png',
|
|
||||||
'.gif',
|
|
||||||
'.webp',
|
|
||||||
'.tiff',
|
|
||||||
'.tif',
|
|
||||||
'.bmp',
|
|
||||||
'.svg',
|
|
||||||
'.heic',
|
|
||||||
'.heif',
|
|
||||||
'.avif'
|
|
||||||
]
|
|
||||||
|
|
||||||
const normalized = ext.toLowerCase()
|
|
||||||
const withDot = normalized.startsWith('.') ? normalized : `.${normalized}`
|
|
||||||
return imageExts.includes(withDot)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,33 +0,0 @@
|
||||||
/**
|
|
||||||
* Intelligent Import Module
|
|
||||||
* Exports main augmentation and types
|
|
||||||
*/
|
|
||||||
|
|
||||||
export { IntelligentImportAugmentation } from './IntelligentImportAugmentation.js'
|
|
||||||
export type {
|
|
||||||
FormatHandler,
|
|
||||||
FormatHandlerOptions,
|
|
||||||
ProcessedData,
|
|
||||||
IntelligentImportConfig,
|
|
||||||
HandlerRegistry
|
|
||||||
} from './types.js'
|
|
||||||
|
|
||||||
// Format Handlers
|
|
||||||
export { CSVHandler } from './handlers/csvHandler.js'
|
|
||||||
export { ExcelHandler } from './handlers/excelHandler.js'
|
|
||||||
export { PDFHandler } from './handlers/pdfHandler.js'
|
|
||||||
export { ImageHandler } from './handlers/imageHandler.js'
|
|
||||||
|
|
||||||
// Format Handler Registry
|
|
||||||
export {
|
|
||||||
FormatHandlerRegistry,
|
|
||||||
globalHandlerRegistry
|
|
||||||
} from './FormatHandlerRegistry.js'
|
|
||||||
export type { HandlerRegistration } from './FormatHandlerRegistry.js'
|
|
||||||
|
|
||||||
// Image Handler Types
|
|
||||||
export type {
|
|
||||||
ImageMetadata,
|
|
||||||
EXIFData,
|
|
||||||
ImageHandlerOptions
|
|
||||||
} from './handlers/imageHandler.js'
|
|
||||||
|
|
@ -1,875 +0,0 @@
|
||||||
/**
|
|
||||||
* Intelligent Verb Scoring Augmentation
|
|
||||||
*
|
|
||||||
* Enhances relationship quality through intelligent semantic scoring
|
|
||||||
* Provides context-aware relationship weights based on:
|
|
||||||
* - Semantic proximity of connected entities
|
|
||||||
* - Frequency-based amplification
|
|
||||||
* - Temporal decay modeling
|
|
||||||
* - Adaptive learning from usage patterns
|
|
||||||
*
|
|
||||||
* Critical for enterprise knowledge graphs with millions of relationships
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { BaseAugmentation, AugmentationContext } from './brainyAugmentation.js'
|
|
||||||
|
|
||||||
interface VerbScoringConfig {
|
|
||||||
enabled?: boolean
|
|
||||||
|
|
||||||
// Semantic Analysis
|
|
||||||
enableSemanticScoring?: boolean // Use entity embeddings for scoring
|
|
||||||
semanticThreshold?: number // Minimum semantic similarity
|
|
||||||
semanticWeight?: number // Weight of semantic component
|
|
||||||
|
|
||||||
// Frequency Analysis
|
|
||||||
enableFrequencyAmplification?: boolean // Amplify frequently used relationships
|
|
||||||
frequencyDecay?: number // How quickly frequency importance decays
|
|
||||||
maxFrequencyBoost?: number // Maximum boost from frequency
|
|
||||||
|
|
||||||
// Temporal Analysis
|
|
||||||
enableTemporalDecay?: boolean // Apply time-based decay
|
|
||||||
temporalDecayRate?: number // Decay rate per day (0-1)
|
|
||||||
temporalWindow?: number // Time window for relevance (days)
|
|
||||||
|
|
||||||
// Learning & Adaptation
|
|
||||||
enableAdaptiveLearning?: boolean // Learn from usage patterns
|
|
||||||
learningRate?: number // How quickly to adapt (0-1)
|
|
||||||
confidenceThreshold?: number // Minimum confidence for relationships
|
|
||||||
|
|
||||||
// Weight Management
|
|
||||||
minWeight?: number // Minimum relationship weight
|
|
||||||
maxWeight?: number // Maximum relationship weight
|
|
||||||
baseWeight?: number // Default weight for new relationships
|
|
||||||
}
|
|
||||||
|
|
||||||
interface RelationshipMetrics {
|
|
||||||
count: number // How many times this relationship was created
|
|
||||||
totalWeight: number // Sum of all weights
|
|
||||||
averageWeight: number // Average weight
|
|
||||||
lastUpdated: number // Last time this relationship was scored
|
|
||||||
semanticScore: number // Semantic similarity score
|
|
||||||
frequencyScore: number // Frequency-based score
|
|
||||||
temporalScore: number // Time-based relevance score
|
|
||||||
confidenceScore: number // Overall confidence
|
|
||||||
}
|
|
||||||
|
|
||||||
interface ScoringMetrics {
|
|
||||||
relationshipsScored: number
|
|
||||||
averageSemanticScore: number
|
|
||||||
averageFrequencyScore: number
|
|
||||||
averageTemporalScore: number
|
|
||||||
averageConfidenceScore: number
|
|
||||||
adaptiveAdjustments: number
|
|
||||||
computationTimeMs: number
|
|
||||||
minScore?: number
|
|
||||||
maxScore?: number
|
|
||||||
averageScore?: number
|
|
||||||
highConfidenceCount?: number
|
|
||||||
}
|
|
||||||
|
|
||||||
export class IntelligentVerbScoringAugmentation extends BaseAugmentation {
|
|
||||||
name = 'IntelligentVerbScoring'
|
|
||||||
timing = 'around' as const
|
|
||||||
readonly metadata = {
|
|
||||||
reads: ['type', 'verb', 'source', 'target'] as string[],
|
|
||||||
writes: ['weight', 'confidence', 'intelligentScoring'] as string[]
|
|
||||||
} // Adds scoring metadata to verbs
|
|
||||||
operations = ['addVerb', 'relate'] as ('addVerb' | 'relate')[]
|
|
||||||
priority = 10 // Enhancement feature - runs after core operations
|
|
||||||
|
|
||||||
// Augmentation metadata
|
|
||||||
readonly category = 'core' as const
|
|
||||||
readonly description = 'AI-powered intelligent scoring for relationship strength analysis'
|
|
||||||
|
|
||||||
protected config: Required<VerbScoringConfig>
|
|
||||||
private relationshipStats: Map<string, RelationshipMetrics> = new Map()
|
|
||||||
private metrics: ScoringMetrics = {
|
|
||||||
relationshipsScored: 0,
|
|
||||||
averageSemanticScore: 0,
|
|
||||||
averageFrequencyScore: 0,
|
|
||||||
averageTemporalScore: 0,
|
|
||||||
averageConfidenceScore: 0,
|
|
||||||
adaptiveAdjustments: 0,
|
|
||||||
computationTimeMs: 0
|
|
||||||
}
|
|
||||||
private scoringInstance: any // Will hold IntelligentVerbScoring instance
|
|
||||||
|
|
||||||
constructor(config: VerbScoringConfig = {}) {
|
|
||||||
super()
|
|
||||||
this.config = {
|
|
||||||
enabled: config.enabled ?? true, // Smart by default!
|
|
||||||
|
|
||||||
// Semantic Analysis
|
|
||||||
enableSemanticScoring: config.enableSemanticScoring ?? true,
|
|
||||||
semanticThreshold: config.semanticThreshold ?? 0.3,
|
|
||||||
semanticWeight: config.semanticWeight ?? 0.4,
|
|
||||||
|
|
||||||
// Frequency Analysis
|
|
||||||
enableFrequencyAmplification: config.enableFrequencyAmplification ?? true,
|
|
||||||
frequencyDecay: config.frequencyDecay ?? 0.95, // 5% decay per occurrence
|
|
||||||
maxFrequencyBoost: config.maxFrequencyBoost ?? 2.0,
|
|
||||||
|
|
||||||
// Temporal Analysis
|
|
||||||
enableTemporalDecay: config.enableTemporalDecay ?? true,
|
|
||||||
temporalDecayRate: config.temporalDecayRate ?? 0.01, // 1% per day
|
|
||||||
temporalWindow: config.temporalWindow ?? 365, // 1 year
|
|
||||||
|
|
||||||
// Learning & Adaptation
|
|
||||||
enableAdaptiveLearning: config.enableAdaptiveLearning ?? true,
|
|
||||||
learningRate: config.learningRate ?? 0.1,
|
|
||||||
confidenceThreshold: config.confidenceThreshold ?? 0.3,
|
|
||||||
|
|
||||||
// Weight Management
|
|
||||||
minWeight: config.minWeight ?? 0.1,
|
|
||||||
maxWeight: config.maxWeight ?? 1.0,
|
|
||||||
baseWeight: config.baseWeight ?? 0.5
|
|
||||||
}
|
|
||||||
|
|
||||||
// Set enabled property based on config
|
|
||||||
this.enabled = this.config.enabled
|
|
||||||
}
|
|
||||||
|
|
||||||
protected async onInitialize(): Promise<void> {
|
|
||||||
if (this.config.enabled) {
|
|
||||||
this.log('Intelligent verb scoring initialized for enhanced relationship quality')
|
|
||||||
} else {
|
|
||||||
this.log('Intelligent verb scoring disabled')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get this augmentation instance for API compatibility
|
|
||||||
* Used by Brainy to access scoring methods
|
|
||||||
*/
|
|
||||||
getScoring(): IntelligentVerbScoringAugmentation {
|
|
||||||
return this
|
|
||||||
}
|
|
||||||
|
|
||||||
shouldExecute(operation: string, params: any): boolean {
|
|
||||||
// For addVerb, params are passed as array: [sourceId, targetId, verbType, metadata, weight]
|
|
||||||
if (operation === 'addVerb' && this.config.enabled) {
|
|
||||||
return Array.isArray(params) && params.length >= 3
|
|
||||||
}
|
|
||||||
// For relate method, params might be an object
|
|
||||||
if (operation === 'relate' && this.config.enabled) {
|
|
||||||
return params.sourceId && params.targetId && params.relationType
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
async execute<T = any>(
|
|
||||||
operation: string,
|
|
||||||
params: any,
|
|
||||||
next: () => Promise<T>
|
|
||||||
): Promise<T> {
|
|
||||||
if (!this.shouldExecute(operation, params)) {
|
|
||||||
return next()
|
|
||||||
}
|
|
||||||
|
|
||||||
const startTime = Date.now()
|
|
||||||
|
|
||||||
try {
|
|
||||||
let sourceId: string, targetId: string, relationType: string, metadata: any
|
|
||||||
let scoringResult: { weight: number; confidence: number; reasoning: string[] } | null = null
|
|
||||||
|
|
||||||
// Extract parameters based on operation type
|
|
||||||
if (operation === 'addVerb' && Array.isArray(params)) {
|
|
||||||
// addVerb params: [sourceId, targetId, verbType, metadata, weight]
|
|
||||||
[sourceId, targetId, relationType, metadata] = params
|
|
||||||
} else if (operation === 'relate') {
|
|
||||||
// relate params might be an object
|
|
||||||
sourceId = params.sourceId
|
|
||||||
targetId = params.targetId
|
|
||||||
relationType = params.relationType
|
|
||||||
metadata = params.metadata
|
|
||||||
} else {
|
|
||||||
return next()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Skip if weight is already provided explicitly
|
|
||||||
if (Array.isArray(params) && params[4] !== undefined && params[4] !== null) {
|
|
||||||
return next()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get the nouns to compute scoring
|
|
||||||
const sourceNoun = await this.context?.brain.get(sourceId)
|
|
||||||
const targetNoun = await this.context?.brain.get(targetId)
|
|
||||||
|
|
||||||
// Compute intelligent scores with reasoning
|
|
||||||
scoringResult = await this.computeVerbScores(
|
|
||||||
sourceNoun,
|
|
||||||
targetNoun,
|
|
||||||
relationType
|
|
||||||
)
|
|
||||||
|
|
||||||
// For addVerb, modify the params array
|
|
||||||
if (operation === 'addVerb' && Array.isArray(params)) {
|
|
||||||
// Set the weight parameter (index 4)
|
|
||||||
params[4] = scoringResult.weight
|
|
||||||
// Enhance metadata with scoring info
|
|
||||||
params[3] = {
|
|
||||||
...params[3],
|
|
||||||
intelligentScoring: {
|
|
||||||
weight: scoringResult.weight,
|
|
||||||
confidence: scoringResult.confidence,
|
|
||||||
reasoning: scoringResult.reasoning,
|
|
||||||
scoringMethod: this.getScoringMethodsUsed(),
|
|
||||||
computedAt: Date.now()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Execute with enhanced parameters
|
|
||||||
const result = await next()
|
|
||||||
|
|
||||||
// Learn from this relationship
|
|
||||||
if (this.config.enableAdaptiveLearning && scoringResult) {
|
|
||||||
await this.updateRelationshipLearning(
|
|
||||||
sourceId,
|
|
||||||
targetId,
|
|
||||||
relationType,
|
|
||||||
scoringResult.weight
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Update metrics
|
|
||||||
const computationTime = Date.now() - startTime
|
|
||||||
if (scoringResult) {
|
|
||||||
this.updateMetrics(scoringResult.weight, computationTime)
|
|
||||||
}
|
|
||||||
|
|
||||||
return result
|
|
||||||
|
|
||||||
} catch (error) {
|
|
||||||
this.log(`Intelligent verb scoring error: ${error}`, 'error')
|
|
||||||
// Fallback to original parameters
|
|
||||||
return next()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private async calculateIntelligentWeight(
|
|
||||||
sourceId: string,
|
|
||||||
targetId: string,
|
|
||||||
relationType: string,
|
|
||||||
metadata?: any
|
|
||||||
): Promise<number> {
|
|
||||||
let finalWeight = this.config.baseWeight
|
|
||||||
let scoreComponents: any = {}
|
|
||||||
|
|
||||||
// 1. Semantic Proximity Score
|
|
||||||
if (this.config.enableSemanticScoring) {
|
|
||||||
const semanticScore = await this.calculateSemanticScore(sourceId, targetId)
|
|
||||||
scoreComponents.semantic = semanticScore
|
|
||||||
finalWeight = finalWeight * (1 + semanticScore * this.config.semanticWeight)
|
|
||||||
}
|
|
||||||
|
|
||||||
// 2. Frequency Amplification Score
|
|
||||||
if (this.config.enableFrequencyAmplification) {
|
|
||||||
const frequencyScore = this.calculateFrequencyScore(sourceId, targetId, relationType)
|
|
||||||
scoreComponents.frequency = frequencyScore
|
|
||||||
finalWeight = finalWeight * (1 + frequencyScore)
|
|
||||||
}
|
|
||||||
|
|
||||||
// 3. Temporal Relevance Score
|
|
||||||
if (this.config.enableTemporalDecay) {
|
|
||||||
const temporalScore = this.calculateTemporalScore(sourceId, targetId, relationType)
|
|
||||||
scoreComponents.temporal = temporalScore
|
|
||||||
finalWeight = finalWeight * temporalScore
|
|
||||||
}
|
|
||||||
|
|
||||||
// 4. Context Awareness (from metadata)
|
|
||||||
const contextScore = this.calculateContextScore(metadata)
|
|
||||||
scoreComponents.context = contextScore
|
|
||||||
finalWeight = finalWeight * (1 + contextScore * 0.2)
|
|
||||||
|
|
||||||
// 5. Apply constraints
|
|
||||||
finalWeight = Math.max(this.config.minWeight,
|
|
||||||
Math.min(this.config.maxWeight, finalWeight))
|
|
||||||
|
|
||||||
// Store detailed scoring for analysis
|
|
||||||
this.storeDetailedScoring(sourceId, targetId, relationType, {
|
|
||||||
finalWeight,
|
|
||||||
components: scoreComponents,
|
|
||||||
timestamp: Date.now()
|
|
||||||
})
|
|
||||||
|
|
||||||
return finalWeight
|
|
||||||
}
|
|
||||||
|
|
||||||
private async calculateSemanticScore(sourceId: string, targetId: string): Promise<number> {
|
|
||||||
try {
|
|
||||||
// Get embeddings for both entities
|
|
||||||
const sourceNoun = await this.context?.brain.get(sourceId)
|
|
||||||
const targetNoun = await this.context?.brain.get(targetId)
|
|
||||||
|
|
||||||
if (!sourceNoun?.vector || !targetNoun?.vector) {
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get noun types using neural detection (taxonomy-based)
|
|
||||||
const sourceType = await this.detectNounType(sourceNoun.vector)
|
|
||||||
const targetType = await this.detectNounType(targetNoun.vector)
|
|
||||||
|
|
||||||
// Calculate direct similarity
|
|
||||||
const directSimilarity = this.calculateCosineSimilarity(sourceNoun.vector, targetNoun.vector)
|
|
||||||
|
|
||||||
// Calculate taxonomy-based similarity boost
|
|
||||||
const taxonomyBoost = await this.calculateTaxonomyBoost(sourceType, targetType)
|
|
||||||
|
|
||||||
// Blend direct similarity with taxonomy guidance
|
|
||||||
// Taxonomy provides consistency while preserving flexibility
|
|
||||||
const semanticScore = directSimilarity * 0.7 + taxonomyBoost * 0.3
|
|
||||||
|
|
||||||
return Math.min(1, Math.max(0, semanticScore))
|
|
||||||
|
|
||||||
} catch (error) {
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Detect noun type using neural taxonomy matching
|
|
||||||
*/
|
|
||||||
private async detectNounType(vector: number[]): Promise<string> {
|
|
||||||
// Use real neural detection from brain's type detector
|
|
||||||
if (!this.context?.brain) return 'unknown'
|
|
||||||
|
|
||||||
try {
|
|
||||||
// Access the brain's neural type detection system
|
|
||||||
const brain = this.context.brain as any
|
|
||||||
|
|
||||||
// Use the actual neural type detection if available
|
|
||||||
if (brain.neuralDetector?.detectType) {
|
|
||||||
const detectedType = await brain.neuralDetector.detectType(vector)
|
|
||||||
return detectedType || 'unknown'
|
|
||||||
}
|
|
||||||
|
|
||||||
// Fallback to pattern-based detection using actual embeddings
|
|
||||||
const patternAnalyzer = brain.patternAnalyzer || brain.neural?.patternAnalyzer
|
|
||||||
if (patternAnalyzer?.analyzeVector) {
|
|
||||||
const analysis = await patternAnalyzer.analyzeVector(vector)
|
|
||||||
return analysis.type || 'unknown'
|
|
||||||
}
|
|
||||||
|
|
||||||
// Use statistical analysis of vector characteristics
|
|
||||||
const stats = this.analyzeVectorStatistics(vector)
|
|
||||||
return this.inferTypeFromStatistics(stats)
|
|
||||||
} catch (error) {
|
|
||||||
this.log(`Type detection failed: ${(error as Error).message}`, 'warn')
|
|
||||||
return 'unknown'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Analyze vector statistics for type inference
|
|
||||||
*/
|
|
||||||
private analyzeVectorStatistics(vector: number[]): {
|
|
||||||
mean: number
|
|
||||||
variance: number
|
|
||||||
sparsity: number
|
|
||||||
entropy: number
|
|
||||||
magnitude: number
|
|
||||||
} {
|
|
||||||
const n = vector.length
|
|
||||||
if (n === 0) return { mean: 0, variance: 0, sparsity: 0, entropy: 0, magnitude: 0 }
|
|
||||||
|
|
||||||
// Calculate mean
|
|
||||||
const mean = vector.reduce((sum, val) => sum + val, 0) / n
|
|
||||||
|
|
||||||
// Calculate variance
|
|
||||||
const variance = vector.reduce((sum, val) => sum + Math.pow(val - mean, 2), 0) / n
|
|
||||||
|
|
||||||
// Calculate sparsity (percentage of non-zero elements)
|
|
||||||
const nonZeroCount = vector.filter(val => Math.abs(val) > 0.001).length
|
|
||||||
const sparsity = 1 - (nonZeroCount / n)
|
|
||||||
|
|
||||||
// Calculate entropy (information content)
|
|
||||||
const absSum = vector.reduce((sum, val) => sum + Math.abs(val), 0) || 1
|
|
||||||
const probs = vector.map(val => Math.abs(val) / absSum)
|
|
||||||
const entropy = -probs.reduce((sum, p) => {
|
|
||||||
return p > 0 ? sum + p * Math.log2(p) : sum
|
|
||||||
}, 0)
|
|
||||||
|
|
||||||
// Calculate magnitude
|
|
||||||
const magnitude = Math.sqrt(vector.reduce((sum, val) => sum + val * val, 0))
|
|
||||||
|
|
||||||
return { mean, variance, sparsity, entropy, magnitude }
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Infer entity type from vector statistics using neural patterns
|
|
||||||
*/
|
|
||||||
private inferTypeFromStatistics(stats: {
|
|
||||||
mean: number
|
|
||||||
variance: number
|
|
||||||
sparsity: number
|
|
||||||
entropy: number
|
|
||||||
magnitude: number
|
|
||||||
}): string {
|
|
||||||
// Neural pattern recognition based on empirical analysis
|
|
||||||
// These thresholds are derived from analyzing actual embeddings
|
|
||||||
|
|
||||||
// High entropy and low sparsity often indicate abstract concepts
|
|
||||||
if (stats.entropy > 4.5 && stats.sparsity < 0.3) {
|
|
||||||
return 'concept'
|
|
||||||
}
|
|
||||||
|
|
||||||
// Moderate entropy with high magnitude indicates concrete entities
|
|
||||||
if (stats.magnitude > 8 && stats.entropy > 3 && stats.entropy < 4.5) {
|
|
||||||
return 'entity'
|
|
||||||
}
|
|
||||||
|
|
||||||
// High sparsity with focused magnitude indicates specific objects
|
|
||||||
if (stats.sparsity > 0.6 && stats.magnitude > 3) {
|
|
||||||
return 'object'
|
|
||||||
}
|
|
||||||
|
|
||||||
// Documents tend to have balanced statistics
|
|
||||||
if (stats.entropy > 3.5 && stats.entropy < 4.2 && stats.variance > 0.1) {
|
|
||||||
return 'document'
|
|
||||||
}
|
|
||||||
|
|
||||||
// Person entities have characteristic patterns
|
|
||||||
if (stats.magnitude > 5 && stats.magnitude < 10 && stats.variance > 0.15) {
|
|
||||||
return 'person'
|
|
||||||
}
|
|
||||||
|
|
||||||
// Tools and functions have lower entropy
|
|
||||||
if (stats.entropy < 3 && stats.magnitude > 4) {
|
|
||||||
return 'tool'
|
|
||||||
}
|
|
||||||
|
|
||||||
// Default to generic item for unclear patterns
|
|
||||||
return 'item'
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Calculate taxonomy-based similarity boost
|
|
||||||
*/
|
|
||||||
private async calculateTaxonomyBoost(sourceType: string, targetType: string): Promise<number> {
|
|
||||||
// Define valid relationship patterns in taxonomy
|
|
||||||
const validPatterns: Record<string, Record<string, number>> = {
|
|
||||||
'person': { 'concept': 0.9, 'skill': 0.85, 'organization': 0.8, 'person': 0.7 },
|
|
||||||
'concept': { 'concept': 0.9, 'example': 0.85, 'application': 0.8 },
|
|
||||||
'entity': { 'entity': 0.8, 'property': 0.85, 'action': 0.75 },
|
|
||||||
'object': { 'object': 0.7, 'property': 0.8, 'location': 0.75 },
|
|
||||||
'document': { 'topic': 0.9, 'author': 0.85, 'document': 0.7 },
|
|
||||||
'tool': { 'output': 0.9, 'input': 0.85, 'user': 0.8 },
|
|
||||||
'unknown': { 'unknown': 0.5 } // Fallback
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get boost from taxonomy patterns
|
|
||||||
const patterns = validPatterns[sourceType] || validPatterns['unknown']
|
|
||||||
const boost = patterns[targetType] || 0.3 // Low score for unrecognized patterns
|
|
||||||
|
|
||||||
return boost
|
|
||||||
}
|
|
||||||
|
|
||||||
private calculateCosineSimilarity(vectorA: number[], vectorB: number[]): number {
|
|
||||||
if (vectorA.length !== vectorB.length) return 0
|
|
||||||
|
|
||||||
let dotProduct = 0
|
|
||||||
let normA = 0
|
|
||||||
let normB = 0
|
|
||||||
|
|
||||||
for (let i = 0; i < vectorA.length; i++) {
|
|
||||||
dotProduct += vectorA[i] * vectorB[i]
|
|
||||||
normA += vectorA[i] * vectorA[i]
|
|
||||||
normB += vectorB[i] * vectorB[i]
|
|
||||||
}
|
|
||||||
|
|
||||||
const magnitude = Math.sqrt(normA) * Math.sqrt(normB)
|
|
||||||
return magnitude ? dotProduct / magnitude : 0
|
|
||||||
}
|
|
||||||
|
|
||||||
private calculateFrequencyScore(sourceId: string, targetId: string, relationType: string): number {
|
|
||||||
const relationshipKey = `${sourceId}:${relationType}:${targetId}`
|
|
||||||
const stats = this.relationshipStats.get(relationshipKey)
|
|
||||||
|
|
||||||
if (!stats || stats.count <= 1) return 0
|
|
||||||
|
|
||||||
// Frequency boost diminishes with each occurrence
|
|
||||||
const frequencyBoost = Math.log(stats.count) * this.config.frequencyDecay
|
|
||||||
return Math.min(this.config.maxFrequencyBoost, frequencyBoost)
|
|
||||||
}
|
|
||||||
|
|
||||||
private calculateTemporalScore(sourceId: string, targetId: string, relationType: string): number {
|
|
||||||
const relationshipKey = `${sourceId}:${relationType}:${targetId}`
|
|
||||||
const stats = this.relationshipStats.get(relationshipKey)
|
|
||||||
|
|
||||||
if (!stats) return 1.0 // New relationship - full temporal score
|
|
||||||
|
|
||||||
const daysSinceUpdate = (Date.now() - stats.lastUpdated) / (1000 * 60 * 60 * 24)
|
|
||||||
const decayFactor = Math.pow(1 - this.config.temporalDecayRate, daysSinceUpdate)
|
|
||||||
|
|
||||||
// Relationships older than temporal window get minimum score
|
|
||||||
if (daysSinceUpdate > this.config.temporalWindow) {
|
|
||||||
return this.config.minWeight / this.config.baseWeight
|
|
||||||
}
|
|
||||||
|
|
||||||
return Math.max(0.1, decayFactor)
|
|
||||||
}
|
|
||||||
|
|
||||||
private calculateContextScore(metadata?: any): number {
|
|
||||||
if (!metadata) return 0
|
|
||||||
|
|
||||||
let contextScore = 0
|
|
||||||
|
|
||||||
// Boost for explicit importance
|
|
||||||
if (metadata.importance) {
|
|
||||||
contextScore += Math.min(0.5, metadata.importance)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Boost for confidence
|
|
||||||
if (metadata.confidence) {
|
|
||||||
contextScore += Math.min(0.3, metadata.confidence)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Boost for source quality
|
|
||||||
if (metadata.sourceQuality) {
|
|
||||||
contextScore += Math.min(0.2, metadata.sourceQuality)
|
|
||||||
}
|
|
||||||
|
|
||||||
return contextScore
|
|
||||||
}
|
|
||||||
|
|
||||||
private async updateRelationshipLearning(
|
|
||||||
sourceId: string,
|
|
||||||
targetId: string,
|
|
||||||
relationType: string,
|
|
||||||
weight: number
|
|
||||||
): Promise<void> {
|
|
||||||
const relationshipKey = `${sourceId}:${relationType}:${targetId}`
|
|
||||||
let stats = this.relationshipStats.get(relationshipKey)
|
|
||||||
|
|
||||||
if (!stats) {
|
|
||||||
stats = {
|
|
||||||
count: 0,
|
|
||||||
totalWeight: 0,
|
|
||||||
averageWeight: this.config.baseWeight,
|
|
||||||
lastUpdated: Date.now(),
|
|
||||||
semanticScore: 0,
|
|
||||||
frequencyScore: 0,
|
|
||||||
temporalScore: 1.0,
|
|
||||||
confidenceScore: this.config.baseWeight
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Update statistics with learning rate
|
|
||||||
stats.count++
|
|
||||||
stats.totalWeight += weight
|
|
||||||
stats.averageWeight = stats.averageWeight * (1 - this.config.learningRate) +
|
|
||||||
weight * this.config.learningRate
|
|
||||||
stats.lastUpdated = Date.now()
|
|
||||||
|
|
||||||
// Update confidence based on consistency
|
|
||||||
const weightVariance = Math.abs(weight - stats.averageWeight)
|
|
||||||
const consistencyScore = 1 - Math.min(1, weightVariance)
|
|
||||||
stats.confidenceScore = stats.confidenceScore * (1 - this.config.learningRate) +
|
|
||||||
consistencyScore * this.config.learningRate
|
|
||||||
|
|
||||||
this.relationshipStats.set(relationshipKey, stats)
|
|
||||||
this.metrics.adaptiveAdjustments++
|
|
||||||
}
|
|
||||||
|
|
||||||
private getConfidenceScore(sourceId: string, targetId: string, relationType: string): number {
|
|
||||||
const relationshipKey = `${sourceId}:${relationType}:${targetId}`
|
|
||||||
const stats = this.relationshipStats.get(relationshipKey)
|
|
||||||
|
|
||||||
return stats ? stats.confidenceScore : this.config.baseWeight
|
|
||||||
}
|
|
||||||
|
|
||||||
private getScoringMethodsUsed(): string[] {
|
|
||||||
const methods = []
|
|
||||||
if (this.config.enableSemanticScoring) methods.push('semantic')
|
|
||||||
if (this.config.enableFrequencyAmplification) methods.push('frequency')
|
|
||||||
if (this.config.enableTemporalDecay) methods.push('temporal')
|
|
||||||
if (this.config.enableAdaptiveLearning) methods.push('adaptive')
|
|
||||||
return methods
|
|
||||||
}
|
|
||||||
|
|
||||||
private storeDetailedScoring(
|
|
||||||
sourceId: string,
|
|
||||||
targetId: string,
|
|
||||||
relationType: string,
|
|
||||||
scoring: any
|
|
||||||
): void {
|
|
||||||
// Store detailed scoring for analysis and debugging
|
|
||||||
// In production, this might be sent to analytics system
|
|
||||||
}
|
|
||||||
|
|
||||||
private updateMetrics(weight: number, computationTime: number): void {
|
|
||||||
this.metrics.relationshipsScored++
|
|
||||||
|
|
||||||
// Update average computation time with exponential moving average
|
|
||||||
const alpha = 0.1 // Smoothing factor
|
|
||||||
this.metrics.computationTimeMs =
|
|
||||||
alpha * computationTime + (1 - alpha) * this.metrics.computationTimeMs
|
|
||||||
|
|
||||||
// Track score distribution for analysis
|
|
||||||
this.updateScoreDistribution(weight)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Update score distribution statistics
|
|
||||||
*/
|
|
||||||
private updateScoreDistribution(weight: number): void {
|
|
||||||
// Track min/max scores
|
|
||||||
if (!this.metrics.minScore || weight < this.metrics.minScore) {
|
|
||||||
this.metrics.minScore = weight
|
|
||||||
}
|
|
||||||
if (!this.metrics.maxScore || weight > this.metrics.maxScore) {
|
|
||||||
this.metrics.maxScore = weight
|
|
||||||
}
|
|
||||||
|
|
||||||
// Update average score with running average
|
|
||||||
const n = this.metrics.relationshipsScored
|
|
||||||
this.metrics.averageScore = ((this.metrics.averageScore || 0) * (n - 1) + weight) / n
|
|
||||||
|
|
||||||
// Track confidence levels
|
|
||||||
if (weight > this.config.baseWeight * 1.5) {
|
|
||||||
this.metrics.highConfidenceCount = (this.metrics.highConfidenceCount || 0) + 1
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get intelligent verb scoring statistics
|
|
||||||
*/
|
|
||||||
getStats(): ScoringMetrics & {
|
|
||||||
totalRelationships: number
|
|
||||||
averageConfidence: number
|
|
||||||
highConfidenceRelationships: number
|
|
||||||
learningEfficiency: number
|
|
||||||
} {
|
|
||||||
let totalConfidence = 0
|
|
||||||
let highConfidenceCount = 0
|
|
||||||
|
|
||||||
for (const stats of this.relationshipStats.values()) {
|
|
||||||
totalConfidence += stats.confidenceScore
|
|
||||||
if (stats.confidenceScore >= this.config.confidenceThreshold * 2) {
|
|
||||||
highConfidenceCount++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const totalRelationships = this.relationshipStats.size
|
|
||||||
const averageConfidence = totalRelationships > 0 ? totalConfidence / totalRelationships : 0
|
|
||||||
const learningEfficiency = this.metrics.adaptiveAdjustments / Math.max(1, this.metrics.relationshipsScored)
|
|
||||||
|
|
||||||
return {
|
|
||||||
...this.metrics,
|
|
||||||
totalRelationships,
|
|
||||||
averageConfidence,
|
|
||||||
highConfidenceRelationships: highConfidenceCount,
|
|
||||||
learningEfficiency
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Export relationship statistics for analysis
|
|
||||||
*/
|
|
||||||
exportRelationshipStats(): Array<{
|
|
||||||
relationship: string
|
|
||||||
metrics: RelationshipMetrics
|
|
||||||
}> {
|
|
||||||
return Array.from(this.relationshipStats.entries()).map(([key, metrics]) => ({
|
|
||||||
relationship: key,
|
|
||||||
metrics
|
|
||||||
}))
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Import relationship statistics from previous sessions
|
|
||||||
*/
|
|
||||||
importRelationshipStats(stats: Array<{ relationship: string, metrics: RelationshipMetrics }>): void {
|
|
||||||
for (const { relationship, metrics } of stats) {
|
|
||||||
this.relationshipStats.set(relationship, metrics)
|
|
||||||
}
|
|
||||||
this.log(`Imported ${stats.length} relationship statistics`)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get learning statistics for monitoring and debugging
|
|
||||||
* Required for Brainy.getVerbScoringStats()
|
|
||||||
*/
|
|
||||||
getLearningStats(): {
|
|
||||||
totalRelationships: number
|
|
||||||
averageConfidence: number
|
|
||||||
feedbackCount: number
|
|
||||||
topRelationships: Array<{
|
|
||||||
relationship: string
|
|
||||||
count: number
|
|
||||||
averageWeight: number
|
|
||||||
}>
|
|
||||||
} {
|
|
||||||
const relationships = Array.from(this.relationshipStats.entries())
|
|
||||||
const totalRelationships = relationships.length
|
|
||||||
const feedbackCount = relationships.reduce((sum, [, stats]) => sum + stats.count, 0)
|
|
||||||
|
|
||||||
const averageWeight = relationships.reduce((sum, [, stats]) => sum + stats.averageWeight, 0) / totalRelationships || 0
|
|
||||||
const averageConfidence = Math.min(averageWeight + 0.2, 1.0)
|
|
||||||
|
|
||||||
const topRelationships = relationships
|
|
||||||
.map(([key, stats]) => ({
|
|
||||||
relationship: key,
|
|
||||||
count: stats.count,
|
|
||||||
averageWeight: stats.averageWeight
|
|
||||||
}))
|
|
||||||
.sort((a, b) => b.count - a.count)
|
|
||||||
.slice(0, 10)
|
|
||||||
|
|
||||||
return {
|
|
||||||
totalRelationships,
|
|
||||||
averageConfidence,
|
|
||||||
feedbackCount,
|
|
||||||
topRelationships
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Export learning data for backup or analysis
|
|
||||||
* Required for Brainy.exportVerbScoringLearningData()
|
|
||||||
*/
|
|
||||||
exportLearningData(): string {
|
|
||||||
const data = {
|
|
||||||
config: this.config,
|
|
||||||
stats: Array.from(this.relationshipStats.entries()).map(([key, stats]) => ({
|
|
||||||
relationship: key,
|
|
||||||
...stats
|
|
||||||
})),
|
|
||||||
exportedAt: new Date().toISOString(),
|
|
||||||
version: '1.0'
|
|
||||||
}
|
|
||||||
return JSON.stringify(data, null, 2)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Import learning data from backup
|
|
||||||
* Required for Brainy.importVerbScoringLearningData()
|
|
||||||
*/
|
|
||||||
importLearningData(jsonData: string): void {
|
|
||||||
try {
|
|
||||||
const data = JSON.parse(jsonData)
|
|
||||||
|
|
||||||
if (data.stats && Array.isArray(data.stats)) {
|
|
||||||
for (const stat of data.stats) {
|
|
||||||
if (stat.relationship) {
|
|
||||||
this.relationshipStats.set(stat.relationship, {
|
|
||||||
count: stat.count || 1,
|
|
||||||
totalWeight: stat.totalWeight || stat.averageWeight || 0.5,
|
|
||||||
averageWeight: stat.averageWeight || 0.5,
|
|
||||||
lastUpdated: stat.lastUpdated || Date.now(),
|
|
||||||
semanticScore: stat.semanticScore || 0.5,
|
|
||||||
frequencyScore: stat.frequencyScore || 0.5,
|
|
||||||
temporalScore: stat.temporalScore || 1.0,
|
|
||||||
confidenceScore: stat.confidenceScore || 0.5
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
this.log(`Imported learning data: ${this.relationshipStats.size} relationships`)
|
|
||||||
} catch (error) {
|
|
||||||
console.error('Failed to import learning data:', error)
|
|
||||||
throw new Error(`Failed to import learning data: ${error}`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Provide feedback on a relationship's weight
|
|
||||||
* Required for Brainy.provideVerbScoringFeedback()
|
|
||||||
*/
|
|
||||||
async provideFeedback(
|
|
||||||
sourceId: string,
|
|
||||||
targetId: string,
|
|
||||||
relationType: string,
|
|
||||||
feedback: number,
|
|
||||||
feedbackType: 'correction' | 'validation' | 'enhancement' = 'correction'
|
|
||||||
): Promise<void> {
|
|
||||||
const key = `${sourceId}-${relationType}-${targetId}`
|
|
||||||
const stats = this.relationshipStats.get(key) || {
|
|
||||||
count: 0,
|
|
||||||
totalWeight: 0,
|
|
||||||
averageWeight: 0.5,
|
|
||||||
lastUpdated: Date.now(),
|
|
||||||
semanticScore: 0.5,
|
|
||||||
frequencyScore: 0.5,
|
|
||||||
temporalScore: 1.0,
|
|
||||||
confidenceScore: 0.5
|
|
||||||
}
|
|
||||||
|
|
||||||
// Update statistics based on feedback
|
|
||||||
if (feedbackType === 'correction') {
|
|
||||||
// Direct correction - heavily weight the feedback
|
|
||||||
stats.averageWeight = stats.averageWeight * 0.3 + feedback * 0.7
|
|
||||||
} else if (feedbackType === 'validation') {
|
|
||||||
// Validation - slightly adjust towards feedback
|
|
||||||
stats.averageWeight = stats.averageWeight * 0.8 + feedback * 0.2
|
|
||||||
} else {
|
|
||||||
// Enhancement - minor adjustment
|
|
||||||
stats.averageWeight = stats.averageWeight * 0.9 + feedback * 0.1
|
|
||||||
}
|
|
||||||
|
|
||||||
stats.count++
|
|
||||||
stats.totalWeight += feedback
|
|
||||||
stats.lastUpdated = Date.now()
|
|
||||||
|
|
||||||
this.relationshipStats.set(key, stats)
|
|
||||||
this.metrics.adaptiveAdjustments++
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Compute intelligent scores for a verb relationship
|
|
||||||
* Used internally during verb creation
|
|
||||||
*/
|
|
||||||
async computeVerbScores(
|
|
||||||
sourceNoun: any,
|
|
||||||
targetNoun: any,
|
|
||||||
relationType: string
|
|
||||||
): Promise<{
|
|
||||||
weight: number
|
|
||||||
confidence: number
|
|
||||||
reasoning: string[]
|
|
||||||
}> {
|
|
||||||
const reasoning: string[] = []
|
|
||||||
let totalScore = 0
|
|
||||||
let components = 0
|
|
||||||
|
|
||||||
// Semantic scoring
|
|
||||||
if (this.config.enableSemanticScoring && sourceNoun?.vector && targetNoun?.vector) {
|
|
||||||
const similarity = this.calculateCosineSimilarity(sourceNoun.vector, targetNoun.vector)
|
|
||||||
const semanticScore = Math.max(similarity, this.config.semanticThreshold)
|
|
||||||
totalScore += semanticScore * this.config.semanticWeight
|
|
||||||
components++
|
|
||||||
reasoning.push(`Semantic similarity: ${(similarity * 100).toFixed(1)}%`)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Frequency scoring
|
|
||||||
const key = `${sourceNoun?.id}-${relationType}-${targetNoun?.id}`
|
|
||||||
const stats = this.relationshipStats.get(key)
|
|
||||||
if (this.config.enableFrequencyAmplification && stats) {
|
|
||||||
const frequencyScore = Math.min(1 + (stats.count - 1) * 0.1, this.config.maxFrequencyBoost)
|
|
||||||
totalScore += frequencyScore * 0.3
|
|
||||||
components++
|
|
||||||
reasoning.push(`Frequency boost: ${frequencyScore.toFixed(2)}x`)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Temporal decay scoring
|
|
||||||
if (this.config.enableTemporalDecay) {
|
|
||||||
reasoning.push(`Temporal decay applied (rate: ${this.config.temporalDecayRate})`)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Calculate final weight
|
|
||||||
const weight = components > 0
|
|
||||||
? Math.min(Math.max(totalScore / components, this.config.minWeight), this.config.maxWeight)
|
|
||||||
: this.config.baseWeight
|
|
||||||
|
|
||||||
const confidence = Math.min(weight + 0.2, 1.0)
|
|
||||||
|
|
||||||
return { weight, confidence, reasoning }
|
|
||||||
}
|
|
||||||
|
|
||||||
protected async onShutdown(): Promise<void> {
|
|
||||||
const stats = this.getStats()
|
|
||||||
this.log(`Intelligent verb scoring shutdown: ${stats.relationshipsScored} relationships scored, ${Math.round(stats.averageConfidence * 100)}% avg confidence`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,207 +0,0 @@
|
||||||
/**
|
|
||||||
* Augmentation Manifest Types
|
|
||||||
*
|
|
||||||
* Defines the manifest structure for augmentation discovery and configuration
|
|
||||||
* Enables tools like brain-cloud to discover and configure augmentations
|
|
||||||
*/
|
|
||||||
|
|
||||||
/**
|
|
||||||
* JSON Schema type for configuration validation
|
|
||||||
*/
|
|
||||||
export interface JSONSchema {
|
|
||||||
type?: 'object' | 'array' | 'string' | 'number' | 'boolean' | 'null'
|
|
||||||
properties?: Record<string, JSONSchema>
|
|
||||||
items?: JSONSchema
|
|
||||||
required?: string[]
|
|
||||||
default?: any
|
|
||||||
description?: string
|
|
||||||
minimum?: number
|
|
||||||
maximum?: number
|
|
||||||
minLength?: number
|
|
||||||
maxLength?: number
|
|
||||||
pattern?: string
|
|
||||||
enum?: any[]
|
|
||||||
additionalProperties?: boolean | JSONSchema
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Augmentation manifest for discovery and configuration
|
|
||||||
*/
|
|
||||||
export interface AugmentationManifest {
|
|
||||||
/**
|
|
||||||
* Unique identifier for the augmentation (e.g., 'wal', 'cache')
|
|
||||||
*/
|
|
||||||
id: string
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Display name for the augmentation
|
|
||||||
*/
|
|
||||||
name: string
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Semantic version (e.g., '2.0.0')
|
|
||||||
*/
|
|
||||||
version: string
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Author or organization
|
|
||||||
*/
|
|
||||||
author?: string
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Short description of what the augmentation does
|
|
||||||
*/
|
|
||||||
description: string
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Detailed description for documentation
|
|
||||||
*/
|
|
||||||
longDescription?: string
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Augmentation category for organization
|
|
||||||
*/
|
|
||||||
category: 'storage' | 'performance' | 'analytics' | 'integration' | 'internal' | 'core' | 'premium' | 'community' | 'external'
|
|
||||||
|
|
||||||
/**
|
|
||||||
* JSON Schema for configuration options
|
|
||||||
* Used to generate configuration UIs and validate configuration
|
|
||||||
*/
|
|
||||||
configSchema?: JSONSchema
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Default configuration values
|
|
||||||
*/
|
|
||||||
configDefaults?: Record<string, any>
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Configuration examples for documentation
|
|
||||||
*/
|
|
||||||
configExamples?: Array<{
|
|
||||||
name: string
|
|
||||||
description: string
|
|
||||||
config: Record<string, any>
|
|
||||||
}>
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Minimum Brainy version required
|
|
||||||
*/
|
|
||||||
minBrainyVersion?: string
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Maximum Brainy version supported
|
|
||||||
*/
|
|
||||||
maxBrainyVersion?: string
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Other augmentations this one depends on
|
|
||||||
*/
|
|
||||||
dependencies?: Array<{
|
|
||||||
id: string
|
|
||||||
version?: string
|
|
||||||
optional?: boolean
|
|
||||||
}>
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Keywords for search and discovery
|
|
||||||
*/
|
|
||||||
keywords?: string[]
|
|
||||||
|
|
||||||
/**
|
|
||||||
* URL to documentation
|
|
||||||
*/
|
|
||||||
documentation?: string
|
|
||||||
|
|
||||||
/**
|
|
||||||
* URL to source code repository
|
|
||||||
*/
|
|
||||||
repository?: string
|
|
||||||
|
|
||||||
/**
|
|
||||||
* License identifier (e.g., 'MIT', 'Apache-2.0')
|
|
||||||
*/
|
|
||||||
license?: string
|
|
||||||
|
|
||||||
/**
|
|
||||||
* UI hints for tools and configuration interfaces
|
|
||||||
*/
|
|
||||||
ui?: {
|
|
||||||
/**
|
|
||||||
* Icon for the augmentation (emoji or URL)
|
|
||||||
*/
|
|
||||||
icon?: string
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Color theme for UI
|
|
||||||
*/
|
|
||||||
color?: string
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Custom React component name for configuration
|
|
||||||
*/
|
|
||||||
configComponent?: string
|
|
||||||
|
|
||||||
/**
|
|
||||||
* URL to dashboard or control panel
|
|
||||||
*/
|
|
||||||
dashboardUrl?: string
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Hide from UI listings
|
|
||||||
*/
|
|
||||||
hidden?: boolean
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Performance characteristics
|
|
||||||
*/
|
|
||||||
performance?: {
|
|
||||||
/**
|
|
||||||
* Estimated memory usage
|
|
||||||
*/
|
|
||||||
memoryUsage?: 'low' | 'medium' | 'high'
|
|
||||||
|
|
||||||
/**
|
|
||||||
* CPU intensity
|
|
||||||
*/
|
|
||||||
cpuUsage?: 'low' | 'medium' | 'high'
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Network usage
|
|
||||||
*/
|
|
||||||
networkUsage?: 'none' | 'low' | 'medium' | 'high'
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Feature flags this augmentation provides
|
|
||||||
*/
|
|
||||||
features?: string[]
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Operations this augmentation enhances
|
|
||||||
*/
|
|
||||||
enhancedOperations?: string[]
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Metrics this augmentation exposes
|
|
||||||
*/
|
|
||||||
metrics?: Array<{
|
|
||||||
name: string
|
|
||||||
type: 'counter' | 'gauge' | 'histogram'
|
|
||||||
description: string
|
|
||||||
}>
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Status of the augmentation
|
|
||||||
*/
|
|
||||||
status?: 'stable' | 'beta' | 'experimental' | 'deprecated'
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Deprecation notice if applicable
|
|
||||||
*/
|
|
||||||
deprecation?: {
|
|
||||||
since: string
|
|
||||||
alternative?: string
|
|
||||||
removalDate?: string
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,204 +0,0 @@
|
||||||
/**
|
|
||||||
* Runtime enforcement of metadata contracts
|
|
||||||
* Ensures augmentations only access declared fields
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { BrainyAugmentation, MetadataAccess } from './brainyAugmentation.js'
|
|
||||||
|
|
||||||
export class MetadataEnforcer {
|
|
||||||
/**
|
|
||||||
* Enforce metadata access based on augmentation contract
|
|
||||||
* Returns a wrapped metadata object that enforces the contract
|
|
||||||
*/
|
|
||||||
static enforce(
|
|
||||||
augmentation: BrainyAugmentation,
|
|
||||||
metadata: any,
|
|
||||||
operation: 'read' | 'write' = 'write'
|
|
||||||
): any {
|
|
||||||
// Handle simple contracts
|
|
||||||
if (augmentation.metadata === 'none') {
|
|
||||||
// No access at all
|
|
||||||
if (operation === 'read') return null
|
|
||||||
throw new Error(`Augmentation '${augmentation.name}' has metadata='none' - cannot access metadata`)
|
|
||||||
}
|
|
||||||
|
|
||||||
if (augmentation.metadata === 'readonly') {
|
|
||||||
if (operation === 'read') {
|
|
||||||
// Return frozen deep clone for read-only access
|
|
||||||
return deepFreeze(deepClone(metadata))
|
|
||||||
}
|
|
||||||
throw new Error(`Augmentation '${augmentation.name}' has metadata='readonly' - cannot write`)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Handle specific field access
|
|
||||||
const access = augmentation.metadata as MetadataAccess
|
|
||||||
|
|
||||||
if (operation === 'read') {
|
|
||||||
// For reads, filter to allowed fields
|
|
||||||
if (access.reads === '*') {
|
|
||||||
return deepClone(metadata) // Can read everything
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!access.reads) {
|
|
||||||
return {} // No read access
|
|
||||||
}
|
|
||||||
|
|
||||||
// Filter to specific fields
|
|
||||||
const filtered: any = {}
|
|
||||||
for (const field of access.reads) {
|
|
||||||
if (field.includes('.')) {
|
|
||||||
// Handle nested fields like '_brainy.deleted'
|
|
||||||
const parts = field.split('.')
|
|
||||||
let source = metadata
|
|
||||||
let target = filtered
|
|
||||||
|
|
||||||
for (let i = 0; i < parts.length - 1; i++) {
|
|
||||||
const part = parts[i]
|
|
||||||
if (!source[part]) break
|
|
||||||
if (!target[part]) target[part] = {}
|
|
||||||
source = source[part]
|
|
||||||
target = target[part]
|
|
||||||
}
|
|
||||||
|
|
||||||
const lastPart = parts[parts.length - 1]
|
|
||||||
if (source && lastPart in source) {
|
|
||||||
target[lastPart] = source[lastPart]
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
// Simple field
|
|
||||||
if (field in metadata) {
|
|
||||||
filtered[field] = metadata[field]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return filtered
|
|
||||||
}
|
|
||||||
|
|
||||||
// For writes, create a proxy that validates
|
|
||||||
return new Proxy(metadata, {
|
|
||||||
set(target, prop, value) {
|
|
||||||
const field = String(prop)
|
|
||||||
|
|
||||||
// Check if write is allowed
|
|
||||||
if (access.writes === '*') {
|
|
||||||
// Can write anything
|
|
||||||
target[prop] = value
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!access.writes || !access.writes.includes(field)) {
|
|
||||||
throw new Error(
|
|
||||||
`Augmentation '${augmentation.name}' cannot write to field '${field}'. ` +
|
|
||||||
`Allowed writes: ${access.writes?.join(', ') || 'none'}`
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check namespace if specified
|
|
||||||
if (access.namespace && !field.startsWith(access.namespace)) {
|
|
||||||
console.warn(
|
|
||||||
`Augmentation '${augmentation.name}' writing outside its namespace. ` +
|
|
||||||
`Expected: ${access.namespace}.*, got: ${field}`
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
target[prop] = value
|
|
||||||
return true
|
|
||||||
},
|
|
||||||
|
|
||||||
deleteProperty(target, prop) {
|
|
||||||
const field = String(prop)
|
|
||||||
|
|
||||||
// Deletion counts as a write
|
|
||||||
if (access.writes === '*' || access.writes?.includes(field)) {
|
|
||||||
delete target[prop]
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
throw new Error(
|
|
||||||
`Augmentation '${augmentation.name}' cannot delete field '${field}'`
|
|
||||||
)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Validate that an augmentation's actual behavior matches its contract
|
|
||||||
* Used in testing to verify contracts are accurate
|
|
||||||
*/
|
|
||||||
static async validateContract(
|
|
||||||
augmentation: BrainyAugmentation,
|
|
||||||
testMetadata: any = { test: 'data', _brainy: { deleted: false } }
|
|
||||||
): Promise<{ valid: boolean; violations: string[] }> {
|
|
||||||
const violations: string[] = []
|
|
||||||
|
|
||||||
// Test read access
|
|
||||||
try {
|
|
||||||
const readable = this.enforce(augmentation, testMetadata, 'read')
|
|
||||||
|
|
||||||
if (augmentation.metadata === 'none' && readable !== null) {
|
|
||||||
violations.push(`Contract says 'none' but got readable metadata`)
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
violations.push(`Read enforcement error: ${error}`)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Test write access
|
|
||||||
try {
|
|
||||||
const writable = this.enforce(augmentation, testMetadata, 'write')
|
|
||||||
|
|
||||||
if (augmentation.metadata === 'none') {
|
|
||||||
violations.push(`Contract says 'none' but got writable metadata`)
|
|
||||||
}
|
|
||||||
|
|
||||||
if (augmentation.metadata === 'readonly') {
|
|
||||||
// Try to write - should fail
|
|
||||||
try {
|
|
||||||
writable.testWrite = 'value'
|
|
||||||
violations.push(`Contract says 'readonly' but write succeeded`)
|
|
||||||
} catch {
|
|
||||||
// Expected to fail
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
// Expected for 'none' and 'readonly' on write
|
|
||||||
if (augmentation.metadata !== 'none' && augmentation.metadata !== 'readonly') {
|
|
||||||
violations.push(`Write enforcement error: ${error}`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
valid: violations.length === 0,
|
|
||||||
violations
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Helper functions
|
|
||||||
function deepClone(obj: any): any {
|
|
||||||
if (obj === null || typeof obj !== 'object') return obj
|
|
||||||
if (obj instanceof Date) return new Date(obj)
|
|
||||||
if (obj instanceof Array) return obj.map(item => deepClone(item))
|
|
||||||
|
|
||||||
const cloned: any = {}
|
|
||||||
for (const key in obj) {
|
|
||||||
if (obj.hasOwnProperty(key)) {
|
|
||||||
cloned[key] = deepClone(obj[key])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return cloned
|
|
||||||
}
|
|
||||||
|
|
||||||
function deepFreeze(obj: any): any {
|
|
||||||
Object.freeze(obj)
|
|
||||||
|
|
||||||
Object.getOwnPropertyNames(obj).forEach(prop => {
|
|
||||||
if (obj[prop] !== null &&
|
|
||||||
(typeof obj[prop] === 'object' || typeof obj[prop] === 'function') &&
|
|
||||||
!Object.isFrozen(obj[prop])) {
|
|
||||||
deepFreeze(obj[prop])
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
return obj
|
|
||||||
}
|
|
||||||
|
|
@ -1,356 +0,0 @@
|
||||||
/**
|
|
||||||
* Metrics Augmentation - Optional Performance & Usage Metrics
|
|
||||||
*
|
|
||||||
* IMPORTANT: This is SEPARATE from core counting (brain.counts.*) which is always enabled.
|
|
||||||
*
|
|
||||||
* Core counting provides O(1) entity/relationship counts for scalability.
|
|
||||||
* This augmentation provides performance analytics and observability.
|
|
||||||
*
|
|
||||||
* Features:
|
|
||||||
* - Search performance tracking (latency, throughput)
|
|
||||||
* - Cache hit/miss rates
|
|
||||||
* - Operation timing analysis
|
|
||||||
* - Usage pattern insights
|
|
||||||
*
|
|
||||||
* Zero-config: Automatically enabled for observability
|
|
||||||
* Can be disabled via augmentation registry without affecting core performance
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { BaseAugmentation, AugmentationContext } from './brainyAugmentation.js'
|
|
||||||
import { StatisticsCollector } from '../utils/statisticsCollector.js'
|
|
||||||
import type { BaseStorageAdapter as StorageAdapter } from '../storage/adapters/baseStorageAdapter.js'
|
|
||||||
|
|
||||||
export interface MetricsConfig {
|
|
||||||
enabled?: boolean
|
|
||||||
trackSearches?: boolean
|
|
||||||
trackContentTypes?: boolean
|
|
||||||
trackVerbTypes?: boolean
|
|
||||||
trackStorageSizes?: boolean
|
|
||||||
persistMetrics?: boolean
|
|
||||||
metricsInterval?: number
|
|
||||||
silent?: boolean // New: Silent mode support
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* MetricsAugmentation - Makes metrics collection optional and pluggable
|
|
||||||
*
|
|
||||||
* Features:
|
|
||||||
* - Performance tracking (search latency, throughput)
|
|
||||||
* - Usage patterns (content types, verb types)
|
|
||||||
* - Storage metrics (sizes, counts)
|
|
||||||
* - Zero-config with smart defaults
|
|
||||||
*/
|
|
||||||
export class MetricsAugmentation extends BaseAugmentation {
|
|
||||||
readonly metadata = 'readonly' as const // Reads metadata for metrics
|
|
||||||
readonly name = 'metrics'
|
|
||||||
readonly timing = 'after' as const
|
|
||||||
operations = ['add', 'update', 'search', 'find', 'similar', 'delete', 'relate', 'unrelate', 'clear', 'all'] as ('add' | 'update' | 'search' | 'find' | 'similar' | 'delete' | 'relate' | 'unrelate' | 'clear' | 'all')[]
|
|
||||||
readonly priority = 40 // Low priority, runs after other augmentations
|
|
||||||
|
|
||||||
private statisticsCollector: StatisticsCollector | null = null
|
|
||||||
protected config: MetricsConfig
|
|
||||||
private metricsTimer: NodeJS.Timeout | null = null
|
|
||||||
|
|
||||||
constructor(config: MetricsConfig = {}) {
|
|
||||||
super()
|
|
||||||
this.config = {
|
|
||||||
enabled: true,
|
|
||||||
trackSearches: true,
|
|
||||||
trackContentTypes: true,
|
|
||||||
trackVerbTypes: true,
|
|
||||||
trackStorageSizes: true,
|
|
||||||
persistMetrics: true,
|
|
||||||
metricsInterval: 60000, // Update metrics every minute
|
|
||||||
...config
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected async onInitialize(): Promise<void> {
|
|
||||||
if (!this.config.enabled) {
|
|
||||||
this.log('Metrics augmentation disabled by configuration')
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Initialize statistics collector
|
|
||||||
this.statisticsCollector = new StatisticsCollector()
|
|
||||||
|
|
||||||
// Load existing metrics from storage if available
|
|
||||||
if (this.config.persistMetrics && this.context?.storage) {
|
|
||||||
try {
|
|
||||||
const storage = this.context.storage as StorageAdapter
|
|
||||||
const existingStats = await storage.getStatistics?.()
|
|
||||||
if (existingStats) {
|
|
||||||
this.statisticsCollector.mergeFromStorage(existingStats)
|
|
||||||
this.log('Loaded existing metrics from storage')
|
|
||||||
}
|
|
||||||
} catch (e) {
|
|
||||||
this.log('Could not load existing metrics', 'info')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Start metrics update timer
|
|
||||||
if (this.config.metricsInterval && this.config.metricsInterval > 0) {
|
|
||||||
this.startMetricsTimer()
|
|
||||||
}
|
|
||||||
|
|
||||||
this.log('Metrics augmentation initialized')
|
|
||||||
}
|
|
||||||
|
|
||||||
protected async onShutdown(): Promise<void> {
|
|
||||||
// Stop metrics timer
|
|
||||||
if (this.metricsTimer) {
|
|
||||||
clearInterval(this.metricsTimer)
|
|
||||||
this.metricsTimer = null
|
|
||||||
}
|
|
||||||
|
|
||||||
// Persist final metrics
|
|
||||||
if (this.config.persistMetrics && this.statisticsCollector && this.context?.storage) {
|
|
||||||
try {
|
|
||||||
await this.persistMetrics()
|
|
||||||
} catch (error) {
|
|
||||||
this.log('Error persisting metrics during shutdown', 'warn')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
this.statisticsCollector = null
|
|
||||||
this.log('Metrics augmentation shut down')
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Execute augmentation - track metrics for operations
|
|
||||||
*/
|
|
||||||
async execute<T = any>(
|
|
||||||
operation: string,
|
|
||||||
params: any,
|
|
||||||
next: () => Promise<T>
|
|
||||||
): Promise<T> {
|
|
||||||
// If metrics disabled, just pass through
|
|
||||||
if (!this.statisticsCollector || !this.config.enabled) {
|
|
||||||
return next()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Track operation timing
|
|
||||||
const startTime = Date.now()
|
|
||||||
|
|
||||||
try {
|
|
||||||
const result = await next()
|
|
||||||
const duration = Date.now() - startTime
|
|
||||||
|
|
||||||
// Track metrics based on operation
|
|
||||||
switch (operation) {
|
|
||||||
case 'add':
|
|
||||||
this.handleAdd(params, duration)
|
|
||||||
break
|
|
||||||
|
|
||||||
case 'search':
|
|
||||||
this.handleSearch(params, duration)
|
|
||||||
break
|
|
||||||
|
|
||||||
case 'delete':
|
|
||||||
this.handleDelete(duration)
|
|
||||||
break
|
|
||||||
|
|
||||||
case 'clear':
|
|
||||||
this.handleClear()
|
|
||||||
break
|
|
||||||
}
|
|
||||||
|
|
||||||
return result
|
|
||||||
} catch (error) {
|
|
||||||
// Error tracking removed - StatisticsCollector doesn't have trackError method
|
|
||||||
// Could be added later if needed
|
|
||||||
throw error
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Handle add operation metrics
|
|
||||||
*/
|
|
||||||
private handleAdd(params: any, duration: number): void {
|
|
||||||
if (!this.statisticsCollector) return
|
|
||||||
|
|
||||||
// Track update
|
|
||||||
this.statisticsCollector.trackUpdate()
|
|
||||||
|
|
||||||
// Track content type if available
|
|
||||||
if (this.config.trackContentTypes && params.metadata?.noun) {
|
|
||||||
this.statisticsCollector.trackContentType(params.metadata.noun)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Track verb type if it's a verb operation
|
|
||||||
if (this.config.trackVerbTypes && params.metadata?.verb) {
|
|
||||||
this.statisticsCollector.trackVerbType(params.metadata.verb)
|
|
||||||
}
|
|
||||||
|
|
||||||
this.log(`Add operation completed in ${duration}ms`, 'info')
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Handle search operation metrics
|
|
||||||
*/
|
|
||||||
private handleSearch(params: any, duration: number): void {
|
|
||||||
if (!this.statisticsCollector || !this.config.trackSearches) return
|
|
||||||
|
|
||||||
const { query } = params
|
|
||||||
this.statisticsCollector.trackSearch(query || '', duration)
|
|
||||||
this.log(`Search completed in ${duration}ms`, 'info')
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Handle delete operation metrics
|
|
||||||
*/
|
|
||||||
private handleDelete(duration: number): void {
|
|
||||||
if (!this.statisticsCollector) return
|
|
||||||
|
|
||||||
this.statisticsCollector.trackUpdate()
|
|
||||||
this.log(`Delete operation completed in ${duration}ms`, 'info')
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Handle clear operation - reset metrics
|
|
||||||
*/
|
|
||||||
private handleClear(): void {
|
|
||||||
if (!this.statisticsCollector) return
|
|
||||||
|
|
||||||
// Reset statistics when all data is cleared
|
|
||||||
this.statisticsCollector = new StatisticsCollector()
|
|
||||||
this.log('Metrics reset due to clear operation')
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Start periodic metrics update timer
|
|
||||||
*/
|
|
||||||
private startMetricsTimer(): void {
|
|
||||||
if (this.metricsTimer) return
|
|
||||||
|
|
||||||
this.metricsTimer = setInterval(async () => {
|
|
||||||
await this.updateStorageMetrics()
|
|
||||||
if (this.config.persistMetrics) {
|
|
||||||
await this.persistMetrics()
|
|
||||||
}
|
|
||||||
}, this.config.metricsInterval!)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Update storage size metrics
|
|
||||||
*/
|
|
||||||
private async updateStorageMetrics(): Promise<void> {
|
|
||||||
if (!this.statisticsCollector || !this.config.trackStorageSizes) return
|
|
||||||
if (!this.context?.storage) return
|
|
||||||
|
|
||||||
try {
|
|
||||||
const storage = this.context.storage as StorageAdapter
|
|
||||||
const stats = await storage.getStatistics?.()
|
|
||||||
|
|
||||||
if (stats) {
|
|
||||||
// Estimate sizes based on counts
|
|
||||||
const avgNounSize = 1024 // 1KB average
|
|
||||||
const avgVerbSize = 256 // 256B average
|
|
||||||
|
|
||||||
this.statisticsCollector.updateStorageSizes({
|
|
||||||
nouns: (stats.totalNodes || 0) * avgNounSize,
|
|
||||||
verbs: (stats.totalEdges || 0) * avgVerbSize,
|
|
||||||
metadata: (stats.totalNodes || 0) * 512, // 512B per metadata
|
|
||||||
index: (stats.hnswIndexSize || 0) // Use HNSW index size from stats
|
|
||||||
})
|
|
||||||
}
|
|
||||||
} catch (e) {
|
|
||||||
this.log('Could not update storage metrics', 'info')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Persist metrics to storage
|
|
||||||
*/
|
|
||||||
private async persistMetrics(): Promise<void> {
|
|
||||||
if (!this.statisticsCollector || !this.context?.storage) return
|
|
||||||
|
|
||||||
try {
|
|
||||||
const stats = this.statisticsCollector.getStatistics()
|
|
||||||
// Storage adapters can optionally store these metrics
|
|
||||||
// This is a no-op for adapters that don't support it
|
|
||||||
this.log('Metrics persisted to storage', 'info')
|
|
||||||
} catch (e) {
|
|
||||||
this.log('Could not persist metrics', 'info')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get current metrics (performance analytics only)
|
|
||||||
*
|
|
||||||
* NOTE: For production counting (entities, relationships), use:
|
|
||||||
* - brain.counts.entities() - O(1) total count
|
|
||||||
* - brain.counts.byType() - O(1) type-specific counts
|
|
||||||
* - brain.counts.relationships() - O(1) relationship counts
|
|
||||||
*/
|
|
||||||
getStatistics() {
|
|
||||||
if (!this.statisticsCollector) {
|
|
||||||
return {
|
|
||||||
enabled: false,
|
|
||||||
note: 'For core counting, use brain.counts.* APIs which are always available',
|
|
||||||
totalSearches: 0,
|
|
||||||
totalUpdates: 0,
|
|
||||||
contentTypes: {},
|
|
||||||
verbTypes: {},
|
|
||||||
searchPerformance: {
|
|
||||||
averageLatency: 0,
|
|
||||||
p95Latency: 0,
|
|
||||||
p99Latency: 0
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
enabled: true,
|
|
||||||
note: 'Performance analytics only. Core counting available via brain.counts.*',
|
|
||||||
...this.statisticsCollector.getStatistics()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Record cache hit (called by cache augmentation)
|
|
||||||
* Note: Cache metrics are tracked internally by StatisticsCollector
|
|
||||||
*/
|
|
||||||
recordCacheHit(): void {
|
|
||||||
// StatisticsCollector doesn't have trackCacheHit method
|
|
||||||
// Cache metrics would need to be implemented if needed
|
|
||||||
this.log('Cache hit recorded', 'info')
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Record cache miss (called by cache augmentation)
|
|
||||||
* Note: Cache metrics are tracked internally by StatisticsCollector
|
|
||||||
*/
|
|
||||||
recordCacheMiss(): void {
|
|
||||||
// StatisticsCollector doesn't have trackCacheMiss method
|
|
||||||
// Cache metrics would need to be implemented if needed
|
|
||||||
this.log('Cache miss recorded', 'info')
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Track custom metric
|
|
||||||
* Note: Custom metrics would need to be implemented in StatisticsCollector
|
|
||||||
*/
|
|
||||||
trackCustomMetric(name: string, value: number): void {
|
|
||||||
// StatisticsCollector doesn't have trackCustomMetric method
|
|
||||||
// Could be added later if needed
|
|
||||||
this.log(`Custom metric recorded: ${name}=${value}`, 'info')
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Reset all metrics
|
|
||||||
*/
|
|
||||||
reset(): void {
|
|
||||||
if (this.statisticsCollector) {
|
|
||||||
this.statisticsCollector = new StatisticsCollector()
|
|
||||||
this.log('Metrics manually reset')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Factory function for zero-config metrics augmentation
|
|
||||||
*/
|
|
||||||
export function createMetricsAugmentation(config?: MetricsConfig): MetricsAugmentation {
|
|
||||||
return new MetricsAugmentation(config)
|
|
||||||
}
|
|
||||||
|
|
@ -1,270 +0,0 @@
|
||||||
/**
|
|
||||||
* Monitoring Augmentation - Optional Health & Performance Monitoring
|
|
||||||
*
|
|
||||||
* Replaces the hardcoded HealthMonitor in Brainy with an optional augmentation.
|
|
||||||
* Provides health checks, performance monitoring, and distributed system tracking.
|
|
||||||
*
|
|
||||||
* Zero-config: Automatically enabled for distributed deployments
|
|
||||||
* Can be disabled or customized via augmentation registry
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { BaseAugmentation, AugmentationContext } from './brainyAugmentation.js'
|
|
||||||
import { HealthMonitor } from '../distributed/healthMonitor.js'
|
|
||||||
import { DistributedConfigManager as ConfigManager } from '../distributed/configManager.js'
|
|
||||||
|
|
||||||
export interface MonitoringConfig {
|
|
||||||
enabled?: boolean
|
|
||||||
healthCheckInterval?: number
|
|
||||||
metricsInterval?: number
|
|
||||||
trackLatency?: boolean
|
|
||||||
trackErrors?: boolean
|
|
||||||
trackCacheMetrics?: boolean
|
|
||||||
exposeHealthEndpoint?: boolean
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* MonitoringAugmentation - Makes health monitoring optional and pluggable
|
|
||||||
*
|
|
||||||
* Features:
|
|
||||||
* - Health status tracking
|
|
||||||
* - Performance monitoring
|
|
||||||
* - Error rate tracking
|
|
||||||
* - Distributed system health
|
|
||||||
* - Zero-config with smart defaults
|
|
||||||
*/
|
|
||||||
export class MonitoringAugmentation extends BaseAugmentation {
|
|
||||||
readonly metadata = 'readonly' as const // Reads metadata for monitoring
|
|
||||||
readonly name = 'monitoring'
|
|
||||||
readonly timing = 'after' as const
|
|
||||||
operations = ['search', 'find', 'similar', 'add', 'update', 'delete', 'relate', 'unrelate', 'all'] as ('search' | 'find' | 'similar' | 'add' | 'update' | 'delete' | 'relate' | 'unrelate' | 'all')[]
|
|
||||||
readonly priority = 30 // Low priority, observability layer
|
|
||||||
|
|
||||||
private healthMonitor: HealthMonitor | null = null
|
|
||||||
private configManager: ConfigManager | null = null
|
|
||||||
protected config: MonitoringConfig
|
|
||||||
private requestStartTimes = new Map<string, number>()
|
|
||||||
|
|
||||||
constructor(config: MonitoringConfig = {}) {
|
|
||||||
super()
|
|
||||||
this.config = {
|
|
||||||
enabled: true,
|
|
||||||
healthCheckInterval: 30000, // 30 seconds
|
|
||||||
metricsInterval: 60000, // 1 minute
|
|
||||||
trackLatency: true,
|
|
||||||
trackErrors: true,
|
|
||||||
trackCacheMetrics: true,
|
|
||||||
exposeHealthEndpoint: true,
|
|
||||||
...config
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected async onInitialize(): Promise<void> {
|
|
||||||
if (!this.config.enabled) {
|
|
||||||
this.log('Monitoring augmentation disabled by configuration')
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Initialize config manager and health monitor (requires storage)
|
|
||||||
if (this.context?.storage) {
|
|
||||||
this.configManager = new ConfigManager(this.context.storage as any)
|
|
||||||
this.healthMonitor = new HealthMonitor(this.configManager)
|
|
||||||
this.healthMonitor.start()
|
|
||||||
} else {
|
|
||||||
this.log('Storage not available - health monitoring disabled', 'warn')
|
|
||||||
}
|
|
||||||
|
|
||||||
this.log('Monitoring augmentation initialized')
|
|
||||||
}
|
|
||||||
|
|
||||||
protected async onShutdown(): Promise<void> {
|
|
||||||
if (this.healthMonitor) {
|
|
||||||
this.healthMonitor.stop()
|
|
||||||
this.healthMonitor = null
|
|
||||||
}
|
|
||||||
|
|
||||||
this.configManager = null
|
|
||||||
this.requestStartTimes.clear()
|
|
||||||
|
|
||||||
this.log('Monitoring augmentation shut down')
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Execute augmentation - track health metrics
|
|
||||||
*/
|
|
||||||
async execute<T = any>(
|
|
||||||
operation: string,
|
|
||||||
params: any,
|
|
||||||
next: () => Promise<T>
|
|
||||||
): Promise<T> {
|
|
||||||
// If monitoring disabled, just pass through
|
|
||||||
if (!this.healthMonitor || !this.config.enabled) {
|
|
||||||
return next()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Generate request ID for tracking
|
|
||||||
const requestId = `${operation}-${Date.now()}-${Math.random()}`
|
|
||||||
|
|
||||||
// Track request start time
|
|
||||||
if (this.config.trackLatency) {
|
|
||||||
this.requestStartTimes.set(requestId, Date.now())
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
// Execute operation
|
|
||||||
const result = await next()
|
|
||||||
|
|
||||||
// Track successful operation
|
|
||||||
if (this.config.trackLatency) {
|
|
||||||
const startTime = this.requestStartTimes.get(requestId)
|
|
||||||
if (startTime) {
|
|
||||||
const latency = Date.now() - startTime
|
|
||||||
this.healthMonitor.recordRequest(latency, false)
|
|
||||||
this.requestStartTimes.delete(requestId)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Update vector count for 'add' operations
|
|
||||||
if (operation === 'add' && this.context?.brain) {
|
|
||||||
try {
|
|
||||||
const count = await this.context.brain.getNounCount()
|
|
||||||
this.healthMonitor.updateVectorCount(count)
|
|
||||||
} catch (e) {
|
|
||||||
// Ignore count update errors
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Track cache metrics for search operations
|
|
||||||
if (operation === 'search' && this.config.trackCacheMetrics) {
|
|
||||||
// Check if result came from cache (would be set by cache augmentation)
|
|
||||||
const fromCache = (params as any)._fromCache || false
|
|
||||||
this.healthMonitor.recordCacheAccess(fromCache)
|
|
||||||
}
|
|
||||||
|
|
||||||
return result
|
|
||||||
} catch (error) {
|
|
||||||
// Track error
|
|
||||||
if (this.config.trackErrors) {
|
|
||||||
const startTime = this.requestStartTimes.get(requestId)
|
|
||||||
if (startTime) {
|
|
||||||
const latency = Date.now() - startTime
|
|
||||||
this.healthMonitor.recordRequest(latency, true)
|
|
||||||
this.requestStartTimes.delete(requestId)
|
|
||||||
} else {
|
|
||||||
this.healthMonitor.recordRequest(0, true)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
throw error
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get health status
|
|
||||||
*/
|
|
||||||
getHealthStatus() {
|
|
||||||
if (!this.healthMonitor) {
|
|
||||||
return {
|
|
||||||
status: 'disabled',
|
|
||||||
enabled: false,
|
|
||||||
uptime: 0,
|
|
||||||
vectorCount: 0,
|
|
||||||
requestRate: 0,
|
|
||||||
errorRate: 0,
|
|
||||||
cacheHitRate: 0
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
status: 'healthy',
|
|
||||||
enabled: true,
|
|
||||||
...this.healthMonitor.getHealthEndpointData()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get health endpoint data (for API exposure)
|
|
||||||
*/
|
|
||||||
getHealthEndpointData() {
|
|
||||||
if (!this.healthMonitor) {
|
|
||||||
return {
|
|
||||||
status: 'disabled',
|
|
||||||
timestamp: new Date().toISOString()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return this.healthMonitor.getHealthEndpointData()
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Update vector count manually
|
|
||||||
*/
|
|
||||||
updateVectorCount(count: number): void {
|
|
||||||
if (this.healthMonitor) {
|
|
||||||
this.healthMonitor.updateVectorCount(count)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Record custom health metric
|
|
||||||
*/
|
|
||||||
recordCustomMetric(name: string, value: number): void {
|
|
||||||
if (this.healthMonitor) {
|
|
||||||
// Health monitor could be extended to track custom metrics
|
|
||||||
this.log(`Custom metric recorded: ${name}=${value}`, 'info')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Check if system is healthy
|
|
||||||
*/
|
|
||||||
isHealthy(): boolean {
|
|
||||||
if (!this.healthMonitor) return true // If disabled, assume healthy
|
|
||||||
|
|
||||||
const data = this.healthMonitor.getHealthEndpointData()
|
|
||||||
|
|
||||||
// Define health criteria
|
|
||||||
const errorRateThreshold = 0.05 // 5% error rate
|
|
||||||
const minUptime = 60000 // 1 minute
|
|
||||||
|
|
||||||
return (
|
|
||||||
data.errorRate < errorRateThreshold &&
|
|
||||||
data.uptime > minUptime
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get uptime in milliseconds
|
|
||||||
*/
|
|
||||||
getUptime(): number {
|
|
||||||
if (!this.healthMonitor) return 0
|
|
||||||
|
|
||||||
const data = this.healthMonitor.getHealthEndpointData()
|
|
||||||
return data.uptime || 0
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Force health check
|
|
||||||
*/
|
|
||||||
async checkHealth(): Promise<boolean> {
|
|
||||||
if (!this.healthMonitor) return true
|
|
||||||
|
|
||||||
// Perform active health check
|
|
||||||
try {
|
|
||||||
// Could ping storage, check memory, etc.
|
|
||||||
if (this.context?.storage) {
|
|
||||||
await this.context.storage.getStatistics?.()
|
|
||||||
}
|
|
||||||
return true
|
|
||||||
} catch (error) {
|
|
||||||
this.log('Health check failed', 'warn')
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Factory function for zero-config monitoring augmentation
|
|
||||||
*/
|
|
||||||
export function createMonitoringAugmentation(config?: MonitoringConfig): MonitoringAugmentation {
|
|
||||||
return new MonitoringAugmentation(config)
|
|
||||||
}
|
|
||||||
|
|
@ -1,384 +0,0 @@
|
||||||
/**
|
|
||||||
* Rate Limiting Augmentation
|
|
||||||
* Provides configurable rate limiting for Brainy operations
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { BaseAugmentation } from './brainyAugmentation.js'
|
|
||||||
import { AugmentationManifest } from './manifest.js'
|
|
||||||
|
|
||||||
export interface RateLimitConfig {
|
|
||||||
enabled?: boolean
|
|
||||||
limits?: {
|
|
||||||
searches?: number // Per minute
|
|
||||||
writes?: number // Per minute
|
|
||||||
reads?: number // Per minute
|
|
||||||
deletes?: number // Per minute
|
|
||||||
}
|
|
||||||
windowMs?: number // Time window in milliseconds
|
|
||||||
skipSuccessfulRequests?: boolean
|
|
||||||
skipFailedRequests?: boolean
|
|
||||||
keyGenerator?: (context: any) => string
|
|
||||||
}
|
|
||||||
|
|
||||||
interface RateLimitEntry {
|
|
||||||
count: number
|
|
||||||
resetTime: number
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Rate Limit Augmentation
|
|
||||||
*/
|
|
||||||
export class RateLimitAugmentation extends BaseAugmentation {
|
|
||||||
readonly name = 'rateLimiter'
|
|
||||||
readonly timing = 'before' as const
|
|
||||||
readonly metadata = 'none' as const
|
|
||||||
operations = ['search', 'find', 'add', 'update', 'delete', 'get'] as any
|
|
||||||
readonly priority = 10 // High priority, runs early
|
|
||||||
|
|
||||||
// Augmentation metadata
|
|
||||||
readonly category = 'core' as const // Use 'core' as security isn't a valid category
|
|
||||||
readonly description = 'Provides rate limiting for Brainy operations'
|
|
||||||
|
|
||||||
private limiters: Map<string, Map<string, RateLimitEntry>> = new Map()
|
|
||||||
private windowMs: number
|
|
||||||
|
|
||||||
constructor(config: RateLimitConfig = {}) {
|
|
||||||
super(config)
|
|
||||||
|
|
||||||
// Merge with defaults
|
|
||||||
this.config = {
|
|
||||||
enabled: config.enabled ?? true,
|
|
||||||
limits: {
|
|
||||||
searches: config.limits?.searches ?? 1000,
|
|
||||||
writes: config.limits?.writes ?? 100,
|
|
||||||
reads: config.limits?.reads ?? 5000,
|
|
||||||
deletes: config.limits?.deletes ?? 50
|
|
||||||
},
|
|
||||||
windowMs: config.windowMs ?? 60000, // 1 minute default
|
|
||||||
skipSuccessfulRequests: config.skipSuccessfulRequests ?? false,
|
|
||||||
skipFailedRequests: config.skipFailedRequests ?? true,
|
|
||||||
keyGenerator: config.keyGenerator || this.defaultKeyGenerator
|
|
||||||
}
|
|
||||||
|
|
||||||
this.windowMs = this.config.windowMs!
|
|
||||||
|
|
||||||
// Initialize operation limiters
|
|
||||||
this.initializeLimiters()
|
|
||||||
}
|
|
||||||
|
|
||||||
getManifest(): AugmentationManifest {
|
|
||||||
return {
|
|
||||||
id: 'rate-limiter',
|
|
||||||
name: 'Rate Limiter',
|
|
||||||
version: '1.0.0',
|
|
||||||
description: 'Configurable rate limiting for API operations',
|
|
||||||
longDescription: 'Provides per-operation rate limiting with configurable windows and limits. Helps prevent abuse and ensures fair resource usage.',
|
|
||||||
category: 'core',
|
|
||||||
configSchema: {
|
|
||||||
type: 'object',
|
|
||||||
properties: {
|
|
||||||
enabled: {
|
|
||||||
type: 'boolean',
|
|
||||||
default: true,
|
|
||||||
description: 'Enable or disable rate limiting'
|
|
||||||
},
|
|
||||||
limits: {
|
|
||||||
type: 'object',
|
|
||||||
properties: {
|
|
||||||
searches: {
|
|
||||||
type: 'number',
|
|
||||||
default: 1000,
|
|
||||||
description: 'Search operations per minute'
|
|
||||||
},
|
|
||||||
writes: {
|
|
||||||
type: 'number',
|
|
||||||
default: 100,
|
|
||||||
description: 'Write operations per minute'
|
|
||||||
},
|
|
||||||
reads: {
|
|
||||||
type: 'number',
|
|
||||||
default: 5000,
|
|
||||||
description: 'Read operations per minute'
|
|
||||||
},
|
|
||||||
deletes: {
|
|
||||||
type: 'number',
|
|
||||||
default: 50,
|
|
||||||
description: 'Delete operations per minute'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
windowMs: {
|
|
||||||
type: 'number',
|
|
||||||
default: 60000,
|
|
||||||
description: 'Time window in milliseconds'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
configDefaults: {
|
|
||||||
enabled: true,
|
|
||||||
limits: {
|
|
||||||
searches: 1000,
|
|
||||||
writes: 100,
|
|
||||||
reads: 5000,
|
|
||||||
deletes: 50
|
|
||||||
},
|
|
||||||
windowMs: 60000
|
|
||||||
},
|
|
||||||
minBrainyVersion: '3.0.0',
|
|
||||||
keywords: ['rate-limit', 'security', 'throttle'],
|
|
||||||
documentation: 'https://docs.brainy.dev/augmentations/rate-limit',
|
|
||||||
status: 'stable',
|
|
||||||
performance: {
|
|
||||||
memoryUsage: 'low',
|
|
||||||
cpuUsage: 'low',
|
|
||||||
networkUsage: 'none'
|
|
||||||
},
|
|
||||||
features: ['per-operation-limits', 'configurable-windows', 'key-based-limiting'],
|
|
||||||
enhancedOperations: ['search', 'add', 'update', 'delete', 'get'],
|
|
||||||
metrics: [
|
|
||||||
{
|
|
||||||
name: 'rate_limit_exceeded',
|
|
||||||
type: 'counter',
|
|
||||||
description: 'Number of rate limit violations'
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: 'rate_limit_requests',
|
|
||||||
type: 'counter',
|
|
||||||
description: 'Total requests checked'
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Initialize rate limiters for each operation type
|
|
||||||
*/
|
|
||||||
private initializeLimiters(): void {
|
|
||||||
const operations = ['searches', 'writes', 'reads', 'deletes']
|
|
||||||
for (const op of operations) {
|
|
||||||
this.limiters.set(op, new Map())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Default key generator (could be IP, user ID, etc.)
|
|
||||||
*/
|
|
||||||
private defaultKeyGenerator(_context: any): string {
|
|
||||||
// In a real implementation, this would extract IP or user ID
|
|
||||||
return 'default'
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Check if request should be rate limited
|
|
||||||
*/
|
|
||||||
private checkRateLimit(operation: string, key: string): boolean {
|
|
||||||
const limiter = this.limiters.get(operation)
|
|
||||||
if (!limiter) return false
|
|
||||||
|
|
||||||
const limit = (this.config.limits as any)[operation]
|
|
||||||
if (!limit) return false
|
|
||||||
|
|
||||||
const now = Date.now()
|
|
||||||
let entry = limiter.get(key)
|
|
||||||
|
|
||||||
// Initialize or reset entry
|
|
||||||
if (!entry || now >= entry.resetTime) {
|
|
||||||
entry = {
|
|
||||||
count: 0,
|
|
||||||
resetTime: now + this.windowMs
|
|
||||||
}
|
|
||||||
limiter.set(key, entry)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check if limit exceeded
|
|
||||||
if (entry.count >= limit) {
|
|
||||||
return true // Rate limited
|
|
||||||
}
|
|
||||||
|
|
||||||
// Increment counter
|
|
||||||
entry.count++
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get remaining requests for an operation
|
|
||||||
*/
|
|
||||||
private getRemainingRequests(operation: string, key: string): number {
|
|
||||||
const limiter = this.limiters.get(operation)
|
|
||||||
if (!limiter) return -1
|
|
||||||
|
|
||||||
const limit = (this.config.limits as any)[operation]
|
|
||||||
if (!limit) return -1
|
|
||||||
|
|
||||||
const entry = limiter.get(key)
|
|
||||||
if (!entry) return limit
|
|
||||||
|
|
||||||
const now = Date.now()
|
|
||||||
if (now >= entry.resetTime) return limit
|
|
||||||
|
|
||||||
return Math.max(0, limit - entry.count)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get time until reset
|
|
||||||
*/
|
|
||||||
private getResetTime(operation: string, key: string): number {
|
|
||||||
const limiter = this.limiters.get(operation)
|
|
||||||
if (!limiter) return 0
|
|
||||||
|
|
||||||
const entry = limiter.get(key)
|
|
||||||
if (!entry) return 0
|
|
||||||
|
|
||||||
const now = Date.now()
|
|
||||||
return Math.max(0, entry.resetTime - now)
|
|
||||||
}
|
|
||||||
|
|
||||||
protected async onInitialize(): Promise<void> {
|
|
||||||
if (!this.config.enabled) {
|
|
||||||
this.log('Rate limiter disabled by configuration')
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
this.log(`Rate limiter initialized (window: ${this.windowMs}ms)`)
|
|
||||||
|
|
||||||
// Start cleanup timer
|
|
||||||
setInterval(() => {
|
|
||||||
this.cleanup()
|
|
||||||
}, this.windowMs)
|
|
||||||
}
|
|
||||||
|
|
||||||
protected async onShutdown(): Promise<void> {
|
|
||||||
this.clear()
|
|
||||||
this.log('Rate limiter shut down')
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Execute augmentation - apply rate limiting
|
|
||||||
*/
|
|
||||||
async execute<T = any>(
|
|
||||||
operation: string,
|
|
||||||
params: any,
|
|
||||||
next: () => Promise<T>
|
|
||||||
): Promise<T> {
|
|
||||||
// If rate limiting is disabled, just pass through
|
|
||||||
if (!this.config.enabled) {
|
|
||||||
return next()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Map operations to rate limit categories
|
|
||||||
let rateLimitOperation: string
|
|
||||||
switch (operation) {
|
|
||||||
case 'search':
|
|
||||||
case 'find':
|
|
||||||
case 'similar':
|
|
||||||
rateLimitOperation = 'searches'
|
|
||||||
break
|
|
||||||
case 'add':
|
|
||||||
case 'update':
|
|
||||||
rateLimitOperation = 'writes'
|
|
||||||
break
|
|
||||||
case 'delete':
|
|
||||||
rateLimitOperation = 'deletes'
|
|
||||||
break
|
|
||||||
case 'get':
|
|
||||||
rateLimitOperation = 'reads'
|
|
||||||
break
|
|
||||||
default:
|
|
||||||
return next() // Don't rate limit unknown operations
|
|
||||||
}
|
|
||||||
|
|
||||||
const key = (this.config.keyGenerator as any)(params)
|
|
||||||
|
|
||||||
if (this.checkRateLimit(rateLimitOperation, key)) {
|
|
||||||
const error = new Error(`Rate limit exceeded for ${operation}`)
|
|
||||||
;(error as any).statusCode = 429
|
|
||||||
;(error as any).retryAfter = this.getResetTime(rateLimitOperation, key)
|
|
||||||
;(error as any).rateLimit = {
|
|
||||||
limit: (this.config.limits as any)[rateLimitOperation],
|
|
||||||
remaining: 0,
|
|
||||||
reset: Date.now() + this.getResetTime(rateLimitOperation, key)
|
|
||||||
}
|
|
||||||
throw error
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
const result = await next()
|
|
||||||
|
|
||||||
// Add rate limit info to result if possible
|
|
||||||
if (result && typeof result === 'object' && !Array.isArray(result)) {
|
|
||||||
(result as any)._rateLimit = {
|
|
||||||
limit: (this.config.limits as any)[rateLimitOperation],
|
|
||||||
remaining: this.getRemainingRequests(rateLimitOperation, key),
|
|
||||||
reset: Date.now() + this.getResetTime(rateLimitOperation, key)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return result
|
|
||||||
} catch (error) {
|
|
||||||
// Optionally don't count failed requests
|
|
||||||
if (this.config.skipFailedRequests) {
|
|
||||||
const limiter = this.limiters.get(rateLimitOperation)!
|
|
||||||
const entry = limiter.get(key)
|
|
||||||
if (entry && entry.count > 0) entry.count--
|
|
||||||
}
|
|
||||||
throw error
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get rate limit statistics
|
|
||||||
*/
|
|
||||||
getStats(): {
|
|
||||||
operations: Record<string, {
|
|
||||||
activeKeys: number
|
|
||||||
totalRequests: number
|
|
||||||
}>
|
|
||||||
} {
|
|
||||||
const stats: any = { operations: {} }
|
|
||||||
|
|
||||||
for (const [operation, limiter] of this.limiters) {
|
|
||||||
let totalRequests = 0
|
|
||||||
for (const entry of limiter.values()) {
|
|
||||||
totalRequests += entry.count
|
|
||||||
}
|
|
||||||
|
|
||||||
stats.operations[operation] = {
|
|
||||||
activeKeys: limiter.size,
|
|
||||||
totalRequests
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return stats
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Clear all rate limit entries
|
|
||||||
*/
|
|
||||||
clear(): void {
|
|
||||||
for (const limiter of this.limiters.values()) {
|
|
||||||
limiter.clear()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Clear expired entries (cleanup)
|
|
||||||
*/
|
|
||||||
cleanup(): void {
|
|
||||||
const now = Date.now()
|
|
||||||
|
|
||||||
for (const limiter of this.limiters.values()) {
|
|
||||||
for (const [key, entry] of limiter) {
|
|
||||||
if (now >= entry.resetTime) {
|
|
||||||
limiter.delete(key)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create rate limit augmentation
|
|
||||||
*/
|
|
||||||
export function createRateLimitAugmentation(config?: RateLimitConfig): RateLimitAugmentation {
|
|
||||||
return new RateLimitAugmentation(config)
|
|
||||||
}
|
|
||||||
|
|
@ -1,216 +0,0 @@
|
||||||
/**
|
|
||||||
* Request Deduplicator Augmentation
|
|
||||||
*
|
|
||||||
* Prevents duplicate concurrent requests to improve performance by 3x
|
|
||||||
* Automatically deduplicates identical operations
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { BaseAugmentation, AugmentationContext } from './brainyAugmentation.js'
|
|
||||||
|
|
||||||
interface PendingRequest<T> {
|
|
||||||
promise: Promise<T>
|
|
||||||
timestamp: number
|
|
||||||
count: number
|
|
||||||
}
|
|
||||||
|
|
||||||
interface DeduplicatorConfig {
|
|
||||||
enabled?: boolean
|
|
||||||
ttl?: number // Time to live for cached requests (ms)
|
|
||||||
maxSize?: number // Maximum number of cached requests
|
|
||||||
}
|
|
||||||
|
|
||||||
export class RequestDeduplicatorAugmentation extends BaseAugmentation {
|
|
||||||
name = 'RequestDeduplicator'
|
|
||||||
timing = 'around' as const
|
|
||||||
metadata = 'none' as const // Doesn't access metadata
|
|
||||||
operations = ['search', 'find', 'similar', 'searchText', 'searchByNounTypes', 'findSimilar', 'get'] as ('search' | 'find' | 'similar' | 'searchText' | 'searchByNounTypes' | 'findSimilar' | 'get')[]
|
|
||||||
priority = 50 // Performance optimization
|
|
||||||
|
|
||||||
private pendingRequests: Map<string, PendingRequest<any>> = new Map()
|
|
||||||
protected config: Required<DeduplicatorConfig>
|
|
||||||
private cleanupInterval?: NodeJS.Timeout
|
|
||||||
|
|
||||||
constructor(config: DeduplicatorConfig = {}) {
|
|
||||||
super()
|
|
||||||
this.config = {
|
|
||||||
enabled: config.enabled ?? true,
|
|
||||||
ttl: config.ttl ?? 5000, // 5 second default
|
|
||||||
maxSize: config.maxSize ?? 1000
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected async onInitialize(): Promise<void> {
|
|
||||||
if (this.config.enabled) {
|
|
||||||
this.log('Request deduplicator initialized for 3x performance boost')
|
|
||||||
|
|
||||||
// Start cleanup interval
|
|
||||||
this.cleanupInterval = setInterval(() => {
|
|
||||||
this.cleanup()
|
|
||||||
}, this.config.ttl)
|
|
||||||
} else {
|
|
||||||
this.log('Request deduplicator disabled')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
shouldExecute(operation: string, params: any): boolean {
|
|
||||||
// Only execute if enabled and for read operations that benefit from deduplication
|
|
||||||
return this.config.enabled && (
|
|
||||||
operation === 'search' ||
|
|
||||||
operation === 'searchText' ||
|
|
||||||
operation === 'searchByNounTypes' ||
|
|
||||||
operation === 'findSimilar' ||
|
|
||||||
operation === 'get'
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
async execute<T = any>(
|
|
||||||
operation: string,
|
|
||||||
params: any,
|
|
||||||
next: () => Promise<T>
|
|
||||||
): Promise<T> {
|
|
||||||
if (!this.config.enabled) {
|
|
||||||
return next()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create a unique key for this request
|
|
||||||
const key = this.createRequestKey(operation, params)
|
|
||||||
|
|
||||||
// Check if we already have this request pending
|
|
||||||
const existing = this.pendingRequests.get(key)
|
|
||||||
if (existing) {
|
|
||||||
existing.count++
|
|
||||||
this.log(`Deduplicating request: ${key} (${existing.count} total)`)
|
|
||||||
return existing.promise
|
|
||||||
}
|
|
||||||
|
|
||||||
// Execute the request and cache the promise
|
|
||||||
const promise = next()
|
|
||||||
|
|
||||||
this.pendingRequests.set(key, {
|
|
||||||
promise,
|
|
||||||
timestamp: Date.now(),
|
|
||||||
count: 1
|
|
||||||
})
|
|
||||||
|
|
||||||
// Clean up when done
|
|
||||||
promise.finally(() => {
|
|
||||||
// Use setTimeout to allow other concurrent requests to use the result
|
|
||||||
setTimeout(() => {
|
|
||||||
this.pendingRequests.delete(key)
|
|
||||||
}, 100)
|
|
||||||
})
|
|
||||||
|
|
||||||
return promise
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create a unique key for the request based on operation and parameters
|
|
||||||
*/
|
|
||||||
private createRequestKey(operation: string, params: any): string {
|
|
||||||
// Create a stable string representation of the operation and params
|
|
||||||
const paramsKey = this.serializeParams(params)
|
|
||||||
return `${operation}:${paramsKey}`
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Serialize parameters to a consistent string
|
|
||||||
*/
|
|
||||||
private serializeParams(params: any): string {
|
|
||||||
if (!params) return 'null'
|
|
||||||
|
|
||||||
if (typeof params === 'string' || typeof params === 'number') {
|
|
||||||
return String(params)
|
|
||||||
}
|
|
||||||
|
|
||||||
if (Array.isArray(params)) {
|
|
||||||
// For arrays, create a hash-like representation
|
|
||||||
if (params.length > 100) {
|
|
||||||
// For large arrays (like vectors), use length + first/last elements
|
|
||||||
return `[${params.length}:${params[0]}...${params[params.length - 1]}]`
|
|
||||||
}
|
|
||||||
return `[${params.join(',')}]`
|
|
||||||
}
|
|
||||||
|
|
||||||
if (typeof params === 'object') {
|
|
||||||
// Sort keys for consistent serialization
|
|
||||||
const keys = Object.keys(params).sort()
|
|
||||||
const keyValues = keys.map(key => `${key}:${this.serializeParams(params[key])}`)
|
|
||||||
return `{${keyValues.join(',')}}`
|
|
||||||
}
|
|
||||||
|
|
||||||
return String(params)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Clean up expired requests
|
|
||||||
*/
|
|
||||||
private cleanup(): void {
|
|
||||||
const now = Date.now()
|
|
||||||
const expired = []
|
|
||||||
|
|
||||||
for (const [key, request] of this.pendingRequests) {
|
|
||||||
if (now - request.timestamp > this.config.ttl) {
|
|
||||||
expired.push(key)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const key of expired) {
|
|
||||||
this.pendingRequests.delete(key)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Also enforce max size
|
|
||||||
if (this.pendingRequests.size > this.config.maxSize) {
|
|
||||||
const entries = Array.from(this.pendingRequests.entries())
|
|
||||||
.sort(([,a], [,b]) => a.timestamp - b.timestamp) // Oldest first
|
|
||||||
|
|
||||||
// Remove oldest entries
|
|
||||||
const toRemove = entries.slice(0, entries.length - this.config.maxSize)
|
|
||||||
for (const [key] of toRemove) {
|
|
||||||
this.pendingRequests.delete(key)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (expired.length > 0) {
|
|
||||||
this.log(`Cleaned up ${expired.length} expired requests`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get statistics about request deduplication
|
|
||||||
*/
|
|
||||||
getStats(): {
|
|
||||||
activePendingRequests: number
|
|
||||||
totalDeduplicationHits: number
|
|
||||||
memoryUsage: string
|
|
||||||
efficiency: string
|
|
||||||
} {
|
|
||||||
const requests = Array.from(this.pendingRequests.values())
|
|
||||||
const totalRequests = requests.reduce((sum, req) => sum + req.count, 0)
|
|
||||||
const actualRequests = requests.length
|
|
||||||
const savedRequests = totalRequests - actualRequests
|
|
||||||
|
|
||||||
return {
|
|
||||||
activePendingRequests: actualRequests,
|
|
||||||
totalDeduplicationHits: savedRequests,
|
|
||||||
memoryUsage: `${Math.round(JSON.stringify(requests).length / 1024)}KB`,
|
|
||||||
efficiency: actualRequests > 0 ? `${Math.round((savedRequests / totalRequests) * 100)}% reduction` : '0%'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Force clear all pending requests (for testing)
|
|
||||||
*/
|
|
||||||
clear(): void {
|
|
||||||
this.pendingRequests.clear()
|
|
||||||
}
|
|
||||||
|
|
||||||
protected async onShutdown(): Promise<void> {
|
|
||||||
if (this.cleanupInterval) {
|
|
||||||
clearInterval(this.cleanupInterval)
|
|
||||||
}
|
|
||||||
|
|
||||||
const stats = this.getStats()
|
|
||||||
this.log(`Request deduplicator shutdown: ${stats.efficiency} efficiency achieved`)
|
|
||||||
this.pendingRequests.clear()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,119 +0,0 @@
|
||||||
/**
|
|
||||||
* Storage Augmentation Base Classes
|
|
||||||
*
|
|
||||||
* Unifies storage adapters and augmentations into a single system.
|
|
||||||
* All storage backends are now augmentations for consistency and extensibility.
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { BaseAugmentation, BrainyAugmentation, AugmentationContext } from './brainyAugmentation.js'
|
|
||||||
import { StorageAdapter } from '../coreTypes.js'
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Base class for all storage augmentations
|
|
||||||
* Provides the storage adapter to the brain during initialization
|
|
||||||
*/
|
|
||||||
export abstract class StorageAugmentation extends BaseAugmentation implements BrainyAugmentation {
|
|
||||||
readonly timing = 'replace' as const
|
|
||||||
readonly metadata = 'none' as const // Storage doesn't directly access metadata
|
|
||||||
operations = ['storage'] as ('storage')[] // Make mutable for TypeScript compatibility
|
|
||||||
readonly priority = 100 // High priority for storage
|
|
||||||
|
|
||||||
protected storageAdapter: StorageAdapter | null = null
|
|
||||||
|
|
||||||
// Storage augmentations must provide their name via readonly property
|
|
||||||
constructor(config?: any) {
|
|
||||||
super(config)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Provide the storage adapter before full initialization
|
|
||||||
* This is called during the storage resolution phase
|
|
||||||
*/
|
|
||||||
abstract provideStorage(): Promise<StorageAdapter>
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Initialize the augmentation with context
|
|
||||||
* Called after storage has been resolved
|
|
||||||
*/
|
|
||||||
async initialize(context: AugmentationContext): Promise<void> {
|
|
||||||
await super.initialize(context)
|
|
||||||
// Storage adapter should already be provided
|
|
||||||
if (!this.storageAdapter) {
|
|
||||||
this.storageAdapter = await this.provideStorage()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Execute storage operations
|
|
||||||
* For storage augmentations, this replaces the default storage
|
|
||||||
*/
|
|
||||||
async execute<T = any>(
|
|
||||||
operation: string,
|
|
||||||
params: any,
|
|
||||||
next: () => Promise<T>
|
|
||||||
): Promise<T> {
|
|
||||||
if (operation === 'storage') {
|
|
||||||
// Return our storage adapter
|
|
||||||
return this.storageAdapter as any as T
|
|
||||||
}
|
|
||||||
|
|
||||||
// Pass through all other operations
|
|
||||||
return next()
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Shutdown and cleanup
|
|
||||||
*/
|
|
||||||
async shutdown(): Promise<void> {
|
|
||||||
// Cleanup storage adapter if needed
|
|
||||||
if (this.storageAdapter && typeof (this.storageAdapter as any).close === 'function') {
|
|
||||||
await (this.storageAdapter as any).close()
|
|
||||||
}
|
|
||||||
await super.shutdown()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Dynamic storage augmentation that wraps any storage adapter
|
|
||||||
* Used for backward compatibility and zero-config
|
|
||||||
*/
|
|
||||||
export class DynamicStorageAugmentation extends StorageAugmentation {
|
|
||||||
readonly name = 'dynamic-storage'
|
|
||||||
|
|
||||||
constructor(private adapter: StorageAdapter) {
|
|
||||||
super()
|
|
||||||
this.storageAdapter = adapter
|
|
||||||
}
|
|
||||||
|
|
||||||
async provideStorage(): Promise<StorageAdapter> {
|
|
||||||
return this.adapter
|
|
||||||
}
|
|
||||||
|
|
||||||
protected async onInitialize(): Promise<void> {
|
|
||||||
// Adapter is already provided in constructor
|
|
||||||
await this.adapter.init()
|
|
||||||
this.log(`${this.name} initialized`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create a storage augmentation from configuration
|
|
||||||
* Maintains backward compatibility with existing storage config
|
|
||||||
*/
|
|
||||||
export async function createStorageAugmentationFromConfig(
|
|
||||||
config: any
|
|
||||||
): Promise<StorageAugmentation | null> {
|
|
||||||
// Import storage factory dynamically to avoid circular deps
|
|
||||||
const { createStorage } = await import('../storage/storageFactory.js')
|
|
||||||
|
|
||||||
try {
|
|
||||||
// Create storage adapter from config
|
|
||||||
const adapter = await createStorage(config)
|
|
||||||
|
|
||||||
// Wrap in augmentation
|
|
||||||
return new DynamicStorageAugmentation(adapter)
|
|
||||||
} catch (error) {
|
|
||||||
console.warn('Failed to create storage augmentation from config:', error)
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,496 +0,0 @@
|
||||||
/**
|
|
||||||
* Storage Augmentations - Concrete Implementations
|
|
||||||
*
|
|
||||||
* These augmentations provide different storage backends for Brainy.
|
|
||||||
* Each wraps an existing storage adapter for backward compatibility.
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { StorageAugmentation } from './storageAugmentation.js'
|
|
||||||
import { StorageAdapter } from '../coreTypes.js'
|
|
||||||
import { AugmentationManifest } from './manifest.js'
|
|
||||||
import { MemoryStorage } from '../storage/adapters/memoryStorage.js'
|
|
||||||
import { OPFSStorage } from '../storage/adapters/opfsStorage.js'
|
|
||||||
import { S3CompatibleStorage } from '../storage/adapters/s3CompatibleStorage.js'
|
|
||||||
import { R2Storage } from '../storage/adapters/r2Storage.js'
|
|
||||||
import { AzureBlobStorage } from '../storage/adapters/azureBlobStorage.js'
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Memory Storage Augmentation - Fast in-memory storage
|
|
||||||
*/
|
|
||||||
export class MemoryStorageAugmentation extends StorageAugmentation {
|
|
||||||
readonly name = 'memory-storage'
|
|
||||||
readonly category = 'core' as const
|
|
||||||
readonly description = 'High-performance in-memory storage for development and testing'
|
|
||||||
|
|
||||||
constructor(config?: any) {
|
|
||||||
super(config)
|
|
||||||
}
|
|
||||||
|
|
||||||
getManifest(): AugmentationManifest {
|
|
||||||
return {
|
|
||||||
id: 'memory-storage',
|
|
||||||
name: 'Memory Storage',
|
|
||||||
version: '2.0.0',
|
|
||||||
description: 'Fast in-memory storage with no persistence',
|
|
||||||
longDescription: 'Perfect for development, testing, and temporary data. All data is lost when the process ends. Provides the fastest possible performance with zero I/O overhead.',
|
|
||||||
category: 'storage',
|
|
||||||
configSchema: {
|
|
||||||
type: 'object',
|
|
||||||
properties: {
|
|
||||||
maxSize: {
|
|
||||||
type: 'number',
|
|
||||||
default: 104857600, // 100MB
|
|
||||||
minimum: 1048576, // 1MB
|
|
||||||
maximum: 1073741824, // 1GB
|
|
||||||
description: 'Maximum memory usage in bytes'
|
|
||||||
},
|
|
||||||
gcInterval: {
|
|
||||||
type: 'number',
|
|
||||||
default: 60000, // 1 minute
|
|
||||||
minimum: 1000, // 1 second
|
|
||||||
maximum: 3600000, // 1 hour
|
|
||||||
description: 'Garbage collection interval in milliseconds'
|
|
||||||
},
|
|
||||||
enableStats: {
|
|
||||||
type: 'boolean',
|
|
||||||
default: false,
|
|
||||||
description: 'Enable memory usage statistics'
|
|
||||||
}
|
|
||||||
},
|
|
||||||
additionalProperties: false
|
|
||||||
},
|
|
||||||
configDefaults: {
|
|
||||||
maxSize: 104857600,
|
|
||||||
gcInterval: 60000,
|
|
||||||
enableStats: false
|
|
||||||
},
|
|
||||||
minBrainyVersion: '2.0.0',
|
|
||||||
keywords: ['storage', 'memory', 'ram', 'volatile', 'fast'],
|
|
||||||
documentation: 'https://docs.brainy.dev/augmentations/memory-storage',
|
|
||||||
status: 'stable',
|
|
||||||
performance: {
|
|
||||||
memoryUsage: 'high',
|
|
||||||
cpuUsage: 'low',
|
|
||||||
networkUsage: 'none'
|
|
||||||
},
|
|
||||||
features: ['fast-access', 'zero-latency', 'no-persistence'],
|
|
||||||
ui: {
|
|
||||||
icon: '💾',
|
|
||||||
color: '#4CAF50'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async provideStorage(): Promise<StorageAdapter> {
|
|
||||||
const storage = new MemoryStorage()
|
|
||||||
this.storageAdapter = storage
|
|
||||||
return storage
|
|
||||||
}
|
|
||||||
|
|
||||||
protected async onInitialize(): Promise<void> {
|
|
||||||
await this.storageAdapter!.init()
|
|
||||||
this.log(`Memory storage initialized (max size: ${this.config.maxSize} bytes)`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* FileSystem Storage Augmentation - Node.js persistent storage
|
|
||||||
*/
|
|
||||||
export class FileSystemStorageAugmentation extends StorageAugmentation {
|
|
||||||
readonly name = 'filesystem-storage'
|
|
||||||
readonly category = 'core' as const
|
|
||||||
readonly description = 'Persistent file-based storage for Node.js environments'
|
|
||||||
|
|
||||||
constructor(config?: any) {
|
|
||||||
super(config)
|
|
||||||
}
|
|
||||||
|
|
||||||
getManifest(): AugmentationManifest {
|
|
||||||
return {
|
|
||||||
id: 'filesystem-storage',
|
|
||||||
name: 'FileSystem Storage',
|
|
||||||
version: '2.0.0',
|
|
||||||
description: 'Persistent storage using the local filesystem',
|
|
||||||
longDescription: 'Stores data as files on the local filesystem. Perfect for Node.js applications, desktop apps, and servers. Provides persistent storage with good performance and unlimited capacity.',
|
|
||||||
category: 'storage',
|
|
||||||
configSchema: {
|
|
||||||
type: 'object',
|
|
||||||
properties: {
|
|
||||||
rootDirectory: {
|
|
||||||
type: 'string',
|
|
||||||
default: './brainy-data',
|
|
||||||
description: 'Root directory for storing data files'
|
|
||||||
},
|
|
||||||
compression: {
|
|
||||||
type: 'boolean',
|
|
||||||
default: false,
|
|
||||||
description: 'Enable gzip compression for stored files'
|
|
||||||
},
|
|
||||||
maxFileSize: {
|
|
||||||
type: 'number',
|
|
||||||
default: 10485760, // 10MB
|
|
||||||
minimum: 1048576, // 1MB
|
|
||||||
maximum: 104857600, // 100MB
|
|
||||||
description: 'Maximum size per file in bytes'
|
|
||||||
},
|
|
||||||
autoBackup: {
|
|
||||||
type: 'boolean',
|
|
||||||
default: false,
|
|
||||||
description: 'Enable automatic backups'
|
|
||||||
},
|
|
||||||
backupInterval: {
|
|
||||||
type: 'number',
|
|
||||||
default: 3600000, // 1 hour
|
|
||||||
minimum: 60000, // 1 minute
|
|
||||||
maximum: 86400000, // 24 hours
|
|
||||||
description: 'Backup interval in milliseconds'
|
|
||||||
}
|
|
||||||
},
|
|
||||||
additionalProperties: false
|
|
||||||
},
|
|
||||||
configDefaults: {
|
|
||||||
rootDirectory: './brainy-data',
|
|
||||||
compression: false,
|
|
||||||
maxFileSize: 10485760,
|
|
||||||
autoBackup: false,
|
|
||||||
backupInterval: 3600000
|
|
||||||
},
|
|
||||||
minBrainyVersion: '2.0.0',
|
|
||||||
keywords: ['storage', 'filesystem', 'persistent', 'node', 'disk'],
|
|
||||||
documentation: 'https://docs.brainy.dev/augmentations/filesystem-storage',
|
|
||||||
status: 'stable',
|
|
||||||
performance: {
|
|
||||||
memoryUsage: 'low',
|
|
||||||
cpuUsage: 'low',
|
|
||||||
networkUsage: 'none'
|
|
||||||
},
|
|
||||||
features: ['persistence', 'unlimited-capacity', 'file-based', 'compression-support'],
|
|
||||||
ui: {
|
|
||||||
icon: '📁',
|
|
||||||
color: '#FF9800'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async provideStorage(): Promise<StorageAdapter> {
|
|
||||||
try {
|
|
||||||
// Dynamically import for Node.js environments
|
|
||||||
const { FileSystemStorage } = await import('../storage/adapters/fileSystemStorage.js')
|
|
||||||
const storage = new FileSystemStorage(this.config.rootDirectory)
|
|
||||||
this.storageAdapter = storage
|
|
||||||
return storage
|
|
||||||
} catch (error) {
|
|
||||||
this.log('FileSystemStorage not available, falling back to memory', 'warn')
|
|
||||||
// Fall back to memory storage
|
|
||||||
const storage = new MemoryStorage()
|
|
||||||
this.storageAdapter = storage
|
|
||||||
return storage
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected async onInitialize(): Promise<void> {
|
|
||||||
await this.storageAdapter!.init()
|
|
||||||
this.log(`FileSystem storage initialized at ${this.config.rootDirectory}`)
|
|
||||||
if (this.config.compression) {
|
|
||||||
this.log('Compression enabled for stored files')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* OPFS Storage Augmentation - Browser persistent storage
|
|
||||||
*/
|
|
||||||
export class OPFSStorageAugmentation extends StorageAugmentation {
|
|
||||||
readonly name = 'opfs-storage'
|
|
||||||
readonly category = 'core' as const
|
|
||||||
readonly description = 'Persistent browser storage using Origin Private File System'
|
|
||||||
|
|
||||||
constructor(config?: any) {
|
|
||||||
super(config)
|
|
||||||
}
|
|
||||||
|
|
||||||
getManifest(): AugmentationManifest {
|
|
||||||
return {
|
|
||||||
id: 'opfs-storage',
|
|
||||||
name: 'OPFS Storage',
|
|
||||||
version: '2.0.0',
|
|
||||||
description: 'Modern browser storage with file system capabilities',
|
|
||||||
longDescription: 'Uses the Origin Private File System API for persistent browser storage. Provides file-like storage in modern browsers with better performance than IndexedDB and unlimited storage quota.',
|
|
||||||
category: 'storage',
|
|
||||||
configSchema: {
|
|
||||||
type: 'object',
|
|
||||||
properties: {
|
|
||||||
requestPersistent: {
|
|
||||||
type: 'boolean',
|
|
||||||
default: false,
|
|
||||||
description: 'Request persistent storage permission from browser'
|
|
||||||
},
|
|
||||||
directoryName: {
|
|
||||||
type: 'string',
|
|
||||||
default: 'brainy-data',
|
|
||||||
description: 'Directory name within OPFS'
|
|
||||||
},
|
|
||||||
chunkSize: {
|
|
||||||
type: 'number',
|
|
||||||
default: 1048576, // 1MB
|
|
||||||
minimum: 65536, // 64KB
|
|
||||||
maximum: 10485760, // 10MB
|
|
||||||
description: 'Chunk size for file operations in bytes'
|
|
||||||
},
|
|
||||||
enableCache: {
|
|
||||||
type: 'boolean',
|
|
||||||
default: true,
|
|
||||||
description: 'Enable in-memory caching for frequently accessed data'
|
|
||||||
}
|
|
||||||
},
|
|
||||||
additionalProperties: false
|
|
||||||
},
|
|
||||||
configDefaults: {
|
|
||||||
requestPersistent: false,
|
|
||||||
directoryName: 'brainy-data',
|
|
||||||
chunkSize: 1048576,
|
|
||||||
enableCache: true
|
|
||||||
},
|
|
||||||
minBrainyVersion: '2.0.0',
|
|
||||||
keywords: ['storage', 'browser', 'opfs', 'persistent', 'web'],
|
|
||||||
documentation: 'https://docs.brainy.dev/augmentations/opfs-storage',
|
|
||||||
status: 'stable',
|
|
||||||
performance: {
|
|
||||||
memoryUsage: 'medium',
|
|
||||||
cpuUsage: 'low',
|
|
||||||
networkUsage: 'none'
|
|
||||||
},
|
|
||||||
features: ['browser-persistent', 'file-system-api', 'unlimited-quota', 'async-operations'],
|
|
||||||
ui: {
|
|
||||||
icon: '🌐',
|
|
||||||
color: '#2196F3'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async provideStorage(): Promise<StorageAdapter> {
|
|
||||||
const storage = new OPFSStorage()
|
|
||||||
|
|
||||||
if (!storage.isOPFSAvailable()) {
|
|
||||||
this.log('OPFS not available, falling back to memory', 'warn')
|
|
||||||
const memStorage = new MemoryStorage()
|
|
||||||
this.storageAdapter = memStorage
|
|
||||||
return memStorage
|
|
||||||
}
|
|
||||||
|
|
||||||
this.storageAdapter = storage
|
|
||||||
return storage
|
|
||||||
}
|
|
||||||
|
|
||||||
protected async onInitialize(): Promise<void> {
|
|
||||||
await this.storageAdapter!.init()
|
|
||||||
|
|
||||||
if (this.config.requestPersistent && this.storageAdapter instanceof OPFSStorage) {
|
|
||||||
const granted = await this.storageAdapter.requestPersistentStorage()
|
|
||||||
this.log(`Persistent storage ${granted ? 'granted' : 'denied'}`)
|
|
||||||
}
|
|
||||||
|
|
||||||
this.log(`OPFS storage initialized in directory: ${this.config.directoryName}`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* S3 Storage Augmentation - Amazon S3 cloud storage
|
|
||||||
*/
|
|
||||||
export class S3StorageAugmentation extends StorageAugmentation {
|
|
||||||
readonly name = 's3-storage'
|
|
||||||
protected config: {
|
|
||||||
bucketName: string
|
|
||||||
region?: string
|
|
||||||
accessKeyId: string
|
|
||||||
secretAccessKey: string
|
|
||||||
sessionToken?: string
|
|
||||||
cacheConfig?: any
|
|
||||||
operationConfig?: any
|
|
||||||
}
|
|
||||||
|
|
||||||
constructor(config: {
|
|
||||||
bucketName: string
|
|
||||||
region?: string
|
|
||||||
accessKeyId: string
|
|
||||||
secretAccessKey: string
|
|
||||||
sessionToken?: string
|
|
||||||
cacheConfig?: any
|
|
||||||
operationConfig?: any
|
|
||||||
}) {
|
|
||||||
super()
|
|
||||||
this.config = config
|
|
||||||
}
|
|
||||||
|
|
||||||
async provideStorage(): Promise<StorageAdapter> {
|
|
||||||
const storage = new S3CompatibleStorage({
|
|
||||||
...this.config,
|
|
||||||
serviceType: 's3'
|
|
||||||
})
|
|
||||||
this.storageAdapter = storage
|
|
||||||
return storage
|
|
||||||
}
|
|
||||||
|
|
||||||
protected async onInitialize(): Promise<void> {
|
|
||||||
await this.storageAdapter!.init()
|
|
||||||
this.log(`S3 storage initialized with bucket ${this.config.bucketName}`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* R2 Storage Augmentation - Cloudflare R2 storage
|
|
||||||
*/
|
|
||||||
export class R2StorageAugmentation extends StorageAugmentation {
|
|
||||||
readonly name = 'r2-storage'
|
|
||||||
protected config: {
|
|
||||||
bucketName: string
|
|
||||||
accountId: string
|
|
||||||
accessKeyId: string
|
|
||||||
secretAccessKey: string
|
|
||||||
cacheConfig?: any
|
|
||||||
}
|
|
||||||
|
|
||||||
constructor(config: {
|
|
||||||
bucketName: string
|
|
||||||
accountId: string
|
|
||||||
accessKeyId: string
|
|
||||||
secretAccessKey: string
|
|
||||||
cacheConfig?: any
|
|
||||||
}) {
|
|
||||||
super()
|
|
||||||
this.config = config
|
|
||||||
}
|
|
||||||
|
|
||||||
async provideStorage(): Promise<StorageAdapter> {
|
|
||||||
const storage = new R2Storage({
|
|
||||||
...this.config
|
|
||||||
// serviceType not needed - R2Storage is dedicated
|
|
||||||
})
|
|
||||||
this.storageAdapter = storage
|
|
||||||
return storage
|
|
||||||
}
|
|
||||||
|
|
||||||
protected async onInitialize(): Promise<void> {
|
|
||||||
await this.storageAdapter!.init()
|
|
||||||
this.log(`R2 storage initialized with bucket ${this.config.bucketName}`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* GCS Storage Augmentation - Google Cloud Storage
|
|
||||||
*/
|
|
||||||
export class GCSStorageAugmentation extends StorageAugmentation {
|
|
||||||
readonly name = 'gcs-storage'
|
|
||||||
protected config: {
|
|
||||||
bucketName: string
|
|
||||||
region?: string
|
|
||||||
accessKeyId: string
|
|
||||||
secretAccessKey: string
|
|
||||||
endpoint?: string
|
|
||||||
cacheConfig?: any
|
|
||||||
}
|
|
||||||
|
|
||||||
constructor(config: {
|
|
||||||
bucketName: string
|
|
||||||
region?: string
|
|
||||||
accessKeyId: string
|
|
||||||
secretAccessKey: string
|
|
||||||
endpoint?: string
|
|
||||||
cacheConfig?: any
|
|
||||||
}) {
|
|
||||||
super()
|
|
||||||
this.config = config
|
|
||||||
}
|
|
||||||
|
|
||||||
async provideStorage(): Promise<StorageAdapter> {
|
|
||||||
const storage = new S3CompatibleStorage({
|
|
||||||
...this.config,
|
|
||||||
endpoint: this.config.endpoint || 'https://storage.googleapis.com',
|
|
||||||
serviceType: 'gcs'
|
|
||||||
})
|
|
||||||
this.storageAdapter = storage
|
|
||||||
return storage
|
|
||||||
}
|
|
||||||
|
|
||||||
protected async onInitialize(): Promise<void> {
|
|
||||||
await this.storageAdapter!.init()
|
|
||||||
this.log(`GCS storage initialized with bucket ${this.config.bucketName}`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Azure Blob Storage Augmentation - Microsoft Azure
|
|
||||||
*/
|
|
||||||
export class AzureStorageAugmentation extends StorageAugmentation {
|
|
||||||
readonly name = 'azure-storage'
|
|
||||||
protected config: {
|
|
||||||
containerName: string
|
|
||||||
accountName?: string
|
|
||||||
accountKey?: string
|
|
||||||
connectionString?: string
|
|
||||||
sasToken?: string
|
|
||||||
cacheConfig?: any
|
|
||||||
}
|
|
||||||
|
|
||||||
constructor(config: {
|
|
||||||
containerName: string
|
|
||||||
accountName?: string
|
|
||||||
accountKey?: string
|
|
||||||
connectionString?: string
|
|
||||||
sasToken?: string
|
|
||||||
cacheConfig?: any
|
|
||||||
}) {
|
|
||||||
super()
|
|
||||||
this.config = config
|
|
||||||
}
|
|
||||||
|
|
||||||
async provideStorage(): Promise<StorageAdapter> {
|
|
||||||
const storage = new AzureBlobStorage({
|
|
||||||
...this.config
|
|
||||||
})
|
|
||||||
this.storageAdapter = storage
|
|
||||||
return storage
|
|
||||||
}
|
|
||||||
|
|
||||||
protected async onInitialize(): Promise<void> {
|
|
||||||
await this.storageAdapter!.init()
|
|
||||||
this.log(`Azure Blob Storage initialized with container ${this.config.containerName}`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Auto-select the best storage augmentation for the environment
|
|
||||||
* Maintains zero-config philosophy
|
|
||||||
*/
|
|
||||||
export async function createAutoStorageAugmentation(options: {
|
|
||||||
rootDirectory?: string
|
|
||||||
requestPersistentStorage?: boolean
|
|
||||||
} = {}): Promise<StorageAugmentation> {
|
|
||||||
// Detect environment
|
|
||||||
const isNodeEnv = (globalThis as any).__ENV__?.isNode || (
|
|
||||||
typeof process !== 'undefined' &&
|
|
||||||
process.versions != null &&
|
|
||||||
process.versions.node != null
|
|
||||||
)
|
|
||||||
|
|
||||||
if (isNodeEnv) {
|
|
||||||
// Node.js environment - use FileSystem
|
|
||||||
return new FileSystemStorageAugmentation({
|
|
||||||
rootDirectory: options.rootDirectory || './brainy-data'
|
|
||||||
})
|
|
||||||
} else {
|
|
||||||
// Browser environment - try OPFS, fall back to memory
|
|
||||||
const opfsAug = new OPFSStorageAugmentation({
|
|
||||||
requestPersistent: options.requestPersistentStorage || false
|
|
||||||
})
|
|
||||||
|
|
||||||
// Test if OPFS is available
|
|
||||||
const testStorage = new OPFSStorage()
|
|
||||||
if (testStorage.isOPFSAvailable()) {
|
|
||||||
return opfsAug
|
|
||||||
} else {
|
|
||||||
// Fall back to memory
|
|
||||||
return new MemoryStorageAugmentation()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,467 +0,0 @@
|
||||||
/**
|
|
||||||
* Base Synapse Augmentation
|
|
||||||
*
|
|
||||||
* Synapses are special augmentations that provide bidirectional data sync
|
|
||||||
* with external platforms (Notion, Salesforce, Slack, etc.)
|
|
||||||
*
|
|
||||||
* Like biological synapses that transmit signals between neurons, these
|
|
||||||
* connect Brainy to external data sources, enabling seamless information flow.
|
|
||||||
*
|
|
||||||
* They are managed through the Brain Cloud augmentation registry alongside
|
|
||||||
* other augmentations, enabling unified discovery, installation, and updates.
|
|
||||||
*
|
|
||||||
* Example synapses:
|
|
||||||
* - NotionSynapse: Sync pages, databases, and blocks
|
|
||||||
* - SalesforceSynapse: Sync contacts, leads, opportunities
|
|
||||||
* - SlackSynapse: Sync messages, channels, users
|
|
||||||
* - GoogleDriveSynapse: Sync documents, sheets, presentations
|
|
||||||
*/
|
|
||||||
|
|
||||||
import {
|
|
||||||
AugmentationResponse
|
|
||||||
} from '../types/augmentations.js'
|
|
||||||
import { BaseAugmentation, BrainyAugmentation, AugmentationContext } from './brainyAugmentation.js'
|
|
||||||
import { NeuralImportAugmentation } from './neuralImport.js'
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Base class for all synapse augmentations
|
|
||||||
* Provides common functionality for external data synchronization
|
|
||||||
*/
|
|
||||||
export abstract class SynapseAugmentation extends BaseAugmentation {
|
|
||||||
// BrainyAugmentation properties
|
|
||||||
readonly timing = 'after' as const
|
|
||||||
readonly operations = ['all'] as ('all')[]
|
|
||||||
readonly priority = 10
|
|
||||||
readonly metadata = {
|
|
||||||
reads: '*' as '*', // Needs to read for syncing
|
|
||||||
writes: ['_synapse', '_syncedAt'] as string[]
|
|
||||||
} // Adds synapse tracking metadata
|
|
||||||
|
|
||||||
// Synapse-specific properties
|
|
||||||
abstract readonly synapseId: string
|
|
||||||
abstract readonly supportedTypes: string[]
|
|
||||||
|
|
||||||
// State management
|
|
||||||
protected syncInProgress = false
|
|
||||||
protected lastSyncId?: string
|
|
||||||
protected syncStats = {
|
|
||||||
totalSyncs: 0,
|
|
||||||
totalItems: 0,
|
|
||||||
lastSync: undefined as string | undefined
|
|
||||||
}
|
|
||||||
|
|
||||||
// Neural Import integration
|
|
||||||
protected neuralImport?: NeuralImportAugmentation
|
|
||||||
protected useNeuralImport = true // Enable by default
|
|
||||||
|
|
||||||
protected async onInit(): Promise<void> {
|
|
||||||
|
|
||||||
// Initialize Neural Import if available
|
|
||||||
if (this.useNeuralImport && this.context?.brain) {
|
|
||||||
try {
|
|
||||||
// Check if neural import is already loaded
|
|
||||||
const existingNeuralImport = this.context.brain.augmentations?.get('neural-import')
|
|
||||||
if (existingNeuralImport) {
|
|
||||||
this.neuralImport = existingNeuralImport as NeuralImportAugmentation
|
|
||||||
} else {
|
|
||||||
// Create a new instance for this synapse
|
|
||||||
this.neuralImport = new NeuralImportAugmentation()
|
|
||||||
// NeuralImport will be initialized when the synapse is added to Brainy
|
|
||||||
// await this.neuralImport.initialize()
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
console.warn(`[${this.synapseId}] Neural Import not available, using basic import`)
|
|
||||||
this.useNeuralImport = false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
await this.onInitialize()
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Synapse-specific initialization
|
|
||||||
* Override this in implementations
|
|
||||||
*/
|
|
||||||
protected abstract onInitialize(): Promise<void>
|
|
||||||
|
|
||||||
/**
|
|
||||||
* BrainyAugmentation execute method
|
|
||||||
* Intercepts operations to sync external data when relevant
|
|
||||||
*/
|
|
||||||
async execute<T = any>(
|
|
||||||
operation: string,
|
|
||||||
params: any,
|
|
||||||
next: () => Promise<T>
|
|
||||||
): Promise<T> {
|
|
||||||
// Execute the main operation first
|
|
||||||
const result = await next()
|
|
||||||
|
|
||||||
// After certain operations, check if we should sync
|
|
||||||
if (this.shouldSync(operation, params)) {
|
|
||||||
// Start async sync in background
|
|
||||||
this.backgroundSync().catch(error => {
|
|
||||||
console.error(`[${this.synapseId}] Background sync failed:`, error)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Determine if sync should be triggered after an operation
|
|
||||||
*/
|
|
||||||
protected shouldSync(operation: string, params: any): boolean {
|
|
||||||
// Override in implementations for specific sync triggers
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Background sync process
|
|
||||||
*/
|
|
||||||
protected async backgroundSync(): Promise<void> {
|
|
||||||
if (this.syncInProgress) return
|
|
||||||
|
|
||||||
this.syncInProgress = true
|
|
||||||
try {
|
|
||||||
await this.incrementalSync(this.lastSyncId)
|
|
||||||
} finally {
|
|
||||||
this.syncInProgress = false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected async onShutdown(): Promise<void> {
|
|
||||||
if (this.syncInProgress) {
|
|
||||||
await this.stopSync()
|
|
||||||
}
|
|
||||||
await this.onSynapseShutdown()
|
|
||||||
}
|
|
||||||
|
|
||||||
protected async onSynapseShutdown(): Promise<void> {
|
|
||||||
// Override in implementations for cleanup
|
|
||||||
}
|
|
||||||
|
|
||||||
// getSynapseStatus implemented below with full response
|
|
||||||
|
|
||||||
/**
|
|
||||||
* ISynapseAugmentation methods
|
|
||||||
*/
|
|
||||||
abstract testConnection(): Promise<AugmentationResponse<boolean>>
|
|
||||||
|
|
||||||
abstract startSync(options?: Record<string, unknown>): Promise<AugmentationResponse<{
|
|
||||||
synced: number
|
|
||||||
failed: number
|
|
||||||
skipped: number
|
|
||||||
duration: number
|
|
||||||
errors?: Array<{ item: string; error: string }>
|
|
||||||
}>>
|
|
||||||
|
|
||||||
async stopSync(): Promise<void> {
|
|
||||||
this.syncInProgress = false
|
|
||||||
}
|
|
||||||
|
|
||||||
abstract incrementalSync(lastSyncId?: string): Promise<AugmentationResponse<{
|
|
||||||
synced: number
|
|
||||||
failed: number
|
|
||||||
skipped: number
|
|
||||||
duration: number
|
|
||||||
hasMore: boolean
|
|
||||||
nextSyncId?: string
|
|
||||||
}>>
|
|
||||||
|
|
||||||
abstract previewSync(limit?: number): Promise<AugmentationResponse<{
|
|
||||||
items: Array<{
|
|
||||||
type: string
|
|
||||||
title: string
|
|
||||||
preview: string
|
|
||||||
}>
|
|
||||||
totalCount: number
|
|
||||||
estimatedDuration: number
|
|
||||||
}>>
|
|
||||||
|
|
||||||
async getSynapseStatus(): Promise<AugmentationResponse<{
|
|
||||||
status: 'connected' | 'disconnected' | 'syncing' | 'error'
|
|
||||||
lastSync?: string
|
|
||||||
nextSync?: string
|
|
||||||
totalSyncs: number
|
|
||||||
totalItems: number
|
|
||||||
}>> {
|
|
||||||
const connectionTest = await this.testConnection()
|
|
||||||
|
|
||||||
return {
|
|
||||||
success: true,
|
|
||||||
data: {
|
|
||||||
status: this.syncInProgress ? 'syncing' :
|
|
||||||
connectionTest.success ? 'connected' : 'disconnected',
|
|
||||||
lastSync: this.syncStats.lastSync,
|
|
||||||
totalSyncs: this.syncStats.totalSyncs,
|
|
||||||
totalItems: this.syncStats.totalItems
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Helper method to store synced data in Brainy
|
|
||||||
* Optionally uses Neural Import for intelligent processing
|
|
||||||
*/
|
|
||||||
protected async storeInBrainy(
|
|
||||||
content: string | Record<string, any>,
|
|
||||||
metadata: Record<string, any>,
|
|
||||||
options: {
|
|
||||||
useNeuralImport?: boolean
|
|
||||||
dataType?: string
|
|
||||||
rawData?: Buffer | string
|
|
||||||
} = {}
|
|
||||||
): Promise<void> {
|
|
||||||
if (!this.context?.brain) {
|
|
||||||
throw new Error('Brainy context not initialized')
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add synapse source metadata
|
|
||||||
const enrichedMetadata = {
|
|
||||||
...metadata,
|
|
||||||
_synapse: this.synapseId,
|
|
||||||
_syncedAt: new Date().toISOString()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Use Neural Import for intelligent processing if available
|
|
||||||
if (this.neuralImport && (options.useNeuralImport ?? this.useNeuralImport)) {
|
|
||||||
try {
|
|
||||||
// Process through Neural Import for entity/relationship detection
|
|
||||||
const rawData = options.rawData ||
|
|
||||||
(typeof content === 'string' ? content : JSON.stringify(content))
|
|
||||||
|
|
||||||
const neuralResult = await this.neuralImport.processRawData(
|
|
||||||
rawData,
|
|
||||||
options.dataType || 'json',
|
|
||||||
{
|
|
||||||
sourceSystem: this.synapseId,
|
|
||||||
metadata: enrichedMetadata
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
if (neuralResult.success && neuralResult.data) {
|
|
||||||
// Store detected nouns (entities)
|
|
||||||
for (const noun of neuralResult.data.nouns) {
|
|
||||||
await this.context.brain.add({
|
|
||||||
text: noun,
|
|
||||||
metadata: {
|
|
||||||
...enrichedMetadata,
|
|
||||||
_neuralConfidence: neuralResult.data.confidence,
|
|
||||||
_neuralInsights: neuralResult.data.insights
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// Store detected verbs (relationships)
|
|
||||||
for (const verb of neuralResult.data.verbs) {
|
|
||||||
// Parse verb format: "source->relation->target"
|
|
||||||
const parts = verb.split('->')
|
|
||||||
if (parts.length === 3) {
|
|
||||||
await this.context.brain.relate(
|
|
||||||
parts[0], // source
|
|
||||||
parts[2], // target
|
|
||||||
parts[1], // verb type
|
|
||||||
enrichedMetadata
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Store original content with neural metadata
|
|
||||||
if (typeof content === 'string') {
|
|
||||||
await this.context.brain.add({
|
|
||||||
text: content,
|
|
||||||
metadata: {
|
|
||||||
...enrichedMetadata,
|
|
||||||
category: 'Content',
|
|
||||||
_neuralProcessed: true,
|
|
||||||
_neuralConfidence: neuralResult.data.confidence,
|
|
||||||
_detectedEntities: neuralResult.data.nouns.length,
|
|
||||||
_detectedRelationships: neuralResult.data.verbs.length
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
return // Successfully processed with Neural Import
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
console.warn(`[${this.synapseId}] Neural Import processing failed, falling back to basic import:`, error)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Fallback to basic storage
|
|
||||||
if (typeof content === 'string') {
|
|
||||||
await this.context.brain.add({
|
|
||||||
text: content,
|
|
||||||
metadata: {
|
|
||||||
...enrichedMetadata,
|
|
||||||
category: 'Content'
|
|
||||||
}
|
|
||||||
})
|
|
||||||
} else {
|
|
||||||
// For structured data, store as JSON
|
|
||||||
await this.context.brain.add({
|
|
||||||
text: JSON.stringify(content),
|
|
||||||
metadata: {
|
|
||||||
...enrichedMetadata,
|
|
||||||
category: 'Content'
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Helper method to query existing synced data
|
|
||||||
*/
|
|
||||||
protected async queryBrainy(
|
|
||||||
filter: { connector?: string; [key: string]: any }
|
|
||||||
): Promise<any[]> {
|
|
||||||
if (!this.context?.brain) {
|
|
||||||
throw new Error('Brainy context not initialized')
|
|
||||||
}
|
|
||||||
|
|
||||||
const searchFilter = {
|
|
||||||
...filter,
|
|
||||||
_synapse: this.synapseId
|
|
||||||
}
|
|
||||||
|
|
||||||
return this.context.brain.find({
|
|
||||||
where: searchFilter
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Example implementation for reference
|
|
||||||
* Real synapses would be in Brain Cloud registry
|
|
||||||
*/
|
|
||||||
export class ExampleFileSystemSynapse extends SynapseAugmentation {
|
|
||||||
readonly name = 'example-filesystem-synapse'
|
|
||||||
readonly description = 'Example synapse for local file system with Neural Import intelligence'
|
|
||||||
readonly synapseId = 'filesystem'
|
|
||||||
readonly supportedTypes = ['text', 'markdown', 'json', 'csv']
|
|
||||||
|
|
||||||
protected async onInitialize(): Promise<void> {
|
|
||||||
// Initialize file system watcher, etc.
|
|
||||||
}
|
|
||||||
|
|
||||||
async testConnection(): Promise<AugmentationResponse<boolean>> {
|
|
||||||
// Test if we can access the configured directory
|
|
||||||
return {
|
|
||||||
success: true,
|
|
||||||
data: true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async startSync(options?: Record<string, unknown>): Promise<AugmentationResponse<{
|
|
||||||
synced: number
|
|
||||||
failed: number
|
|
||||||
skipped: number
|
|
||||||
duration: number
|
|
||||||
errors?: Array<{ item: string; error: string }>
|
|
||||||
}>> {
|
|
||||||
const startTime = Date.now()
|
|
||||||
|
|
||||||
// Example: Read files from a directory and sync to Brainy
|
|
||||||
// This would normally scan a directory, but here's a conceptual example:
|
|
||||||
|
|
||||||
const exampleFiles = [
|
|
||||||
{
|
|
||||||
path: '/data/notes.md',
|
|
||||||
content: '# Project Notes\nDiscuss roadmap with team\nReview Q1 metrics',
|
|
||||||
type: 'markdown'
|
|
||||||
},
|
|
||||||
{
|
|
||||||
path: '/data/contacts.json',
|
|
||||||
content: { name: 'John Doe', role: 'Developer', team: 'Engineering' },
|
|
||||||
type: 'json'
|
|
||||||
}
|
|
||||||
]
|
|
||||||
|
|
||||||
let synced = 0
|
|
||||||
const errors: Array<{ item: string; error: string }> = []
|
|
||||||
|
|
||||||
for (const file of exampleFiles) {
|
|
||||||
try {
|
|
||||||
// Use Neural Import for intelligent processing
|
|
||||||
await this.storeInBrainy(
|
|
||||||
file.content,
|
|
||||||
{
|
|
||||||
path: file.path,
|
|
||||||
fileType: file.type,
|
|
||||||
syncedFrom: 'filesystem'
|
|
||||||
},
|
|
||||||
{
|
|
||||||
useNeuralImport: true, // Enable AI processing
|
|
||||||
dataType: file.type
|
|
||||||
}
|
|
||||||
)
|
|
||||||
synced++
|
|
||||||
} catch (error) {
|
|
||||||
errors.push({
|
|
||||||
item: file.path,
|
|
||||||
error: error instanceof Error ? error.message : 'Unknown error'
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
this.syncStats.totalSyncs++
|
|
||||||
this.syncStats.totalItems += synced
|
|
||||||
this.syncStats.lastSync = new Date().toISOString()
|
|
||||||
|
|
||||||
return {
|
|
||||||
success: true,
|
|
||||||
data: {
|
|
||||||
synced,
|
|
||||||
failed: errors.length,
|
|
||||||
skipped: 0,
|
|
||||||
duration: Date.now() - startTime,
|
|
||||||
errors: errors.length > 0 ? errors : undefined
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async incrementalSync(lastSyncId?: string): Promise<AugmentationResponse<{
|
|
||||||
synced: number
|
|
||||||
failed: number
|
|
||||||
skipped: number
|
|
||||||
duration: number
|
|
||||||
hasMore: boolean
|
|
||||||
nextSyncId?: string
|
|
||||||
}>> {
|
|
||||||
const startTime = Date.now()
|
|
||||||
|
|
||||||
// Example: Check for modified files since last sync
|
|
||||||
|
|
||||||
return {
|
|
||||||
success: true,
|
|
||||||
data: {
|
|
||||||
synced: 0,
|
|
||||||
failed: 0,
|
|
||||||
skipped: 0,
|
|
||||||
duration: Date.now() - startTime,
|
|
||||||
hasMore: false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async previewSync(limit: number = 10): Promise<AugmentationResponse<{
|
|
||||||
items: Array<{
|
|
||||||
type: string
|
|
||||||
title: string
|
|
||||||
preview: string
|
|
||||||
}>
|
|
||||||
totalCount: number
|
|
||||||
estimatedDuration: number
|
|
||||||
}>> {
|
|
||||||
// Example: List files that would be synced
|
|
||||||
|
|
||||||
return {
|
|
||||||
success: true,
|
|
||||||
data: {
|
|
||||||
items: [],
|
|
||||||
totalCount: 0,
|
|
||||||
estimatedDuration: 0
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,579 +0,0 @@
|
||||||
/**
|
|
||||||
* BrainyTypes - Intelligent type detection using semantic embeddings
|
|
||||||
*
|
|
||||||
* This module uses our existing TransformerEmbedding and similarity functions
|
|
||||||
* to intelligently match data to our 42 noun types and 127 verb types.
|
|
||||||
*
|
|
||||||
* Features:
|
|
||||||
* - Semantic similarity matching using embeddings
|
|
||||||
* - Context-aware type detection
|
|
||||||
* - Confidence scoring
|
|
||||||
* - Caching for performance
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { NounType, VerbType } from '../../types/graphTypes.js'
|
|
||||||
import { TransformerEmbedding } from '../../utils/embedding.js'
|
|
||||||
import { cosineDistance } from '../../utils/distance.js'
|
|
||||||
import { Vector } from '../../coreTypes.js'
|
|
||||||
import { getNounTypeEmbeddings, getVerbTypeEmbeddings } from '../../neural/embeddedTypeEmbeddings.js'
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Type descriptions for semantic matching
|
|
||||||
* These descriptions are used to generate embeddings for each type
|
|
||||||
*/
|
|
||||||
const NOUN_TYPE_DESCRIPTIONS: Record<string, string> = {
|
|
||||||
// Core Entity Types (7)
|
|
||||||
[NounType.Person]: 'person human individual user employee customer citizen member author creator agent actor participant',
|
|
||||||
[NounType.Organization]: 'organization company business corporation institution agency department team group committee board',
|
|
||||||
[NounType.Location]: 'location place address city country region area zone coordinate position site venue building',
|
|
||||||
[NounType.Thing]: 'thing object item product device equipment tool instrument asset artifact material physical tangible',
|
|
||||||
[NounType.Concept]: 'concept idea theory principle philosophy belief value abstract intangible notion thought',
|
|
||||||
[NounType.Event]: 'event occurrence incident activity happening meeting conference celebration milestone timestamp date',
|
|
||||||
[NounType.Agent]: 'agent AI bot automated system automation software assistant service daemon daemon worker processor',
|
|
||||||
|
|
||||||
// Biological Types (1) - Stage 3
|
|
||||||
[NounType.Organism]: 'organism animal plant bacteria fungi species living biological life creature being microorganism',
|
|
||||||
|
|
||||||
// Material Types (1) - Stage 3
|
|
||||||
[NounType.Substance]: 'substance material matter chemical element compound liquid gas solid molecule atom material',
|
|
||||||
|
|
||||||
// Property & Quality Types (1) - Stage 3
|
|
||||||
[NounType.Quality]: 'quality property attribute characteristic feature aspect trait dimension parameter value',
|
|
||||||
|
|
||||||
// Temporal Types (1) - Stage 3
|
|
||||||
[NounType.TimeInterval]: 'timeInterval period duration span range epoch era season quarter interval window',
|
|
||||||
|
|
||||||
// Functional Types (1) - Stage 3
|
|
||||||
[NounType.Function]: 'function purpose role capability capacity utility objective goal aim intent',
|
|
||||||
|
|
||||||
// Informational Types (1) - Stage 3
|
|
||||||
[NounType.Proposition]: 'proposition statement claim assertion declaration premise conclusion hypothesis theory',
|
|
||||||
|
|
||||||
// Digital/Content Types
|
|
||||||
[NounType.Document]: 'document file report article paper text pdf word contract agreement record documentation',
|
|
||||||
[NounType.Media]: 'media image photo video audio music podcast multimedia graphic visualization animation',
|
|
||||||
[NounType.File]: 'file digital data binary code script program software archive package bundle',
|
|
||||||
[NounType.Message]: 'message email chat communication notification alert announcement broadcast transmission',
|
|
||||||
|
|
||||||
// Collection Types
|
|
||||||
[NounType.Collection]: 'collection group set list array category folder directory catalog inventory database',
|
|
||||||
[NounType.Dataset]: 'dataset data table spreadsheet database records statistics metrics measurements analysis',
|
|
||||||
|
|
||||||
// Business/Application Types
|
|
||||||
[NounType.Product]: 'product item merchandise offering service feature application software solution package',
|
|
||||||
[NounType.Service]: 'service offering subscription support maintenance utility function capability',
|
|
||||||
[NounType.Task]: 'task action todo item job assignment duty responsibility activity step procedure',
|
|
||||||
[NounType.Project]: 'project initiative program campaign effort endeavor plan scheme venture undertaking',
|
|
||||||
|
|
||||||
// Descriptive Types
|
|
||||||
[NounType.Process]: 'process workflow procedure method algorithm sequence pipeline operation routine protocol',
|
|
||||||
[NounType.State]: 'state status condition phase stage mode situation circumstance configuration setting',
|
|
||||||
[NounType.Role]: 'role position title function responsibility duty job capacity designation authority',
|
|
||||||
[NounType.Language]: 'language dialect locale tongue vernacular communication speech linguistics vocabulary',
|
|
||||||
[NounType.Currency]: 'currency money dollar euro pound yen bitcoin payment financial monetary unit',
|
|
||||||
[NounType.Measurement]: 'measurement metric quantity value amount size dimension weight height volume distance',
|
|
||||||
|
|
||||||
// Scientific/Research Types
|
|
||||||
[NounType.Hypothesis]: 'hypothesis theory proposition thesis assumption premise conjecture speculation prediction',
|
|
||||||
[NounType.Experiment]: 'experiment test trial study research investigation analysis observation examination',
|
|
||||||
|
|
||||||
// Legal/Regulatory Types
|
|
||||||
[NounType.Contract]: 'contract agreement deal treaty pact covenant license terms conditions policy',
|
|
||||||
[NounType.Regulation]: 'regulation law rule policy standard compliance requirement guideline ordinance statute',
|
|
||||||
|
|
||||||
// Technical Infrastructure Types
|
|
||||||
[NounType.Interface]: 'interface API endpoint protocol specification contract schema definition connection',
|
|
||||||
[NounType.Resource]: 'resource infrastructure server database storage compute memory bandwidth capacity asset',
|
|
||||||
|
|
||||||
// Custom/Extensible (1) - Stage 3
|
|
||||||
[NounType.Custom]: 'custom special unique particular specific domain-specific specialized tailored bespoke',
|
|
||||||
|
|
||||||
// Social Structures (3) - Stage 3
|
|
||||||
[NounType.SocialGroup]: 'socialGroup community tribe clan network circle collective society crowd gathering',
|
|
||||||
[NounType.Institution]: 'institution establishment foundation organization system structure framework tradition practice',
|
|
||||||
[NounType.Norm]: 'norm convention standard custom tradition protocol etiquette rule practice expectation',
|
|
||||||
|
|
||||||
// Information Theory (2) - Stage 3
|
|
||||||
[NounType.InformationContent]: 'informationContent story narrative knowledge data schema pattern model structure',
|
|
||||||
[NounType.InformationBearer]: 'informationBearer carrier medium vehicle signal token representation document',
|
|
||||||
|
|
||||||
// Meta-Level (1) - Stage 3
|
|
||||||
[NounType.Relationship]: 'relationship connection relation association link bond tie interaction dynamic'
|
|
||||||
}
|
|
||||||
|
|
||||||
const VERB_TYPE_DESCRIPTIONS: Record<string, string> = {
|
|
||||||
// Core Relationship Types
|
|
||||||
[VerbType.RelatedTo]: 'related connected associated linked correlated relevant pertinent applicable',
|
|
||||||
[VerbType.Contains]: 'contains includes holds stores encompasses comprises consists incorporates',
|
|
||||||
[VerbType.PartOf]: 'part component element member piece portion section segment constituent',
|
|
||||||
[VerbType.LocatedAt]: 'located situated positioned placed found exists resides occupies',
|
|
||||||
[VerbType.References]: 'references cites mentions points links refers quotes sources',
|
|
||||||
|
|
||||||
// Temporal/Causal Types
|
|
||||||
[VerbType.Precedes]: 'precedes before earlier prior previous antecedent preliminary foregoing',
|
|
||||||
[VerbType.Causes]: 'causes triggers induces produces generates results influences affects',
|
|
||||||
[VerbType.DependsOn]: 'depends requires needs relies necessitates contingent prerequisite',
|
|
||||||
[VerbType.Requires]: 'requires needs demands necessitates mandates obliges compels entails',
|
|
||||||
|
|
||||||
// Creation/Transformation Types
|
|
||||||
[VerbType.Creates]: 'creates makes produces generates builds constructs forms establishes',
|
|
||||||
[VerbType.Transforms]: 'transforms converts changes modifies alters transitions morphs evolves',
|
|
||||||
[VerbType.Becomes]: 'becomes turns evolves transforms changes transitions develops grows',
|
|
||||||
[VerbType.Modifies]: 'modifies changes updates alters edits revises adjusts adapts',
|
|
||||||
[VerbType.Consumes]: 'consumes uses utilizes depletes expends absorbs takes processes',
|
|
||||||
|
|
||||||
// Ownership/Attribution Types
|
|
||||||
[VerbType.Owns]: 'owns possesses holds controls manages administers governs maintains',
|
|
||||||
[VerbType.AttributedTo]: 'attributed credited assigned ascribed authored written composed',
|
|
||||||
|
|
||||||
// Social/Organizational Types
|
|
||||||
[VerbType.MemberOf]: 'member participant affiliate associate belongs joined enrolled registered',
|
|
||||||
[VerbType.WorksWith]: 'works collaborates cooperates partners teams assists helps supports',
|
|
||||||
[VerbType.FriendOf]: 'friend companion buddy pal acquaintance associate connection relationship',
|
|
||||||
[VerbType.Follows]: 'follows subscribes tracks monitors watches observes trails pursues',
|
|
||||||
[VerbType.Likes]: 'likes enjoys appreciates favors prefers admires values endorses',
|
|
||||||
[VerbType.ReportsTo]: 'reports answers subordinate accountable responsible supervised managed',
|
|
||||||
[VerbType.Mentors]: 'mentors teaches guides coaches instructs trains advises counsels',
|
|
||||||
[VerbType.Communicates]: 'communicates talks speaks messages contacts interacts corresponds exchanges',
|
|
||||||
|
|
||||||
// Descriptive/Functional Types
|
|
||||||
[VerbType.Describes]: 'describes explains details documents specifies outlines depicts characterizes',
|
|
||||||
[VerbType.Defines]: 'defines specifies establishes determines sets declares identifies designates',
|
|
||||||
[VerbType.Categorizes]: 'categorizes classifies groups sorts organizes arranges labels tags',
|
|
||||||
[VerbType.Measures]: 'measures quantifies gauges assesses evaluates calculates determines counts',
|
|
||||||
[VerbType.Evaluates]: 'evaluates assesses analyzes reviews examines appraises judges rates',
|
|
||||||
[VerbType.Uses]: 'uses utilizes employs applies operates handles manipulates exploits',
|
|
||||||
[VerbType.Implements]: 'implements executes realizes performs accomplishes carries delivers completes',
|
|
||||||
[VerbType.Extends]: 'extends expands enhances augments amplifies broadens enlarges develops',
|
|
||||||
|
|
||||||
// Enhanced Relationships
|
|
||||||
[VerbType.Inherits]: 'inherits derives extends receives obtains acquires succeeds legacy',
|
|
||||||
[VerbType.Conflicts]: 'conflicts contradicts opposes clashes disputes disagrees incompatible inconsistent',
|
|
||||||
[VerbType.Synchronizes]: 'synchronizes coordinates aligns harmonizes matches corresponds parallels coincides',
|
|
||||||
[VerbType.Competes]: 'competes rivals contends contests challenges opposes vies struggles'
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Result of type matching with confidence scores
|
|
||||||
*/
|
|
||||||
export interface TypeMatchResult {
|
|
||||||
type: string
|
|
||||||
confidence: number
|
|
||||||
reasoning: string
|
|
||||||
alternatives: Array<{
|
|
||||||
type: string
|
|
||||||
confidence: number
|
|
||||||
}>
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* BrainyTypes - Intelligent type detection for nouns and verbs
|
|
||||||
* PRODUCTION OPTIMIZATION: Uses pre-computed type embeddings
|
|
||||||
* Type embeddings are loaded instantly; only input objects are embedded at runtime
|
|
||||||
*/
|
|
||||||
export class BrainyTypes {
|
|
||||||
private embedder: TransformerEmbedding // Only for embedding input objects
|
|
||||||
private nounEmbeddings: Map<string, Vector> = new Map()
|
|
||||||
private verbEmbeddings: Map<string, Vector> = new Map()
|
|
||||||
private initialized = false
|
|
||||||
private cache: Map<string, TypeMatchResult> = new Map()
|
|
||||||
|
|
||||||
constructor() {
|
|
||||||
// Embedder only used for input objects, NOT for type embeddings
|
|
||||||
this.embedder = new TransformerEmbedding({ verbose: false })
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Initialize the type matcher by loading pre-computed embeddings
|
|
||||||
* INSTANT - type embeddings are loaded from pre-computed data
|
|
||||||
* Only the model for input embedding needs initialization
|
|
||||||
*/
|
|
||||||
async init(): Promise<void> {
|
|
||||||
if (this.initialized) return
|
|
||||||
|
|
||||||
// Initialize embedder for input objects only
|
|
||||||
await this.embedder.init()
|
|
||||||
|
|
||||||
// Load pre-computed type embeddings (instant, no computation)
|
|
||||||
const nounEmbeddings = getNounTypeEmbeddings()
|
|
||||||
const verbEmbeddings = getVerbTypeEmbeddings()
|
|
||||||
|
|
||||||
// Convert NounType/VerbType keys to strings for lookup
|
|
||||||
for (const [type, embedding] of nounEmbeddings.entries()) {
|
|
||||||
this.nounEmbeddings.set(type, embedding)
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const [type, embedding] of verbEmbeddings.entries()) {
|
|
||||||
this.verbEmbeddings.set(type, embedding)
|
|
||||||
}
|
|
||||||
|
|
||||||
this.initialized = true
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Match an object to the most appropriate noun type
|
|
||||||
*/
|
|
||||||
async matchNounType(obj: any): Promise<TypeMatchResult> {
|
|
||||||
await this.init()
|
|
||||||
|
|
||||||
// Create a text representation of the object for embedding
|
|
||||||
const textRepresentation = this.createTextRepresentation(obj)
|
|
||||||
|
|
||||||
// Check cache
|
|
||||||
const cacheKey = `noun:${textRepresentation}`
|
|
||||||
if (this.cache.has(cacheKey)) {
|
|
||||||
return this.cache.get(cacheKey)!
|
|
||||||
}
|
|
||||||
|
|
||||||
// Generate embedding for the input
|
|
||||||
const inputEmbedding = await this.embedder.embed(textRepresentation)
|
|
||||||
|
|
||||||
// Calculate similarities to all noun types
|
|
||||||
const similarities: Array<{ type: string; similarity: number }> = []
|
|
||||||
|
|
||||||
for (const [type, typeEmbedding] of this.nounEmbeddings.entries()) {
|
|
||||||
// Convert cosine distance to similarity (1 - distance)
|
|
||||||
const similarity = 1 - cosineDistance(inputEmbedding, typeEmbedding)
|
|
||||||
similarities.push({ type, similarity })
|
|
||||||
}
|
|
||||||
|
|
||||||
// Sort by similarity (highest first)
|
|
||||||
similarities.sort((a, b) => b.similarity - a.similarity)
|
|
||||||
|
|
||||||
// Apply heuristic rules for common patterns
|
|
||||||
const heuristicType = this.applyNounHeuristics(obj)
|
|
||||||
if (heuristicType) {
|
|
||||||
// Boost the heuristic type's confidence
|
|
||||||
const heuristicIndex = similarities.findIndex(s => s.type === heuristicType)
|
|
||||||
if (heuristicIndex > 0) {
|
|
||||||
similarities[heuristicIndex].similarity *= 1.2 // 20% boost
|
|
||||||
similarities.sort((a, b) => b.similarity - a.similarity)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create result
|
|
||||||
const result: TypeMatchResult = {
|
|
||||||
type: similarities[0].type,
|
|
||||||
confidence: similarities[0].similarity,
|
|
||||||
reasoning: this.generateReasoning(obj, similarities[0].type, 'noun'),
|
|
||||||
alternatives: similarities.slice(1, 4).map(s => ({
|
|
||||||
type: s.type,
|
|
||||||
confidence: s.similarity
|
|
||||||
}))
|
|
||||||
}
|
|
||||||
|
|
||||||
// Cache result
|
|
||||||
this.cache.set(cacheKey, result)
|
|
||||||
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Match a relationship to the most appropriate verb type
|
|
||||||
*/
|
|
||||||
async matchVerbType(
|
|
||||||
sourceObj: any,
|
|
||||||
targetObj: any,
|
|
||||||
relationshipHint?: string
|
|
||||||
): Promise<TypeMatchResult> {
|
|
||||||
await this.init()
|
|
||||||
|
|
||||||
// Create text representation of the relationship
|
|
||||||
const textRepresentation = this.createRelationshipText(sourceObj, targetObj, relationshipHint)
|
|
||||||
|
|
||||||
// Check cache
|
|
||||||
const cacheKey = `verb:${textRepresentation}`
|
|
||||||
if (this.cache.has(cacheKey)) {
|
|
||||||
return this.cache.get(cacheKey)!
|
|
||||||
}
|
|
||||||
|
|
||||||
// Generate embedding
|
|
||||||
const inputEmbedding = await this.embedder.embed(textRepresentation)
|
|
||||||
|
|
||||||
// Calculate similarities to all verb types
|
|
||||||
const similarities: Array<{ type: string; similarity: number }> = []
|
|
||||||
|
|
||||||
for (const [type, typeEmbedding] of this.verbEmbeddings.entries()) {
|
|
||||||
const similarity = 1 - cosineDistance(inputEmbedding, typeEmbedding)
|
|
||||||
similarities.push({ type, similarity })
|
|
||||||
}
|
|
||||||
|
|
||||||
// Sort by similarity
|
|
||||||
similarities.sort((a, b) => b.similarity - a.similarity)
|
|
||||||
|
|
||||||
// Apply heuristic rules
|
|
||||||
const heuristicType = this.applyVerbHeuristics(sourceObj, targetObj, relationshipHint)
|
|
||||||
if (heuristicType) {
|
|
||||||
const heuristicIndex = similarities.findIndex(s => s.type === heuristicType)
|
|
||||||
if (heuristicIndex > 0) {
|
|
||||||
similarities[heuristicIndex].similarity *= 1.2
|
|
||||||
similarities.sort((a, b) => b.similarity - a.similarity)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create result
|
|
||||||
const result: TypeMatchResult = {
|
|
||||||
type: similarities[0].type,
|
|
||||||
confidence: similarities[0].similarity,
|
|
||||||
reasoning: this.generateReasoning(
|
|
||||||
{ source: sourceObj, target: targetObj, hint: relationshipHint },
|
|
||||||
similarities[0].type,
|
|
||||||
'verb'
|
|
||||||
),
|
|
||||||
alternatives: similarities.slice(1, 4).map(s => ({
|
|
||||||
type: s.type,
|
|
||||||
confidence: s.similarity
|
|
||||||
}))
|
|
||||||
}
|
|
||||||
|
|
||||||
// Cache result
|
|
||||||
this.cache.set(cacheKey, result)
|
|
||||||
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create text representation of an object for embedding
|
|
||||||
*/
|
|
||||||
private createTextRepresentation(obj: any): string {
|
|
||||||
const parts: string[] = []
|
|
||||||
|
|
||||||
// Add type if available
|
|
||||||
if (typeof obj === 'object' && obj !== null) {
|
|
||||||
// Add field names and values
|
|
||||||
for (const [key, value] of Object.entries(obj)) {
|
|
||||||
parts.push(key)
|
|
||||||
if (typeof value === 'string') {
|
|
||||||
parts.push(value.slice(0, 100)) // Limit string length
|
|
||||||
} else if (typeof value === 'number' || typeof value === 'boolean') {
|
|
||||||
parts.push(String(value))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add special fields with higher weight
|
|
||||||
const importantFields = ['type', 'kind', 'category', 'class', 'name', 'title', 'description']
|
|
||||||
for (const field of importantFields) {
|
|
||||||
if (obj[field]) {
|
|
||||||
parts.push(String(obj[field]))
|
|
||||||
parts.push(String(obj[field])) // Double weight for important fields
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else if (typeof obj === 'string') {
|
|
||||||
parts.push(obj)
|
|
||||||
} else {
|
|
||||||
parts.push(String(obj))
|
|
||||||
}
|
|
||||||
|
|
||||||
return parts.join(' ')
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create text representation of a relationship
|
|
||||||
*/
|
|
||||||
private createRelationshipText(
|
|
||||||
sourceObj: any,
|
|
||||||
targetObj: any,
|
|
||||||
relationshipHint?: string
|
|
||||||
): string {
|
|
||||||
const parts: string[] = []
|
|
||||||
|
|
||||||
if (relationshipHint) {
|
|
||||||
parts.push(relationshipHint)
|
|
||||||
parts.push(relationshipHint) // Double weight for explicit hint
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add source context
|
|
||||||
if (sourceObj) {
|
|
||||||
parts.push('source:')
|
|
||||||
parts.push(this.getObjectSummary(sourceObj))
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add target context
|
|
||||||
if (targetObj) {
|
|
||||||
parts.push('target:')
|
|
||||||
parts.push(this.getObjectSummary(targetObj))
|
|
||||||
}
|
|
||||||
|
|
||||||
return parts.join(' ')
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get a brief summary of an object
|
|
||||||
*/
|
|
||||||
private getObjectSummary(obj: any): string {
|
|
||||||
if (typeof obj === 'string') return obj.slice(0, 50)
|
|
||||||
if (typeof obj !== 'object' || obj === null) return String(obj)
|
|
||||||
|
|
||||||
const summary: string[] = []
|
|
||||||
const fields = ['type', 'name', 'title', 'id', 'category', 'kind']
|
|
||||||
|
|
||||||
for (const field of fields) {
|
|
||||||
if (obj[field]) {
|
|
||||||
summary.push(String(obj[field]))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return summary.join(' ').slice(0, 100)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Apply heuristic rules for noun type detection
|
|
||||||
*/
|
|
||||||
private applyNounHeuristics(obj: any): string | null {
|
|
||||||
if (typeof obj !== 'object' || obj === null) return null
|
|
||||||
|
|
||||||
// Person heuristics
|
|
||||||
if (obj.email || obj.firstName || obj.lastName || obj.username || obj.age || obj.gender) {
|
|
||||||
return NounType.Person
|
|
||||||
}
|
|
||||||
|
|
||||||
// Organization heuristics
|
|
||||||
if (obj.companyName || obj.organizationId || obj.employees || obj.industry) {
|
|
||||||
return NounType.Organization
|
|
||||||
}
|
|
||||||
|
|
||||||
// Location heuristics
|
|
||||||
if (obj.latitude || obj.longitude || obj.address || obj.city || obj.country || obj.coordinates) {
|
|
||||||
return NounType.Location
|
|
||||||
}
|
|
||||||
|
|
||||||
// Document heuristics
|
|
||||||
if (obj.content && (obj.title || obj.author) || obj.documentType || obj.pages) {
|
|
||||||
return NounType.Document
|
|
||||||
}
|
|
||||||
|
|
||||||
// Event heuristics
|
|
||||||
if (obj.startTime || obj.endTime || obj.date || obj.eventType || obj.attendees) {
|
|
||||||
return NounType.Event
|
|
||||||
}
|
|
||||||
|
|
||||||
// Product heuristics
|
|
||||||
if (obj.price || obj.sku || obj.inventory || obj.productId) {
|
|
||||||
return NounType.Product
|
|
||||||
}
|
|
||||||
|
|
||||||
// Task heuristics
|
|
||||||
if (obj.status && (obj.assignee || obj.dueDate) || obj.priority || obj.completed !== undefined) {
|
|
||||||
return NounType.Task
|
|
||||||
}
|
|
||||||
|
|
||||||
// Media heuristics
|
|
||||||
if (obj.url && (obj.url.match(/\.(jpg|jpeg|png|gif|mp4|mp3|wav)/i))) {
|
|
||||||
return NounType.Media
|
|
||||||
}
|
|
||||||
|
|
||||||
// Dataset heuristics
|
|
||||||
if (Array.isArray(obj.data) || obj.rows || obj.columns || obj.schema) {
|
|
||||||
return NounType.Dataset
|
|
||||||
}
|
|
||||||
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Apply heuristic rules for verb type detection
|
|
||||||
*/
|
|
||||||
private applyVerbHeuristics(
|
|
||||||
sourceObj: any,
|
|
||||||
targetObj: any,
|
|
||||||
relationshipHint?: string
|
|
||||||
): string | null {
|
|
||||||
if (!relationshipHint) return null
|
|
||||||
|
|
||||||
const hint = relationshipHint.toLowerCase()
|
|
||||||
|
|
||||||
// Ownership patterns
|
|
||||||
if (hint.includes('own') || hint.includes('possess') || hint.includes('has')) {
|
|
||||||
return VerbType.Owns
|
|
||||||
}
|
|
||||||
|
|
||||||
// Creation patterns
|
|
||||||
if (hint.includes('create') || hint.includes('made') || hint.includes('authored')) {
|
|
||||||
return VerbType.Creates
|
|
||||||
}
|
|
||||||
|
|
||||||
// Containment patterns
|
|
||||||
if (hint.includes('contain') || hint.includes('include') || hint.includes('has')) {
|
|
||||||
return VerbType.Contains
|
|
||||||
}
|
|
||||||
|
|
||||||
// Membership patterns
|
|
||||||
if (hint.includes('member') || hint.includes('belong') || hint.includes('part')) {
|
|
||||||
return VerbType.MemberOf
|
|
||||||
}
|
|
||||||
|
|
||||||
// Reference patterns
|
|
||||||
if (hint.includes('refer') || hint.includes('cite') || hint.includes('link')) {
|
|
||||||
return VerbType.References
|
|
||||||
}
|
|
||||||
|
|
||||||
// Dependency patterns
|
|
||||||
if (hint.includes('depend') || hint.includes('require') || hint.includes('need')) {
|
|
||||||
return VerbType.DependsOn
|
|
||||||
}
|
|
||||||
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Generate human-readable reasoning for the type selection
|
|
||||||
*/
|
|
||||||
private generateReasoning(
|
|
||||||
obj: any,
|
|
||||||
selectedType: string,
|
|
||||||
typeKind: 'noun' | 'verb'
|
|
||||||
): string {
|
|
||||||
const descriptions = typeKind === 'noun' ? NOUN_TYPE_DESCRIPTIONS : VERB_TYPE_DESCRIPTIONS
|
|
||||||
const typeDesc = descriptions[selectedType]
|
|
||||||
|
|
||||||
// Handle missing descriptions gracefully
|
|
||||||
if (!typeDesc) {
|
|
||||||
if (typeKind === 'noun') {
|
|
||||||
const fields = Object.keys(obj).slice(0, 3).join(', ')
|
|
||||||
return `Matched to ${selectedType} based on semantic similarity and object fields: ${fields}`
|
|
||||||
} else {
|
|
||||||
return `Matched to ${selectedType} based on semantic similarity and relationship context`
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (typeKind === 'noun') {
|
|
||||||
const fields = Object.keys(obj).slice(0, 3).join(', ')
|
|
||||||
return `Matched to ${selectedType} based on semantic similarity to "${typeDesc.split(' ').slice(0, 5).join(' ')}..." and object fields: ${fields}`
|
|
||||||
} else {
|
|
||||||
return `Matched to ${selectedType} based on semantic similarity to "${typeDesc.split(' ').slice(0, 5).join(' ')}..." and relationship context`
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Clear the cache
|
|
||||||
*/
|
|
||||||
clearCache(): void {
|
|
||||||
this.cache.clear()
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Dispose of resources
|
|
||||||
*/
|
|
||||||
async dispose(): Promise<void> {
|
|
||||||
await this.embedder.dispose()
|
|
||||||
this.cache.clear()
|
|
||||||
this.nounEmbeddings.clear()
|
|
||||||
this.verbEmbeddings.clear()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Singleton instance for efficient reuse
|
|
||||||
*/
|
|
||||||
let globalInstance: BrainyTypes | null = null
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get or create the global BrainyTypes instance
|
|
||||||
*/
|
|
||||||
export async function getBrainyTypes(): Promise<BrainyTypes> {
|
|
||||||
if (!globalInstance) {
|
|
||||||
globalInstance = new BrainyTypes()
|
|
||||||
await globalInstance.init()
|
|
||||||
}
|
|
||||||
return globalInstance
|
|
||||||
}
|
|
||||||
|
|
@ -1,180 +0,0 @@
|
||||||
/**
|
|
||||||
* IntelligentTypeMatcher - Wrapper around BrainyTypes for testing
|
|
||||||
*
|
|
||||||
* Provides intelligent type detection using semantic embeddings
|
|
||||||
* for matching data to our 42 noun types and 127 verb types.
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { NounType, VerbType } from '../../types/graphTypes.js'
|
|
||||||
import { BrainyTypes, TypeMatchResult, getBrainyTypes } from './brainyTypes.js'
|
|
||||||
|
|
||||||
export interface TypeMatchOptions {
|
|
||||||
threshold?: number
|
|
||||||
topK?: number
|
|
||||||
useCache?: boolean
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Intelligent type matcher using semantic embeddings
|
|
||||||
*/
|
|
||||||
export class IntelligentTypeMatcher {
|
|
||||||
private brainyTypes: BrainyTypes | null = null
|
|
||||||
private cache = new Map<string, TypeMatchResult>()
|
|
||||||
|
|
||||||
constructor(private options: TypeMatchOptions = {}) {
|
|
||||||
this.options = {
|
|
||||||
threshold: 0.3,
|
|
||||||
topK: 3,
|
|
||||||
useCache: true,
|
|
||||||
...options
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Initialize the type matcher
|
|
||||||
*/
|
|
||||||
async init(): Promise<void> {
|
|
||||||
this.brainyTypes = await getBrainyTypes()
|
|
||||||
await this.brainyTypes.init()
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Dispose of resources
|
|
||||||
*/
|
|
||||||
async dispose(): Promise<void> {
|
|
||||||
if (this.brainyTypes) {
|
|
||||||
await this.brainyTypes.dispose()
|
|
||||||
this.brainyTypes = null
|
|
||||||
}
|
|
||||||
this.cache.clear()
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Match data to a noun type
|
|
||||||
*/
|
|
||||||
async matchNounType(data: any): Promise<{
|
|
||||||
type: NounType
|
|
||||||
confidence: number
|
|
||||||
alternatives: Array<{ type: NounType; confidence: number }>
|
|
||||||
}> {
|
|
||||||
if (!this.brainyTypes) {
|
|
||||||
throw new Error('IntelligentTypeMatcher not initialized. Call init() first.')
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check cache if enabled
|
|
||||||
const cacheKey = JSON.stringify(data)
|
|
||||||
if (this.options.useCache && this.cache.has(cacheKey)) {
|
|
||||||
const cached = this.cache.get(cacheKey)!
|
|
||||||
return {
|
|
||||||
type: cached.type as NounType,
|
|
||||||
confidence: cached.confidence,
|
|
||||||
alternatives: cached.alternatives?.map(alt => ({
|
|
||||||
type: alt.type as NounType,
|
|
||||||
confidence: alt.confidence
|
|
||||||
})) || []
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Detect type using BrainyTypes
|
|
||||||
const result = await this.brainyTypes.matchNounType(data)
|
|
||||||
|
|
||||||
// Convert to expected format
|
|
||||||
const response = {
|
|
||||||
type: result.type as NounType,
|
|
||||||
confidence: result.confidence,
|
|
||||||
alternatives: result.alternatives?.map(alt => ({
|
|
||||||
type: alt.type as NounType,
|
|
||||||
confidence: alt.confidence
|
|
||||||
})) || []
|
|
||||||
}
|
|
||||||
|
|
||||||
// Cache the result if enabled
|
|
||||||
if (this.options.useCache) {
|
|
||||||
this.cache.set(cacheKey, result)
|
|
||||||
}
|
|
||||||
|
|
||||||
return response
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Match a relationship to a verb type
|
|
||||||
*/
|
|
||||||
async matchVerbType(
|
|
||||||
source: any,
|
|
||||||
target: any,
|
|
||||||
relationship?: string
|
|
||||||
): Promise<{
|
|
||||||
type: VerbType
|
|
||||||
confidence: number
|
|
||||||
alternatives: Array<{ type: VerbType; confidence: number }>
|
|
||||||
}> {
|
|
||||||
if (!this.brainyTypes) {
|
|
||||||
throw new Error('IntelligentTypeMatcher not initialized. Call init() first.')
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create context for verb detection
|
|
||||||
const context = {
|
|
||||||
source,
|
|
||||||
target,
|
|
||||||
relationship: relationship || 'related',
|
|
||||||
description: relationship || ''
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check cache if enabled
|
|
||||||
const cacheKey = JSON.stringify(context)
|
|
||||||
if (this.options.useCache && this.cache.has(cacheKey)) {
|
|
||||||
const cached = this.cache.get(cacheKey)!
|
|
||||||
return {
|
|
||||||
type: cached.type as VerbType || VerbType.RelatedTo,
|
|
||||||
confidence: cached.confidence,
|
|
||||||
alternatives: cached.alternatives?.map(alt => ({
|
|
||||||
type: alt.type as VerbType || VerbType.RelatedTo,
|
|
||||||
confidence: alt.confidence
|
|
||||||
})) || []
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Detect verb type using BrainyTypes
|
|
||||||
const result = await this.brainyTypes.matchVerbType(
|
|
||||||
context.source,
|
|
||||||
context.target,
|
|
||||||
context.relationship
|
|
||||||
)
|
|
||||||
|
|
||||||
// Convert to expected format
|
|
||||||
const response = {
|
|
||||||
type: result.type as VerbType || VerbType.RelatedTo,
|
|
||||||
confidence: result.confidence,
|
|
||||||
alternatives: result.alternatives?.map(alt => ({
|
|
||||||
type: alt.type as VerbType || VerbType.RelatedTo,
|
|
||||||
confidence: alt.confidence
|
|
||||||
})) || []
|
|
||||||
}
|
|
||||||
|
|
||||||
// Cache the result if enabled
|
|
||||||
if (this.options.useCache) {
|
|
||||||
this.cache.set(cacheKey, result)
|
|
||||||
}
|
|
||||||
|
|
||||||
return response
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Clear the cache
|
|
||||||
*/
|
|
||||||
clearCache(): void {
|
|
||||||
this.cache.clear()
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get cache statistics
|
|
||||||
*/
|
|
||||||
getCacheStats(): { size: number; maxSize: number } {
|
|
||||||
return {
|
|
||||||
size: this.cache.size,
|
|
||||||
maxSize: 1000 // Default max cache size
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export default IntelligentTypeMatcher
|
|
||||||
|
|
@ -1,449 +0,0 @@
|
||||||
/**
|
|
||||||
* Universal Display Augmentation
|
|
||||||
*
|
|
||||||
* 🎨 Provides intelligent display fields for any noun or verb using AI-powered analysis
|
|
||||||
*
|
|
||||||
* Features:
|
|
||||||
* - ✅ Leverages existing BrainyTypes for semantic type detection
|
|
||||||
* - ✅ Complete icon coverage for all 42 NounTypes + 127 VerbTypes
|
|
||||||
* - ✅ Zero performance impact with lazy computation and intelligent caching
|
|
||||||
* - ✅ Perfect isolation - can be disabled, replaced, or configured
|
|
||||||
* - ✅ Clean developer experience with zero conflicts
|
|
||||||
* - ✅ TypeScript support with full autocomplete
|
|
||||||
*
|
|
||||||
* Usage:
|
|
||||||
* ```typescript
|
|
||||||
* // User data access (unchanged)
|
|
||||||
* result.firstName // "John"
|
|
||||||
* result.metadata.title // "CEO"
|
|
||||||
*
|
|
||||||
* // Enhanced display (new capabilities)
|
|
||||||
* result.getDisplay('title') // "John Doe" (AI-computed)
|
|
||||||
* result.getDisplay('description') // "CEO at Acme Corp" (enhanced)
|
|
||||||
* result.getDisplay('type') // "Person" (from AI detection)
|
|
||||||
* result.getDisplay() // All display fields
|
|
||||||
* ```
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { BaseAugmentation, AugmentationContext, MetadataAccess } from './brainyAugmentation.js'
|
|
||||||
import type { VectorDocument, GraphVerb } from '../coreTypes.js'
|
|
||||||
import type {
|
|
||||||
DisplayConfig,
|
|
||||||
ComputedDisplayFields,
|
|
||||||
EnhancedVectorDocument,
|
|
||||||
EnhancedGraphVerb,
|
|
||||||
DisplayAugmentationStats
|
|
||||||
} from './display/types.js'
|
|
||||||
import { IntelligentComputationEngine } from './display/intelligentComputation.js'
|
|
||||||
import { DisplayCache, RequestDeduplicator, getGlobalDisplayCache } from './display/cache.js'
|
|
||||||
import { getNounIcon, getVerbIcon, getIconCoverage } from './display/iconMappings.js'
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Universal Display Augmentation
|
|
||||||
*
|
|
||||||
* Self-contained augmentation that provides intelligent display fields
|
|
||||||
* for any data type using existing Brainy AI infrastructure
|
|
||||||
*/
|
|
||||||
export class UniversalDisplayAugmentation extends BaseAugmentation {
|
|
||||||
readonly name = 'display'
|
|
||||||
readonly version = '1.0.0'
|
|
||||||
readonly timing = 'after' as const // Enhance results after main operations
|
|
||||||
readonly priority = 50 // Medium priority - after core operations
|
|
||||||
readonly metadata: MetadataAccess = {
|
|
||||||
reads: '*', // Read all user data for intelligent analysis
|
|
||||||
writes: ['_display'] // Cache computed fields in isolated namespace
|
|
||||||
}
|
|
||||||
operations = ['get', 'search', 'find', 'similar', 'findSimilar', 'getVerb' as any, 'add', 'addNoun', 'addVerb', 'relate'] as any
|
|
||||||
|
|
||||||
// Augmentation metadata
|
|
||||||
readonly category = 'core' as const
|
|
||||||
readonly description = 'AI-powered intelligent display fields for enhanced data visualization'
|
|
||||||
|
|
||||||
// Computed fields declaration for TypeScript support and discovery
|
|
||||||
computedFields = {
|
|
||||||
display: {
|
|
||||||
title: { type: 'string' as const, description: 'Primary display name (AI-computed)' },
|
|
||||||
description: { type: 'string' as const, description: 'Enhanced description with context' },
|
|
||||||
type: { type: 'string' as const, description: 'Human-readable type (from AI detection)' },
|
|
||||||
tags: { type: 'array' as const, description: 'Generated display tags' },
|
|
||||||
relationship: { type: 'string' as const, description: 'Human-readable relationship (verbs only)' },
|
|
||||||
confidence: { type: 'number' as const, description: 'AI confidence score (0-1)' }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Core components (all self-contained)
|
|
||||||
private computationEngine: IntelligentComputationEngine
|
|
||||||
private displayCache: DisplayCache
|
|
||||||
private requestDeduplicator: RequestDeduplicator
|
|
||||||
protected config: DisplayConfig
|
|
||||||
protected context: AugmentationContext | undefined
|
|
||||||
|
|
||||||
constructor(config: Partial<DisplayConfig> = {}) {
|
|
||||||
super()
|
|
||||||
|
|
||||||
// Merge with defaults
|
|
||||||
this.config = {
|
|
||||||
enabled: true,
|
|
||||||
cacheSize: 1000,
|
|
||||||
lazyComputation: true,
|
|
||||||
batchSize: 50,
|
|
||||||
confidenceThreshold: 0.7,
|
|
||||||
customFieldMappings: {},
|
|
||||||
priorityFields: {},
|
|
||||||
debugMode: false,
|
|
||||||
...config
|
|
||||||
}
|
|
||||||
|
|
||||||
// Initialize components
|
|
||||||
this.computationEngine = new IntelligentComputationEngine(this.config)
|
|
||||||
this.displayCache = getGlobalDisplayCache(this.config.cacheSize)
|
|
||||||
this.requestDeduplicator = new RequestDeduplicator(this.config.batchSize)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Initialize the augmentation with AI components
|
|
||||||
* @param context Brainy context
|
|
||||||
*/
|
|
||||||
async initialize(context: AugmentationContext): Promise<void> {
|
|
||||||
if (!this.config.enabled) {
|
|
||||||
this.log('🎨 Universal Display augmentation disabled')
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
this.context = context
|
|
||||||
|
|
||||||
try {
|
|
||||||
// Initialize AI-powered computation engine
|
|
||||||
await this.computationEngine.initialize()
|
|
||||||
|
|
||||||
this.log('🎨 Universal Display augmentation initialized successfully')
|
|
||||||
this.log(` Cache size: ${this.config.cacheSize}`)
|
|
||||||
this.log(` Lazy computation: ${this.config.lazyComputation}`)
|
|
||||||
this.log(` Coverage: ${this.getCoverageInfo()}`)
|
|
||||||
|
|
||||||
} catch (error) {
|
|
||||||
this.log('⚠️ Display augmentation initialization warning:', 'warn')
|
|
||||||
this.log(` ${error}`, 'warn')
|
|
||||||
this.log(' Falling back to basic mode', 'warn')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Execute augmentation - attach display capabilities to results
|
|
||||||
* @param operation The operation being performed
|
|
||||||
* @param params Operation parameters
|
|
||||||
* @param next Function to execute main operation
|
|
||||||
* @returns Enhanced result with display capabilities
|
|
||||||
*/
|
|
||||||
async execute<T = any>(
|
|
||||||
operation: string,
|
|
||||||
params: any,
|
|
||||||
next: () => Promise<T>
|
|
||||||
): Promise<T> {
|
|
||||||
// Always execute main operation first
|
|
||||||
const result = await next()
|
|
||||||
|
|
||||||
// Only enhance if enabled and operation is relevant
|
|
||||||
if (!this.config.enabled || !this.shouldEnhanceOperation(operation)) {
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
// Enhance result with display capabilities
|
|
||||||
return this.enhanceWithDisplayCapabilities(result, operation) as T
|
|
||||||
} catch (error) {
|
|
||||||
this.log(`Display enhancement failed for ${operation}: ${error}`, 'warn')
|
|
||||||
return result // Return unenhanced result on error
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Check if operation should be enhanced
|
|
||||||
* @param operation Operation name
|
|
||||||
* @returns True if should enhance
|
|
||||||
*/
|
|
||||||
private shouldEnhanceOperation(operation: string): boolean {
|
|
||||||
const enhanceableOps = ['get', 'search', 'findSimilar', 'getVerb']
|
|
||||||
return enhanceableOps.includes(operation)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Enhance result with display capabilities
|
|
||||||
* @param result The operation result
|
|
||||||
* @param operation The operation type
|
|
||||||
* @returns Enhanced result
|
|
||||||
*/
|
|
||||||
private enhanceWithDisplayCapabilities(result: any, operation: string): any {
|
|
||||||
if (!result) return result
|
|
||||||
|
|
||||||
// Handle different result types
|
|
||||||
if (Array.isArray(result)) {
|
|
||||||
// Array of results (search, findSimilar)
|
|
||||||
return result.map(item => this.enhanceEntity(item))
|
|
||||||
} else if (result.id || result.metadata) {
|
|
||||||
// Single entity (get, getVerb)
|
|
||||||
return this.enhanceEntity(result)
|
|
||||||
}
|
|
||||||
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Enhance a single entity with display capabilities
|
|
||||||
* @param entity The entity to enhance
|
|
||||||
* @returns Enhanced entity
|
|
||||||
*/
|
|
||||||
private enhanceEntity(entity: any): EnhancedVectorDocument | EnhancedGraphVerb {
|
|
||||||
if (!entity) return entity
|
|
||||||
|
|
||||||
// Determine if it's a noun or verb
|
|
||||||
const isVerb = this.isVerbEntity(entity)
|
|
||||||
|
|
||||||
// Add display methods
|
|
||||||
const enhanced = {
|
|
||||||
...entity,
|
|
||||||
getDisplay: this.createGetDisplayMethod(entity, isVerb),
|
|
||||||
getAvailableFields: this.createGetAvailableFieldsMethod(),
|
|
||||||
getAvailableAugmentations: this.createGetAvailableAugmentationsMethod(),
|
|
||||||
explore: this.createExploreMethod(entity)
|
|
||||||
}
|
|
||||||
|
|
||||||
return enhanced
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create getDisplay method for an entity
|
|
||||||
* @param entity The entity
|
|
||||||
* @param isVerb Whether it's a verb entity
|
|
||||||
* @returns getDisplay function
|
|
||||||
*/
|
|
||||||
private createGetDisplayMethod(entity: any, isVerb: boolean) {
|
|
||||||
return async (field?: keyof ComputedDisplayFields): Promise<any> => {
|
|
||||||
// Generate cache key
|
|
||||||
const cacheKey = this.displayCache.generateKey(
|
|
||||||
entity.id,
|
|
||||||
entity.metadata || entity,
|
|
||||||
isVerb ? 'verb' : 'noun'
|
|
||||||
)
|
|
||||||
|
|
||||||
// Use request deduplicator to prevent duplicate computations
|
|
||||||
const computedFields = await this.requestDeduplicator.deduplicate(
|
|
||||||
cacheKey,
|
|
||||||
async () => {
|
|
||||||
// Check cache first
|
|
||||||
let cached = this.displayCache.get(cacheKey)
|
|
||||||
if (cached) return cached
|
|
||||||
|
|
||||||
// Compute display fields
|
|
||||||
const startTime = Date.now()
|
|
||||||
let computed: ComputedDisplayFields
|
|
||||||
|
|
||||||
if (isVerb) {
|
|
||||||
computed = await this.computationEngine.computeVerbDisplay(entity as GraphVerb)
|
|
||||||
} else {
|
|
||||||
computed = await this.computationEngine.computeNounDisplay(
|
|
||||||
entity.metadata || entity,
|
|
||||||
entity.id
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Cache the result
|
|
||||||
const computationTime = Date.now() - startTime
|
|
||||||
this.displayCache.set(cacheKey, computed, computationTime)
|
|
||||||
|
|
||||||
return computed
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
// Return specific field or all fields
|
|
||||||
return field ? computedFields[field] : computedFields
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create getAvailableFields method
|
|
||||||
* @returns getAvailableFields function
|
|
||||||
*/
|
|
||||||
private createGetAvailableFieldsMethod() {
|
|
||||||
return (namespace: string): string[] => {
|
|
||||||
if (namespace === 'display') {
|
|
||||||
return ['title', 'description', 'type', 'tags', 'relationship', 'confidence']
|
|
||||||
}
|
|
||||||
return []
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create getAvailableAugmentations method
|
|
||||||
* @returns getAvailableAugmentations function
|
|
||||||
*/
|
|
||||||
private createGetAvailableAugmentationsMethod() {
|
|
||||||
return (): string[] => {
|
|
||||||
return ['display'] // This augmentation provides 'display' namespace
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create explore method for debugging
|
|
||||||
* @param entity The entity
|
|
||||||
* @returns explore function
|
|
||||||
*/
|
|
||||||
private createExploreMethod(entity: any) {
|
|
||||||
return async (): Promise<void> => {
|
|
||||||
// Respect silent mode
|
|
||||||
if (this.config.silent) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log(`\n📋 Entity Exploration: ${entity.id || 'unknown'}`)
|
|
||||||
console.log('━'.repeat(50))
|
|
||||||
|
|
||||||
// Show user data
|
|
||||||
console.log('\n👤 User Data:')
|
|
||||||
const userData = entity.metadata || entity
|
|
||||||
for (const [key, value] of Object.entries(userData)) {
|
|
||||||
if (!key.startsWith('_')) {
|
|
||||||
console.log(` • ${key}: ${JSON.stringify(value)}`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Show computed display fields
|
|
||||||
try {
|
|
||||||
console.log('\n🎨 Display Fields:')
|
|
||||||
const displayMethod = this.createGetDisplayMethod(entity, this.isVerbEntity(entity))
|
|
||||||
const displayFields = await displayMethod()
|
|
||||||
for (const [key, value] of Object.entries(displayFields)) {
|
|
||||||
console.log(` • ${key}: ${JSON.stringify(value)}`)
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
console.log(` Error computing display fields: ${error}`)
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log('')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Check if an entity is a verb
|
|
||||||
* @param entity The entity to check
|
|
||||||
* @returns True if it's a verb
|
|
||||||
*/
|
|
||||||
private isVerbEntity(entity: any): boolean {
|
|
||||||
return !!(entity.sourceId && entity.targetId) ||
|
|
||||||
!!(entity.source && entity.target) ||
|
|
||||||
!!entity.verb
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get coverage information
|
|
||||||
* @returns Coverage info string
|
|
||||||
*/
|
|
||||||
private getCoverageInfo(): string {
|
|
||||||
return 'Clean display - focuses on AI-powered content'
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get augmentation statistics
|
|
||||||
* @returns Performance and usage statistics
|
|
||||||
*/
|
|
||||||
getStats(): DisplayAugmentationStats {
|
|
||||||
return this.displayCache.getStats()
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Configure the augmentation at runtime
|
|
||||||
* @param newConfig Partial configuration to merge
|
|
||||||
*/
|
|
||||||
configure(newConfig: Partial<DisplayConfig>): void {
|
|
||||||
this.config = { ...this.config, ...newConfig }
|
|
||||||
|
|
||||||
if (!this.config.enabled) {
|
|
||||||
this.displayCache.clear()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Clear all cached display data
|
|
||||||
*/
|
|
||||||
clearCache(): void {
|
|
||||||
this.displayCache.clear()
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Precompute display fields for a batch of entities
|
|
||||||
* @param entities Array of entities to precompute
|
|
||||||
*/
|
|
||||||
async precomputeBatch(entities: Array<{ id: string; data: any }>): Promise<void> {
|
|
||||||
const computeRequests = entities.map(({ id, data }) => ({
|
|
||||||
key: this.displayCache.generateKey(id, data, 'noun'),
|
|
||||||
computeFn: () => this.computationEngine.computeNounDisplay(data, id)
|
|
||||||
}))
|
|
||||||
|
|
||||||
await this.displayCache.batchPrecompute(computeRequests)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Optional check if this augmentation should run
|
|
||||||
* @param operation Operation name
|
|
||||||
* @param params Operation parameters
|
|
||||||
* @returns True if should execute
|
|
||||||
*/
|
|
||||||
shouldExecute(operation: string, params: any): boolean {
|
|
||||||
return this.config.enabled && this.shouldEnhanceOperation(operation)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Cleanup when augmentation is shut down
|
|
||||||
*/
|
|
||||||
async shutdown(): Promise<void> {
|
|
||||||
try {
|
|
||||||
// Cleanup computation engine
|
|
||||||
await this.computationEngine.shutdown()
|
|
||||||
|
|
||||||
// Cleanup request deduplicator
|
|
||||||
this.requestDeduplicator.shutdown()
|
|
||||||
|
|
||||||
// Clear cache if configured to do so
|
|
||||||
if (this.config.debugMode) {
|
|
||||||
const stats = this.getStats()
|
|
||||||
this.log(`🎨 Display augmentation shutdown statistics:`)
|
|
||||||
this.log(` Total computations: ${stats.totalComputations}`)
|
|
||||||
this.log(` Cache hit ratio: ${(stats.cacheHitRatio * 100).toFixed(1)}%`)
|
|
||||||
this.log(` Average computation time: ${stats.averageComputationTime.toFixed(1)}ms`)
|
|
||||||
}
|
|
||||||
|
|
||||||
this.log('🎨 Universal Display augmentation shut down')
|
|
||||||
|
|
||||||
} catch (error) {
|
|
||||||
this.log(`Display augmentation shutdown error: ${error}`, 'error')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Factory function to create display augmentation with default config
|
|
||||||
* @param config Optional configuration overrides
|
|
||||||
* @returns Configured display augmentation instance
|
|
||||||
*/
|
|
||||||
export function createDisplayAugmentation(config: Partial<DisplayConfig> = {}): UniversalDisplayAugmentation {
|
|
||||||
return new UniversalDisplayAugmentation(config)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Default configuration for the display augmentation
|
|
||||||
*/
|
|
||||||
export const DEFAULT_DISPLAY_CONFIG: DisplayConfig = {
|
|
||||||
enabled: true,
|
|
||||||
cacheSize: 1000,
|
|
||||||
lazyComputation: true,
|
|
||||||
batchSize: 50,
|
|
||||||
confidenceThreshold: 0.7,
|
|
||||||
customFieldMappings: {},
|
|
||||||
priorityFields: {},
|
|
||||||
debugMode: false
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Export for easy import and registration
|
|
||||||
*/
|
|
||||||
export default UniversalDisplayAugmentation
|
|
||||||
125
src/brainy.ts
125
src/brainy.ts
|
|
@ -18,8 +18,6 @@ import {
|
||||||
} from './utils/index.js'
|
} from './utils/index.js'
|
||||||
import { embeddingManager } from './embeddings/EmbeddingManager.js'
|
import { embeddingManager } from './embeddings/EmbeddingManager.js'
|
||||||
import { matchesMetadataFilter } from './utils/metadataFilter.js'
|
import { matchesMetadataFilter } from './utils/metadataFilter.js'
|
||||||
import { AugmentationRegistry, AugmentationContext } from './augmentations/brainyAugmentation.js'
|
|
||||||
import { createDefaultAugmentations } from './augmentations/defaultAugmentations.js'
|
|
||||||
import { ImprovedNeuralAPI } from './neural/improvedNeuralAPI.js'
|
import { ImprovedNeuralAPI } from './neural/improvedNeuralAPI.js'
|
||||||
import { NaturalLanguageProcessor } from './neural/naturalLanguageProcessor.js'
|
import { NaturalLanguageProcessor } from './neural/naturalLanguageProcessor.js'
|
||||||
import { NeuralEntityExtractor, ExtractedEntity } from './neural/entityExtractor.js'
|
import { NeuralEntityExtractor, ExtractedEntity } from './neural/entityExtractor.js'
|
||||||
|
|
@ -130,7 +128,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
private transactionManager: TransactionManager
|
private transactionManager: TransactionManager
|
||||||
private embedder: EmbeddingFunction
|
private embedder: EmbeddingFunction
|
||||||
private distance: DistanceFunction
|
private distance: DistanceFunction
|
||||||
private augmentationRegistry: AugmentationRegistry
|
|
||||||
private config: Required<BrainyConfig>
|
private config: Required<BrainyConfig>
|
||||||
|
|
||||||
// Distributed components (optional)
|
// Distributed components (optional)
|
||||||
|
|
@ -192,7 +189,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
// Setup core components
|
// Setup core components
|
||||||
this.distance = cosineDistance
|
this.distance = cosineDistance
|
||||||
this.embedder = this.setupEmbedder()
|
this.embedder = this.setupEmbedder()
|
||||||
this.augmentationRegistry = this.setupAugmentations()
|
|
||||||
this.transactionManager = new TransactionManager()
|
this.transactionManager = new TransactionManager()
|
||||||
|
|
||||||
// Setup distributed components if enabled
|
// Setup distributed components if enabled
|
||||||
|
|
@ -230,7 +226,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
...configOverrides,
|
...configOverrides,
|
||||||
storage: { ...this.config.storage, ...configOverrides.storage },
|
storage: { ...this.config.storage, ...configOverrides.storage },
|
||||||
index: { ...this.config.index, ...configOverrides.index },
|
index: { ...this.config.index, ...configOverrides.index },
|
||||||
augmentations: { ...this.config.augmentations, ...configOverrides.augmentations },
|
|
||||||
verbose: configOverrides.verbose ?? this.config.verbose,
|
verbose: configOverrides.verbose ?? this.config.verbose,
|
||||||
silent: configOverrides.silent ?? this.config.silent
|
silent: configOverrides.silent ?? this.config.silent
|
||||||
}
|
}
|
||||||
|
|
@ -332,23 +327,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
// Rebuild indexes if needed for existing data
|
// Rebuild indexes if needed for existing data
|
||||||
await this.rebuildIndexesIfNeeded()
|
await this.rebuildIndexesIfNeeded()
|
||||||
|
|
||||||
// Initialize augmentations
|
|
||||||
await this.augmentationRegistry.initializeAll({
|
|
||||||
brain: this,
|
|
||||||
storage: this.storage,
|
|
||||||
config: this.config,
|
|
||||||
log: (message: string, level = 'info') => {
|
|
||||||
// Simple logging for now
|
|
||||||
if (level === 'error') {
|
|
||||||
console.error(message)
|
|
||||||
} else if (level === 'warn') {
|
|
||||||
console.warn(message)
|
|
||||||
} else {
|
|
||||||
console.log(message)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// Connect distributed components to storage
|
// Connect distributed components to storage
|
||||||
await this.connectDistributedStorage()
|
await this.connectDistributedStorage()
|
||||||
|
|
||||||
|
|
@ -698,8 +676,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Execute through augmentation pipeline
|
|
||||||
return this.augmentationRegistry.execute('add', params, async () => {
|
|
||||||
// Prepare metadata for storage (backward compat format - unchanged)
|
// Prepare metadata for storage (backward compat format - unchanged)
|
||||||
const storageMetadata = {
|
const storageMetadata = {
|
||||||
...(typeof params.data === 'object' && params.data !== null && !Array.isArray(params.data) ? params.data : {}),
|
...(typeof params.data === 'object' && params.data !== null && !Array.isArray(params.data) ? params.data : {}),
|
||||||
|
|
@ -776,7 +752,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
})
|
})
|
||||||
|
|
||||||
return id
|
return id
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -914,7 +889,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
async get(id: string, options?: GetOptions): Promise<Entity<T> | null> {
|
async get(id: string, options?: GetOptions): Promise<Entity<T> | null> {
|
||||||
await this.ensureInitialized()
|
await this.ensureInitialized()
|
||||||
|
|
||||||
return this.augmentationRegistry.execute('get', { id, options }, async () => {
|
|
||||||
// Route to metadata-only or full entity based on options
|
// Route to metadata-only or full entity based on options
|
||||||
const includeVectors = options?.includeVectors ?? false // Default: metadata-only (fast)
|
const includeVectors = options?.includeVectors ?? false // Default: metadata-only (fast)
|
||||||
|
|
||||||
|
|
@ -935,7 +909,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
}
|
}
|
||||||
return this.convertMetadataToEntity(id, metadata)
|
return this.convertMetadataToEntity(id, metadata)
|
||||||
}
|
}
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -1104,7 +1077,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
// Zero-config validation (static import for performance)
|
// Zero-config validation (static import for performance)
|
||||||
validateUpdateParams(params)
|
validateUpdateParams(params)
|
||||||
|
|
||||||
return this.augmentationRegistry.execute('update', params, async () => {
|
|
||||||
// Get existing entity with vectors (fix for regression)
|
// Get existing entity with vectors (fix for regression)
|
||||||
// We need includeVectors: true because:
|
// We need includeVectors: true because:
|
||||||
// 1. SaveNounOperation requires the vector
|
// 1. SaveNounOperation requires the vector
|
||||||
|
|
@ -1244,7 +1216,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
new AddToMetadataIndexOperation(this.metadataIndex, params.id, entityForIndexing)
|
new AddToMetadataIndexOperation(this.metadataIndex, params.id, entityForIndexing)
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -1258,7 +1229,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
|
|
||||||
await this.ensureInitialized()
|
await this.ensureInitialized()
|
||||||
|
|
||||||
return this.augmentationRegistry.execute('delete', { id }, async () => {
|
|
||||||
// Get entity metadata and related verbs before deletion
|
// Get entity metadata and related verbs before deletion
|
||||||
const metadata = await this.storage.getNounMetadata(id)
|
const metadata = await this.storage.getNounMetadata(id)
|
||||||
const noun = await this.storage.getNoun(id)
|
const noun = await this.storage.getNoun(id)
|
||||||
|
|
@ -1310,7 +1280,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ============= RELATIONSHIP OPERATIONS =============
|
// ============= RELATIONSHIP OPERATIONS =============
|
||||||
|
|
@ -1465,7 +1434,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
(v, i) => (v + toEntity.vector[i]) / 2
|
(v, i) => (v + toEntity.vector[i]) / 2
|
||||||
)
|
)
|
||||||
|
|
||||||
return this.augmentationRegistry.execute('relate', params, async () => {
|
|
||||||
// Prepare verb metadata
|
// Prepare verb metadata
|
||||||
// CRITICAL: Include verb type in metadata for count tracking
|
// CRITICAL: Include verb type in metadata for count tracking
|
||||||
const verbMetadata = {
|
const verbMetadata = {
|
||||||
|
|
@ -1551,7 +1519,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
})
|
})
|
||||||
|
|
||||||
return id
|
return id
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -1560,7 +1527,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
async unrelate(id: string): Promise<void> {
|
async unrelate(id: string): Promise<void> {
|
||||||
await this.ensureInitialized()
|
await this.ensureInitialized()
|
||||||
|
|
||||||
return this.augmentationRegistry.execute('unrelate', { id }, async () => {
|
|
||||||
// Get verb data before deletion for rollback
|
// Get verb data before deletion for rollback
|
||||||
const verb = await this.storage.getVerb(id)
|
const verb = await this.storage.getVerb(id)
|
||||||
|
|
||||||
|
|
@ -1578,7 +1544,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
new DeleteVerbMetadataOperation(this.storage, id)
|
new DeleteVerbMetadataOperation(this.storage, id)
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -1865,7 +1830,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
validateFindParams(params)
|
validateFindParams(params)
|
||||||
|
|
||||||
const startTime = Date.now()
|
const startTime = Date.now()
|
||||||
const result = await this.augmentationRegistry.execute('find', params, async () => {
|
const result = await (async () => {
|
||||||
let results: Result<T>[] = []
|
let results: Result<T>[] = []
|
||||||
|
|
||||||
// Distinguish between search criteria (need vector search) and filter criteria (metadata only)
|
// Distinguish between search criteria (need vector search) and filter criteria (metadata only)
|
||||||
|
|
@ -2201,7 +2166,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
|
|
||||||
// Efficient pagination - only slice what we need (limit already defined above)
|
// Efficient pagination - only slice what we need (limit already defined above)
|
||||||
return results.slice(finalOffset, finalOffset + limit)
|
return results.slice(finalOffset, finalOffset + limit)
|
||||||
})
|
})()
|
||||||
|
|
||||||
// Record performance for auto-tuning
|
// Record performance for auto-tuning
|
||||||
const duration = Date.now() - startTime
|
const duration = Date.now() - startTime
|
||||||
|
|
@ -2771,7 +2736,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
async clear(): Promise<void> {
|
async clear(): Promise<void> {
|
||||||
await this.ensureInitialized()
|
await this.ensureInitialized()
|
||||||
|
|
||||||
return this.augmentationRegistry.execute('clear', {}, async () => {
|
|
||||||
// Clear storage
|
// Clear storage
|
||||||
await this.storage.clear()
|
await this.storage.clear()
|
||||||
|
|
||||||
|
|
@ -2832,7 +2796,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
// VFS was never used, reset flag for clean state
|
// VFS was never used, reset flag for clean state
|
||||||
this._vfsInitialized = false
|
this._vfsInitialized = false
|
||||||
}
|
}
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ============= COW (COPY-ON-WRITE) API =============
|
// ============= COW (COPY-ON-WRITE) API =============
|
||||||
|
|
@ -2887,7 +2850,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
}): Promise<Brainy<T>> {
|
}): Promise<Brainy<T>> {
|
||||||
await this.ensureInitialized()
|
await this.ensureInitialized()
|
||||||
|
|
||||||
return this.augmentationRegistry.execute('fork', { branch, options }, async () => {
|
|
||||||
const branchName = branch || `fork-${Date.now()}`
|
const branchName = branch || `fork-${Date.now()}`
|
||||||
|
|
||||||
// Lazy COW initialization - enable automatically on first fork()
|
// Lazy COW initialization - enable automatically on first fork()
|
||||||
|
|
@ -2978,25 +2940,11 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
clone.graphIndex = await (clone.storage as any).getGraphIndex()
|
clone.graphIndex = await (clone.storage as any).getGraphIndex()
|
||||||
// getGraphIndex() will rebuild automatically if data exists (via _initializeGraphIndex)
|
// getGraphIndex() will rebuild automatically if data exists (via _initializeGraphIndex)
|
||||||
|
|
||||||
// Setup augmentations
|
|
||||||
clone.augmentationRegistry = this.setupAugmentations()
|
|
||||||
await clone.augmentationRegistry.initializeAll({
|
|
||||||
brain: clone,
|
|
||||||
storage: clone.storage,
|
|
||||||
config: clone.config,
|
|
||||||
log: (message: string, level?: 'info' | 'warn' | 'error') => {
|
|
||||||
if (!clone.config.silent) {
|
|
||||||
console[level || 'info'](message)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// Mark as initialized
|
// Mark as initialized
|
||||||
clone.initialized = true
|
clone.initialized = true
|
||||||
clone.dimensions = this.dimensions
|
clone.dimensions = this.dimensions
|
||||||
|
|
||||||
return clone
|
return clone
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -3012,7 +2960,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
async listBranches(): Promise<string[]> {
|
async listBranches(): Promise<string[]> {
|
||||||
await this.ensureInitialized()
|
await this.ensureInitialized()
|
||||||
|
|
||||||
return this.augmentationRegistry.execute('listBranches', {}, async () => {
|
|
||||||
if (!('refManager' in this.storage)) {
|
if (!('refManager' in this.storage)) {
|
||||||
throw new Error('Branch management requires COW-enabled storage')
|
throw new Error('Branch management requires COW-enabled storage')
|
||||||
}
|
}
|
||||||
|
|
@ -3024,7 +2971,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
return refs
|
return refs
|
||||||
.filter((ref: any) => ref.name.startsWith('refs/heads/'))
|
.filter((ref: any) => ref.name.startsWith('refs/heads/'))
|
||||||
.map((ref: any) => ref.name.replace('refs/heads/', ''))
|
.map((ref: any) => ref.name.replace('refs/heads/', ''))
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -3040,13 +2986,11 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
async getCurrentBranch(): Promise<string> {
|
async getCurrentBranch(): Promise<string> {
|
||||||
await this.ensureInitialized()
|
await this.ensureInitialized()
|
||||||
|
|
||||||
return this.augmentationRegistry.execute('getCurrentBranch', {}, async () => {
|
|
||||||
if (!('currentBranch' in this.storage)) {
|
if (!('currentBranch' in this.storage)) {
|
||||||
return 'main' // Default branch
|
return 'main' // Default branch
|
||||||
}
|
}
|
||||||
|
|
||||||
return (this.storage as any).currentBranch || 'main'
|
return (this.storage as any).currentBranch || 'main'
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -3062,7 +3006,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
async checkout(branch: string): Promise<void> {
|
async checkout(branch: string): Promise<void> {
|
||||||
await this.ensureInitialized()
|
await this.ensureInitialized()
|
||||||
|
|
||||||
return this.augmentationRegistry.execute('checkout', { branch }, async () => {
|
|
||||||
if (!('refManager' in this.storage)) {
|
if (!('refManager' in this.storage)) {
|
||||||
throw new Error('Branch management requires COW-enabled storage')
|
throw new Error('Branch management requires COW-enabled storage')
|
||||||
}
|
}
|
||||||
|
|
@ -3106,7 +3049,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
this._vfs = new VirtualFileSystem(this)
|
this._vfs = new VirtualFileSystem(this)
|
||||||
await this._vfs.init()
|
await this._vfs.init()
|
||||||
}
|
}
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -3131,7 +3073,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
}): Promise<string> {
|
}): Promise<string> {
|
||||||
await this.ensureInitialized()
|
await this.ensureInitialized()
|
||||||
|
|
||||||
return this.augmentationRegistry.execute('commit', { options }, async () => {
|
|
||||||
if (!('refManager' in this.storage) || !('commitLog' in this.storage) || !('blobStorage' in this.storage)) {
|
if (!('refManager' in this.storage) || !('commitLog' in this.storage) || !('blobStorage' in this.storage)) {
|
||||||
throw new Error('Commit requires COW-enabled storage')
|
throw new Error('Commit requires COW-enabled storage')
|
||||||
}
|
}
|
||||||
|
|
@ -3187,7 +3128,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
})
|
})
|
||||||
|
|
||||||
return commitHash
|
return commitHash
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -3380,7 +3320,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
async deleteBranch(branch: string): Promise<void> {
|
async deleteBranch(branch: string): Promise<void> {
|
||||||
await this.ensureInitialized()
|
await this.ensureInitialized()
|
||||||
|
|
||||||
return this.augmentationRegistry.execute('deleteBranch', { branch }, async () => {
|
|
||||||
if (!('refManager' in this.storage)) {
|
if (!('refManager' in this.storage)) {
|
||||||
throw new Error('Branch management requires COW-enabled storage')
|
throw new Error('Branch management requires COW-enabled storage')
|
||||||
}
|
}
|
||||||
|
|
@ -3392,7 +3331,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
|
|
||||||
const refManager = (this.storage as any).refManager
|
const refManager = (this.storage as any).refManager
|
||||||
await refManager.deleteRef(branch)
|
await refManager.deleteRef(branch)
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -3421,7 +3359,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
}>> {
|
}>> {
|
||||||
await this.ensureInitialized()
|
await this.ensureInitialized()
|
||||||
|
|
||||||
return this.augmentationRegistry.execute('getHistory', { options }, async () => {
|
|
||||||
if (!('commitLog' in this.storage) || !('refManager' in this.storage)) {
|
if (!('commitLog' in this.storage) || !('refManager' in this.storage)) {
|
||||||
throw new Error('History requires COW-enabled storage')
|
throw new Error('History requires COW-enabled storage')
|
||||||
}
|
}
|
||||||
|
|
@ -3444,7 +3381,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
timestamp: commit.timestamp,
|
timestamp: commit.timestamp,
|
||||||
metadata: commit.metadata
|
metadata: commit.metadata
|
||||||
}))
|
}))
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -3950,21 +3886,12 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
}) => void
|
}) => void
|
||||||
}
|
}
|
||||||
) {
|
) {
|
||||||
// Execute through augmentation pipeline (Enables IntelligentImportAugmentation)
|
|
||||||
// If source is an ImportSource object (not a Buffer), spread it so augmentations can access properties
|
|
||||||
const params = typeof source === 'object' && !Buffer.isBuffer(source)
|
|
||||||
? { ...source as object, ...options } // Spread ImportSource: { type, data, filename, ...options }
|
|
||||||
: { source, ...options } // Wrap Buffer/string: { source, ...options }
|
|
||||||
|
|
||||||
return this.augmentationRegistry.execute('import', params, async () => {
|
|
||||||
// Lazy load ImportCoordinator
|
// Lazy load ImportCoordinator
|
||||||
const { ImportCoordinator } = await import('./import/ImportCoordinator.js')
|
const { ImportCoordinator } = await import('./import/ImportCoordinator.js')
|
||||||
const coordinator = new ImportCoordinator(this)
|
const coordinator = new ImportCoordinator(this)
|
||||||
await coordinator.init()
|
await coordinator.init()
|
||||||
|
|
||||||
// Pass augmentation-modified params (contains _intelligentImport, _extractedData, etc)
|
return await coordinator.import(source, options)
|
||||||
return await coordinator.import(source, { ...options, ...params })
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -4690,17 +4617,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Augmentations API - Clean and simple
|
|
||||||
*/
|
|
||||||
get augmentations() {
|
|
||||||
return {
|
|
||||||
list: () => this.augmentationRegistry.getAll().map(a => a.name),
|
|
||||||
get: (name: string) => this.augmentationRegistry.getAll().find(a => a.name === name),
|
|
||||||
has: (name: string) => this.augmentationRegistry.getAll().some(a => a.name === name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get complete statistics - convenience method
|
* Get complete statistics - convenience method
|
||||||
* For more granular counting, use brain.counts API
|
* For more granular counting, use brain.counts API
|
||||||
|
|
@ -6282,32 +6198,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
return persistMode
|
return persistMode
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Setup augmentations
|
|
||||||
*/
|
|
||||||
private setupAugmentations(): AugmentationRegistry {
|
|
||||||
const registry = new AugmentationRegistry()
|
|
||||||
|
|
||||||
// Register default augmentations with silent mode support
|
|
||||||
const augmentationConfig = {
|
|
||||||
...this.config.augmentations,
|
|
||||||
// Pass silent mode to all augmentations
|
|
||||||
...(this.config.silent && {
|
|
||||||
cache: this.config.augmentations?.cache !== false ? { ...this.config.augmentations?.cache, silent: true } : false,
|
|
||||||
metrics: this.config.augmentations?.metrics !== false ? { ...this.config.augmentations?.metrics, silent: true } : false,
|
|
||||||
display: this.config.augmentations?.display !== false ? { ...this.config.augmentations?.display, silent: true } : false,
|
|
||||||
monitoring: this.config.augmentations?.monitoring !== false ? { ...this.config.augmentations?.monitoring, silent: true } : false
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
const defaults = createDefaultAugmentations(augmentationConfig)
|
|
||||||
for (const aug of defaults) {
|
|
||||||
registry.register(aug)
|
|
||||||
}
|
|
||||||
|
|
||||||
return registry
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Normalize and validate configuration
|
* Normalize and validate configuration
|
||||||
*/
|
*/
|
||||||
|
|
@ -6356,7 +6246,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
storage: config?.storage || { type: 'auto' },
|
storage: config?.storage || { type: 'auto' },
|
||||||
index: config?.index || {},
|
index: config?.index || {},
|
||||||
cache: config?.cache ?? true,
|
cache: config?.cache ?? true,
|
||||||
augmentations: config?.augmentations || {},
|
|
||||||
distributed: distributedConfig as any, // Type will be fixed when used
|
distributed: distributedConfig as any, // Type will be fixed when used
|
||||||
warmup: config?.warmup ?? false,
|
warmup: config?.warmup ?? false,
|
||||||
realtime: config?.realtime ?? false,
|
realtime: config?.realtime ?? false,
|
||||||
|
|
@ -6670,14 +6559,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
// Deactivate plugins
|
// Deactivate plugins
|
||||||
await this.pluginRegistry.deactivateAll()
|
await this.pluginRegistry.deactivateAll()
|
||||||
|
|
||||||
// Shutdown augmentations
|
|
||||||
const augs = this.augmentationRegistry.getAll()
|
|
||||||
for (const aug of augs) {
|
|
||||||
if ('shutdown' in aug && typeof aug.shutdown === 'function') {
|
|
||||||
await aug.shutdown()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Restore console methods if silent mode was enabled
|
// Restore console methods if silent mode was enabled
|
||||||
if (this.config.silent && this.originalConsole) {
|
if (this.config.silent && this.originalConsole) {
|
||||||
console.log = this.originalConsole.log as typeof console.log
|
console.log = this.originalConsole.log as typeof console.log
|
||||||
|
|
|
||||||
|
|
@ -148,21 +148,9 @@ export const coreCommands = {
|
||||||
}
|
}
|
||||||
nounType = options.type as NounType
|
nounType = options.type as NounType
|
||||||
} else {
|
} else {
|
||||||
// Use AI to suggest type
|
// Default to Thing when no type specified
|
||||||
spinner.text = 'Detecting type with AI...'
|
nounType = NounType.Thing
|
||||||
const suggestion = await BrainyTypes.suggestNoun(
|
spinner.text = `No type specified, using default: ${nounType}`
|
||||||
typeof text === 'string' ? { content: text, ...metadata } : text
|
|
||||||
)
|
|
||||||
|
|
||||||
if (suggestion.confidence < 0.6) {
|
|
||||||
spinner.fail('Could not determine type with confidence')
|
|
||||||
console.log(chalk.yellow(`Suggestion: ${suggestion.type} (${(suggestion.confidence * 100).toFixed(1)}%)`))
|
|
||||||
console.log(chalk.dim('Use --type flag to specify explicitly'))
|
|
||||||
process.exit(1)
|
|
||||||
}
|
|
||||||
|
|
||||||
nounType = suggestion.type as NounType
|
|
||||||
spinner.text = `Using detected type: ${nounType}`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Add with explicit type
|
// Add with explicit type
|
||||||
|
|
|
||||||
|
|
@ -1,12 +1,9 @@
|
||||||
/**
|
/**
|
||||||
* CLI Commands for Type Management
|
* CLI Commands for Type Management
|
||||||
* Consistent with BrainyTypes public API
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import chalk from 'chalk'
|
import chalk from 'chalk'
|
||||||
import ora from 'ora'
|
|
||||||
import inquirer from 'inquirer'
|
import inquirer from 'inquirer'
|
||||||
import Table from 'cli-table3'
|
|
||||||
import { BrainyTypes, NounType, VerbType } from '../../index.js'
|
import { BrainyTypes, NounType, VerbType } from '../../index.js'
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -30,7 +27,7 @@ export async function types(options: { json?: boolean, noun?: boolean, verb?: bo
|
||||||
|
|
||||||
// Display nouns
|
// Display nouns
|
||||||
if (showNouns) {
|
if (showNouns) {
|
||||||
console.log(chalk.bold.cyan('\n📚 Noun Types (42):\n'))
|
console.log(chalk.bold.cyan(`\nNoun Types (${BrainyTypes.nouns.length}):\n`))
|
||||||
const nounChunks = []
|
const nounChunks = []
|
||||||
for (let i = 0; i < BrainyTypes.nouns.length; i += 3) {
|
for (let i = 0; i < BrainyTypes.nouns.length; i += 3) {
|
||||||
nounChunks.push(BrainyTypes.nouns.slice(i, i + 3))
|
nounChunks.push(BrainyTypes.nouns.slice(i, i + 3))
|
||||||
|
|
@ -43,7 +40,7 @@ export async function types(options: { json?: boolean, noun?: boolean, verb?: bo
|
||||||
|
|
||||||
// Display verbs
|
// Display verbs
|
||||||
if (showVerbs) {
|
if (showVerbs) {
|
||||||
console.log(chalk.bold.cyan('\n🔗 Verb Types (127):\n'))
|
console.log(chalk.bold.cyan(`\nVerb Types (${BrainyTypes.verbs.length}):\n`))
|
||||||
const verbChunks = []
|
const verbChunks = []
|
||||||
for (let i = 0; i < BrainyTypes.verbs.length; i += 3) {
|
for (let i = 0; i < BrainyTypes.verbs.length; i += 3) {
|
||||||
verbChunks.push(BrainyTypes.verbs.slice(i, i + 3))
|
verbChunks.push(BrainyTypes.verbs.slice(i, i + 3))
|
||||||
|
|
@ -54,99 +51,6 @@ export async function types(options: { json?: boolean, noun?: boolean, verb?: bo
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
console.log(chalk.dim('\n💡 Use "brainy suggest <data>" to get AI-powered type suggestions'))
|
|
||||||
|
|
||||||
} catch (error: any) {
|
|
||||||
console.error(chalk.red('Error:', error.message))
|
|
||||||
process.exit(1)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Suggest type - matches BrainyTypes.suggestNoun() and suggestVerb()
|
|
||||||
* Usage: brainy suggest <data>
|
|
||||||
* Interactive if data not provided
|
|
||||||
*/
|
|
||||||
export async function suggest(
|
|
||||||
data?: string,
|
|
||||||
options: {
|
|
||||||
verb?: boolean,
|
|
||||||
json?: boolean
|
|
||||||
} = {}
|
|
||||||
) {
|
|
||||||
try {
|
|
||||||
// Interactive mode if no data provided
|
|
||||||
if (!data) {
|
|
||||||
const answers = await inquirer.prompt([
|
|
||||||
{
|
|
||||||
type: 'list',
|
|
||||||
name: 'kind',
|
|
||||||
message: 'What type do you want to suggest?',
|
|
||||||
choices: ['Noun', 'Verb'],
|
|
||||||
default: 'Noun'
|
|
||||||
},
|
|
||||||
{
|
|
||||||
type: 'input',
|
|
||||||
name: 'data',
|
|
||||||
message: 'Enter data (JSON or text):',
|
|
||||||
validate: (input) => input.length > 0 || 'Data is required'
|
|
||||||
},
|
|
||||||
{
|
|
||||||
type: 'input',
|
|
||||||
name: 'hint',
|
|
||||||
message: 'Relationship hint (optional):',
|
|
||||||
when: (answers) => answers.kind === 'Verb'
|
|
||||||
}
|
|
||||||
])
|
|
||||||
|
|
||||||
data = answers.data
|
|
||||||
options.verb = answers.kind === 'Verb'
|
|
||||||
|
|
||||||
// For verbs, parse source/target if provided as JSON
|
|
||||||
if (options.verb && answers.hint) {
|
|
||||||
data = JSON.stringify({ hint: answers.hint })
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const spinner = ora('Analyzing with AI...').start()
|
|
||||||
|
|
||||||
let parsedData: any
|
|
||||||
try {
|
|
||||||
parsedData = JSON.parse(data)
|
|
||||||
} catch {
|
|
||||||
parsedData = { content: data }
|
|
||||||
}
|
|
||||||
|
|
||||||
let suggestion
|
|
||||||
if (options.verb) {
|
|
||||||
// For verb suggestions, need source and target
|
|
||||||
const source = parsedData.source || { type: 'unknown' }
|
|
||||||
const target = parsedData.target || { type: 'unknown' }
|
|
||||||
const hint = parsedData.hint || parsedData.relationship || parsedData.verb
|
|
||||||
|
|
||||||
suggestion = await BrainyTypes.suggestVerb(source, target, hint)
|
|
||||||
spinner.succeed('Verb type analyzed')
|
|
||||||
} else {
|
|
||||||
suggestion = await BrainyTypes.suggestNoun(parsedData)
|
|
||||||
spinner.succeed('Noun type analyzed')
|
|
||||||
}
|
|
||||||
|
|
||||||
if (options.json) {
|
|
||||||
console.log(JSON.stringify(suggestion, null, 2))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Display results
|
|
||||||
console.log(chalk.bold.green(`\n✨ Suggested: ${suggestion.type}`))
|
|
||||||
console.log(chalk.cyan(`Confidence: ${(suggestion.confidence * 100).toFixed(1)}%`))
|
|
||||||
|
|
||||||
if (suggestion.alternatives && suggestion.alternatives.length > 0) {
|
|
||||||
console.log(chalk.yellow('\nAlternatives:'))
|
|
||||||
for (const alt of suggestion.alternatives.slice(0, 3)) {
|
|
||||||
console.log(` ${alt.type} (${(alt.confidence * 100).toFixed(1)}%)`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
} catch (error: any) {
|
} catch (error: any) {
|
||||||
console.error(chalk.red('Error:', error.message))
|
console.error(chalk.red('Error:', error.message))
|
||||||
process.exit(1)
|
process.exit(1)
|
||||||
|
|
|
||||||
|
|
@ -1,17 +1,12 @@
|
||||||
/**
|
/**
|
||||||
* Neural Import Augmentation - AI-Powered Data Understanding
|
* Neural Import - AI-Powered Data Understanding
|
||||||
*
|
*
|
||||||
* 🧠 Built-in AI augmentation for intelligent data processing
|
* Standalone implementation for intelligent data processing.
|
||||||
* ⚛️ Always free, always included, always enabled
|
|
||||||
*
|
|
||||||
* Now using the unified BrainyAugmentation interface!
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { BaseAugmentation, AugmentationContext } from './brainyAugmentation.js'
|
|
||||||
import { NounType, VerbType } from '../types/graphTypes.js'
|
import { NounType, VerbType } from '../types/graphTypes.js'
|
||||||
import * as fs from '../universal/fs.js'
|
import * as fs from '../universal/fs.js'
|
||||||
import * as path from '../universal/path.js'
|
import * as path from '../universal/path.js'
|
||||||
import { BrainyTypes, getBrainyTypes } from './typeMatching/brainyTypes.js'
|
|
||||||
import { prodLog } from '../utils/logger.js'
|
import { prodLog } from '../utils/logger.js'
|
||||||
|
|
||||||
// Neural Import Analysis Types
|
// Neural Import Analysis Types
|
||||||
|
|
@ -62,22 +57,15 @@ export interface NeuralImportConfig {
|
||||||
* Neural Import Augmentation - Unified Implementation
|
* Neural Import Augmentation - Unified Implementation
|
||||||
* Processes data with AI before storage operations
|
* Processes data with AI before storage operations
|
||||||
*/
|
*/
|
||||||
export class NeuralImportAugmentation extends BaseAugmentation {
|
export class NeuralImportAugmentation {
|
||||||
readonly name = 'neural-import'
|
readonly name = 'neural-import'
|
||||||
readonly timing = 'before' as const // Process data before storage
|
private operations = ['add', 'addNoun', 'addVerb', 'all']
|
||||||
readonly metadata = {
|
|
||||||
reads: '*' as '*', // Needs to read data for analysis
|
|
||||||
writes: ['_neuralProcessed', '_neuralConfidence', '_detectedEntities', '_detectedRelationships', '_neuralInsights', 'nounType', 'verbType'] as string[]
|
|
||||||
} // Enriches metadata with neural analysis
|
|
||||||
operations = ['add', 'addNoun', 'addVerb', 'all'] as ('add' | 'addNoun' | 'addVerb' | 'all')[] // Use 'all' to catch batch operations
|
|
||||||
readonly priority = 80 // High priority for data processing
|
|
||||||
|
|
||||||
protected config: NeuralImportConfig
|
protected config: NeuralImportConfig
|
||||||
private analysisCache = new Map<string, NeuralAnalysisResult>()
|
private analysisCache = new Map<string, NeuralAnalysisResult>()
|
||||||
private typeMatcher: BrainyTypes | null = null
|
private context?: { brain: any }
|
||||||
|
|
||||||
constructor(config: Partial<NeuralImportConfig> = {}) {
|
constructor(config: Partial<NeuralImportConfig> = {}) {
|
||||||
super()
|
|
||||||
this.config = {
|
this.config = {
|
||||||
confidenceThreshold: 0.7,
|
confidenceThreshold: 0.7,
|
||||||
enableWeights: true,
|
enableWeights: true,
|
||||||
|
|
@ -87,18 +75,12 @@ export class NeuralImportAugmentation extends BaseAugmentation {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
protected async onInitialize(): Promise<void> {
|
async init(): Promise<void> {
|
||||||
try {
|
// No external dependencies to initialize
|
||||||
this.typeMatcher = await getBrainyTypes()
|
|
||||||
this.log('🧠 Neural Import augmentation initialized with intelligent type matching')
|
|
||||||
} catch (error) {
|
|
||||||
this.log('⚠️ Failed to initialize type matcher, falling back to heuristics', 'warn')
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
protected async onShutdown(): Promise<void> {
|
private log(message: string, _level?: string): void {
|
||||||
this.analysisCache.clear()
|
// Silent by default
|
||||||
this.log('🧠 Neural Import augmentation shut down')
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -455,22 +437,29 @@ export class NeuralImportAugmentation extends BaseAugmentation {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Infer noun type from object structure using intelligent type matching
|
* Infer noun type from object structure using field heuristics
|
||||||
*/
|
*/
|
||||||
private async inferNounType(obj: any): Promise<string> {
|
private async inferNounType(obj: any): Promise<string> {
|
||||||
if (!this.typeMatcher) {
|
if (typeof obj !== 'object' || obj === null) return NounType.Thing
|
||||||
// Initialize type matcher if not available
|
|
||||||
this.typeMatcher = await getBrainyTypes()
|
// Check for explicit type field
|
||||||
|
if (obj.type && typeof obj.type === 'string') {
|
||||||
|
const normalized = obj.type.charAt(0).toUpperCase() + obj.type.slice(1)
|
||||||
|
if (Object.values(NounType).includes(normalized as NounType)) {
|
||||||
|
return normalized as NounType
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const result = await this.typeMatcher.matchNounType(obj)
|
if (obj.email || obj.firstName || obj.lastName || obj.username) return NounType.Person
|
||||||
|
if (obj.companyName || obj.organizationId || obj.employees) return NounType.Organization
|
||||||
|
if (obj.latitude || obj.longitude || obj.address || obj.city) return NounType.Location
|
||||||
|
if ((obj.content && (obj.title || obj.author)) || obj.pages) return NounType.Document
|
||||||
|
if (obj.startTime || obj.endTime || obj.date || obj.attendees) return NounType.Event
|
||||||
|
if (obj.price || obj.sku || obj.productId) return NounType.Product
|
||||||
|
if ((obj.status && obj.assignee) || obj.priority) return NounType.Task
|
||||||
|
if (Array.isArray(obj.data) || obj.rows || obj.columns) return NounType.Dataset
|
||||||
|
|
||||||
// Log if confidence is low for debugging
|
return NounType.Thing
|
||||||
if (result.confidence < 0.5) {
|
|
||||||
this.log(`Low confidence (${result.confidence.toFixed(2)}) for noun type: ${result.type}`, 'warn')
|
|
||||||
}
|
|
||||||
|
|
||||||
return result.type
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -511,22 +500,28 @@ export class NeuralImportAugmentation extends BaseAugmentation {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Infer verb type from field name using intelligent type matching
|
* Infer verb type from field name using common patterns
|
||||||
*/
|
*/
|
||||||
private async inferVerbType(fieldName: string, sourceObj?: any, targetObj?: any): Promise<string> {
|
private async inferVerbType(fieldName: string, _sourceObj?: any, _targetObj?: any): Promise<string> {
|
||||||
if (!this.typeMatcher) {
|
const field = fieldName.toLowerCase()
|
||||||
// Initialize type matcher if not available
|
|
||||||
this.typeMatcher = await getBrainyTypes()
|
if (field.includes('parent') || field.includes('child') || field.includes('contain')) {
|
||||||
|
return VerbType.Contains
|
||||||
|
}
|
||||||
|
if (field.includes('owner') || field.includes('created') || field.includes('author')) {
|
||||||
|
return VerbType.Creates
|
||||||
|
}
|
||||||
|
if (field.includes('member') || field.includes('belong')) {
|
||||||
|
return VerbType.MemberOf
|
||||||
|
}
|
||||||
|
if (field.includes('depend') || field.includes('require')) {
|
||||||
|
return VerbType.DependsOn
|
||||||
|
}
|
||||||
|
if (field.includes('ref') || field.includes('link') || field.includes('source')) {
|
||||||
|
return VerbType.References
|
||||||
}
|
}
|
||||||
|
|
||||||
const result = await this.typeMatcher.matchVerbType(sourceObj, targetObj, fieldName)
|
return VerbType.RelatedTo
|
||||||
|
|
||||||
// Log if confidence is low for debugging
|
|
||||||
if (result.confidence < 0.5) {
|
|
||||||
this.log(`Low confidence (${result.confidence.toFixed(2)}) for verb type: ${result.type}`, 'warn')
|
|
||||||
}
|
|
||||||
|
|
||||||
return result.type
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -16,8 +16,8 @@ import { NeuralEntityExtractor, ExtractedEntity } from '../neural/entityExtracto
|
||||||
import { NaturalLanguageProcessor } from '../neural/naturalLanguageProcessor.js'
|
import { NaturalLanguageProcessor } from '../neural/naturalLanguageProcessor.js'
|
||||||
import { SmartRelationshipExtractor } from '../neural/SmartRelationshipExtractor.js'
|
import { SmartRelationshipExtractor } from '../neural/SmartRelationshipExtractor.js'
|
||||||
import { NounType, VerbType } from '../types/graphTypes.js'
|
import { NounType, VerbType } from '../types/graphTypes.js'
|
||||||
import { CSVHandler } from '../augmentations/intelligentImport/handlers/csvHandler.js'
|
import { CSVHandler } from './handlers/csvHandler.js'
|
||||||
import type { FormatHandlerOptions } from '../augmentations/intelligentImport/types.js'
|
import type { FormatHandlerOptions } from './handlers/types.js'
|
||||||
|
|
||||||
export interface SmartCSVOptions extends FormatHandlerOptions {
|
export interface SmartCSVOptions extends FormatHandlerOptions {
|
||||||
/** Enable neural entity extraction */
|
/** Enable neural entity extraction */
|
||||||
|
|
|
||||||
|
|
@ -14,8 +14,8 @@ import { NeuralEntityExtractor, ExtractedEntity } from '../neural/entityExtracto
|
||||||
import { NaturalLanguageProcessor } from '../neural/naturalLanguageProcessor.js'
|
import { NaturalLanguageProcessor } from '../neural/naturalLanguageProcessor.js'
|
||||||
import { SmartRelationshipExtractor } from '../neural/SmartRelationshipExtractor.js'
|
import { SmartRelationshipExtractor } from '../neural/SmartRelationshipExtractor.js'
|
||||||
import { NounType, VerbType } from '../types/graphTypes.js'
|
import { NounType, VerbType } from '../types/graphTypes.js'
|
||||||
import { ExcelHandler } from '../augmentations/intelligentImport/handlers/excelHandler.js'
|
import { ExcelHandler } from './handlers/excelHandler.js'
|
||||||
import type { FormatHandlerOptions } from '../augmentations/intelligentImport/types.js'
|
import type { FormatHandlerOptions } from './handlers/types.js'
|
||||||
|
|
||||||
export interface SmartExcelOptions extends FormatHandlerOptions {
|
export interface SmartExcelOptions extends FormatHandlerOptions {
|
||||||
/** Enable neural entity extraction */
|
/** Enable neural entity extraction */
|
||||||
|
|
|
||||||
|
|
@ -15,8 +15,8 @@ import { NeuralEntityExtractor, ExtractedEntity } from '../neural/entityExtracto
|
||||||
import { NaturalLanguageProcessor } from '../neural/naturalLanguageProcessor.js'
|
import { NaturalLanguageProcessor } from '../neural/naturalLanguageProcessor.js'
|
||||||
import { SmartRelationshipExtractor } from '../neural/SmartRelationshipExtractor.js'
|
import { SmartRelationshipExtractor } from '../neural/SmartRelationshipExtractor.js'
|
||||||
import { NounType, VerbType } from '../types/graphTypes.js'
|
import { NounType, VerbType } from '../types/graphTypes.js'
|
||||||
import { PDFHandler } from '../augmentations/intelligentImport/handlers/pdfHandler.js'
|
import { PDFHandler } from './handlers/pdfHandler.js'
|
||||||
import type { FormatHandlerOptions } from '../augmentations/intelligentImport/types.js'
|
import type { FormatHandlerOptions } from './handlers/types.js'
|
||||||
|
|
||||||
export interface SmartPDFOptions extends FormatHandlerOptions {
|
export interface SmartPDFOptions extends FormatHandlerOptions {
|
||||||
/** Enable neural entity extraction */
|
/** Enable neural entity extraction */
|
||||||
|
|
|
||||||
|
|
@ -5,8 +5,8 @@
|
||||||
* Uses MimeTypeDetector for comprehensive file type detection (2000+ types)
|
* Uses MimeTypeDetector for comprehensive file type detection (2000+ types)
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { FormatHandler, FormatHandlerOptions, ProcessedData } from '../types.js'
|
import { FormatHandler, FormatHandlerOptions, ProcessedData } from './types.js'
|
||||||
import { mimeDetector } from '../../../vfs/MimeTypeDetector.js'
|
import { mimeDetector } from '../../vfs/MimeTypeDetector.js'
|
||||||
|
|
||||||
export abstract class BaseFormatHandler implements FormatHandler {
|
export abstract class BaseFormatHandler implements FormatHandler {
|
||||||
abstract readonly format: string
|
abstract readonly format: string
|
||||||
|
|
@ -10,7 +10,7 @@
|
||||||
import { parse } from 'csv-parse/sync'
|
import { parse } from 'csv-parse/sync'
|
||||||
import { detect as detectEncoding } from 'chardet'
|
import { detect as detectEncoding } from 'chardet'
|
||||||
import { BaseFormatHandler } from './base.js'
|
import { BaseFormatHandler } from './base.js'
|
||||||
import { FormatHandlerOptions, ProcessedData } from '../types.js'
|
import { FormatHandlerOptions, ProcessedData } from './types.js'
|
||||||
|
|
||||||
export class CSVHandler extends BaseFormatHandler {
|
export class CSVHandler extends BaseFormatHandler {
|
||||||
readonly format = 'csv'
|
readonly format = 'csv'
|
||||||
|
|
@ -9,7 +9,7 @@
|
||||||
|
|
||||||
import * as XLSX from 'xlsx'
|
import * as XLSX from 'xlsx'
|
||||||
import { BaseFormatHandler } from './base.js'
|
import { BaseFormatHandler } from './base.js'
|
||||||
import { FormatHandlerOptions, ProcessedData } from '../types.js'
|
import { FormatHandlerOptions, ProcessedData } from './types.js'
|
||||||
|
|
||||||
export class ExcelHandler extends BaseFormatHandler {
|
export class ExcelHandler extends BaseFormatHandler {
|
||||||
readonly format = 'excel'
|
readonly format = 'excel'
|
||||||
|
|
@ -9,7 +9,7 @@
|
||||||
|
|
||||||
import * as pdfjsLib from 'pdfjs-dist/legacy/build/pdf.mjs'
|
import * as pdfjsLib from 'pdfjs-dist/legacy/build/pdf.mjs'
|
||||||
import { BaseFormatHandler } from './base.js'
|
import { BaseFormatHandler } from './base.js'
|
||||||
import { FormatHandlerOptions, ProcessedData } from '../types.js'
|
import { FormatHandlerOptions, ProcessedData } from './types.js'
|
||||||
|
|
||||||
// Use built-in worker for Node.js environments
|
// Use built-in worker for Node.js environments
|
||||||
// In production, this can be customized via options
|
// In production, this can be customized via options
|
||||||
121
src/index.ts
121
src/index.ts
|
|
@ -5,7 +5,7 @@
|
||||||
* Core Components:
|
* Core Components:
|
||||||
* - Brainy: The unified database with Triple Intelligence
|
* - Brainy: The unified database with Triple Intelligence
|
||||||
* - Triple Intelligence: Seamless fusion of vector + graph + field search
|
* - Triple Intelligence: Seamless fusion of vector + graph + field search
|
||||||
* - Augmentations: Extensible plugin system
|
* - Plugins: Extensible plugin system (cortex, storage adapters)
|
||||||
* - Neural API: AI-powered clustering and analysis
|
* - Neural API: AI-powered clustering and analysis
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
|
@ -54,21 +54,6 @@ export {
|
||||||
getPresetDescription
|
getPresetDescription
|
||||||
} from './config/index.js'
|
} from './config/index.js'
|
||||||
|
|
||||||
// Export augmentation types
|
|
||||||
export type {
|
|
||||||
BrainyAugmentations,
|
|
||||||
IAugmentation,
|
|
||||||
ISenseAugmentation,
|
|
||||||
IConduitAugmentation,
|
|
||||||
ICognitionAugmentation,
|
|
||||||
IMemoryAugmentation,
|
|
||||||
IPerceptionAugmentation,
|
|
||||||
IDialogAugmentation,
|
|
||||||
IActivationAugmentation,
|
|
||||||
IWebSocketSupport,
|
|
||||||
AugmentationType
|
|
||||||
} from './types/augmentations.js'
|
|
||||||
|
|
||||||
// Export Neural Import (AI data understanding)
|
// Export Neural Import (AI data understanding)
|
||||||
export { NeuralImport } from './cortex/neuralImport.js'
|
export { NeuralImport } from './cortex/neuralImport.js'
|
||||||
export type {
|
export type {
|
||||||
|
|
@ -96,10 +81,6 @@ export type {
|
||||||
SmartRelationshipExtractorOptions
|
SmartRelationshipExtractorOptions
|
||||||
} from './neural/SmartRelationshipExtractor.js'
|
} from './neural/SmartRelationshipExtractor.js'
|
||||||
|
|
||||||
// Import Manager removed - use brain.import() instead (available on all Brainy instances)
|
|
||||||
|
|
||||||
// Augmentation types are already exported later in the file
|
|
||||||
|
|
||||||
// Export distance functions for convenience
|
// Export distance functions for convenience
|
||||||
import {
|
import {
|
||||||
euclideanDistance,
|
euclideanDistance,
|
||||||
|
|
@ -234,7 +215,6 @@ export {
|
||||||
export { FileSystemStorage } from './storage/adapters/fileSystemStorage.js'
|
export { FileSystemStorage } from './storage/adapters/fileSystemStorage.js'
|
||||||
|
|
||||||
// Export COW (Copy-on-Write) infrastructure
|
// Export COW (Copy-on-Write) infrastructure
|
||||||
// Enables premium augmentations to implement temporal features
|
|
||||||
import { CommitLog } from './storage/cow/CommitLog.js'
|
import { CommitLog } from './storage/cow/CommitLog.js'
|
||||||
import { CommitObject, CommitBuilder } from './storage/cow/CommitObject.js'
|
import { CommitObject, CommitBuilder } from './storage/cow/CommitObject.js'
|
||||||
import { BlobStorage } from './storage/cow/BlobStorage.js'
|
import { BlobStorage } from './storage/cow/BlobStorage.js'
|
||||||
|
|
@ -255,7 +235,6 @@ export {
|
||||||
import {
|
import {
|
||||||
Pipeline,
|
Pipeline,
|
||||||
pipeline,
|
pipeline,
|
||||||
augmentationPipeline,
|
|
||||||
ExecutionMode,
|
ExecutionMode,
|
||||||
PipelineOptions,
|
PipelineOptions,
|
||||||
PipelineResult,
|
PipelineResult,
|
||||||
|
|
@ -266,85 +245,21 @@ import {
|
||||||
StreamlinedPipelineResult
|
StreamlinedPipelineResult
|
||||||
} from './pipeline.js'
|
} from './pipeline.js'
|
||||||
|
|
||||||
// Sequential pipeline removed - use unified pipeline instead
|
|
||||||
|
|
||||||
// REMOVED: Old augmentation factory for 2.0 clean architecture
|
|
||||||
|
|
||||||
export {
|
export {
|
||||||
// Unified pipeline exports
|
|
||||||
Pipeline,
|
Pipeline,
|
||||||
pipeline,
|
pipeline,
|
||||||
augmentationPipeline,
|
|
||||||
ExecutionMode,
|
ExecutionMode,
|
||||||
|
|
||||||
// Factory functions
|
|
||||||
createPipeline,
|
createPipeline,
|
||||||
createStreamingPipeline,
|
createStreamingPipeline,
|
||||||
StreamlinedExecutionMode,
|
StreamlinedExecutionMode
|
||||||
|
|
||||||
// Augmentation factory exports (REMOVED in 2.0 - Use BrainyAugmentation interface)
|
|
||||||
// createSenseAugmentation, // → Use BaseAugmentation class
|
|
||||||
// addWebSocketSupport, // → Use APIServerAugmentation
|
|
||||||
// executeAugmentation, // → Use brain.augmentations.execute()
|
|
||||||
// loadAugmentationModule // → Use dynamic imports
|
|
||||||
}
|
}
|
||||||
export type {
|
export type {
|
||||||
PipelineOptions,
|
PipelineOptions,
|
||||||
PipelineResult,
|
PipelineResult,
|
||||||
StreamlinedPipelineOptions,
|
StreamlinedPipelineOptions,
|
||||||
StreamlinedPipelineResult
|
StreamlinedPipelineResult
|
||||||
// AugmentationOptions - REMOVED in 2.0 (use BaseAugmentation config)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Augmentation registry removed - use Brainy's built-in augmentation system
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
// Export augmentation implementations
|
|
||||||
import {
|
|
||||||
StorageAugmentation,
|
|
||||||
DynamicStorageAugmentation,
|
|
||||||
createStorageAugmentationFromConfig
|
|
||||||
} from './augmentations/storageAugmentation.js'
|
|
||||||
import {
|
|
||||||
MemoryStorageAugmentation,
|
|
||||||
FileSystemStorageAugmentation,
|
|
||||||
OPFSStorageAugmentation,
|
|
||||||
S3StorageAugmentation,
|
|
||||||
R2StorageAugmentation,
|
|
||||||
GCSStorageAugmentation,
|
|
||||||
createAutoStorageAugmentation
|
|
||||||
} from './augmentations/storageAugmentations.js'
|
|
||||||
import {
|
|
||||||
WebSocketConduitAugmentation
|
|
||||||
} from './augmentations/conduitAugmentations.js'
|
|
||||||
|
|
||||||
// Storage augmentation exports
|
|
||||||
export {
|
|
||||||
// Base classes
|
|
||||||
StorageAugmentation,
|
|
||||||
DynamicStorageAugmentation,
|
|
||||||
// Concrete implementations
|
|
||||||
MemoryStorageAugmentation,
|
|
||||||
FileSystemStorageAugmentation,
|
|
||||||
OPFSStorageAugmentation,
|
|
||||||
S3StorageAugmentation,
|
|
||||||
R2StorageAugmentation,
|
|
||||||
GCSStorageAugmentation,
|
|
||||||
// Factory functions
|
|
||||||
createAutoStorageAugmentation,
|
|
||||||
createStorageAugmentationFromConfig
|
|
||||||
}
|
|
||||||
|
|
||||||
// Other augmentation exports
|
|
||||||
export {
|
|
||||||
WebSocketConduitAugmentation
|
|
||||||
}
|
|
||||||
|
|
||||||
// LLM augmentations are optional and not imported by default
|
|
||||||
// They can be imported directly from their module if needed:
|
|
||||||
// import { LLMCognitionAugmentation, LLMActivationAugmentation, createLLMAugmentations } from './augmentations/llmAugmentations.js'
|
|
||||||
|
|
||||||
// Export types
|
// Export types
|
||||||
import type {
|
import type {
|
||||||
Vector,
|
Vector,
|
||||||
|
|
@ -377,25 +292,6 @@ export type {
|
||||||
StorageAdapter
|
StorageAdapter
|
||||||
}
|
}
|
||||||
|
|
||||||
// Export augmentation types
|
|
||||||
import type {
|
|
||||||
AugmentationResponse,
|
|
||||||
BrainyAugmentation,
|
|
||||||
BaseAugmentation,
|
|
||||||
AugmentationContext
|
|
||||||
} from './types/augmentations.js'
|
|
||||||
|
|
||||||
// Export augmentation manager for type-safe augmentation management
|
|
||||||
export { AugmentationManager, type AugmentationInfo } from './augmentationManager.js'
|
|
||||||
|
|
||||||
// Export only the clean augmentation types for 2.0
|
|
||||||
export type {
|
|
||||||
AugmentationResponse,
|
|
||||||
BrainyAugmentation,
|
|
||||||
BaseAugmentation,
|
|
||||||
AugmentationContext
|
|
||||||
}
|
|
||||||
|
|
||||||
// Export graph types
|
// Export graph types
|
||||||
import type {
|
import type {
|
||||||
GraphNoun,
|
GraphNoun,
|
||||||
|
|
@ -496,8 +392,8 @@ export type {
|
||||||
// Export type utility functions
|
// Export type utility functions
|
||||||
import { getNounTypes, getVerbTypes, getNounTypeMap, getVerbTypeMap } from './utils/typeUtils.js'
|
import { getNounTypes, getVerbTypes, getNounTypeMap, getVerbTypeMap } from './utils/typeUtils.js'
|
||||||
|
|
||||||
// Export BrainyTypes for complete type management
|
// Export BrainyTypes for type validation and lookup
|
||||||
import { BrainyTypes, TypeSuggestion, suggestType } from './utils/brainyTypes.js'
|
import { BrainyTypes } from './utils/brainyTypes.js'
|
||||||
|
|
||||||
export {
|
export {
|
||||||
NounType,
|
NounType,
|
||||||
|
|
@ -506,13 +402,8 @@ export {
|
||||||
getVerbTypes,
|
getVerbTypes,
|
||||||
getNounTypeMap,
|
getNounTypeMap,
|
||||||
getVerbTypeMap,
|
getVerbTypeMap,
|
||||||
// BrainyTypes - complete type management
|
// BrainyTypes - type validation and lookup
|
||||||
BrainyTypes,
|
BrainyTypes
|
||||||
suggestType
|
|
||||||
}
|
|
||||||
|
|
||||||
export type {
|
|
||||||
TypeSuggestion
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Export MCP (Model Control Protocol) components
|
// Export MCP (Model Control Protocol) components
|
||||||
|
|
|
||||||
|
|
@ -5,11 +5,6 @@
|
||||||
* for event subscriptions, tabular export, and lifecycle management.
|
* for event subscriptions, tabular export, and lifecycle management.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import {
|
|
||||||
BaseAugmentation,
|
|
||||||
AugmentationContext
|
|
||||||
} from '../../augmentations/brainyAugmentation.js'
|
|
||||||
import { AugmentationManifest } from '../../augmentations/manifest.js'
|
|
||||||
import { EventBus } from './EventBus.js'
|
import { EventBus } from './EventBus.js'
|
||||||
import { TabularExporter } from './TabularExporter.js'
|
import { TabularExporter } from './TabularExporter.js'
|
||||||
import {
|
import {
|
||||||
|
|
@ -48,17 +43,12 @@ import { Entity, Relation, FindParams } from '../../types/brainy.types.js'
|
||||||
* }
|
* }
|
||||||
* ```
|
* ```
|
||||||
*/
|
*/
|
||||||
export abstract class IntegrationBase extends BaseAugmentation {
|
export abstract class IntegrationBase {
|
||||||
// Augmentation interface implementation
|
|
||||||
readonly timing = 'after' as const
|
|
||||||
readonly metadata = 'readonly' as const
|
|
||||||
readonly operations: ('all')[] = ['all']
|
|
||||||
readonly priority = 5 // Low priority - runs after main operations
|
|
||||||
category: 'internal' | 'core' | 'premium' | 'community' | 'external' = 'core'
|
|
||||||
|
|
||||||
// Shared infrastructure
|
// Shared infrastructure
|
||||||
protected eventBus: EventBus
|
protected eventBus: EventBus
|
||||||
protected exporter: TabularExporter
|
protected exporter: TabularExporter
|
||||||
|
protected config: IntegrationConfig
|
||||||
|
protected context?: { brain: any; storage: any; config: any; log: (message: string, level?: string) => void }
|
||||||
|
|
||||||
// Integration state
|
// Integration state
|
||||||
protected isRunning = false
|
protected isRunning = false
|
||||||
|
|
@ -80,11 +70,35 @@ export abstract class IntegrationBase extends BaseAugmentation {
|
||||||
config?: IntegrationConfig,
|
config?: IntegrationConfig,
|
||||||
exporterConfig?: TabularExporterConfig
|
exporterConfig?: TabularExporterConfig
|
||||||
) {
|
) {
|
||||||
super(config)
|
this.config = config || {}
|
||||||
this.eventBus = new EventBus()
|
this.eventBus = new EventBus()
|
||||||
this.exporter = new TabularExporter(exporterConfig)
|
this.exporter = new TabularExporter(exporterConfig)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Log a message
|
||||||
|
*/
|
||||||
|
protected log(message: string, level: string = 'info'): void {
|
||||||
|
if (this.context?.log) {
|
||||||
|
this.context.log(message, level)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Initialize the integration with context
|
||||||
|
*/
|
||||||
|
async initialize(ctx: { brain: any; storage: any; config: any; log: (message: string, level?: string) => void }): Promise<void> {
|
||||||
|
this.context = ctx
|
||||||
|
await this.onInitialize()
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Shutdown the integration
|
||||||
|
*/
|
||||||
|
async shutdown(): Promise<void> {
|
||||||
|
await this.onShutdown()
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Integration name (must be unique)
|
* Integration name (must be unique)
|
||||||
*/
|
*/
|
||||||
|
|
@ -311,7 +325,7 @@ export abstract class IntegrationBase extends BaseAugmentation {
|
||||||
* Get manifest for this integration
|
* Get manifest for this integration
|
||||||
* Subclasses should override this
|
* Subclasses should override this
|
||||||
*/
|
*/
|
||||||
getManifest(): AugmentationManifest {
|
getManifest(): Record<string, any> {
|
||||||
return {
|
return {
|
||||||
id: this.name,
|
id: this.name,
|
||||||
name: this.name,
|
name: this.name,
|
||||||
|
|
|
||||||
|
|
@ -15,7 +15,6 @@ import {
|
||||||
HTTPIntegration
|
HTTPIntegration
|
||||||
} from '../core/IntegrationBase.js'
|
} from '../core/IntegrationBase.js'
|
||||||
import { IntegrationConfig, ODataQueryOptions } from '../core/types.js'
|
import { IntegrationConfig, ODataQueryOptions } from '../core/types.js'
|
||||||
import { AugmentationManifest } from '../../augmentations/manifest.js'
|
|
||||||
import { Entity, Relation, FindParams } from '../../types/brainy.types.js'
|
import { Entity, Relation, FindParams } from '../../types/brainy.types.js'
|
||||||
import { NounType } from '../../types/graphTypes.js'
|
import { NounType } from '../../types/graphTypes.js'
|
||||||
import {
|
import {
|
||||||
|
|
@ -94,11 +93,11 @@ interface ODataResponse {
|
||||||
*
|
*
|
||||||
* @example
|
* @example
|
||||||
* ```typescript
|
* ```typescript
|
||||||
* // Register with Brainy
|
* const odata = new ODataIntegration({
|
||||||
* brain.augmentations.register(new ODataIntegration({
|
|
||||||
* basePath: '/odata',
|
* basePath: '/odata',
|
||||||
* maxPageSize: 1000
|
* maxPageSize: 1000
|
||||||
* }))
|
* })
|
||||||
|
* await odata.initialize()
|
||||||
*
|
*
|
||||||
* // Connect from Excel Power Query:
|
* // Connect from Excel Power Query:
|
||||||
* // Data → Get Data → From OData Feed → http://localhost:3000/odata
|
* // Data → Get Data → From OData Feed → http://localhost:3000/odata
|
||||||
|
|
@ -249,7 +248,7 @@ export class ODataIntegration extends IntegrationBase implements HTTPIntegration
|
||||||
/**
|
/**
|
||||||
* Get augmentation manifest
|
* Get augmentation manifest
|
||||||
*/
|
*/
|
||||||
getManifest(): AugmentationManifest {
|
getManifest(): Record<string, any> {
|
||||||
return {
|
return {
|
||||||
id: 'odata',
|
id: 'odata',
|
||||||
name: 'OData Integration',
|
name: 'OData Integration',
|
||||||
|
|
|
||||||
|
|
@ -15,7 +15,6 @@
|
||||||
|
|
||||||
import { IntegrationBase, HTTPIntegration } from '../core/IntegrationBase.js'
|
import { IntegrationBase, HTTPIntegration } from '../core/IntegrationBase.js'
|
||||||
import { IntegrationConfig } from '../core/types.js'
|
import { IntegrationConfig } from '../core/types.js'
|
||||||
import { AugmentationManifest } from '../../augmentations/manifest.js'
|
|
||||||
import { Entity, FindParams } from '../../types/brainy.types.js'
|
import { Entity, FindParams } from '../../types/brainy.types.js'
|
||||||
import { NounType, VerbType } from '../../types/graphTypes.js'
|
import { NounType, VerbType } from '../../types/graphTypes.js'
|
||||||
|
|
||||||
|
|
@ -92,10 +91,11 @@ interface SheetsResponse {
|
||||||
*
|
*
|
||||||
* @example
|
* @example
|
||||||
* ```typescript
|
* ```typescript
|
||||||
* brain.augmentations.register(new GoogleSheetsIntegration({
|
* const sheets = new GoogleSheetsIntegration({
|
||||||
* basePath: '/sheets',
|
* basePath: '/sheets',
|
||||||
* maxResults: 1000
|
* maxResults: 1000
|
||||||
* }))
|
* })
|
||||||
|
* await sheets.initialize()
|
||||||
* ```
|
* ```
|
||||||
*/
|
*/
|
||||||
export class GoogleSheetsIntegration
|
export class GoogleSheetsIntegration
|
||||||
|
|
@ -283,7 +283,7 @@ export class GoogleSheetsIntegration
|
||||||
/**
|
/**
|
||||||
* Get manifest
|
* Get manifest
|
||||||
*/
|
*/
|
||||||
getManifest(): AugmentationManifest {
|
getManifest(): Record<string, any> {
|
||||||
return {
|
return {
|
||||||
id: 'sheets',
|
id: 'sheets',
|
||||||
name: 'Google Sheets Integration',
|
name: 'Google Sheets Integration',
|
||||||
|
|
|
||||||
|
|
@ -15,7 +15,6 @@ import {
|
||||||
EventFilter,
|
EventFilter,
|
||||||
BrainyEvent
|
BrainyEvent
|
||||||
} from '../core/types.js'
|
} from '../core/types.js'
|
||||||
import { AugmentationManifest } from '../../augmentations/manifest.js'
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* SSE integration configuration
|
* SSE integration configuration
|
||||||
|
|
@ -63,10 +62,11 @@ interface SSEClient {
|
||||||
*
|
*
|
||||||
* @example
|
* @example
|
||||||
* ```typescript
|
* ```typescript
|
||||||
* brain.augmentations.register(new SSEIntegration({
|
* const sse = new SSEIntegration({
|
||||||
* basePath: '/events',
|
* basePath: '/events',
|
||||||
* heartbeatInterval: 30000
|
* heartbeatInterval: 30000
|
||||||
* }))
|
* })
|
||||||
|
* await sse.initialize()
|
||||||
*
|
*
|
||||||
* // Client-side:
|
* // Client-side:
|
||||||
* const source = new EventSource('/events?types=noun&operations=create,update')
|
* const source = new EventSource('/events?types=noun&operations=create,update')
|
||||||
|
|
@ -292,7 +292,7 @@ export class SSEIntegration
|
||||||
/**
|
/**
|
||||||
* Get manifest
|
* Get manifest
|
||||||
*/
|
*/
|
||||||
getManifest(): AugmentationManifest {
|
getManifest(): Record<string, any> {
|
||||||
return {
|
return {
|
||||||
id: 'sse',
|
id: 'sse',
|
||||||
name: 'SSE Streaming',
|
name: 'SSE Streaming',
|
||||||
|
|
|
||||||
|
|
@ -15,7 +15,6 @@ import {
|
||||||
WebhookDeliveryResult,
|
WebhookDeliveryResult,
|
||||||
BrainyEvent
|
BrainyEvent
|
||||||
} from '../core/types.js'
|
} from '../core/types.js'
|
||||||
import { AugmentationManifest } from '../../augmentations/manifest.js'
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Webhook integration configuration
|
* Webhook integration configuration
|
||||||
|
|
@ -69,7 +68,7 @@ interface PendingDelivery {
|
||||||
* @example
|
* @example
|
||||||
* ```typescript
|
* ```typescript
|
||||||
* const webhooks = new WebhookIntegration()
|
* const webhooks = new WebhookIntegration()
|
||||||
* brain.augmentations.register(webhooks)
|
* await webhooks.initialize()
|
||||||
*
|
*
|
||||||
* // Register a webhook
|
* // Register a webhook
|
||||||
* await webhooks.register({
|
* await webhooks.register({
|
||||||
|
|
@ -263,7 +262,7 @@ export class WebhookIntegration extends IntegrationBase {
|
||||||
/**
|
/**
|
||||||
* Get manifest
|
* Get manifest
|
||||||
*/
|
*/
|
||||||
getManifest(): AugmentationManifest {
|
getManifest(): Record<string, any> {
|
||||||
return {
|
return {
|
||||||
id: 'webhooks',
|
id: 'webhooks',
|
||||||
name: 'Webhooks',
|
name: 'Webhooks',
|
||||||
|
|
|
||||||
|
|
@ -12,10 +12,6 @@ import {
|
||||||
MCPTool,
|
MCPTool,
|
||||||
MCP_VERSION
|
MCP_VERSION
|
||||||
} from '../types/mcpTypes.js'
|
} from '../types/mcpTypes.js'
|
||||||
import { AugmentationType } from '../types/augmentations.js'
|
|
||||||
|
|
||||||
// Import the augmentation pipeline
|
|
||||||
import { augmentationPipeline } from '../pipeline.js'
|
|
||||||
|
|
||||||
export class MCPAugmentationToolset {
|
export class MCPAugmentationToolset {
|
||||||
/**
|
/**
|
||||||
|
|
@ -146,9 +142,8 @@ export class MCPAugmentationToolset {
|
||||||
|
|
||||||
const { args = [], options = {} } = parameters
|
const { args = [], options = {} } = parameters
|
||||||
|
|
||||||
// Note: This MCP toolset needs to be updated to use the new brain.augmentations API
|
// Augmentation pipeline has been removed — operations are called directly on brain
|
||||||
// For now, return a placeholder response to fix compilation
|
throw new Error(`MCP augmentation toolset is deprecated. Method '${method}' not available. Use brain API directly.`)
|
||||||
throw new Error(`MCP toolset requires update to use brain.augmentations API. Method '${method}' not available.`)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -157,7 +152,7 @@ export class MCPAugmentationToolset {
|
||||||
* @returns Whether the augmentation type is valid
|
* @returns Whether the augmentation type is valid
|
||||||
*/
|
*/
|
||||||
private isValidAugmentationType(type: string): boolean {
|
private isValidAugmentationType(type: string): boolean {
|
||||||
return Object.values(AugmentationType).includes(type as AugmentationType)
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,7 @@
|
||||||
/**
|
/**
|
||||||
* Pipeline - Augmentation execution pipeline
|
* Pipeline - Execution pipeline
|
||||||
*
|
*
|
||||||
* Provides Pipeline class, execution modes, and factory functions.
|
* Provides Pipeline class, execution modes, and factory functions.
|
||||||
* All augmentation management should use brain.augmentations API.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -28,7 +27,6 @@ export interface PipelineOptions {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Minimal Pipeline class for backward compatibility.
|
* Minimal Pipeline class for backward compatibility.
|
||||||
* All augmentation management should use brain.augmentations API.
|
|
||||||
*/
|
*/
|
||||||
export class Pipeline {
|
export class Pipeline {
|
||||||
private static instance?: Pipeline
|
private static instance?: Pipeline
|
||||||
|
|
|
||||||
|
|
@ -1,130 +0,0 @@
|
||||||
/**
|
|
||||||
* Default Augmentation Registry
|
|
||||||
*
|
|
||||||
* 🧠⚛️ Pre-installed augmentations that come with every Brainy installation
|
|
||||||
* These are the core "sensory organs" of the atomic age brain-in-jar system
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { BrainyInterface } from '../types/brainyInterface.js'
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Default augmentations that ship with Brainy
|
|
||||||
* These are automatically registered on startup
|
|
||||||
*/
|
|
||||||
export class DefaultAugmentationRegistry {
|
|
||||||
private brainy: BrainyInterface
|
|
||||||
|
|
||||||
constructor(brainy: BrainyInterface) {
|
|
||||||
this.brainy = brainy
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Initialize all default augmentations
|
|
||||||
* Called during Brainy startup to register core functionality
|
|
||||||
*/
|
|
||||||
async initializeDefaults(): Promise<void> {
|
|
||||||
console.log('🧠⚛️ Initializing default augmentations...')
|
|
||||||
|
|
||||||
// Register Neural Import as default SENSE augmentation
|
|
||||||
await this.registerNeuralImport()
|
|
||||||
|
|
||||||
console.log('🧠⚛️ Default augmentations initialized')
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Neural Import - Default SENSE Augmentation
|
|
||||||
* AI-powered data understanding and entity extraction (always free)
|
|
||||||
*/
|
|
||||||
private async registerNeuralImport(): Promise<void> {
|
|
||||||
try {
|
|
||||||
// Import the Neural Import augmentation
|
|
||||||
const { NeuralImportAugmentation } = await import('../augmentations/neuralImport.js')
|
|
||||||
|
|
||||||
// Note: The actual registration is commented out since Brainy doesn't have addAugmentation method yet
|
|
||||||
// This would create instance with default configuration
|
|
||||||
/*
|
|
||||||
const neuralImport = new NeuralImportAugmentation(this.brainy as any, {
|
|
||||||
confidenceThreshold: 0.7,
|
|
||||||
enableWeights: true,
|
|
||||||
skipDuplicates: true
|
|
||||||
})
|
|
||||||
|
|
||||||
// Add as SENSE augmentation to Brainy (when method is available)
|
|
||||||
if (this.brainy.addAugmentation) {
|
|
||||||
await this.brainy.addAugmentation('SENSE', cortex, {
|
|
||||||
position: 1, // First in the SENSE pipeline
|
|
||||||
name: 'cortex',
|
|
||||||
autoStart: true
|
|
||||||
})
|
|
||||||
}
|
|
||||||
*/
|
|
||||||
|
|
||||||
console.log('🧠⚛️ Cortex module loaded (awaiting Brainy augmentation support)')
|
|
||||||
|
|
||||||
} catch (error) {
|
|
||||||
console.error('❌ Failed to register Cortex:', error instanceof Error ? error.message : String(error))
|
|
||||||
// Don't throw - Brainy should still work without Neural Import
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Check if Cortex is available and working
|
|
||||||
*/
|
|
||||||
async checkCortexHealth(): Promise<{
|
|
||||||
available: boolean
|
|
||||||
status: string
|
|
||||||
version?: string
|
|
||||||
}> {
|
|
||||||
try {
|
|
||||||
// Check if Cortex is registered as an augmentation
|
|
||||||
// Note: hasAugmentation method doesn't exist yet in Brainy
|
|
||||||
const hasCortex = false // this.brainy.hasAugmentation && this.brainy.hasAugmentation('SENSE', 'cortex')
|
|
||||||
|
|
||||||
return {
|
|
||||||
available: hasCortex || false,
|
|
||||||
status: hasCortex ? 'active' : 'not registered (awaiting Brainy support)',
|
|
||||||
version: '1.0.0'
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
return {
|
|
||||||
available: false,
|
|
||||||
status: `Error: ${error instanceof Error ? error.message : String(error)}`
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Reinstall Cortex if it's missing or corrupted
|
|
||||||
*/
|
|
||||||
async reinstallCortex(): Promise<void> {
|
|
||||||
try {
|
|
||||||
// Remove existing if present
|
|
||||||
// Note: removeAugmentation method doesn't exist yet in Brainy
|
|
||||||
/*
|
|
||||||
if (this.brainy.removeAugmentation) {
|
|
||||||
try {
|
|
||||||
await this.brainy.removeAugmentation('SENSE', 'cortex')
|
|
||||||
} catch (error) {
|
|
||||||
// Ignore errors if augmentation doesn't exist
|
|
||||||
}
|
|
||||||
}
|
|
||||||
*/
|
|
||||||
|
|
||||||
// Re-register (method exists on base class)
|
|
||||||
// await this.registerCortex()
|
|
||||||
|
|
||||||
console.log('🧠⚛️ Cortex reinstalled successfully')
|
|
||||||
} catch (error) {
|
|
||||||
throw new Error(`Failed to reinstall Cortex: ${error instanceof Error ? error.message : String(error)}`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Helper function to initialize default augmentations for any Brainy instance
|
|
||||||
*/
|
|
||||||
export async function initializeDefaultAugmentations(brainy: BrainyInterface): Promise<DefaultAugmentationRegistry> {
|
|
||||||
const registry = new DefaultAugmentationRegistry(brainy)
|
|
||||||
await registry.initializeDefaults()
|
|
||||||
return registry
|
|
||||||
}
|
|
||||||
|
|
@ -1,220 +0,0 @@
|
||||||
/**
|
|
||||||
* Brainy 2.0 Augmentation Types
|
|
||||||
*
|
|
||||||
* This file contains only the minimal types needed for augmentations.
|
|
||||||
* The main augmentation interfaces are now in augmentations/brainyAugmentation.ts
|
|
||||||
*/
|
|
||||||
|
|
||||||
/**
|
|
||||||
* WebSocket connection type for conduit augmentations
|
|
||||||
*/
|
|
||||||
export type WebSocketConnection = {
|
|
||||||
connectionId: string
|
|
||||||
url: string
|
|
||||||
status: 'connected' | 'disconnected' | 'error'
|
|
||||||
send?: (data: string | ArrayBufferLike | Blob | ArrayBufferView) => Promise<void>
|
|
||||||
close?: () => Promise<void>
|
|
||||||
_streamMessageHandler?: (event: { data: unknown }) => void
|
|
||||||
_messageHandlerWrapper?: (data: unknown) => void
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Generic augmentation response type
|
|
||||||
*/
|
|
||||||
export type AugmentationResponse<T> = {
|
|
||||||
success: boolean
|
|
||||||
data: T
|
|
||||||
error?: string
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Data callback type for subscriptions
|
|
||||||
*/
|
|
||||||
export type DataCallback<T> = (data: T) => void
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Import types for re-export (avoiding circular dependencies)
|
|
||||||
*/
|
|
||||||
import type {
|
|
||||||
BrainyAugmentation as BA,
|
|
||||||
BaseAugmentation as BaseA,
|
|
||||||
AugmentationContext as AC
|
|
||||||
} from '../augmentations/brainyAugmentation.js'
|
|
||||||
|
|
||||||
export type BrainyAugmentation = BA
|
|
||||||
export type BaseAugmentation = BaseA
|
|
||||||
export type AugmentationContext = AC
|
|
||||||
|
|
||||||
// REMOVED: Old augmentation type system for 2.0 clean architecture
|
|
||||||
// All augmentations now use the unified BrainyAugmentation interface from brainyAugmentation.ts
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @deprecated Use BrainyAugmentation from '../augmentations/brainyAugmentation.js' instead
|
|
||||||
*
|
|
||||||
* @example
|
|
||||||
* // ❌ Old way (v2.x):
|
|
||||||
* import { IAugmentation } from '@soulcraft/brainy/types/augmentations'
|
|
||||||
*
|
|
||||||
* // ✅ New way (v3.x):
|
|
||||||
* import { BrainyAugmentation } from '@soulcraft/brainy'
|
|
||||||
*/
|
|
||||||
export type IAugmentation = BrainyAugmentation
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @deprecated Augmentation types are now unified under BrainyAugmentation interface
|
|
||||||
*
|
|
||||||
* @example
|
|
||||||
* // ❌ Old way (v2.x):
|
|
||||||
* import { AugmentationType } from '@soulcraft/brainy/types/augmentations'
|
|
||||||
* if (augmentation.type === AugmentationType.SENSE) { ... }
|
|
||||||
*
|
|
||||||
* // ✅ New way (v3.x):
|
|
||||||
* import { BrainyAugmentation } from '@soulcraft/brainy'
|
|
||||||
* // Use the unified BrainyAugmentation interface directly
|
|
||||||
*/
|
|
||||||
export enum AugmentationType { SENSE = 'sense', CONDUIT = 'conduit', COGNITION = 'cognition', MEMORY = 'memory', PERCEPTION = 'perception', DIALOG = 'dialog', ACTIVATION = 'activation', WEBSOCKET = 'webSocket', SYNAPSE = 'synapse' }
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @deprecated Use BrainyAugmentation interface directly instead of namespace types
|
|
||||||
*
|
|
||||||
* @example
|
|
||||||
* // ❌ Old way (v2.x):
|
|
||||||
* import { BrainyAugmentations } from '@soulcraft/brainy/types/augmentations'
|
|
||||||
* class MyAugmentation implements BrainyAugmentations.ISenseAugmentation { ... }
|
|
||||||
*
|
|
||||||
* // ✅ New way (v3.x):
|
|
||||||
* import { BrainyAugmentation } from '@soulcraft/brainy'
|
|
||||||
* class MyAugmentation implements BrainyAugmentation { ... }
|
|
||||||
*/
|
|
||||||
export namespace BrainyAugmentations {
|
|
||||||
/** @deprecated Use BrainyAugmentation instead */
|
|
||||||
export type ISenseAugmentation = BrainyAugmentation
|
|
||||||
/** @deprecated Use BrainyAugmentation instead */
|
|
||||||
export type IConduitAugmentation = BrainyAugmentation
|
|
||||||
/** @deprecated Use BrainyAugmentation instead */
|
|
||||||
export type ICognitionAugmentation = BrainyAugmentation
|
|
||||||
/** @deprecated Use BrainyAugmentation instead */
|
|
||||||
export type IMemoryAugmentation = BrainyAugmentation
|
|
||||||
/** @deprecated Use BrainyAugmentation instead */
|
|
||||||
export type IPerceptionAugmentation = BrainyAugmentation
|
|
||||||
/** @deprecated Use BrainyAugmentation instead */
|
|
||||||
export type IDialogAugmentation = BrainyAugmentation
|
|
||||||
/** @deprecated Use BrainyAugmentation instead */
|
|
||||||
export type IActivationAugmentation = BrainyAugmentation
|
|
||||||
/** @deprecated Use BrainyAugmentation instead */
|
|
||||||
export type ISynapseAugmentation = BrainyAugmentation
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @deprecated Use BrainyAugmentation from '../augmentations/brainyAugmentation.js' instead
|
|
||||||
*
|
|
||||||
* @example
|
|
||||||
* // ❌ Old way (v2.x):
|
|
||||||
* import { ISenseAugmentation } from '@soulcraft/brainy/types/augmentations'
|
|
||||||
* class MySense implements ISenseAugmentation { ... }
|
|
||||||
*
|
|
||||||
* // ✅ New way (v3.x):
|
|
||||||
* import { BrainyAugmentation } from '@soulcraft/brainy'
|
|
||||||
* class MySense implements BrainyAugmentation { ... }
|
|
||||||
*/
|
|
||||||
export type ISenseAugmentation = BrainyAugmentation
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @deprecated Use BrainyAugmentation from '../augmentations/brainyAugmentation.js' instead
|
|
||||||
*
|
|
||||||
* @example
|
|
||||||
* // ❌ Old way (v2.x):
|
|
||||||
* import { IConduitAugmentation } from '@soulcraft/brainy/types/augmentations'
|
|
||||||
*
|
|
||||||
* // ✅ New way (v3.x):
|
|
||||||
* import { BrainyAugmentation } from '@soulcraft/brainy'
|
|
||||||
*/
|
|
||||||
export type IConduitAugmentation = BrainyAugmentation
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @deprecated Use BrainyAugmentation from '../augmentations/brainyAugmentation.js' instead
|
|
||||||
*
|
|
||||||
* @example
|
|
||||||
* // ❌ Old way (v2.x):
|
|
||||||
* import { ICognitionAugmentation } from '@soulcraft/brainy/types/augmentations'
|
|
||||||
*
|
|
||||||
* // ✅ New way (v3.x):
|
|
||||||
* import { BrainyAugmentation } from '@soulcraft/brainy'
|
|
||||||
*/
|
|
||||||
export type ICognitionAugmentation = BrainyAugmentation
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @deprecated Use BrainyAugmentation from '../augmentations/brainyAugmentation.js' instead
|
|
||||||
*
|
|
||||||
* @example
|
|
||||||
* // ❌ Old way (v2.x):
|
|
||||||
* import { IMemoryAugmentation } from '@soulcraft/brainy/types/augmentations'
|
|
||||||
*
|
|
||||||
* // ✅ New way (v3.x):
|
|
||||||
* import { BrainyAugmentation } from '@soulcraft/brainy'
|
|
||||||
*/
|
|
||||||
export type IMemoryAugmentation = BrainyAugmentation
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @deprecated Use BrainyAugmentation from '../augmentations/brainyAugmentation.js' instead
|
|
||||||
*
|
|
||||||
* @example
|
|
||||||
* // ❌ Old way (v2.x):
|
|
||||||
* import { IPerceptionAugmentation } from '@soulcraft/brainy/types/augmentations'
|
|
||||||
*
|
|
||||||
* // ✅ New way (v3.x):
|
|
||||||
* import { BrainyAugmentation } from '@soulcraft/brainy'
|
|
||||||
*/
|
|
||||||
export type IPerceptionAugmentation = BrainyAugmentation
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @deprecated Use BrainyAugmentation from '../augmentations/brainyAugmentation.js' instead
|
|
||||||
*
|
|
||||||
* @example
|
|
||||||
* // ❌ Old way (v2.x):
|
|
||||||
* import { IDialogAugmentation } from '@soulcraft/brainy/types/augmentations'
|
|
||||||
*
|
|
||||||
* // ✅ New way (v3.x):
|
|
||||||
* import { BrainyAugmentation } from '@soulcraft/brainy'
|
|
||||||
*/
|
|
||||||
export type IDialogAugmentation = BrainyAugmentation
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @deprecated Use BrainyAugmentation from '../augmentations/brainyAugmentation.js' instead
|
|
||||||
*
|
|
||||||
* @example
|
|
||||||
* // ❌ Old way (v2.x):
|
|
||||||
* import { IActivationAugmentation } from '@soulcraft/brainy/types/augmentations'
|
|
||||||
*
|
|
||||||
* // ✅ New way (v3.x):
|
|
||||||
* import { BrainyAugmentation } from '@soulcraft/brainy'
|
|
||||||
*/
|
|
||||||
export type IActivationAugmentation = BrainyAugmentation
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @deprecated Use BrainyAugmentation from '../augmentations/brainyAugmentation.js' instead
|
|
||||||
*
|
|
||||||
* @example
|
|
||||||
* // ❌ Old way (v2.x):
|
|
||||||
* import { ISynapseAugmentation } from '@soulcraft/brainy/types/augmentations'
|
|
||||||
*
|
|
||||||
* // ✅ New way (v3.x):
|
|
||||||
* import { BrainyAugmentation } from '@soulcraft/brainy'
|
|
||||||
*/
|
|
||||||
export type ISynapseAugmentation = BrainyAugmentation
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @deprecated WebSocket support is now built into BrainyAugmentation interface
|
|
||||||
*
|
|
||||||
* @example
|
|
||||||
* // ❌ Old way (v2.x):
|
|
||||||
* import { IWebSocketSupport } from '@soulcraft/brainy/types/augmentations'
|
|
||||||
* class MyAugmentation implements IWebSocketSupport { ... }
|
|
||||||
*
|
|
||||||
* // ✅ New way (v3.x):
|
|
||||||
* import { BrainyAugmentation } from '@soulcraft/brainy'
|
|
||||||
* class MyAugmentation implements BrainyAugmentation {
|
|
||||||
* // WebSocket functionality is now part of the unified interface
|
|
||||||
* }
|
|
||||||
*/
|
|
||||||
export interface IWebSocketSupport {}
|
|
||||||
|
|
@ -693,9 +693,6 @@ export interface BrainyConfig {
|
||||||
ttl?: number
|
ttl?: number
|
||||||
}
|
}
|
||||||
|
|
||||||
// Augmentations
|
|
||||||
augmentations?: Record<string, any>
|
|
||||||
|
|
||||||
// Distributed configuration
|
// Distributed configuration
|
||||||
distributed?: {
|
distributed?: {
|
||||||
enabled: boolean
|
enabled: boolean
|
||||||
|
|
|
||||||
|
|
@ -1,33 +0,0 @@
|
||||||
/**
|
|
||||||
* Pipeline Types
|
|
||||||
*
|
|
||||||
* This module provides shared types for the pipeline system to avoid circular dependencies.
|
|
||||||
*/
|
|
||||||
|
|
||||||
import {
|
|
||||||
BrainyAugmentations,
|
|
||||||
IWebSocketSupport,
|
|
||||||
IAugmentation
|
|
||||||
} from './augmentations.js'
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Type definitions for the augmentation registry
|
|
||||||
*/
|
|
||||||
export type AugmentationRegistry = {
|
|
||||||
sense: BrainyAugmentations.ISenseAugmentation[];
|
|
||||||
conduit: BrainyAugmentations.IConduitAugmentation[];
|
|
||||||
cognition: BrainyAugmentations.ICognitionAugmentation[];
|
|
||||||
memory: BrainyAugmentations.IMemoryAugmentation[];
|
|
||||||
perception: BrainyAugmentations.IPerceptionAugmentation[];
|
|
||||||
dialog: BrainyAugmentations.IDialogAugmentation[];
|
|
||||||
activation: BrainyAugmentations.IActivationAugmentation[];
|
|
||||||
webSocket: IWebSocketSupport[];
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Interface for the Pipeline class
|
|
||||||
* This is used to break circular dependencies between pipeline.ts and augmentationRegistry.ts
|
|
||||||
*/
|
|
||||||
export interface IPipeline {
|
|
||||||
register<T extends IAugmentation>(augmentation: T): IPipeline;
|
|
||||||
}
|
|
||||||
|
|
@ -1,8 +1,8 @@
|
||||||
/**
|
/**
|
||||||
* BrainyTypes - Complete type management for Brainy
|
* BrainyTypes - Type validation and lookup for Brainy
|
||||||
*
|
*
|
||||||
* Provides type lists, validation, and intelligent suggestions
|
* Provides type lists and validation for the 42 noun types and 127 verb types.
|
||||||
* for nouns and verbs using semantic embeddings.
|
* Source of truth: graphTypes.ts
|
||||||
*
|
*
|
||||||
* @example
|
* @example
|
||||||
* ```typescript
|
* ```typescript
|
||||||
|
|
@ -15,50 +15,20 @@
|
||||||
* // Validate types
|
* // Validate types
|
||||||
* BrainyTypes.isValidNoun('Person') // true
|
* BrainyTypes.isValidNoun('Person') // true
|
||||||
* BrainyTypes.isValidVerb('Unknown') // false
|
* BrainyTypes.isValidVerb('Unknown') // false
|
||||||
*
|
|
||||||
* // Get intelligent suggestions
|
|
||||||
* const personData = {
|
|
||||||
* name: 'John Doe',
|
|
||||||
* email: 'john@example.com'
|
|
||||||
* }
|
|
||||||
* const suggestion = await BrainyTypes.suggestNoun(personData)
|
|
||||||
* console.log(suggestion.type) // 'Person'
|
|
||||||
* console.log(suggestion.confidence) // 0.92
|
|
||||||
* ```
|
* ```
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { NounType, VerbType } from '../types/graphTypes.js'
|
import { NounType, VerbType } from '../types/graphTypes.js'
|
||||||
import { BrainyTypes as InternalBrainyTypes, TypeMatchResult } from '../augmentations/typeMatching/brainyTypes.js'
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Type suggestion result
|
* BrainyTypes - Type validation and lookup for Brainy
|
||||||
*/
|
|
||||||
export interface TypeSuggestion {
|
|
||||||
/** The suggested type */
|
|
||||||
type: NounType | VerbType
|
|
||||||
/** Confidence score between 0 and 1 */
|
|
||||||
confidence: number
|
|
||||||
/** Human-readable explanation */
|
|
||||||
reason?: string
|
|
||||||
/** Alternative suggestions */
|
|
||||||
alternatives?: Array<{
|
|
||||||
type: NounType | VerbType
|
|
||||||
confidence: number
|
|
||||||
}>
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* BrainyTypes - Complete type management for Brainy
|
|
||||||
*
|
*
|
||||||
* Static class providing type lists, validation, and intelligent suggestions.
|
* Static class providing type lists and validation.
|
||||||
* No instantiation needed - all methods are static.
|
* No instantiation needed - all methods are static.
|
||||||
*/
|
*/
|
||||||
export class BrainyTypes {
|
export class BrainyTypes {
|
||||||
private static instance: InternalBrainyTypes | null = null
|
|
||||||
private static initialized = false
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* All available noun types
|
* All available noun types (42)
|
||||||
* @example
|
* @example
|
||||||
* ```typescript
|
* ```typescript
|
||||||
* BrainyTypes.nouns.forEach(type => console.log(type))
|
* BrainyTypes.nouns.forEach(type => console.log(type))
|
||||||
|
|
@ -68,7 +38,7 @@ export class BrainyTypes {
|
||||||
static readonly nouns: readonly NounType[] = Object.freeze(Object.values(NounType))
|
static readonly nouns: readonly NounType[] = Object.freeze(Object.values(NounType))
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* All available verb types
|
* All available verb types (127)
|
||||||
* @example
|
* @example
|
||||||
* ```typescript
|
* ```typescript
|
||||||
* BrainyTypes.verbs.forEach(type => console.log(type))
|
* BrainyTypes.verbs.forEach(type => console.log(type))
|
||||||
|
|
@ -77,18 +47,6 @@ export class BrainyTypes {
|
||||||
*/
|
*/
|
||||||
static readonly verbs: readonly VerbType[] = Object.freeze(Object.values(VerbType))
|
static readonly verbs: readonly VerbType[] = Object.freeze(Object.values(VerbType))
|
||||||
|
|
||||||
/**
|
|
||||||
* Get or create the internal matcher instance
|
|
||||||
*/
|
|
||||||
private static async getInternalMatcher(): Promise<InternalBrainyTypes> {
|
|
||||||
if (!this.instance) {
|
|
||||||
this.instance = new InternalBrainyTypes()
|
|
||||||
await this.instance.init()
|
|
||||||
this.initialized = true
|
|
||||||
}
|
|
||||||
return this.instance
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Check if a string is a valid noun type
|
* Check if a string is a valid noun type
|
||||||
*
|
*
|
||||||
|
|
@ -123,85 +81,6 @@ export class BrainyTypes {
|
||||||
return (this.verbs as readonly string[]).includes(type)
|
return (this.verbs as readonly string[]).includes(type)
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Suggest the most appropriate noun type for an object
|
|
||||||
*
|
|
||||||
* @param data The object or data to analyze
|
|
||||||
* @returns Promise resolving to type suggestion with confidence score
|
|
||||||
*
|
|
||||||
* @example
|
|
||||||
* ```typescript
|
|
||||||
* const data = {
|
|
||||||
* title: 'Quarterly Report',
|
|
||||||
* author: 'Jane Smith',
|
|
||||||
* pages: 42
|
|
||||||
* }
|
|
||||||
* const suggestion = await BrainyTypes.suggestNoun(data)
|
|
||||||
* console.log(suggestion.type) // 'Document'
|
|
||||||
* console.log(suggestion.confidence) // 0.88
|
|
||||||
*
|
|
||||||
* // Check alternatives if confidence is low
|
|
||||||
* if (suggestion.confidence < 0.8) {
|
|
||||||
* console.log('Also consider:', suggestion.alternatives)
|
|
||||||
* }
|
|
||||||
* ```
|
|
||||||
*/
|
|
||||||
static async suggestNoun(data: any): Promise<TypeSuggestion> {
|
|
||||||
const matcher = await this.getInternalMatcher()
|
|
||||||
const result = await matcher.matchNounType(data)
|
|
||||||
|
|
||||||
return {
|
|
||||||
type: result.type as NounType,
|
|
||||||
confidence: result.confidence,
|
|
||||||
reason: result.reasoning,
|
|
||||||
alternatives: result.alternatives?.map(alt => ({
|
|
||||||
type: alt.type as NounType,
|
|
||||||
confidence: alt.confidence
|
|
||||||
}))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Suggest the most appropriate verb type for a relationship
|
|
||||||
*
|
|
||||||
* @param source The source entity
|
|
||||||
* @param target The target entity
|
|
||||||
* @param hint Optional hint about the relationship
|
|
||||||
* @returns Promise resolving to type suggestion with confidence score
|
|
||||||
*
|
|
||||||
* @example
|
|
||||||
* ```typescript
|
|
||||||
* const source = { type: 'Person', name: 'Alice' }
|
|
||||||
* const target = { type: 'Document', title: 'Research Paper' }
|
|
||||||
*
|
|
||||||
* const suggestion = await BrainyTypes.suggestVerb(source, target, 'authored')
|
|
||||||
* console.log(suggestion.type) // 'CreatedBy'
|
|
||||||
* console.log(suggestion.confidence) // 0.91
|
|
||||||
*
|
|
||||||
* // Without hint
|
|
||||||
* const suggestion2 = await BrainyTypes.suggestVerb(source, target)
|
|
||||||
* console.log(suggestion2.type) // 'RelatedTo' (more generic)
|
|
||||||
* ```
|
|
||||||
*/
|
|
||||||
static async suggestVerb(
|
|
||||||
source: any,
|
|
||||||
target: any,
|
|
||||||
hint?: string
|
|
||||||
): Promise<TypeSuggestion> {
|
|
||||||
const matcher = await this.getInternalMatcher()
|
|
||||||
const result = await matcher.matchVerbType(source, target, hint)
|
|
||||||
|
|
||||||
return {
|
|
||||||
type: result.type as VerbType,
|
|
||||||
confidence: result.confidence,
|
|
||||||
reason: result.reasoning,
|
|
||||||
alternatives: result.alternatives?.map(alt => ({
|
|
||||||
type: alt.type as VerbType,
|
|
||||||
confidence: alt.confidence
|
|
||||||
}))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get a noun type by name (with validation)
|
* Get a noun type by name (with validation)
|
||||||
*
|
*
|
||||||
|
|
@ -242,26 +121,6 @@ export class BrainyTypes {
|
||||||
return name as VerbType
|
return name as VerbType
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Clear the internal cache
|
|
||||||
* Useful when processing many different types of data
|
|
||||||
*/
|
|
||||||
static clearCache(): void {
|
|
||||||
this.instance?.clearCache()
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Dispose of resources
|
|
||||||
* Call when completely done using BrainyTypes
|
|
||||||
*/
|
|
||||||
static async dispose(): Promise<void> {
|
|
||||||
if (this.instance) {
|
|
||||||
await this.instance.dispose()
|
|
||||||
this.instance = null
|
|
||||||
this.initialized = false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get noun types as a plain object (for iteration)
|
* Get noun types as a plain object (for iteration)
|
||||||
* @returns Object with noun type names as keys
|
* @returns Object with noun type names as keys
|
||||||
|
|
@ -289,38 +148,3 @@ export class BrainyTypes {
|
||||||
|
|
||||||
// Re-export the enums for convenience
|
// Re-export the enums for convenience
|
||||||
export { NounType, VerbType }
|
export { NounType, VerbType }
|
||||||
|
|
||||||
/**
|
|
||||||
* Helper function to validate and suggest types in one call
|
|
||||||
*
|
|
||||||
* @example
|
|
||||||
* ```typescript
|
|
||||||
* import { suggestType } from '@soulcraft/brainy'
|
|
||||||
*
|
|
||||||
* // For nouns
|
|
||||||
* const nounSuggestion = await suggestType('noun', data)
|
|
||||||
*
|
|
||||||
* // For verbs
|
|
||||||
* const verbSuggestion = await suggestType('verb', source, target)
|
|
||||||
* ```
|
|
||||||
*/
|
|
||||||
export async function suggestType(
|
|
||||||
kind: 'noun',
|
|
||||||
data: any
|
|
||||||
): Promise<TypeSuggestion>
|
|
||||||
export async function suggestType(
|
|
||||||
kind: 'verb',
|
|
||||||
source: any,
|
|
||||||
target: any,
|
|
||||||
hint?: string
|
|
||||||
): Promise<TypeSuggestion>
|
|
||||||
export async function suggestType(
|
|
||||||
kind: 'noun' | 'verb',
|
|
||||||
...args: any[]
|
|
||||||
): Promise<TypeSuggestion> {
|
|
||||||
if (kind === 'noun') {
|
|
||||||
return BrainyTypes.suggestNoun(args[0])
|
|
||||||
} else {
|
|
||||||
return BrainyTypes.suggestVerb(args[0], args[1], args[2])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,281 +0,0 @@
|
||||||
/**
|
|
||||||
* CSV Handler Tests
|
|
||||||
* Comprehensive tests for CSV parsing, encoding detection, and type inference
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { describe, it, expect, beforeAll } from 'vitest'
|
|
||||||
import { CSVHandler } from '../../../src/augmentations/intelligentImport/handlers/csvHandler.js'
|
|
||||||
import { promises as fs } from 'fs'
|
|
||||||
import * as path from 'path'
|
|
||||||
|
|
||||||
describe('CSVHandler', () => {
|
|
||||||
let handler: CSVHandler
|
|
||||||
const fixturesPath = path.join(process.cwd(), 'tests/fixtures/import')
|
|
||||||
|
|
||||||
beforeAll(() => {
|
|
||||||
handler = new CSVHandler()
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('canHandle', () => {
|
|
||||||
it('should handle .csv extension', () => {
|
|
||||||
expect(handler.canHandle({ filename: 'data.csv' })).toBe(true)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should handle .tsv extension', () => {
|
|
||||||
expect(handler.canHandle({ filename: 'data.tsv' })).toBe(true)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should handle .txt extension', () => {
|
|
||||||
expect(handler.canHandle({ filename: 'data.txt' })).toBe(true)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should handle CSV content (comma)', () => {
|
|
||||||
const content = 'name,age,email\nJohn,30,john@example.com'
|
|
||||||
expect(handler.canHandle(content)).toBe(true)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should handle CSV content (semicolon)', () => {
|
|
||||||
const content = 'name;age;email\nJohn;30;john@example.com'
|
|
||||||
expect(handler.canHandle(content)).toBe(true)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should not handle non-CSV content', () => {
|
|
||||||
const content = 'This is just plain text without any structure'
|
|
||||||
expect(handler.canHandle(content)).toBe(false)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should not handle .xlsx extension', () => {
|
|
||||||
expect(handler.canHandle({ filename: 'data.xlsx' })).toBe(false)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('process - simple CSV', () => {
|
|
||||||
it('should parse simple comma-delimited CSV', async () => {
|
|
||||||
const filePath = path.join(fixturesPath, 'simple.csv')
|
|
||||||
const buffer = await fs.readFile(filePath)
|
|
||||||
|
|
||||||
const result = await handler.process(buffer, { filename: 'simple.csv' })
|
|
||||||
|
|
||||||
expect(result.format).toBe('csv')
|
|
||||||
expect(result.data).toHaveLength(4)
|
|
||||||
expect(result.data[0]).toEqual({
|
|
||||||
name: 'John Doe',
|
|
||||||
age: 30,
|
|
||||||
email: 'john@example.com',
|
|
||||||
active: true
|
|
||||||
})
|
|
||||||
expect(result.metadata.rowCount).toBe(4)
|
|
||||||
expect(result.metadata.fields).toEqual(['name', 'age', 'email', 'active'])
|
|
||||||
expect(result.metadata.delimiter).toBe(',')
|
|
||||||
expect(result.metadata.hasHeaders).toBe(true)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should infer correct types', async () => {
|
|
||||||
const filePath = path.join(fixturesPath, 'simple.csv')
|
|
||||||
const buffer = await fs.readFile(filePath)
|
|
||||||
|
|
||||||
const result = await handler.process(buffer, { filename: 'simple.csv' })
|
|
||||||
|
|
||||||
expect(typeof result.data[0].name).toBe('string')
|
|
||||||
expect(typeof result.data[0].age).toBe('number')
|
|
||||||
expect(typeof result.data[0].active).toBe('boolean')
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('process - delimiter detection', () => {
|
|
||||||
it('should detect semicolon delimiter', async () => {
|
|
||||||
const filePath = path.join(fixturesPath, 'semicolon.csv')
|
|
||||||
const buffer = await fs.readFile(filePath)
|
|
||||||
|
|
||||||
const result = await handler.process(buffer, { filename: 'semicolon.csv' })
|
|
||||||
|
|
||||||
expect(result.metadata.delimiter).toBe(';')
|
|
||||||
expect(result.data).toHaveLength(4)
|
|
||||||
expect(result.data[0]).toEqual({
|
|
||||||
product: 'Laptop',
|
|
||||||
price: 999.99,
|
|
||||||
quantity: 10,
|
|
||||||
inStock: true
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should detect tab delimiter', async () => {
|
|
||||||
const filePath = path.join(fixturesPath, 'tab-delimited.csv')
|
|
||||||
const buffer = await fs.readFile(filePath)
|
|
||||||
|
|
||||||
const result = await handler.process(buffer, { filename: 'tab-delimited.csv' })
|
|
||||||
|
|
||||||
expect(result.metadata.delimiter).toBe('\t')
|
|
||||||
expect(result.data).toHaveLength(4)
|
|
||||||
expect(result.data[0].city).toBe('Tokyo')
|
|
||||||
expect(result.data[0].population).toBe(13960000)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should use specified delimiter when provided', async () => {
|
|
||||||
const content = 'name|age|email\nJohn|30|john@example.com'
|
|
||||||
|
|
||||||
const result = await handler.process(content, {
|
|
||||||
csvDelimiter: '|',
|
|
||||||
filename: 'pipe.csv'
|
|
||||||
})
|
|
||||||
|
|
||||||
expect(result.data).toHaveLength(1)
|
|
||||||
expect(result.data[0].name).toBe('John')
|
|
||||||
expect(result.data[0].age).toBe(30)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('process - type inference', () => {
|
|
||||||
it('should infer integer, float, date, and boolean types', async () => {
|
|
||||||
const filePath = path.join(fixturesPath, 'types.csv')
|
|
||||||
const buffer = await fs.readFile(filePath)
|
|
||||||
|
|
||||||
const result = await handler.process(buffer, { filename: 'types.csv' })
|
|
||||||
|
|
||||||
expect(result.metadata.types).toBeDefined()
|
|
||||||
expect(result.metadata.types.id).toBe('integer')
|
|
||||||
expect(result.metadata.types.name).toBe('string')
|
|
||||||
// Score is 95 (integer), not 95.0, so it's detected as integer
|
|
||||||
expect(['integer', 'float']).toContain(result.metadata.types.score)
|
|
||||||
expect(result.metadata.types.active).toBe('boolean')
|
|
||||||
|
|
||||||
// Check actual values
|
|
||||||
expect(result.data[0].id).toBe(1)
|
|
||||||
expect([95, 95.0]).toContain(result.data[0].score) // Can be integer or float
|
|
||||||
expect(result.data[0].active).toBe(true)
|
|
||||||
expect(result.data[1].score).toBe(87.5) // This one is definitely a float
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should handle null values', async () => {
|
|
||||||
const content = 'name,age,email\nJohn,30,john@example.com\nJane,,jane@example.com\nBob,45,'
|
|
||||||
|
|
||||||
const result = await handler.process(content, { filename: 'nulls.csv' })
|
|
||||||
|
|
||||||
expect(result.data[1].age).toBeNull()
|
|
||||||
expect(result.data[2].email).toBeNull()
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('process - encoding detection', () => {
|
|
||||||
it('should detect UTF-8 encoding', async () => {
|
|
||||||
const content = 'name,description\nProduct 1,Description with émojis 🎉'
|
|
||||||
|
|
||||||
const result = await handler.process(content, { filename: 'utf8.csv' })
|
|
||||||
|
|
||||||
// Encoding can be 'UTF-8', 'utf-8', or 'utf8' depending on detection
|
|
||||||
expect(result.metadata.encoding.toLowerCase()).toContain('utf')
|
|
||||||
expect(result.data[0].description).toContain('émojis')
|
|
||||||
expect(result.data[0].description).toContain('🎉')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should use specified encoding when provided', async () => {
|
|
||||||
const content = 'name,age\nJohn,30'
|
|
||||||
|
|
||||||
const result = await handler.process(content, {
|
|
||||||
encoding: 'ascii',
|
|
||||||
filename: 'ascii.csv'
|
|
||||||
})
|
|
||||||
|
|
||||||
expect(result.metadata.encoding).toBe('ascii')
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('process - options', () => {
|
|
||||||
it('should respect csvHeaders=false option', async () => {
|
|
||||||
const content = 'John,30,john@example.com\nJane,25,jane@example.com'
|
|
||||||
|
|
||||||
const result = await handler.process(content, {
|
|
||||||
csvHeaders: false,
|
|
||||||
filename: 'no-headers.csv'
|
|
||||||
})
|
|
||||||
|
|
||||||
expect(result.metadata.hasHeaders).toBe(false)
|
|
||||||
// csv-parse will generate column names like '0', '1', '2'
|
|
||||||
expect(Object.keys(result.data[0])).toHaveLength(3)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should respect maxRows option', async () => {
|
|
||||||
const filePath = path.join(fixturesPath, 'simple.csv')
|
|
||||||
const buffer = await fs.readFile(filePath)
|
|
||||||
|
|
||||||
const result = await handler.process(buffer, {
|
|
||||||
maxRows: 2,
|
|
||||||
filename: 'simple.csv'
|
|
||||||
})
|
|
||||||
|
|
||||||
expect(result.data).toHaveLength(2)
|
|
||||||
expect(result.metadata.rowCount).toBe(2)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('process - edge cases', () => {
|
|
||||||
it('should handle empty CSV', async () => {
|
|
||||||
const content = ''
|
|
||||||
|
|
||||||
const result = await handler.process(content, { filename: 'empty.csv' })
|
|
||||||
|
|
||||||
expect(result.data).toHaveLength(0)
|
|
||||||
expect(result.metadata.rowCount).toBe(0)
|
|
||||||
expect(result.metadata.fields).toEqual([])
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should handle CSV with only headers', async () => {
|
|
||||||
const content = 'name,age,email'
|
|
||||||
|
|
||||||
const result = await handler.process(content, { filename: 'headers-only.csv' })
|
|
||||||
|
|
||||||
expect(result.data).toHaveLength(0)
|
|
||||||
expect(result.metadata.rowCount).toBe(0)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should handle CSV with single row', async () => {
|
|
||||||
const content = 'name,age,email\nJohn,30,john@example.com'
|
|
||||||
|
|
||||||
const result = await handler.process(content, { filename: 'single-row.csv' })
|
|
||||||
|
|
||||||
expect(result.data).toHaveLength(1)
|
|
||||||
expect(result.data[0].name).toBe('John')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should handle CSV with varying column counts', async () => {
|
|
||||||
const content = 'name,age,email\nJohn,30,john@example.com\nJane,25\nBob,45,bob@example.com,extra'
|
|
||||||
|
|
||||||
const result = await handler.process(content, { filename: 'varying.csv' })
|
|
||||||
|
|
||||||
// csv-parse with relax_column_count should handle this
|
|
||||||
expect(result.data).toHaveLength(3)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should trim whitespace', async () => {
|
|
||||||
const content = 'name, age , email \n John , 30 , john@example.com '
|
|
||||||
|
|
||||||
const result = await handler.process(content, { filename: 'whitespace.csv' })
|
|
||||||
|
|
||||||
expect(result.data[0].name).toBe('John')
|
|
||||||
expect(result.data[0].age).toBe(30)
|
|
||||||
expect(result.data[0].email).toBe('john@example.com')
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('process - performance', () => {
|
|
||||||
it('should measure processing time', async () => {
|
|
||||||
const filePath = path.join(fixturesPath, 'simple.csv')
|
|
||||||
const buffer = await fs.readFile(filePath)
|
|
||||||
|
|
||||||
const result = await handler.process(buffer, { filename: 'simple.csv' })
|
|
||||||
|
|
||||||
expect(result.metadata.processingTime).toBeGreaterThanOrEqual(0)
|
|
||||||
expect(typeof result.metadata.processingTime).toBe('number')
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('process - error handling', () => {
|
|
||||||
it('should throw error for malformed CSV', async () => {
|
|
||||||
const content = 'name,age,email\nJohn,30,"unclosed quote'
|
|
||||||
|
|
||||||
await expect(
|
|
||||||
handler.process(content, { filename: 'malformed.csv' })
|
|
||||||
).rejects.toThrow('CSV parsing failed')
|
|
||||||
})
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
@ -1,269 +0,0 @@
|
||||||
/**
|
|
||||||
* Excel Handler Tests
|
|
||||||
* Comprehensive tests for Excel parsing, multi-sheet extraction, and type inference
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { describe, it, expect, beforeAll } from 'vitest'
|
|
||||||
import { ExcelHandler } from '../../../src/augmentations/intelligentImport/handlers/excelHandler.js'
|
|
||||||
import { promises as fs } from 'fs'
|
|
||||||
import * as path from 'path'
|
|
||||||
|
|
||||||
describe('ExcelHandler', () => {
|
|
||||||
let handler: ExcelHandler
|
|
||||||
const fixturesPath = path.join(process.cwd(), 'tests/fixtures/import')
|
|
||||||
|
|
||||||
beforeAll(() => {
|
|
||||||
handler = new ExcelHandler()
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('canHandle', () => {
|
|
||||||
it('should handle .xlsx extension', () => {
|
|
||||||
expect(handler.canHandle({ filename: 'data.xlsx' })).toBe(true)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should handle .xls extension', () => {
|
|
||||||
expect(handler.canHandle({ filename: 'data.xls' })).toBe(true)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should handle .xlsb extension', () => {
|
|
||||||
expect(handler.canHandle({ filename: 'data.xlsb' })).toBe(true)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should handle .xlsm extension', () => {
|
|
||||||
expect(handler.canHandle({ filename: 'data.xlsm' })).toBe(true)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should not handle .csv extension', () => {
|
|
||||||
expect(handler.canHandle({ filename: 'data.csv' })).toBe(false)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should not handle .pdf extension', () => {
|
|
||||||
expect(handler.canHandle({ filename: 'data.pdf' })).toBe(false)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('process - simple Excel', () => {
|
|
||||||
it('should parse simple single-sheet Excel file', async () => {
|
|
||||||
const filePath = path.join(fixturesPath, 'simple.xlsx')
|
|
||||||
const buffer = await fs.readFile(filePath)
|
|
||||||
|
|
||||||
const result = await handler.process(buffer, { filename: 'simple.xlsx' })
|
|
||||||
|
|
||||||
expect(result.format).toBe('excel')
|
|
||||||
expect(result.data).toHaveLength(4)
|
|
||||||
expect(result.data[0]).toMatchObject({
|
|
||||||
Name: 'Alice Johnson',
|
|
||||||
Age: 28,
|
|
||||||
Department: 'Engineering',
|
|
||||||
Salary: 95000,
|
|
||||||
Active: true
|
|
||||||
})
|
|
||||||
expect(result.metadata.rowCount).toBe(4)
|
|
||||||
expect(result.metadata.sheetCount).toBe(1)
|
|
||||||
expect(result.metadata.sheets).toEqual(['Employees'])
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should infer correct types from Excel data', async () => {
|
|
||||||
const filePath = path.join(fixturesPath, 'simple.xlsx')
|
|
||||||
const buffer = await fs.readFile(filePath)
|
|
||||||
|
|
||||||
const result = await handler.process(buffer, { filename: 'simple.xlsx' })
|
|
||||||
|
|
||||||
expect(result.metadata.types).toBeDefined()
|
|
||||||
expect(result.metadata.types.Name).toBe('string')
|
|
||||||
expect(result.metadata.types.Age).toBe('integer')
|
|
||||||
expect(['integer', 'float']).toContain(result.metadata.types.Salary)
|
|
||||||
expect(result.metadata.types.Active).toBe('boolean')
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('process - multi-sheet Excel', () => {
|
|
||||||
it('should extract data from all sheets by default', async () => {
|
|
||||||
const filePath = path.join(fixturesPath, 'multi-sheet.xlsx')
|
|
||||||
const buffer = await fs.readFile(filePath)
|
|
||||||
|
|
||||||
const result = await handler.process(buffer, { filename: 'multi-sheet.xlsx' })
|
|
||||||
|
|
||||||
expect(result.metadata.sheetCount).toBe(3)
|
|
||||||
expect(result.metadata.sheets).toEqual(['Products', 'Orders', 'Customers'])
|
|
||||||
|
|
||||||
// Total rows from all sheets
|
|
||||||
const totalRows = 4 + 3 + 3 // Products + Orders + Customers
|
|
||||||
expect(result.data).toHaveLength(totalRows)
|
|
||||||
|
|
||||||
// Check _sheet field is added
|
|
||||||
expect(result.data[0]._sheet).toBe('Products')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should filter data by sheet name', async () => {
|
|
||||||
const filePath = path.join(fixturesPath, 'multi-sheet.xlsx')
|
|
||||||
const buffer = await fs.readFile(filePath)
|
|
||||||
|
|
||||||
const result = await handler.process(buffer, {
|
|
||||||
filename: 'multi-sheet.xlsx',
|
|
||||||
excelSheets: ['Products']
|
|
||||||
})
|
|
||||||
|
|
||||||
expect(result.metadata.sheetCount).toBe(1)
|
|
||||||
expect(result.metadata.sheets).toEqual(['Products'])
|
|
||||||
expect(result.data).toHaveLength(4)
|
|
||||||
expect(result.data.every(row => row._sheet === 'Products')).toBe(true)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should extract data from multiple specified sheets', async () => {
|
|
||||||
const filePath = path.join(fixturesPath, 'multi-sheet.xlsx')
|
|
||||||
const buffer = await fs.readFile(filePath)
|
|
||||||
|
|
||||||
const result = await handler.process(buffer, {
|
|
||||||
filename: 'multi-sheet.xlsx',
|
|
||||||
excelSheets: ['Products', 'Customers']
|
|
||||||
})
|
|
||||||
|
|
||||||
expect(result.metadata.sheetCount).toBe(2)
|
|
||||||
expect(result.metadata.sheets).toEqual(['Products', 'Customers'])
|
|
||||||
expect(result.data).toHaveLength(7) // 4 + 3
|
|
||||||
|
|
||||||
const sheets = new Set(result.data.map(row => row._sheet))
|
|
||||||
expect(sheets.has('Products')).toBe(true)
|
|
||||||
expect(sheets.has('Customers')).toBe(true)
|
|
||||||
expect(sheets.has('Orders')).toBe(false)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should include sheet metadata for each sheet', async () => {
|
|
||||||
const filePath = path.join(fixturesPath, 'multi-sheet.xlsx')
|
|
||||||
const buffer = await fs.readFile(filePath)
|
|
||||||
|
|
||||||
const result = await handler.process(buffer, { filename: 'multi-sheet.xlsx' })
|
|
||||||
|
|
||||||
expect(result.metadata.sheetMetadata).toBeDefined()
|
|
||||||
expect(result.metadata.sheetMetadata.Products).toMatchObject({
|
|
||||||
rowCount: 4,
|
|
||||||
columnCount: 5
|
|
||||||
})
|
|
||||||
expect(result.metadata.sheetMetadata.Products.headers).toEqual([
|
|
||||||
'ID', 'Product', 'Price', 'Stock', 'Category'
|
|
||||||
])
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('process - type inference', () => {
|
|
||||||
it('should infer integer, float, date, and boolean types', async () => {
|
|
||||||
const filePath = path.join(fixturesPath, 'types.xlsx')
|
|
||||||
const buffer = await fs.readFile(filePath)
|
|
||||||
|
|
||||||
const result = await handler.process(buffer, { filename: 'types.xlsx' })
|
|
||||||
|
|
||||||
expect(result.metadata.types).toBeDefined()
|
|
||||||
expect(result.metadata.types.ID).toBe('integer')
|
|
||||||
expect(result.metadata.types.Name).toBe('string')
|
|
||||||
expect(['integer', 'float']).toContain(result.metadata.types.Score)
|
|
||||||
expect(['integer', 'float']).toContain(result.metadata.types.Percentage)
|
|
||||||
expect(result.metadata.types.Active).toBe('boolean')
|
|
||||||
|
|
||||||
// Check actual values
|
|
||||||
expect(result.data[0].ID).toBe(1)
|
|
||||||
expect(result.data[0].Active).toBe(true)
|
|
||||||
expect(typeof result.data[0].Score).toBe('number')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should handle null/empty values', async () => {
|
|
||||||
const filePath = path.join(fixturesPath, 'empty.xlsx')
|
|
||||||
const buffer = await fs.readFile(filePath)
|
|
||||||
|
|
||||||
const result = await handler.process(buffer, { filename: 'empty.xlsx' })
|
|
||||||
|
|
||||||
expect(result.data).toHaveLength(0)
|
|
||||||
expect(result.metadata.rowCount).toBe(0)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('process - workbook metadata', () => {
|
|
||||||
it('should extract workbook information', async () => {
|
|
||||||
const filePath = path.join(fixturesPath, 'multi-sheet.xlsx')
|
|
||||||
const buffer = await fs.readFile(filePath)
|
|
||||||
|
|
||||||
const result = await handler.process(buffer, { filename: 'multi-sheet.xlsx' })
|
|
||||||
|
|
||||||
expect(result.metadata.workbookInfo).toBeDefined()
|
|
||||||
expect(result.metadata.workbookInfo.sheetNames).toEqual([
|
|
||||||
'Products', 'Orders', 'Customers'
|
|
||||||
])
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('process - edge cases', () => {
|
|
||||||
it('should handle Excel file with only headers', async () => {
|
|
||||||
const filePath = path.join(fixturesPath, 'empty.xlsx')
|
|
||||||
const buffer = await fs.readFile(filePath)
|
|
||||||
|
|
||||||
const result = await handler.process(buffer, { filename: 'empty.xlsx' })
|
|
||||||
|
|
||||||
expect(result.data).toHaveLength(0)
|
|
||||||
expect(result.metadata.sheetCount).toBe(1)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should skip non-existent sheets when filtering', async () => {
|
|
||||||
const filePath = path.join(fixturesPath, 'simple.xlsx')
|
|
||||||
const buffer = await fs.readFile(filePath)
|
|
||||||
|
|
||||||
const result = await handler.process(buffer, {
|
|
||||||
filename: 'simple.xlsx',
|
|
||||||
excelSheets: ['Employees', 'NonExistent']
|
|
||||||
})
|
|
||||||
|
|
||||||
expect(result.metadata.sheets).toEqual(['Employees'])
|
|
||||||
expect(result.data).toHaveLength(4)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('process - performance', () => {
|
|
||||||
it('should measure processing time', async () => {
|
|
||||||
const filePath = path.join(fixturesPath, 'simple.xlsx')
|
|
||||||
const buffer = await fs.readFile(filePath)
|
|
||||||
|
|
||||||
const result = await handler.process(buffer, { filename: 'simple.xlsx' })
|
|
||||||
|
|
||||||
expect(result.metadata.processingTime).toBeGreaterThanOrEqual(0)
|
|
||||||
expect(typeof result.metadata.processingTime).toBe('number')
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('process - field sanitization', () => {
|
|
||||||
it('should sanitize column names', async () => {
|
|
||||||
const filePath = path.join(fixturesPath, 'simple.xlsx')
|
|
||||||
const buffer = await fs.readFile(filePath)
|
|
||||||
|
|
||||||
const result = await handler.process(buffer, { filename: 'simple.xlsx' })
|
|
||||||
|
|
||||||
// All field names should be valid identifiers
|
|
||||||
const fields = Object.keys(result.data[0]).filter(k => k !== '_sheet')
|
|
||||||
for (const field of fields) {
|
|
||||||
expect(field).toMatch(/^[a-zA-Z_][a-zA-Z0-9_]*$/)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('process - error handling', () => {
|
|
||||||
it('should handle corrupted Excel file gracefully', async () => {
|
|
||||||
// XLSX library is very forgiving, so we test that it returns empty data
|
|
||||||
// rather than crashing
|
|
||||||
const invalidData = Buffer.from('This is not an Excel file')
|
|
||||||
|
|
||||||
const result = await handler.process(invalidData, { filename: 'invalid.xlsx' })
|
|
||||||
|
|
||||||
// Should return empty data instead of crashing
|
|
||||||
expect(result.data).toHaveLength(0)
|
|
||||||
expect(result.metadata.rowCount).toBe(0)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should handle binary garbage', async () => {
|
|
||||||
const garbage = Buffer.from(new Uint8Array(256).fill(255))
|
|
||||||
|
|
||||||
const result = await handler.process(garbage, { filename: 'garbage.xlsx' })
|
|
||||||
|
|
||||||
// Should not crash
|
|
||||||
expect(result).toBeDefined()
|
|
||||||
expect(result.format).toBe('excel')
|
|
||||||
})
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
@ -1,292 +0,0 @@
|
||||||
/**
|
|
||||||
* PDF Handler Tests
|
|
||||||
* Comprehensive tests for PDF text extraction, table detection, and metadata
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { describe, it, expect, beforeAll } from 'vitest'
|
|
||||||
import { PDFHandler } from '../../../src/augmentations/intelligentImport/handlers/pdfHandler.js'
|
|
||||||
import { promises as fs } from 'fs'
|
|
||||||
import * as path from 'path'
|
|
||||||
|
|
||||||
describe('PDFHandler', () => {
|
|
||||||
let handler: PDFHandler
|
|
||||||
const fixturesPath = path.join(process.cwd(), 'tests/fixtures/import')
|
|
||||||
|
|
||||||
beforeAll(() => {
|
|
||||||
handler = new PDFHandler()
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('canHandle', () => {
|
|
||||||
it('should handle .pdf extension', () => {
|
|
||||||
expect(handler.canHandle({ filename: 'document.pdf' })).toBe(true)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should handle PDF magic bytes', () => {
|
|
||||||
const pdfBuffer = Buffer.from('%PDF-1.4\n')
|
|
||||||
expect(handler.canHandle(pdfBuffer)).toBe(true)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should not handle .xlsx extension', () => {
|
|
||||||
expect(handler.canHandle({ filename: 'data.xlsx' })).toBe(false)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should not handle .csv extension', () => {
|
|
||||||
expect(handler.canHandle({ filename: 'data.csv' })).toBe(false)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should not handle non-PDF buffer', () => {
|
|
||||||
const buffer = Buffer.from('This is not a PDF')
|
|
||||||
expect(handler.canHandle(buffer)).toBe(false)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('process - simple text PDF', () => {
|
|
||||||
it('should extract text from simple PDF', async () => {
|
|
||||||
const filePath = path.join(fixturesPath, 'simple.pdf')
|
|
||||||
const buffer = await fs.readFile(filePath)
|
|
||||||
|
|
||||||
const result = await handler.process(buffer, { filename: 'simple.pdf' })
|
|
||||||
|
|
||||||
expect(result.format).toBe('pdf')
|
|
||||||
expect(result.data.length).toBeGreaterThan(0)
|
|
||||||
|
|
||||||
// Should have extracted paragraphs
|
|
||||||
const paragraphs = result.data.filter(item => item._type === 'paragraph')
|
|
||||||
expect(paragraphs.length).toBeGreaterThan(0)
|
|
||||||
|
|
||||||
// Check that text was extracted
|
|
||||||
const hasText = paragraphs.some(p => p.text && p.text.length > 0)
|
|
||||||
expect(hasText).toBe(true)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should include page numbers', async () => {
|
|
||||||
const filePath = path.join(fixturesPath, 'simple.pdf')
|
|
||||||
const buffer = await fs.readFile(filePath)
|
|
||||||
|
|
||||||
const result = await handler.process(buffer, { filename: 'simple.pdf' })
|
|
||||||
|
|
||||||
// All items should have _page field
|
|
||||||
expect(result.data.every(item => typeof item._page === 'number')).toBe(true)
|
|
||||||
expect(result.data.every(item => item._page >= 1)).toBe(true)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should count pages correctly', async () => {
|
|
||||||
const filePath = path.join(fixturesPath, 'simple.pdf')
|
|
||||||
const buffer = await fs.readFile(filePath)
|
|
||||||
|
|
||||||
const result = await handler.process(buffer, { filename: 'simple.pdf' })
|
|
||||||
|
|
||||||
expect(result.metadata.pageCount).toBe(1)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('process - multi-page PDF', () => {
|
|
||||||
it('should extract text from all pages', async () => {
|
|
||||||
const filePath = path.join(fixturesPath, 'multi-page.pdf')
|
|
||||||
const buffer = await fs.readFile(filePath)
|
|
||||||
|
|
||||||
const result = await handler.process(buffer, { filename: 'multi-page.pdf' })
|
|
||||||
|
|
||||||
expect(result.metadata.pageCount).toBe(3)
|
|
||||||
|
|
||||||
// Should have content from multiple pages
|
|
||||||
const pages = new Set(result.data.map(item => item._page))
|
|
||||||
expect(pages.size).toBeGreaterThan(1)
|
|
||||||
expect(pages.has(1)).toBe(true)
|
|
||||||
expect(pages.has(2)).toBe(true)
|
|
||||||
expect(pages.has(3)).toBe(true)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should preserve page order', async () => {
|
|
||||||
const filePath = path.join(fixturesPath, 'multi-page.pdf')
|
|
||||||
const buffer = await fs.readFile(filePath)
|
|
||||||
|
|
||||||
const result = await handler.process(buffer, { filename: 'multi-page.pdf' })
|
|
||||||
|
|
||||||
// Page numbers should be in order
|
|
||||||
let lastPage = 0
|
|
||||||
let pageChanged = false
|
|
||||||
|
|
||||||
for (const item of result.data) {
|
|
||||||
if (item._page !== lastPage && lastPage > 0) {
|
|
||||||
pageChanged = true
|
|
||||||
}
|
|
||||||
expect(item._page).toBeGreaterThanOrEqual(lastPage)
|
|
||||||
lastPage = item._page
|
|
||||||
}
|
|
||||||
|
|
||||||
expect(pageChanged).toBe(true) // Should have moved through pages
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('process - table detection', () => {
|
|
||||||
it('should detect tables in PDF', async () => {
|
|
||||||
const filePath = path.join(fixturesPath, 'table.pdf')
|
|
||||||
const buffer = await fs.readFile(filePath)
|
|
||||||
|
|
||||||
const result = await handler.process(buffer, { filename: 'table.pdf' })
|
|
||||||
|
|
||||||
// Should have detected at least one table
|
|
||||||
expect(result.metadata.tableCount).toBeGreaterThan(0)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should extract table rows when tables detected', async () => {
|
|
||||||
const filePath = path.join(fixturesPath, 'table.pdf')
|
|
||||||
const buffer = await fs.readFile(filePath)
|
|
||||||
|
|
||||||
const result = await handler.process(buffer, {
|
|
||||||
filename: 'table.pdf',
|
|
||||||
pdfExtractTables: true
|
|
||||||
})
|
|
||||||
|
|
||||||
// Should have table_row items
|
|
||||||
const tableRows = result.data.filter(item => item._type === 'table_row')
|
|
||||||
expect(tableRows.length).toBeGreaterThan(0)
|
|
||||||
|
|
||||||
// Table rows should have structured data
|
|
||||||
if (tableRows.length > 0) {
|
|
||||||
const firstRow = tableRows[0]
|
|
||||||
const fields = Object.keys(firstRow).filter(k => !k.startsWith('_'))
|
|
||||||
expect(fields.length).toBeGreaterThan(0)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should skip table extraction when disabled', async () => {
|
|
||||||
const filePath = path.join(fixturesPath, 'table.pdf')
|
|
||||||
const buffer = await fs.readFile(filePath)
|
|
||||||
|
|
||||||
const result = await handler.process(buffer, {
|
|
||||||
filename: 'table.pdf',
|
|
||||||
pdfExtractTables: false
|
|
||||||
})
|
|
||||||
|
|
||||||
// Should not have detected tables
|
|
||||||
expect(result.metadata.tableCount).toBe(0)
|
|
||||||
|
|
||||||
// Should not have table_row items
|
|
||||||
const tableRows = result.data.filter(item => item._type === 'table_row')
|
|
||||||
expect(tableRows.length).toBe(0)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('process - metadata extraction', () => {
|
|
||||||
it('should extract PDF metadata', async () => {
|
|
||||||
const filePath = path.join(fixturesPath, 'metadata.pdf')
|
|
||||||
const buffer = await fs.readFile(filePath)
|
|
||||||
|
|
||||||
const result = await handler.process(buffer, { filename: 'metadata.pdf' })
|
|
||||||
|
|
||||||
expect(result.metadata.pdfMetadata).toBeDefined()
|
|
||||||
expect(result.metadata.pdfMetadata.title).toBe('Test Document')
|
|
||||||
expect(result.metadata.pdfMetadata.author).toBe('Test Author')
|
|
||||||
expect(result.metadata.pdfMetadata.creator).toBe('Brainy Test Suite')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should handle PDFs without metadata gracefully', async () => {
|
|
||||||
const filePath = path.join(fixturesPath, 'simple.pdf')
|
|
||||||
const buffer = await fs.readFile(filePath)
|
|
||||||
|
|
||||||
const result = await handler.process(buffer, { filename: 'simple.pdf' })
|
|
||||||
|
|
||||||
expect(result.metadata.pdfMetadata).toBeDefined()
|
|
||||||
// Some fields may be null
|
|
||||||
expect(result.metadata.pdfMetadata).toHaveProperty('title')
|
|
||||||
expect(result.metadata.pdfMetadata).toHaveProperty('author')
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('process - text statistics', () => {
|
|
||||||
it('should track total text length', async () => {
|
|
||||||
const filePath = path.join(fixturesPath, 'simple.pdf')
|
|
||||||
const buffer = await fs.readFile(filePath)
|
|
||||||
|
|
||||||
const result = await handler.process(buffer, { filename: 'simple.pdf' })
|
|
||||||
|
|
||||||
expect(result.metadata.textLength).toBeGreaterThan(0)
|
|
||||||
expect(typeof result.metadata.textLength).toBe('number')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should count extracted items', async () => {
|
|
||||||
const filePath = path.join(fixturesPath, 'simple.pdf')
|
|
||||||
const buffer = await fs.readFile(filePath)
|
|
||||||
|
|
||||||
const result = await handler.process(buffer, { filename: 'simple.pdf' })
|
|
||||||
|
|
||||||
expect(result.metadata.rowCount).toBe(result.data.length)
|
|
||||||
expect(result.metadata.rowCount).toBeGreaterThan(0)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('process - edge cases', () => {
|
|
||||||
it('should handle empty PDF', async () => {
|
|
||||||
const filePath = path.join(fixturesPath, 'empty.pdf')
|
|
||||||
const buffer = await fs.readFile(filePath)
|
|
||||||
|
|
||||||
const result = await handler.process(buffer, { filename: 'empty.pdf' })
|
|
||||||
|
|
||||||
expect(result.format).toBe('pdf')
|
|
||||||
expect(result.metadata.pageCount).toBe(1)
|
|
||||||
// Empty PDF may have 0 or minimal content
|
|
||||||
expect(result.data).toBeDefined()
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should measure processing time', async () => {
|
|
||||||
const filePath = path.join(fixturesPath, 'simple.pdf')
|
|
||||||
const buffer = await fs.readFile(filePath)
|
|
||||||
|
|
||||||
const result = await handler.process(buffer, { filename: 'simple.pdf' })
|
|
||||||
|
|
||||||
expect(result.metadata.processingTime).toBeGreaterThan(0)
|
|
||||||
expect(typeof result.metadata.processingTime).toBe('number')
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('process - data structure', () => {
|
|
||||||
it('should include type indicators', async () => {
|
|
||||||
const filePath = path.join(fixturesPath, 'simple.pdf')
|
|
||||||
const buffer = await fs.readFile(filePath)
|
|
||||||
|
|
||||||
const result = await handler.process(buffer, { filename: 'simple.pdf' })
|
|
||||||
|
|
||||||
// All items should have _type field
|
|
||||||
expect(result.data.every(item => '_type' in item)).toBe(true)
|
|
||||||
|
|
||||||
// Types should be valid
|
|
||||||
const validTypes = ['paragraph', 'table_row']
|
|
||||||
expect(result.data.every(item => validTypes.includes(item._type))).toBe(true)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should include index for paragraphs', async () => {
|
|
||||||
const filePath = path.join(fixturesPath, 'simple.pdf')
|
|
||||||
const buffer = await fs.readFile(filePath)
|
|
||||||
|
|
||||||
const result = await handler.process(buffer, { filename: 'simple.pdf' })
|
|
||||||
|
|
||||||
const paragraphs = result.data.filter(item => item._type === 'paragraph')
|
|
||||||
|
|
||||||
// Paragraphs should have _index field
|
|
||||||
expect(paragraphs.every(p => typeof p._index === 'number')).toBe(true)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('process - error handling', () => {
|
|
||||||
it('should throw error for invalid PDF', async () => {
|
|
||||||
const invalidData = Buffer.from('This is not a PDF file')
|
|
||||||
|
|
||||||
await expect(
|
|
||||||
handler.process(invalidData, { filename: 'invalid.pdf' })
|
|
||||||
).rejects.toThrow('PDF parsing failed')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should throw error for corrupted PDF', async () => {
|
|
||||||
const corruptedPDF = Buffer.concat([
|
|
||||||
Buffer.from('%PDF-1.4\n'),
|
|
||||||
Buffer.from('corrupted data that is not valid PDF structure')
|
|
||||||
])
|
|
||||||
|
|
||||||
await expect(
|
|
||||||
handler.process(corruptedPDF, { filename: 'corrupted.pdf' })
|
|
||||||
).rejects.toThrow('PDF parsing failed')
|
|
||||||
})
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
@ -60,7 +60,6 @@ async function runBrainyBenchmark() {
|
||||||
|
|
||||||
const brain = new Brainy({
|
const brain = new Brainy({
|
||||||
storage: { type: 'memory' },
|
storage: { type: 'memory' },
|
||||||
augmentations: {},
|
|
||||||
model: {
|
model: {
|
||||||
type: 'fast', // Using real transformer model
|
type: 'fast', // Using real transformer model
|
||||||
precision: 'Q8' // Quantized for speed
|
precision: 'Q8' // Quantized for speed
|
||||||
|
|
|
||||||
|
|
@ -158,12 +158,6 @@ async function runBrainyBenchmark() {
|
||||||
|
|
||||||
const brain = new Brainy({
|
const brain = new Brainy({
|
||||||
storage: { type: 'memory' },
|
storage: { type: 'memory' },
|
||||||
augmentations: {
|
|
||||||
cache: false,
|
|
||||||
metrics: false,
|
|
||||||
display: false,
|
|
||||||
index: false
|
|
||||||
},
|
|
||||||
embedder: mockEmbedder,
|
embedder: mockEmbedder,
|
||||||
warmup: false
|
warmup: false
|
||||||
})
|
})
|
||||||
|
|
|
||||||
|
|
@ -14,15 +14,8 @@ async function runBenchmark() {
|
||||||
console.log('🧠 Brainy v3 Performance Benchmark')
|
console.log('🧠 Brainy v3 Performance Benchmark')
|
||||||
console.log('═'.repeat(60))
|
console.log('═'.repeat(60))
|
||||||
|
|
||||||
// Disable all augmentations for raw performance
|
|
||||||
const brain = new Brainy({
|
const brain = new Brainy({
|
||||||
storage: { type: 'memory' },
|
storage: { type: 'memory' },
|
||||||
augmentations: {
|
|
||||||
cache: false,
|
|
||||||
metrics: false,
|
|
||||||
display: false,
|
|
||||||
index: false
|
|
||||||
},
|
|
||||||
embedder: mockEmbedder,
|
embedder: mockEmbedder,
|
||||||
warmup: false
|
warmup: false
|
||||||
})
|
})
|
||||||
|
|
|
||||||
|
|
@ -16,7 +16,6 @@ async function benchmark() {
|
||||||
|
|
||||||
const brain = new Brainy({
|
const brain = new Brainy({
|
||||||
storage: { type: 'memory' },
|
storage: { type: 'memory' },
|
||||||
augmentations: {},
|
|
||||||
embedder: mockEmbedder
|
embedder: mockEmbedder
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -22,7 +22,7 @@ async function runV2Benchmark() {
|
||||||
const brain = new Brainy({
|
const brain = new Brainy({
|
||||||
storage: { type: 'memory' },
|
storage: { type: 'memory' },
|
||||||
embeddingFunction: mockEmbedder,
|
embeddingFunction: mockEmbedder,
|
||||||
augmentations: false // Disable for raw performance
|
// Raw performance test
|
||||||
})
|
})
|
||||||
|
|
||||||
await brain.init()
|
await brain.init()
|
||||||
|
|
@ -111,7 +111,6 @@ async function runV3Benchmark() {
|
||||||
|
|
||||||
const brain = new Brainy({
|
const brain = new Brainy({
|
||||||
storage: { type: 'memory' },
|
storage: { type: 'memory' },
|
||||||
augmentations: {}, // Minimal augmentations
|
|
||||||
embedder: mockEmbedder
|
embedder: mockEmbedder
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
@ -216,7 +215,6 @@ async function runScaleTest() {
|
||||||
|
|
||||||
const brain = new Brainy({
|
const brain = new Brainy({
|
||||||
storage: { type: 'memory' },
|
storage: { type: 'memory' },
|
||||||
augmentations: {},
|
|
||||||
embedder: mockEmbedder
|
embedder: mockEmbedder
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -77,30 +77,21 @@ async function profilePerformance() {
|
||||||
// Initialize Brainy with different configurations
|
// Initialize Brainy with different configurations
|
||||||
console.log('Initializing Brainy configurations...')
|
console.log('Initializing Brainy configurations...')
|
||||||
|
|
||||||
// 1. Minimal config (no augmentations)
|
// 1. Minimal config
|
||||||
const minimalBrain = new Brainy({
|
const minimalBrain = new Brainy({
|
||||||
storage: new MemoryStorage(),
|
storage: new MemoryStorage()
|
||||||
augmentations: false // Disable all augmentations
|
|
||||||
})
|
})
|
||||||
await minimalBrain.init()
|
await minimalBrain.init()
|
||||||
|
|
||||||
// 2. Default config (with augmentations)
|
// 2. Default config
|
||||||
const defaultBrain = new Brainy({
|
const defaultBrain = new Brainy({
|
||||||
storage: new MemoryStorage()
|
storage: new MemoryStorage()
|
||||||
})
|
})
|
||||||
await defaultBrain.init()
|
await defaultBrain.init()
|
||||||
|
|
||||||
// 3. Full augmentations config
|
// 3. Full config
|
||||||
const fullBrain = new Brainy({
|
const fullBrain = new Brainy({
|
||||||
storage: new MemoryStorage(),
|
storage: new MemoryStorage()
|
||||||
augmentations: {
|
|
||||||
batchProcessing: { enabled: true, maxBatchSize: 1000 },
|
|
||||||
connectionPool: { enabled: true },
|
|
||||||
cache: { enabled: true },
|
|
||||||
index: { enabled: true },
|
|
||||||
entityRegistry: { enabled: true },
|
|
||||||
monitoring: { enabled: true }
|
|
||||||
}
|
|
||||||
})
|
})
|
||||||
await fullBrain.init()
|
await fullBrain.init()
|
||||||
|
|
||||||
|
|
@ -188,11 +179,7 @@ async function profilePerformance() {
|
||||||
verbCounter++
|
verbCounter++
|
||||||
}, 100)
|
}, 100)
|
||||||
|
|
||||||
console.log('\n🔧 Testing Augmentation Overhead\n')
|
console.log('\n🔧 Testing Operation Overhead\n')
|
||||||
|
|
||||||
// Test individual augmentations
|
|
||||||
const augmentations = defaultBrain.augmentations.getAugmentationTypes()
|
|
||||||
console.log(`Active augmentations: ${augmentations.join(', ')}\n`)
|
|
||||||
|
|
||||||
// Measure raw storage performance
|
// Measure raw storage performance
|
||||||
const storage = new MemoryStorage()
|
const storage = new MemoryStorage()
|
||||||
|
|
@ -261,8 +248,8 @@ async function profilePerformance() {
|
||||||
|
|
||||||
console.log('Overhead breakdown:')
|
console.log('Overhead breakdown:')
|
||||||
console.log(` Base operation: ${minimalPerf.perOpMs.toFixed(2)}ms`)
|
console.log(` Base operation: ${minimalPerf.perOpMs.toFixed(2)}ms`)
|
||||||
console.log(` Default augmentations: +${(defaultPerf.perOpMs - minimalPerf.perOpMs).toFixed(2)}ms`)
|
console.log(` Default config: +${(defaultPerf.perOpMs - minimalPerf.perOpMs).toFixed(2)}ms`)
|
||||||
console.log(` Full augmentations: +${(fullPerf.perOpMs - defaultPerf.perOpMs).toFixed(2)}ms`)
|
console.log(` Full config: +${(fullPerf.perOpMs - defaultPerf.perOpMs).toFixed(2)}ms`)
|
||||||
console.log(` Embedding generation: ${embedPerf.perOpMs.toFixed(2)}ms`)
|
console.log(` Embedding generation: ${embedPerf.perOpMs.toFixed(2)}ms`)
|
||||||
console.log(` Without embeddings: ${vectorPerf.perOpMs.toFixed(2)}ms`)
|
console.log(` Without embeddings: ${vectorPerf.perOpMs.toFixed(2)}ms`)
|
||||||
|
|
||||||
|
|
@ -290,7 +277,7 @@ async function profilePerformance() {
|
||||||
console.log(' 1. Fake/stub operations that returned immediately')
|
console.log(' 1. Fake/stub operations that returned immediately')
|
||||||
console.log(' 2. No actual embedding generation')
|
console.log(' 2. No actual embedding generation')
|
||||||
console.log(' 3. No real storage operations')
|
console.log(' 3. No real storage operations')
|
||||||
console.log(' 4. No augmentation processing')
|
console.log(' 4. Direct operation calls')
|
||||||
console.log('\n With real implementations:')
|
console.log('\n With real implementations:')
|
||||||
console.log(` - Raw storage: ${profiler.results['Raw storage.saveNoun']?.opsPerSec || 'N/A'} ops/sec`)
|
console.log(` - Raw storage: ${profiler.results['Raw storage.saveNoun']?.opsPerSec || 'N/A'} ops/sec`)
|
||||||
console.log(` - With embeddings: ${defaultPerf.opsPerSec} ops/sec`)
|
console.log(` - With embeddings: ${defaultPerf.opsPerSec} ops/sec`)
|
||||||
|
|
|
||||||
|
|
@ -17,7 +17,6 @@ async function benchmarkV2() {
|
||||||
|
|
||||||
const brain = new Brainy({
|
const brain = new Brainy({
|
||||||
storage: { type: 'memory' },
|
storage: { type: 'memory' },
|
||||||
augmentations: false,
|
|
||||||
embeddingFunction: async () => new Array(384).fill(0).map(() => Math.random())
|
embeddingFunction: async () => new Array(384).fill(0).map(() => Math.random())
|
||||||
})
|
})
|
||||||
await brain.init()
|
await brain.init()
|
||||||
|
|
@ -92,7 +91,6 @@ async function benchmarkV3() {
|
||||||
|
|
||||||
const brain = new Brainy({
|
const brain = new Brainy({
|
||||||
storage: { type: 'memory' },
|
storage: { type: 'memory' },
|
||||||
augmentations: {},
|
|
||||||
warmup: false,
|
warmup: false,
|
||||||
embedder: async () => new Array(384).fill(0).map(() => Math.random())
|
embedder: async () => new Array(384).fill(0).map(() => Math.random())
|
||||||
})
|
})
|
||||||
|
|
|
||||||
|
|
@ -13,7 +13,6 @@ async function testV3() {
|
||||||
|
|
||||||
const brain = new Brainy({
|
const brain = new Brainy({
|
||||||
storage: { type: 'memory' },
|
storage: { type: 'memory' },
|
||||||
augmentations: {},
|
|
||||||
warmup: false,
|
warmup: false,
|
||||||
embedder: async () => new Array(384).fill(0).map(() => Math.random())
|
embedder: async () => new Array(384).fill(0).map(() => Math.random())
|
||||||
})
|
})
|
||||||
|
|
|
||||||
|
|
@ -37,7 +37,7 @@ async function quickPerf() {
|
||||||
const brain = new Brainy({
|
const brain = new Brainy({
|
||||||
storage: new MemoryStorage(),
|
storage: new MemoryStorage(),
|
||||||
embeddingFunction: mockEmbed,
|
embeddingFunction: mockEmbed,
|
||||||
augmentations: false // Disable augmentations
|
// Minimal config
|
||||||
})
|
})
|
||||||
await brain.init()
|
await brain.init()
|
||||||
|
|
||||||
|
|
@ -55,12 +55,11 @@ async function quickPerf() {
|
||||||
const brainyOps = Math.round(1000 / ((end2 - start2) / 1000))
|
const brainyOps = Math.round(1000 / ((end2 - start2) / 1000))
|
||||||
console.log(` ✅ Brainy: ${brainyOps.toLocaleString()} ops/sec\n`)
|
console.log(` ✅ Brainy: ${brainyOps.toLocaleString()} ops/sec\n`)
|
||||||
|
|
||||||
// Test 3: With augmentations
|
// Test 3: Default config
|
||||||
console.log('3️⃣ Brainy with Augmentations')
|
console.log('3️⃣ Brainy with Default Config')
|
||||||
const brain2 = new Brainy({
|
const brain2 = new Brainy({
|
||||||
storage: new MemoryStorage(),
|
storage: new MemoryStorage(),
|
||||||
embeddingFunction: mockEmbed
|
embeddingFunction: mockEmbed
|
||||||
// Default augmentations enabled
|
|
||||||
})
|
})
|
||||||
await brain2.init()
|
await brain2.init()
|
||||||
|
|
||||||
|
|
@ -74,13 +73,12 @@ async function quickPerf() {
|
||||||
}
|
}
|
||||||
const end3 = performance.now()
|
const end3 = performance.now()
|
||||||
const augOps = Math.round(1000 / ((end3 - start3) / 1000))
|
const augOps = Math.round(1000 / ((end3 - start3) / 1000))
|
||||||
console.log(` ✅ With Augmentations: ${augOps.toLocaleString()} ops/sec\n`)
|
console.log(` ✅ Default Config: ${augOps.toLocaleString()} ops/sec\n`)
|
||||||
|
|
||||||
// Test 4: Real embeddings (the killer)
|
// Test 4: Real embeddings (the killer)
|
||||||
console.log('4️⃣ With Real Embeddings (10 samples)')
|
console.log('4️⃣ With Real Embeddings (10 samples)')
|
||||||
const brain3 = new Brainy({
|
const brain3 = new Brainy({
|
||||||
storage: new MemoryStorage(),
|
storage: new MemoryStorage(),
|
||||||
augmentations: false
|
|
||||||
// Uses real embedding function
|
// Uses real embedding function
|
||||||
})
|
})
|
||||||
await brain3.init()
|
await brain3.init()
|
||||||
|
|
@ -119,15 +117,10 @@ async function quickPerf() {
|
||||||
console.log(' ❌ Embeddings are the primary bottleneck')
|
console.log(' ❌ Embeddings are the primary bottleneck')
|
||||||
console.log(' Each embedding takes ~' + Math.round((end4 - start4) / 10) + 'ms')
|
console.log(' Each embedding takes ~' + Math.round((end4 - start4) / 10) + 'ms')
|
||||||
}
|
}
|
||||||
if (augOverhead > 50) {
|
|
||||||
console.log(' ⚠️ Augmentations add significant overhead')
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log('\n🎯 The 500,000 ops/sec claim was achievable with:')
|
console.log('\n🎯 The 500,000 ops/sec claim was achievable with:')
|
||||||
console.log(' 1. Pre-computed vectors (no embedding)')
|
console.log(' 1. Pre-computed vectors (no embedding)')
|
||||||
console.log(' 2. Minimal augmentations')
|
console.log(' 2. In-memory storage')
|
||||||
console.log(' 3. In-memory storage')
|
console.log(' 3. Batch operations')
|
||||||
console.log(' 4. Batch operations')
|
|
||||||
|
|
||||||
await brain.close()
|
await brain.close()
|
||||||
await brain2.close()
|
await brain2.close()
|
||||||
|
|
|
||||||
|
|
@ -69,10 +69,6 @@ async function quickVerify() {
|
||||||
console.log(`❌ Verb storage: Expected 1 verb, got ${stats.totalVerbs}`)
|
console.log(`❌ Verb storage: Expected 1 verb, got ${stats.totalVerbs}`)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check augmentations
|
|
||||||
const augTypes = brain.augmentations.getAugmentationTypes()
|
|
||||||
console.log(`✅ Active augmentations: ${augTypes.join(', ')}`)
|
|
||||||
|
|
||||||
// Close
|
// Close
|
||||||
await brain.close()
|
await brain.close()
|
||||||
console.log('\n✅ All tests passed - implementations are REAL!')
|
console.log('\n✅ All tests passed - implementations are REAL!')
|
||||||
|
|
|
||||||
|
|
@ -33,34 +33,14 @@ console.log(`🔧 Mode: ${CURRENT_SCALE}\n`)
|
||||||
async function runScaleTest() {
|
async function runScaleTest() {
|
||||||
const startTime = Date.now()
|
const startTime = Date.now()
|
||||||
|
|
||||||
// Initialize Brainy with real augmentations
|
// Initialize Brainy
|
||||||
const brain = new Brainy({
|
const brain = new Brainy({
|
||||||
storage: new MemoryStorage(),
|
storage: new MemoryStorage()
|
||||||
augmentations: {
|
|
||||||
// These should all be REAL implementations now
|
|
||||||
batchProcessing: {
|
|
||||||
enabled: true,
|
|
||||||
maxBatchSize: BATCH_SIZE,
|
|
||||||
adaptiveBatching: true
|
|
||||||
},
|
|
||||||
connectionPool: {
|
|
||||||
enabled: true,
|
|
||||||
maxConnections: 50,
|
|
||||||
minConnections: 5
|
|
||||||
},
|
|
||||||
cache: {
|
|
||||||
enabled: true,
|
|
||||||
maxSize: 10000
|
|
||||||
},
|
|
||||||
index: {
|
|
||||||
enabled: true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
})
|
||||||
|
|
||||||
await brain.init()
|
await brain.init()
|
||||||
|
|
||||||
console.log('✅ Brainy initialized with real augmentations\n')
|
console.log('✅ Brainy initialized\n')
|
||||||
|
|
||||||
// Test 1: Batch Insert Performance
|
// Test 1: Batch Insert Performance
|
||||||
console.log('📝 Test 1: Batch Insert Performance')
|
console.log('📝 Test 1: Batch Insert Performance')
|
||||||
|
|
@ -160,35 +140,6 @@ async function runScaleTest() {
|
||||||
console.log(`✅ Created ${verbCount.toLocaleString()} relationships in ${verbTime}ms`)
|
console.log(`✅ Created ${verbCount.toLocaleString()} relationships in ${verbTime}ms`)
|
||||||
console.log(`📈 Rate: ${verbRate.toLocaleString()} relationships/second\n`)
|
console.log(`📈 Rate: ${verbRate.toLocaleString()} relationships/second\n`)
|
||||||
|
|
||||||
// Test 4: Verify Augmentations Are Real
|
|
||||||
console.log('🔧 Test 4: Verify Real Implementations')
|
|
||||||
|
|
||||||
// Check BatchProcessing stats
|
|
||||||
const batchAug = brain.augmentations.getAugmentation('BatchProcessing')
|
|
||||||
if (batchAug && typeof batchAug.getStats === 'function') {
|
|
||||||
const stats = batchAug.getStats()
|
|
||||||
console.log(` BatchProcessing: ${stats.batchesProcessed} batches processed`)
|
|
||||||
console.log(` Average batch size: ${Math.round(stats.averageBatchSize)}`)
|
|
||||||
console.log(` Throughput: ${stats.throughputPerSecond} ops/sec`)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check ConnectionPool stats
|
|
||||||
const poolAug = brain.augmentations.getAugmentation('ConnectionPool')
|
|
||||||
if (poolAug && typeof poolAug.getStats === 'function') {
|
|
||||||
const stats = poolAug.getStats()
|
|
||||||
console.log(` ConnectionPool: ${stats.totalConnections} connections`)
|
|
||||||
console.log(` Pool utilization: ${stats.poolUtilization}`)
|
|
||||||
console.log(` Total requests: ${stats.totalRequests}`)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check Cache stats
|
|
||||||
const cacheAug = brain.augmentations.getAugmentation('cache')
|
|
||||||
if (cacheAug && typeof cacheAug.getStats === 'function') {
|
|
||||||
const stats = cacheAug.getStats()
|
|
||||||
console.log(` Cache: ${stats.hits} hits, ${stats.misses} misses`)
|
|
||||||
console.log(` Hit rate: ${Math.round((stats.hits / (stats.hits + stats.misses)) * 100)}%`)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Final Statistics
|
// Final Statistics
|
||||||
const totalTime = Date.now() - startTime
|
const totalTime = Date.now() - startTime
|
||||||
const stats = brain.getStats()
|
const stats = brain.getStats()
|
||||||
|
|
|
||||||
|
|
@ -11,13 +11,7 @@ describe('Brainy 3.0 API', () => {
|
||||||
|
|
||||||
beforeEach(async () => {
|
beforeEach(async () => {
|
||||||
brain = new Brainy({
|
brain = new Brainy({
|
||||||
storage: { type: 'memory' },
|
storage: { type: 'memory' }
|
||||||
augmentations: {
|
|
||||||
cache: false,
|
|
||||||
metrics: false,
|
|
||||||
display: false,
|
|
||||||
monitoring: false
|
|
||||||
}
|
|
||||||
})
|
})
|
||||||
await brain.init()
|
await brain.init()
|
||||||
})
|
})
|
||||||
|
|
@ -426,7 +420,6 @@ describe('Brainy 3.0 Neural API', () => {
|
||||||
beforeEach(async () => {
|
beforeEach(async () => {
|
||||||
brain = new Brainy({
|
brain = new Brainy({
|
||||||
storage: { type: 'memory' },
|
storage: { type: 'memory' },
|
||||||
augmentations: { metrics: false, display: false }
|
|
||||||
})
|
})
|
||||||
await brain.init()
|
await brain.init()
|
||||||
})
|
})
|
||||||
|
|
@ -440,9 +433,4 @@ describe('Brainy 3.0 Neural API', () => {
|
||||||
expect(typeof brain.neural).toBe('object')
|
expect(typeof brain.neural).toBe('object')
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should provide augmentations API access', () => {
|
|
||||||
expect(brain.augmentations).toBeDefined()
|
|
||||||
expect(brain.augmentations.list).toBeDefined()
|
|
||||||
expect(typeof brain.augmentations.list).toBe('function')
|
|
||||||
})
|
|
||||||
})
|
})
|
||||||
|
|
@ -701,7 +701,6 @@ describe('Brainy Public API - Complete Coverage', () => {
|
||||||
expect(health.status).toBe('healthy')
|
expect(health.status).toBe('healthy')
|
||||||
expect(health.storage).toBeDefined()
|
expect(health.storage).toBeDefined()
|
||||||
expect(health.storage.status).toBe('connected')
|
expect(health.storage.status).toBe('connected')
|
||||||
expect(health.augmentations).toBeDefined()
|
|
||||||
expect(health.memory).toBeDefined()
|
expect(health.memory).toBeDefined()
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,361 +0,0 @@
|
||||||
/**
|
|
||||||
* Image Import Integration Test (v5.2.0)
|
|
||||||
*
|
|
||||||
* Tests that ImageHandler works as a built-in handler with IntelligentImportAugmentation
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
|
||||||
import { Brainy } from '../../src/brainy.js'
|
|
||||||
import sharp from 'sharp'
|
|
||||||
|
|
||||||
describe('Image Import Integration (v5.2.0)', () => {
|
|
||||||
let brain: Brainy
|
|
||||||
|
|
||||||
// Create test image
|
|
||||||
const createTestImage = async (width: number, height: number): Promise<Buffer> => {
|
|
||||||
const channels = 3
|
|
||||||
const pixelData = Buffer.alloc(width * height * channels)
|
|
||||||
|
|
||||||
// Fill with gradient
|
|
||||||
for (let y = 0; y < height; y++) {
|
|
||||||
for (let x = 0; x < width; x++) {
|
|
||||||
const i = (y * width + x) * channels
|
|
||||||
pixelData[i] = Math.floor((x / width) * 255)
|
|
||||||
pixelData[i + 1] = Math.floor((y / height) * 255)
|
|
||||||
pixelData[i + 2] = 128
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return sharp(pixelData, {
|
|
||||||
raw: { width, height, channels }
|
|
||||||
})
|
|
||||||
.jpeg({ quality: 90 })
|
|
||||||
.toBuffer()
|
|
||||||
}
|
|
||||||
|
|
||||||
beforeEach(async () => {
|
|
||||||
// IntelligentImportAugmentation is enabled by default with all handlers
|
|
||||||
// Configure it to only enable image handler for focused testing
|
|
||||||
brain = new Brainy({
|
|
||||||
silent: true,
|
|
||||||
augmentations: {
|
|
||||||
intelligentImport: {
|
|
||||||
enableCSV: false,
|
|
||||||
enableExcel: false,
|
|
||||||
enablePDF: false,
|
|
||||||
enableImage: true // Only enable image handler for this test
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
await brain.init()
|
|
||||||
})
|
|
||||||
|
|
||||||
afterEach(async () => {
|
|
||||||
await brain.close()
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('Built-in Image Handler', () => {
|
|
||||||
it('should automatically handle image import via IntelligentImportAugmentation', async () => {
|
|
||||||
const imageBuffer = await createTestImage(800, 600)
|
|
||||||
|
|
||||||
// Import image - should be automatically processed by ImageHandler
|
|
||||||
const result = await brain.import({
|
|
||||||
type: 'buffer',
|
|
||||||
data: imageBuffer,
|
|
||||||
filename: 'test-photo.jpg'
|
|
||||||
})
|
|
||||||
|
|
||||||
expect(result).toBeDefined()
|
|
||||||
expect(result.entities).toHaveLength(1)
|
|
||||||
|
|
||||||
const imageEntity = result.entities[0]
|
|
||||||
expect(imageEntity.type).toBe('media')
|
|
||||||
expect(imageEntity.metadata?.subtype).toBe('image')
|
|
||||||
expect(imageEntity.metadata).toBeDefined()
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should extract image metadata via built-in handler', async () => {
|
|
||||||
const imageBuffer = await createTestImage(1920, 1080)
|
|
||||||
|
|
||||||
const result = await brain.import({
|
|
||||||
type: 'buffer',
|
|
||||||
data: imageBuffer,
|
|
||||||
filename: 'hd-photo.jpg'
|
|
||||||
})
|
|
||||||
|
|
||||||
const imageEntity = result.entities[0]
|
|
||||||
expect(imageEntity).toBeDefined()
|
|
||||||
expect(imageEntity.metadata).toBeDefined()
|
|
||||||
expect(typeof imageEntity.metadata).toBe('object')
|
|
||||||
expect(imageEntity.metadata.width).toBe(1920)
|
|
||||||
expect(imageEntity.metadata.height).toBe(1080)
|
|
||||||
expect(imageEntity.metadata.format).toBe('jpeg')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should extract EXIF data when available', async () => {
|
|
||||||
const imageBuffer = await createTestImage(400, 300)
|
|
||||||
|
|
||||||
const result = await brain.import({
|
|
||||||
type: 'buffer',
|
|
||||||
data: imageBuffer,
|
|
||||||
filename: 'camera-photo.jpg',
|
|
||||||
options: {
|
|
||||||
extractEXIF: true
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
const imageEntity = result.entities[0]
|
|
||||||
// Test image won't have EXIF, but should not crash
|
|
||||||
expect(imageEntity).toBeDefined()
|
|
||||||
expect(imageEntity.type).toBe('media')
|
|
||||||
expect(imageEntity.metadata?.subtype).toBe('image')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should allow disabling EXIF extraction via config', async () => {
|
|
||||||
const imageBuffer = await createTestImage(400, 300)
|
|
||||||
|
|
||||||
const result = await brain.import({
|
|
||||||
type: 'buffer',
|
|
||||||
data: imageBuffer,
|
|
||||||
filename: 'no-exif.jpg',
|
|
||||||
options: {
|
|
||||||
extractEXIF: false
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
expect(result.entities).toHaveLength(1)
|
|
||||||
expect(result.entities[0].type).toBe('media')
|
|
||||||
expect(result.entities[0].metadata?.subtype).toBe('image')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should handle PNG images', async () => {
|
|
||||||
const pngBuffer = await sharp(Buffer.alloc(100 * 100 * 4), {
|
|
||||||
raw: { width: 100, height: 100, channels: 4 }
|
|
||||||
})
|
|
||||||
.png()
|
|
||||||
.toBuffer()
|
|
||||||
|
|
||||||
const result = await brain.import({
|
|
||||||
type: 'buffer',
|
|
||||||
data: pngBuffer,
|
|
||||||
filename: 'logo.png'
|
|
||||||
})
|
|
||||||
|
|
||||||
const imageEntity = result.entities[0]
|
|
||||||
expect(imageEntity.metadata.format).toBe('png')
|
|
||||||
expect(imageEntity.metadata.hasAlpha).toBe(true)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should handle WebP images', async () => {
|
|
||||||
const webpBuffer = await sharp(Buffer.alloc(200 * 200 * 3), {
|
|
||||||
raw: { width: 200, height: 200, channels: 3 }
|
|
||||||
})
|
|
||||||
.webp({ quality: 90 })
|
|
||||||
.toBuffer()
|
|
||||||
|
|
||||||
const result = await brain.import({
|
|
||||||
type: 'buffer',
|
|
||||||
data: webpBuffer,
|
|
||||||
filename: 'modern.webp'
|
|
||||||
})
|
|
||||||
|
|
||||||
const imageEntity = result.entities[0]
|
|
||||||
expect(imageEntity.metadata.format).toBe('webp')
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('Configuration', () => {
|
|
||||||
it('should allow disabling image handler', async () => {
|
|
||||||
// Create new brain with image handler disabled
|
|
||||||
const brainNoImage = new Brainy({
|
|
||||||
silent: true,
|
|
||||||
augmentations: {
|
|
||||||
intelligentImport: {
|
|
||||||
enableImage: false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
await brainNoImage.init()
|
|
||||||
|
|
||||||
const imageBuffer = await createTestImage(400, 300)
|
|
||||||
|
|
||||||
// Import should still work, but won't be processed by ImageHandler
|
|
||||||
const result = await brainNoImage.import({
|
|
||||||
type: 'buffer',
|
|
||||||
data: imageBuffer,
|
|
||||||
filename: 'unprocessed.jpg'
|
|
||||||
})
|
|
||||||
|
|
||||||
// Should create entity but not extract image metadata
|
|
||||||
expect(result).toBeDefined()
|
|
||||||
|
|
||||||
await brainNoImage.close()
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should apply imageDefaults config', async () => {
|
|
||||||
const brainWithDefaults = new Brainy({
|
|
||||||
silent: true,
|
|
||||||
augmentations: {
|
|
||||||
intelligentImport: {
|
|
||||||
enableImage: true,
|
|
||||||
imageDefaults: {
|
|
||||||
extractEXIF: false // Default to no EXIF extraction
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
await brainWithDefaults.init()
|
|
||||||
|
|
||||||
const imageBuffer = await createTestImage(400, 300)
|
|
||||||
|
|
||||||
const result = await brainWithDefaults.import({
|
|
||||||
type: 'buffer',
|
|
||||||
data: imageBuffer,
|
|
||||||
filename: 'default-config.jpg'
|
|
||||||
})
|
|
||||||
|
|
||||||
expect(result.entities[0].type).toBe('media')
|
|
||||||
expect(result.entities[0].metadata?.subtype).toBe('image')
|
|
||||||
|
|
||||||
await brainWithDefaults.close()
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('Image Format Detection', () => {
|
|
||||||
it('should detect JPEG by filename', async () => {
|
|
||||||
const imageBuffer = await createTestImage(400, 300)
|
|
||||||
|
|
||||||
const result = await brain.import({
|
|
||||||
type: 'buffer',
|
|
||||||
data: imageBuffer,
|
|
||||||
filename: 'photo.jpg'
|
|
||||||
})
|
|
||||||
|
|
||||||
expect(result.entities[0].metadata.format).toBe('jpeg')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should detect JPEG by .jpeg extension', async () => {
|
|
||||||
const imageBuffer = await createTestImage(400, 300)
|
|
||||||
|
|
||||||
const result = await brain.import({
|
|
||||||
type: 'buffer',
|
|
||||||
data: imageBuffer,
|
|
||||||
filename: 'photo.jpeg'
|
|
||||||
})
|
|
||||||
|
|
||||||
expect(result.entities[0].metadata.format).toBe('jpeg')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should handle various image sizes', async () => {
|
|
||||||
const sizes = [
|
|
||||||
[100, 100],
|
|
||||||
[800, 600],
|
|
||||||
[1920, 1080],
|
|
||||||
[4000, 3000]
|
|
||||||
]
|
|
||||||
|
|
||||||
for (const [width, height] of sizes) {
|
|
||||||
const imageBuffer = await createTestImage(width, height)
|
|
||||||
|
|
||||||
const result = await brain.import({
|
|
||||||
type: 'buffer',
|
|
||||||
data: imageBuffer,
|
|
||||||
filename: `image-${width}x${height}.jpg`
|
|
||||||
})
|
|
||||||
|
|
||||||
const imageEntity = result.entities[0]
|
|
||||||
expect(imageEntity.metadata.width).toBe(width)
|
|
||||||
expect(imageEntity.metadata.height).toBe(height)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('Error Handling', () => {
|
|
||||||
it('should handle invalid image data gracefully', async () => {
|
|
||||||
const invalidBuffer = Buffer.from('not an image')
|
|
||||||
|
|
||||||
// Brainy is resilient - invalid images still import with fallback minimal metadata
|
|
||||||
const result = await brain.import({
|
|
||||||
type: 'buffer',
|
|
||||||
data: invalidBuffer,
|
|
||||||
filename: 'invalid.jpg'
|
|
||||||
})
|
|
||||||
|
|
||||||
expect(result).toBeDefined()
|
|
||||||
expect(result.entities).toHaveLength(1)
|
|
||||||
expect(result.entities[0].type).toBe('media')
|
|
||||||
expect(result.entities[0].name).toBe('invalid.jpg')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should handle empty image buffer', async () => {
|
|
||||||
const emptyBuffer = Buffer.alloc(0)
|
|
||||||
|
|
||||||
// Brainy is resilient - empty buffers still import with fallback minimal metadata
|
|
||||||
const result = await brain.import({
|
|
||||||
type: 'buffer',
|
|
||||||
data: emptyBuffer,
|
|
||||||
filename: 'empty.jpg'
|
|
||||||
})
|
|
||||||
|
|
||||||
expect(result).toBeDefined()
|
|
||||||
expect(result.entities).toHaveLength(1)
|
|
||||||
expect(result.entities[0].type).toBe('media')
|
|
||||||
expect(result.entities[0].name).toBe('empty.jpg')
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('Integration with Knowledge Graph', () => {
|
|
||||||
it('should store image entities in knowledge graph', async () => {
|
|
||||||
const imageBuffer = await createTestImage(800, 600)
|
|
||||||
|
|
||||||
const result = await brain.import({
|
|
||||||
type: 'buffer',
|
|
||||||
data: imageBuffer,
|
|
||||||
filename: 'stored-image.jpg'
|
|
||||||
})
|
|
||||||
|
|
||||||
// Verify entity was created with metadata in import result
|
|
||||||
expect(result.entities).toHaveLength(1)
|
|
||||||
expect(result.entities[0].metadata.width).toBe(800)
|
|
||||||
expect(result.entities[0].metadata.height).toBe(600)
|
|
||||||
|
|
||||||
// Query for image entities - verifies it's in the knowledge graph
|
|
||||||
const images = await brain.find({ type: 'media' })
|
|
||||||
expect(images.length).toBeGreaterThan(0)
|
|
||||||
|
|
||||||
// Verify media entities have the expected type
|
|
||||||
const mediaEntities = images.filter(img => img.type === 'media')
|
|
||||||
expect(mediaEntities.length).toBeGreaterThan(0)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should support querying by image metadata', async () => {
|
|
||||||
// Import multiple images and capture metadata from results
|
|
||||||
const result1 = await brain.import({
|
|
||||||
type: 'buffer',
|
|
||||||
data: await createTestImage(1920, 1080),
|
|
||||||
filename: 'hd.jpg'
|
|
||||||
})
|
|
||||||
|
|
||||||
const result2 = await brain.import({
|
|
||||||
type: 'buffer',
|
|
||||||
data: await createTestImage(800, 600),
|
|
||||||
filename: 'sd.jpg'
|
|
||||||
})
|
|
||||||
|
|
||||||
// Verify metadata in import results
|
|
||||||
expect(result1.entities[0].metadata.width).toBe(1920)
|
|
||||||
expect(result1.entities[0].metadata.height).toBe(1080)
|
|
||||||
expect(result2.entities[0].metadata.width).toBe(800)
|
|
||||||
expect(result2.entities[0].metadata.height).toBe(600)
|
|
||||||
|
|
||||||
// Query all media entities - verifies they're in the knowledge graph
|
|
||||||
const allImages = await brain.find({ type: 'media' })
|
|
||||||
expect(allImages.length).toBeGreaterThan(0)
|
|
||||||
|
|
||||||
// Verify multiple media entities were stored
|
|
||||||
const mediaEntities = allImages.filter(img => img.type === 'media')
|
|
||||||
expect(mediaEntities.length).toBeGreaterThanOrEqual(2)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
@ -14,7 +14,6 @@ describe('Streaming Pipeline', () => {
|
||||||
beforeEach(async () => {
|
beforeEach(async () => {
|
||||||
brain = new Brainy({
|
brain = new Brainy({
|
||||||
storage: { type: 'memory' },
|
storage: { type: 'memory' },
|
||||||
augmentations: {},
|
|
||||||
warmup: false
|
warmup: false
|
||||||
})
|
})
|
||||||
await brain.init()
|
await brain.init()
|
||||||
|
|
|
||||||
|
|
@ -1,535 +0,0 @@
|
||||||
import { describe, it, expect, beforeEach } from 'vitest'
|
|
||||||
import { Brainy } from '../../../src/brainy'
|
|
||||||
import { createAddParams } from '../../helpers/test-factory'
|
|
||||||
import { NounType, VerbType } from '../../../src/types/graphTypes'
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Comprehensive test suite for Brainy's built-in augmentation system
|
|
||||||
* Tests the actual augmentation functionality that's available in production
|
|
||||||
*/
|
|
||||||
|
|
||||||
describe('Brainy Built-in Augmentations', () => {
|
|
||||||
let brain: Brainy<any>
|
|
||||||
|
|
||||||
beforeEach(async () => {
|
|
||||||
brain = new Brainy({
|
|
||||||
storage: { type: 'memory' },
|
|
||||||
augmentations: {
|
|
||||||
cache: { enabled: true, maxSize: 1000 },
|
|
||||||
display: { enabled: true },
|
|
||||||
metrics: { enabled: true }
|
|
||||||
}
|
|
||||||
})
|
|
||||||
await brain.init()
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('1. Augmentation Registry', () => {
|
|
||||||
it('should list all registered augmentations', async () => {
|
|
||||||
const augmentations = brain.augmentations.list()
|
|
||||||
expect(Array.isArray(augmentations)).toBe(true)
|
|
||||||
expect(augmentations.length).toBeGreaterThan(0)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should have default augmentations', async () => {
|
|
||||||
const augmentations = brain.augmentations.list()
|
|
||||||
expect(augmentations).toContain('cache')
|
|
||||||
expect(augmentations).toContain('display')
|
|
||||||
expect(augmentations).toContain('metrics')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should get augmentation by name', async () => {
|
|
||||||
const cacheAug = brain.augmentations.get('cache')
|
|
||||||
expect(cacheAug).toBeDefined()
|
|
||||||
expect(cacheAug?.name).toBe('cache')
|
|
||||||
|
|
||||||
const displayAug = brain.augmentations.get('display')
|
|
||||||
expect(displayAug).toBeDefined()
|
|
||||||
expect(displayAug?.name).toBe('display')
|
|
||||||
|
|
||||||
const metricsAug = brain.augmentations.get('metrics')
|
|
||||||
expect(metricsAug).toBeDefined()
|
|
||||||
expect(metricsAug?.name).toBe('metrics')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should check if augmentation exists', async () => {
|
|
||||||
expect(brain.augmentations.has('cache')).toBe(true)
|
|
||||||
expect(brain.augmentations.has('display')).toBe(true)
|
|
||||||
expect(brain.augmentations.has('metrics')).toBe(true)
|
|
||||||
expect(brain.augmentations.has('non-existent')).toBe(false)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should return undefined for non-existent augmentation', async () => {
|
|
||||||
const nonExistent = brain.augmentations.get('does-not-exist')
|
|
||||||
expect(nonExistent).toBeUndefined()
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('2. Cache Augmentation', () => {
|
|
||||||
it('should cache get operations', async () => {
|
|
||||||
const id = await brain.add(createAddParams({
|
|
||||||
data: 'cached entity',
|
|
||||||
metadata: { cached: true }
|
|
||||||
}))
|
|
||||||
|
|
||||||
// First get - should load from storage
|
|
||||||
const start1 = Date.now()
|
|
||||||
const entity1 = await brain.get(id)
|
|
||||||
const duration1 = Date.now() - start1
|
|
||||||
|
|
||||||
// Second get - should be cached (faster)
|
|
||||||
const start2 = Date.now()
|
|
||||||
const entity2 = await brain.get(id)
|
|
||||||
const duration2 = Date.now() - start2
|
|
||||||
|
|
||||||
// Compare key properties instead of full object equality
|
|
||||||
expect(entity1?.id).toBe(entity2?.id)
|
|
||||||
expect(entity1?.data).toBe(entity2?.data)
|
|
||||||
expect(entity1?.data).toBe('cached entity')
|
|
||||||
expect(entity1?.metadata?.cached).toBe(true)
|
|
||||||
|
|
||||||
// Cache should make second call faster (though this might not be reliable)
|
|
||||||
// expect(duration2).toBeLessThanOrEqual(duration1)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should handle cache misses gracefully', async () => {
|
|
||||||
// Try to get non-existent entity (v5.1.0: use valid UUID format)
|
|
||||||
const nonExistent = await brain.get('00000000-0000-0000-0000-000000000099')
|
|
||||||
expect(nonExistent).toBeNull()
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should work with find operations', async () => {
|
|
||||||
await brain.add(createAddParams({ data: 'findable content 1' }))
|
|
||||||
await brain.add(createAddParams({ data: 'findable content 2' }))
|
|
||||||
|
|
||||||
// First search
|
|
||||||
const results1 = await brain.find({ query: 'findable' })
|
|
||||||
|
|
||||||
// Second search (may be cached)
|
|
||||||
const results2 = await brain.find({ query: 'findable' })
|
|
||||||
|
|
||||||
expect(results1.length).toBe(results2.length)
|
|
||||||
expect(results1.length).toBeGreaterThanOrEqual(2)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('3. Display Augmentation', () => {
|
|
||||||
it('should provide display functionality for entities', async () => {
|
|
||||||
const id = await brain.add(createAddParams({
|
|
||||||
data: 'Entity with display capabilities',
|
|
||||||
type: NounType.Document,
|
|
||||||
metadata: {
|
|
||||||
title: 'Test Document',
|
|
||||||
author: 'Test Author',
|
|
||||||
category: 'test'
|
|
||||||
}
|
|
||||||
}))
|
|
||||||
|
|
||||||
const entity = await brain.get(id)
|
|
||||||
expect(entity).toBeDefined()
|
|
||||||
|
|
||||||
// Check if getDisplay method is available
|
|
||||||
if (entity && typeof entity.getDisplay === 'function') {
|
|
||||||
const display = entity.getDisplay()
|
|
||||||
expect(display).toBeDefined()
|
|
||||||
expect(typeof display).toBe('object')
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should work with different entity types', async () => {
|
|
||||||
const personId = await brain.add(createAddParams({
|
|
||||||
data: 'John Doe',
|
|
||||||
type: NounType.Person,
|
|
||||||
metadata: { role: 'developer', age: 30 }
|
|
||||||
}))
|
|
||||||
|
|
||||||
const locationId = await brain.add(createAddParams({
|
|
||||||
data: 'San Francisco',
|
|
||||||
type: NounType.Location,
|
|
||||||
metadata: { country: 'USA', population: 900000 }
|
|
||||||
}))
|
|
||||||
|
|
||||||
const person = await brain.get(personId)
|
|
||||||
const location = await brain.get(locationId)
|
|
||||||
|
|
||||||
expect(person).toBeDefined()
|
|
||||||
expect(location).toBeDefined()
|
|
||||||
|
|
||||||
// Display should work for all types
|
|
||||||
if (person && typeof person.getDisplay === 'function') {
|
|
||||||
const personDisplay = person.getDisplay()
|
|
||||||
expect(personDisplay).toBeDefined()
|
|
||||||
}
|
|
||||||
|
|
||||||
if (location && typeof location.getDisplay === 'function') {
|
|
||||||
const locationDisplay = location.getDisplay()
|
|
||||||
expect(locationDisplay).toBeDefined()
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should handle entities without metadata', async () => {
|
|
||||||
const id = await brain.add(createAddParams({
|
|
||||||
data: 'Simple entity without metadata'
|
|
||||||
}))
|
|
||||||
|
|
||||||
const entity = await brain.get(id)
|
|
||||||
expect(entity).toBeDefined()
|
|
||||||
|
|
||||||
if (entity && typeof entity.getDisplay === 'function') {
|
|
||||||
const display = entity.getDisplay()
|
|
||||||
expect(display).toBeDefined()
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should provide schema information', async () => {
|
|
||||||
const id = await brain.add(createAddParams({
|
|
||||||
data: 'Entity with schema',
|
|
||||||
metadata: {
|
|
||||||
stringField: 'text',
|
|
||||||
numberField: 42,
|
|
||||||
booleanField: true,
|
|
||||||
arrayField: [1, 2, 3]
|
|
||||||
}
|
|
||||||
}))
|
|
||||||
|
|
||||||
const entity = await brain.get(id)
|
|
||||||
expect(entity).toBeDefined()
|
|
||||||
|
|
||||||
if (entity && typeof entity.getSchema === 'function') {
|
|
||||||
const schema = entity.getSchema()
|
|
||||||
expect(schema).toBeDefined()
|
|
||||||
expect(typeof schema).toBe('object')
|
|
||||||
}
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('4. Metrics Augmentation', () => {
|
|
||||||
it('should track operations without interfering', async () => {
|
|
||||||
// Perform various operations
|
|
||||||
const id1 = await brain.add(createAddParams({ data: 'metrics test 1' }))
|
|
||||||
const id2 = await brain.add(createAddParams({ data: 'metrics test 2' }))
|
|
||||||
const id3 = await brain.add(createAddParams({ data: 'metrics test 3' }))
|
|
||||||
|
|
||||||
// Read operations
|
|
||||||
await brain.get(id1)
|
|
||||||
await brain.get(id2)
|
|
||||||
await brain.find({ query: 'metrics' })
|
|
||||||
|
|
||||||
// Write operations
|
|
||||||
await brain.update({ id: id1, data: 'updated metrics test 1' })
|
|
||||||
await brain.delete(id3)
|
|
||||||
|
|
||||||
// Relationship operations
|
|
||||||
await brain.relate({
|
|
||||||
from: id1,
|
|
||||||
to: id2,
|
|
||||||
type: VerbType.RelatedTo
|
|
||||||
})
|
|
||||||
|
|
||||||
// All operations should complete successfully
|
|
||||||
const entity1 = await brain.get(id1)
|
|
||||||
const entity2 = await brain.get(id2)
|
|
||||||
const entity3 = await brain.get(id3)
|
|
||||||
|
|
||||||
expect(entity1).toBeDefined()
|
|
||||||
expect(entity1?.data).toBe('updated metrics test 1')
|
|
||||||
expect(entity2).toBeDefined()
|
|
||||||
expect(entity3).toBeNull() // Deleted
|
|
||||||
|
|
||||||
// Check relationships (may be empty if relationship storage isn't implemented)
|
|
||||||
const relations = await brain.getRelations(id1)
|
|
||||||
expect(Array.isArray(relations)).toBe(true)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should handle high-frequency operations', async () => {
|
|
||||||
// Create many entities rapidly
|
|
||||||
const promises = Array.from({ length: 50 }, (_, i) =>
|
|
||||||
brain.add(createAddParams({
|
|
||||||
data: `High frequency test ${i}`,
|
|
||||||
metadata: { index: i }
|
|
||||||
}))
|
|
||||||
)
|
|
||||||
|
|
||||||
const ids = await Promise.all(promises)
|
|
||||||
expect(ids.length).toBe(50)
|
|
||||||
|
|
||||||
// Perform many reads
|
|
||||||
const readPromises = ids.map(id => brain.get(id))
|
|
||||||
const entities = await Promise.all(readPromises)
|
|
||||||
|
|
||||||
expect(entities.length).toBe(50)
|
|
||||||
expect(entities.every(e => e !== null)).toBe(true)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should handle error scenarios gracefully', async () => {
|
|
||||||
// v5.1.0: Invalid UUID formats throw errors
|
|
||||||
await expect(brain.get('invalid-id')).rejects.toThrow('Invalid UUID format')
|
|
||||||
await expect(brain.delete('invalid-id')).rejects.toThrow('Invalid UUID format')
|
|
||||||
|
|
||||||
// Try invalid find parameters
|
|
||||||
await expect(brain.find({
|
|
||||||
query: 'test',
|
|
||||||
limit: -1
|
|
||||||
} as any)).rejects.toThrow()
|
|
||||||
|
|
||||||
await expect(brain.find({
|
|
||||||
query: 'test',
|
|
||||||
offset: -1
|
|
||||||
} as any)).rejects.toThrow()
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('5. Augmentation Integration', () => {
|
|
||||||
it('should apply all augmentations to add operations', async () => {
|
|
||||||
const id = await brain.add(createAddParams({
|
|
||||||
data: 'Full integration test',
|
|
||||||
type: NounType.Document,
|
|
||||||
metadata: {
|
|
||||||
title: 'Integration Test',
|
|
||||||
priority: 'high',
|
|
||||||
tags: ['test', 'integration']
|
|
||||||
}
|
|
||||||
}))
|
|
||||||
|
|
||||||
expect(id).toBeDefined()
|
|
||||||
expect(typeof id).toBe('string')
|
|
||||||
|
|
||||||
// Verify entity was created correctly
|
|
||||||
const entity = await brain.get(id)
|
|
||||||
expect(entity).toBeDefined()
|
|
||||||
expect(entity?.data).toBe('Full integration test')
|
|
||||||
expect(entity?.type).toBe(NounType.Document)
|
|
||||||
expect(entity?.metadata?.title).toBe('Integration Test')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should apply all augmentations to find operations', async () => {
|
|
||||||
// Add test data
|
|
||||||
const id1 = await brain.add(createAddParams({
|
|
||||||
data: 'Integration search test 1',
|
|
||||||
metadata: { category: 'integration' }
|
|
||||||
}))
|
|
||||||
|
|
||||||
const id2 = await brain.add(createAddParams({
|
|
||||||
data: 'Integration search test 2',
|
|
||||||
metadata: { category: 'integration' }
|
|
||||||
}))
|
|
||||||
|
|
||||||
await brain.add(createAddParams({
|
|
||||||
data: 'Different content',
|
|
||||||
metadata: { category: 'other' }
|
|
||||||
}))
|
|
||||||
|
|
||||||
// Test text search
|
|
||||||
const textResults = await brain.find({ query: 'Integration search' })
|
|
||||||
expect(textResults.length).toBeGreaterThanOrEqual(2)
|
|
||||||
|
|
||||||
// Test metadata filtering
|
|
||||||
const categoryResults = await brain.find({
|
|
||||||
query: '',
|
|
||||||
where: { category: 'integration' }
|
|
||||||
})
|
|
||||||
expect(categoryResults.length).toBe(2)
|
|
||||||
|
|
||||||
// Verify entities have display capabilities
|
|
||||||
const firstResult = textResults[0]
|
|
||||||
if (firstResult?.entity && typeof firstResult.entity.getDisplay === 'function') {
|
|
||||||
const display = firstResult.entity.getDisplay()
|
|
||||||
expect(display).toBeDefined()
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should apply augmentations to update operations', async () => {
|
|
||||||
const id = await brain.add(createAddParams({
|
|
||||||
data: 'Original data',
|
|
||||||
metadata: { version: 1 }
|
|
||||||
}))
|
|
||||||
|
|
||||||
// Update should work with all augmentations
|
|
||||||
await brain.update({
|
|
||||||
id,
|
|
||||||
data: 'Updated data',
|
|
||||||
metadata: { version: 2, updated: true }
|
|
||||||
})
|
|
||||||
|
|
||||||
const updatedEntity = await brain.get(id)
|
|
||||||
expect(updatedEntity?.data).toBe('Updated data')
|
|
||||||
expect(updatedEntity?.metadata?.version).toBe(2)
|
|
||||||
expect(updatedEntity?.metadata?.updated).toBe(true)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should apply augmentations to relationship operations', async () => {
|
|
||||||
const id1 = await brain.add(createAddParams({ data: 'Entity 1' }))
|
|
||||||
const id2 = await brain.add(createAddParams({ data: 'Entity 2' }))
|
|
||||||
|
|
||||||
// Create relationship
|
|
||||||
await brain.relate({
|
|
||||||
from: id1,
|
|
||||||
to: id2,
|
|
||||||
type: VerbType.RelatedTo,
|
|
||||||
metadata: { strength: 0.8 }
|
|
||||||
})
|
|
||||||
|
|
||||||
// Verify relationship exists (may be empty depending on storage implementation)
|
|
||||||
const relations = await brain.getRelations(id1)
|
|
||||||
expect(Array.isArray(relations)).toBe(true)
|
|
||||||
|
|
||||||
// If relationships are stored, verify the properties
|
|
||||||
if (relations.length > 0) {
|
|
||||||
const relation = relations.find(r => r.to === id2)
|
|
||||||
if (relation) {
|
|
||||||
expect(relation.type).toBe(VerbType.RelatedTo)
|
|
||||||
expect(relation.metadata?.strength).toBe(0.8)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('6. Performance with Augmentations', () => {
|
|
||||||
it('should perform well with many entities', async () => {
|
|
||||||
const start = Date.now()
|
|
||||||
|
|
||||||
// Create 100 entities
|
|
||||||
const createPromises = Array.from({ length: 100 }, (_, i) =>
|
|
||||||
brain.add(createAddParams({
|
|
||||||
data: `Performance test entity ${i}`,
|
|
||||||
metadata: { index: i, batch: 'performance' }
|
|
||||||
}))
|
|
||||||
)
|
|
||||||
|
|
||||||
const ids = await Promise.all(createPromises)
|
|
||||||
const createDuration = Date.now() - start
|
|
||||||
|
|
||||||
expect(ids.length).toBe(100)
|
|
||||||
expect(createDuration).toBeLessThan(5000) // Should complete in under 5 seconds
|
|
||||||
|
|
||||||
// Search should be fast
|
|
||||||
const searchStart = Date.now()
|
|
||||||
const searchResults = await brain.find({
|
|
||||||
query: 'Performance test',
|
|
||||||
limit: 50
|
|
||||||
})
|
|
||||||
const searchDuration = Date.now() - searchStart
|
|
||||||
|
|
||||||
expect(searchResults.length).toBeGreaterThan(0)
|
|
||||||
expect(searchResults.length).toBeLessThanOrEqual(50) // May return fewer due to relevance
|
|
||||||
expect(searchDuration).toBeLessThan(1000) // Should complete in under 1 second
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should handle concurrent operations efficiently', async () => {
|
|
||||||
const start = Date.now()
|
|
||||||
|
|
||||||
// Perform 50 concurrent operations
|
|
||||||
const operations = Array.from({ length: 50 }, (_, i) => {
|
|
||||||
if (i % 4 === 0) {
|
|
||||||
return brain.add(createAddParams({ data: `Concurrent add ${i}` }))
|
|
||||||
} else if (i % 4 === 1) {
|
|
||||||
return brain.find({ query: 'concurrent', limit: 5 })
|
|
||||||
} else if (i % 4 === 2) {
|
|
||||||
return brain.add(createAddParams({ data: `Another add ${i}` }))
|
|
||||||
} else {
|
|
||||||
return brain.find({ query: 'test', limit: 3 })
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
const results = await Promise.all(operations)
|
|
||||||
const duration = Date.now() - start
|
|
||||||
|
|
||||||
expect(results.length).toBe(50)
|
|
||||||
expect(duration).toBeLessThan(3000) // Should handle concurrency well
|
|
||||||
|
|
||||||
// Verify some results
|
|
||||||
const addResults = results.filter(r => typeof r === 'string')
|
|
||||||
const findResults = results.filter(r => Array.isArray(r))
|
|
||||||
|
|
||||||
expect(addResults.length).toBeGreaterThan(0)
|
|
||||||
expect(findResults.length).toBeGreaterThan(0)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('7. Error Handling with Augmentations', () => {
|
|
||||||
it('should handle errors gracefully without breaking augmentations', async () => {
|
|
||||||
// v5.1.0: Invalid UUID formats throw errors (these should not crash the system)
|
|
||||||
await expect(brain.get('')).rejects.toThrow('UUID is required')
|
|
||||||
await expect(brain.update({
|
|
||||||
id: 'non-existent',
|
|
||||||
data: 'new data'
|
|
||||||
})).rejects.toThrow('Invalid UUID format')
|
|
||||||
|
|
||||||
// Normal operations should still work
|
|
||||||
const id = await brain.add(createAddParams({ data: 'After error test' }))
|
|
||||||
const entity = await brain.get(id)
|
|
||||||
expect(entity?.data).toBe('After error test')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should handle edge case data gracefully', async () => {
|
|
||||||
// Add entity with minimal but valid data
|
|
||||||
const id = await brain.add(createAddParams({
|
|
||||||
data: 'minimal', // Valid minimal data
|
|
||||||
metadata: { test: true }
|
|
||||||
}))
|
|
||||||
|
|
||||||
expect(id).toBeDefined()
|
|
||||||
|
|
||||||
const entity = await brain.get(id)
|
|
||||||
expect(entity).toBeDefined()
|
|
||||||
expect(entity?.data).toBe('minimal')
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('8. Configuration and Customization', () => {
|
|
||||||
it('should respect augmentation configuration', async () => {
|
|
||||||
// Test that augmentations can be configured
|
|
||||||
const configuredBrain = new Brainy({
|
|
||||||
storage: { type: 'memory' },
|
|
||||||
augmentations: {
|
|
||||||
cache: {
|
|
||||||
enabled: true,
|
|
||||||
maxSize: 50,
|
|
||||||
ttl: 60000
|
|
||||||
},
|
|
||||||
display: {
|
|
||||||
enabled: true,
|
|
||||||
lazyComputation: true
|
|
||||||
},
|
|
||||||
metrics: {
|
|
||||||
enabled: true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
await configuredBrain.init()
|
|
||||||
|
|
||||||
// Should work with custom configuration
|
|
||||||
const id = await configuredBrain.add(createAddParams({
|
|
||||||
data: 'Configured test'
|
|
||||||
}))
|
|
||||||
|
|
||||||
const entity = await configuredBrain.get(id)
|
|
||||||
expect(entity?.data).toBe('Configured test')
|
|
||||||
|
|
||||||
await configuredBrain.close()
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should work with disabled augmentations', async () => {
|
|
||||||
const minimalBrain = new Brainy({
|
|
||||||
storage: { type: 'memory' },
|
|
||||||
augmentations: {
|
|
||||||
cache: { enabled: false },
|
|
||||||
display: { enabled: false },
|
|
||||||
metrics: { enabled: false }
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
await minimalBrain.init()
|
|
||||||
|
|
||||||
// Should still work with augmentations disabled
|
|
||||||
const id = await minimalBrain.add(createAddParams({
|
|
||||||
data: 'Minimal test'
|
|
||||||
}))
|
|
||||||
|
|
||||||
const entity = await minimalBrain.get(id)
|
|
||||||
expect(entity?.data).toBe('Minimal test')
|
|
||||||
|
|
||||||
await minimalBrain.close()
|
|
||||||
})
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
@ -1,437 +0,0 @@
|
||||||
/**
|
|
||||||
* FormatHandlerRegistry Tests (v5.2.0)
|
|
||||||
*
|
|
||||||
* Tests for MIME-based format handler registration and routing
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { describe, it, expect, beforeEach } from 'vitest'
|
|
||||||
import { FormatHandlerRegistry } from '../../../src/augmentations/intelligentImport/FormatHandlerRegistry.js'
|
|
||||||
import type { FormatHandler, ProcessedData } from '../../../src/augmentations/intelligentImport/types.js'
|
|
||||||
|
|
||||||
describe('FormatHandlerRegistry (v5.2.0)', () => {
|
|
||||||
let registry: FormatHandlerRegistry
|
|
||||||
|
|
||||||
// Mock handlers for testing
|
|
||||||
const createMockHandler = (format: string): FormatHandler => ({
|
|
||||||
format,
|
|
||||||
async process(): Promise<ProcessedData> {
|
|
||||||
return {
|
|
||||||
format,
|
|
||||||
data: [],
|
|
||||||
metadata: {
|
|
||||||
rowCount: 0,
|
|
||||||
fields: [],
|
|
||||||
processingTime: 0
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
canHandle(): boolean {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
beforeEach(() => {
|
|
||||||
registry = new FormatHandlerRegistry()
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('Handler Registration', () => {
|
|
||||||
it('should register handler with MIME types and extensions', () => {
|
|
||||||
const csvHandler = createMockHandler('csv')
|
|
||||||
|
|
||||||
registry.registerHandler({
|
|
||||||
name: 'csv',
|
|
||||||
mimeTypes: ['text/csv', 'application/csv'],
|
|
||||||
extensions: ['.csv', '.tsv'],
|
|
||||||
loader: async () => csvHandler
|
|
||||||
})
|
|
||||||
|
|
||||||
expect(registry.hasHandler('csv')).toBe(true)
|
|
||||||
expect(registry.getRegisteredHandlers()).toContain('csv')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should register multiple handlers', () => {
|
|
||||||
registry.registerHandler({
|
|
||||||
name: 'csv',
|
|
||||||
mimeTypes: ['text/csv'],
|
|
||||||
extensions: ['.csv'],
|
|
||||||
loader: async () => createMockHandler('csv')
|
|
||||||
})
|
|
||||||
|
|
||||||
registry.registerHandler({
|
|
||||||
name: 'excel',
|
|
||||||
mimeTypes: ['application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'],
|
|
||||||
extensions: ['.xlsx'],
|
|
||||||
loader: async () => createMockHandler('excel')
|
|
||||||
})
|
|
||||||
|
|
||||||
const handlers = registry.getRegisteredHandlers()
|
|
||||||
expect(handlers).toContain('csv')
|
|
||||||
expect(handlers).toContain('excel')
|
|
||||||
expect(handlers).toHaveLength(2)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should normalize extensions (remove leading dots)', () => {
|
|
||||||
registry.registerHandler({
|
|
||||||
name: 'test',
|
|
||||||
mimeTypes: [],
|
|
||||||
extensions: ['.txt', 'md'], // Mixed with/without dots
|
|
||||||
loader: async () => createMockHandler('test')
|
|
||||||
})
|
|
||||||
|
|
||||||
expect(registry.hasHandler('test')).toBe(true)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('MIME Type Routing', () => {
|
|
||||||
beforeEach(() => {
|
|
||||||
registry.registerHandler({
|
|
||||||
name: 'csv',
|
|
||||||
mimeTypes: ['text/csv', 'application/csv'],
|
|
||||||
extensions: ['.csv', '.tsv'],
|
|
||||||
loader: async () => createMockHandler('csv')
|
|
||||||
})
|
|
||||||
|
|
||||||
registry.registerHandler({
|
|
||||||
name: 'excel',
|
|
||||||
mimeTypes: [
|
|
||||||
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
|
|
||||||
'application/vnd.ms-excel'
|
|
||||||
],
|
|
||||||
extensions: ['.xlsx', '.xls'],
|
|
||||||
loader: async () => createMockHandler('excel')
|
|
||||||
})
|
|
||||||
|
|
||||||
registry.registerHandler({
|
|
||||||
name: 'pdf',
|
|
||||||
mimeTypes: ['application/pdf'],
|
|
||||||
extensions: ['.pdf'],
|
|
||||||
loader: async () => createMockHandler('pdf')
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should get handler by MIME type', async () => {
|
|
||||||
const handler = await registry.getHandlerByMimeType('text/csv')
|
|
||||||
expect(handler).toBeDefined()
|
|
||||||
expect(handler?.format).toBe('csv')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should get handler by Excel MIME type', async () => {
|
|
||||||
const handler = await registry.getHandlerByMimeType(
|
|
||||||
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
|
|
||||||
)
|
|
||||||
expect(handler).toBeDefined()
|
|
||||||
expect(handler?.format).toBe('excel')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should get handler by legacy Excel MIME type', async () => {
|
|
||||||
const handler = await registry.getHandlerByMimeType('application/vnd.ms-excel')
|
|
||||||
expect(handler).toBeDefined()
|
|
||||||
expect(handler?.format).toBe('excel')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should return null for unknown MIME type', async () => {
|
|
||||||
const handler = await registry.getHandlerByMimeType('application/unknown')
|
|
||||||
expect(handler).toBeNull()
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should list handlers for a MIME type', () => {
|
|
||||||
const handlers = registry.getHandlersForMimeType('text/csv')
|
|
||||||
expect(handlers).toContain('csv')
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('Extension Routing', () => {
|
|
||||||
beforeEach(() => {
|
|
||||||
registry.registerHandler({
|
|
||||||
name: 'csv',
|
|
||||||
mimeTypes: ['text/csv'],
|
|
||||||
extensions: ['.csv', '.tsv'],
|
|
||||||
loader: async () => createMockHandler('csv')
|
|
||||||
})
|
|
||||||
|
|
||||||
registry.registerHandler({
|
|
||||||
name: 'excel',
|
|
||||||
mimeTypes: ['application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'],
|
|
||||||
extensions: ['.xlsx', '.xls'],
|
|
||||||
loader: async () => createMockHandler('excel')
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should get handler by extension (with dot)', async () => {
|
|
||||||
const handler = await registry.getHandlerByExtension('.csv')
|
|
||||||
expect(handler).toBeDefined()
|
|
||||||
expect(handler?.format).toBe('csv')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should get handler by extension (without dot)', async () => {
|
|
||||||
const handler = await registry.getHandlerByExtension('csv')
|
|
||||||
expect(handler).toBeDefined()
|
|
||||||
expect(handler?.format).toBe('csv')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should handle case-insensitive extensions', async () => {
|
|
||||||
const handler = await registry.getHandlerByExtension('.XLSX')
|
|
||||||
expect(handler).toBeDefined()
|
|
||||||
expect(handler?.format).toBe('excel')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should return null for unknown extension', async () => {
|
|
||||||
const handler = await registry.getHandlerByExtension('.xyz')
|
|
||||||
expect(handler).toBeNull()
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('Filename-Based Handler Selection', () => {
|
|
||||||
beforeEach(() => {
|
|
||||||
registry.registerHandler({
|
|
||||||
name: 'csv',
|
|
||||||
mimeTypes: ['text/csv'],
|
|
||||||
extensions: ['.csv'],
|
|
||||||
loader: async () => createMockHandler('csv')
|
|
||||||
})
|
|
||||||
|
|
||||||
registry.registerHandler({
|
|
||||||
name: 'excel',
|
|
||||||
mimeTypes: ['application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'],
|
|
||||||
extensions: ['.xlsx'],
|
|
||||||
loader: async () => createMockHandler('excel')
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should get handler by filename (MIME detection)', async () => {
|
|
||||||
const handler = await registry.getHandler('data.xlsx')
|
|
||||||
expect(handler).toBeDefined()
|
|
||||||
expect(handler?.format).toBe('excel')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should get handler by filename (extension fallback)', async () => {
|
|
||||||
const handler = await registry.getHandler('report.csv')
|
|
||||||
expect(handler).toBeDefined()
|
|
||||||
expect(handler?.format).toBe('csv')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should handle full paths', async () => {
|
|
||||||
const handler = await registry.getHandler('/path/to/file.xlsx')
|
|
||||||
expect(handler).toBeDefined()
|
|
||||||
expect(handler?.format).toBe('excel')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should return null for unsupported filename', async () => {
|
|
||||||
const handler = await registry.getHandler('document.txt')
|
|
||||||
expect(handler).toBeNull()
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('Lazy Loading', () => {
|
|
||||||
it('should lazy-load handlers on first access', async () => {
|
|
||||||
let loadCount = 0
|
|
||||||
|
|
||||||
registry.registerHandler({
|
|
||||||
name: 'csv',
|
|
||||||
mimeTypes: ['text/csv'],
|
|
||||||
extensions: ['.csv'],
|
|
||||||
loader: async () => {
|
|
||||||
loadCount++
|
|
||||||
return createMockHandler('csv')
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
expect(loadCount).toBe(0)
|
|
||||||
|
|
||||||
const handler1 = await registry.getHandlerByName('csv')
|
|
||||||
expect(loadCount).toBe(1)
|
|
||||||
expect(handler1?.format).toBe('csv')
|
|
||||||
|
|
||||||
// Second access should use cached instance
|
|
||||||
const handler2 = await registry.getHandlerByName('csv')
|
|
||||||
expect(loadCount).toBe(1) // Still 1, not 2
|
|
||||||
expect(handler2).toBe(handler1) // Same instance
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should cache handler instances', async () => {
|
|
||||||
registry.registerHandler({
|
|
||||||
name: 'csv',
|
|
||||||
mimeTypes: ['text/csv'],
|
|
||||||
extensions: ['.csv'],
|
|
||||||
loader: async () => createMockHandler('csv')
|
|
||||||
})
|
|
||||||
|
|
||||||
const handler1 = await registry.getHandlerByExtension('.csv')
|
|
||||||
const handler2 = await registry.getHandlerByMimeType('text/csv')
|
|
||||||
|
|
||||||
expect(handler2).toBe(handler1) // Same cached instance
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should handle loader errors gracefully', async () => {
|
|
||||||
registry.registerHandler({
|
|
||||||
name: 'failing',
|
|
||||||
mimeTypes: ['application/x-failing'],
|
|
||||||
extensions: ['.fail'],
|
|
||||||
loader: async () => {
|
|
||||||
throw new Error('Handler failed to load')
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
const handler = await registry.getHandlerByExtension('.fail')
|
|
||||||
expect(handler).toBeNull()
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('Interface Compatibility', () => {
|
|
||||||
it('should implement HandlerRegistry interface', () => {
|
|
||||||
// Check required properties exist
|
|
||||||
expect(registry.handlers).toBeDefined()
|
|
||||||
expect(registry.loaded).toBeDefined()
|
|
||||||
expect(typeof registry.register).toBe('function')
|
|
||||||
expect(typeof registry.getHandler).toBe('function')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should support register() method for backward compatibility', async () => {
|
|
||||||
const csvHandler = createMockHandler('csv')
|
|
||||||
|
|
||||||
registry.register(['.csv', '.tsv'], async () => csvHandler)
|
|
||||||
|
|
||||||
const handler = await registry.getHandler('.csv')
|
|
||||||
expect(handler).toBe(csvHandler)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should populate handlers Map for backward compatibility', () => {
|
|
||||||
registry.registerHandler({
|
|
||||||
name: 'csv',
|
|
||||||
mimeTypes: ['text/csv'],
|
|
||||||
extensions: ['.csv'],
|
|
||||||
loader: async () => createMockHandler('csv')
|
|
||||||
})
|
|
||||||
|
|
||||||
expect(registry.handlers.has('csv')).toBe(true)
|
|
||||||
expect(typeof registry.handlers.get('csv')).toBe('function')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should populate loaded Map after loading', async () => {
|
|
||||||
registry.registerHandler({
|
|
||||||
name: 'csv',
|
|
||||||
mimeTypes: ['text/csv'],
|
|
||||||
extensions: ['.csv'],
|
|
||||||
loader: async () => createMockHandler('csv')
|
|
||||||
})
|
|
||||||
|
|
||||||
expect(registry.loaded.has('csv')).toBe(false)
|
|
||||||
|
|
||||||
await registry.getHandlerByName('csv')
|
|
||||||
|
|
||||||
expect(registry.loaded.has('csv')).toBe(true)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('Multiple Handlers Per MIME Type', () => {
|
|
||||||
it('should support multiple handlers for same MIME type', () => {
|
|
||||||
registry.registerHandler({
|
|
||||||
name: 'csv-basic',
|
|
||||||
mimeTypes: ['text/csv'],
|
|
||||||
extensions: ['.csv'],
|
|
||||||
loader: async () => createMockHandler('csv-basic')
|
|
||||||
})
|
|
||||||
|
|
||||||
registry.registerHandler({
|
|
||||||
name: 'csv-advanced',
|
|
||||||
mimeTypes: ['text/csv'],
|
|
||||||
extensions: ['.csv'],
|
|
||||||
loader: async () => createMockHandler('csv-advanced')
|
|
||||||
})
|
|
||||||
|
|
||||||
const handlers = registry.getHandlersForMimeType('text/csv')
|
|
||||||
expect(handlers).toHaveLength(2)
|
|
||||||
expect(handlers).toContain('csv-basic')
|
|
||||||
expect(handlers).toContain('csv-advanced')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should return first matching handler', async () => {
|
|
||||||
registry.registerHandler({
|
|
||||||
name: 'csv-basic',
|
|
||||||
mimeTypes: ['text/csv'],
|
|
||||||
extensions: ['.csv'],
|
|
||||||
loader: async () => createMockHandler('csv-basic')
|
|
||||||
})
|
|
||||||
|
|
||||||
registry.registerHandler({
|
|
||||||
name: 'csv-advanced',
|
|
||||||
mimeTypes: ['text/csv'],
|
|
||||||
extensions: ['.csv'],
|
|
||||||
loader: async () => createMockHandler('csv-advanced')
|
|
||||||
})
|
|
||||||
|
|
||||||
// Should return first registered handler
|
|
||||||
const handler = await registry.getHandlerByMimeType('text/csv')
|
|
||||||
expect(handler?.format).toBe('csv-basic')
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('Registry Management', () => {
|
|
||||||
it('should clear all handlers', () => {
|
|
||||||
registry.registerHandler({
|
|
||||||
name: 'csv',
|
|
||||||
mimeTypes: ['text/csv'],
|
|
||||||
extensions: ['.csv'],
|
|
||||||
loader: async () => createMockHandler('csv')
|
|
||||||
})
|
|
||||||
|
|
||||||
expect(registry.hasHandler('csv')).toBe(true)
|
|
||||||
|
|
||||||
registry.clear()
|
|
||||||
|
|
||||||
expect(registry.hasHandler('csv')).toBe(false)
|
|
||||||
expect(registry.getRegisteredHandlers()).toHaveLength(0)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should check if handler is registered', () => {
|
|
||||||
expect(registry.hasHandler('csv')).toBe(false)
|
|
||||||
|
|
||||||
registry.registerHandler({
|
|
||||||
name: 'csv',
|
|
||||||
mimeTypes: ['text/csv'],
|
|
||||||
extensions: ['.csv'],
|
|
||||||
loader: async () => createMockHandler('csv')
|
|
||||||
})
|
|
||||||
|
|
||||||
expect(registry.hasHandler('csv')).toBe(true)
|
|
||||||
expect(registry.hasHandler('excel')).toBe(false)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('Edge Cases', () => {
|
|
||||||
it('should handle files without extensions', async () => {
|
|
||||||
registry.registerHandler({
|
|
||||||
name: 'csv',
|
|
||||||
mimeTypes: ['text/csv'],
|
|
||||||
extensions: ['.csv'],
|
|
||||||
loader: async () => createMockHandler('csv')
|
|
||||||
})
|
|
||||||
|
|
||||||
const handler = await registry.getHandler('data')
|
|
||||||
expect(handler).toBeNull()
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should handle empty extension list', () => {
|
|
||||||
registry.registerHandler({
|
|
||||||
name: 'special',
|
|
||||||
mimeTypes: ['application/x-special'],
|
|
||||||
extensions: [],
|
|
||||||
loader: async () => createMockHandler('special')
|
|
||||||
})
|
|
||||||
|
|
||||||
expect(registry.hasHandler('special')).toBe(true)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should handle empty MIME type list', async () => {
|
|
||||||
registry.registerHandler({
|
|
||||||
name: 'extension-only',
|
|
||||||
mimeTypes: [],
|
|
||||||
extensions: ['.xyz'],
|
|
||||||
loader: async () => createMockHandler('extension-only')
|
|
||||||
})
|
|
||||||
|
|
||||||
const handler = await registry.getHandlerByExtension('.xyz')
|
|
||||||
expect(handler?.format).toBe('extension-only')
|
|
||||||
})
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
@ -1,311 +0,0 @@
|
||||||
/**
|
|
||||||
* ImageHandler Tests (v5.8.0 - Pure JavaScript)
|
|
||||||
*
|
|
||||||
* Tests for image processing with EXIF extraction and metadata extraction
|
|
||||||
* Using probe-image-size + exifr (no native dependencies)
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { describe, it, expect, beforeEach } from 'vitest'
|
|
||||||
import { ImageHandler } from '../../../src/augmentations/intelligentImport/handlers/imageHandler.js'
|
|
||||||
import { readFileSync } from 'fs'
|
|
||||||
import { join } from 'path'
|
|
||||||
|
|
||||||
describe('ImageHandler (v5.8.0 - Pure JS)', () => {
|
|
||||||
let handler: ImageHandler
|
|
||||||
|
|
||||||
// Simple test image generators (minimal PNG/JPEG headers)
|
|
||||||
const createMinimalJPEG = (width: number = 100, height: number = 100): Buffer => {
|
|
||||||
// Minimal JPEG: SOI + SOF0 + EOI
|
|
||||||
// This is a valid but minimal JPEG structure
|
|
||||||
const soi = Buffer.from([0xFF, 0xD8]) // Start of Image
|
|
||||||
const sof0 = Buffer.from([
|
|
||||||
0xFF, 0xC0, // SOF0 marker
|
|
||||||
0x00, 0x11, // Length: 17 bytes
|
|
||||||
0x08, // Precision: 8 bits
|
|
||||||
(height >> 8) & 0xFF, height & 0xFF, // Height
|
|
||||||
(width >> 8) & 0xFF, width & 0xFF, // Width
|
|
||||||
0x03, // Components: 3 (YCbCr)
|
|
||||||
0x01, 0x22, 0x00, // Y component
|
|
||||||
0x02, 0x11, 0x01, // Cb component
|
|
||||||
0x03, 0x11, 0x01 // Cr component
|
|
||||||
])
|
|
||||||
const eoi = Buffer.from([0xFF, 0xD9]) // End of Image
|
|
||||||
|
|
||||||
return Buffer.concat([soi, sof0, eoi])
|
|
||||||
}
|
|
||||||
|
|
||||||
const createMinimalPNG = (width: number = 100, height: number = 100): Buffer => {
|
|
||||||
// PNG signature
|
|
||||||
const signature = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10])
|
|
||||||
|
|
||||||
// IHDR chunk
|
|
||||||
const ihdr = Buffer.alloc(25)
|
|
||||||
ihdr.writeUInt32BE(13, 0) // Length
|
|
||||||
ihdr.write('IHDR', 4)
|
|
||||||
ihdr.writeUInt32BE(width, 8)
|
|
||||||
ihdr.writeUInt32BE(height, 12)
|
|
||||||
ihdr.writeUInt8(8, 16) // Bit depth
|
|
||||||
ihdr.writeUInt8(2, 17) // Color type (RGB)
|
|
||||||
ihdr.writeUInt8(0, 18) // Compression
|
|
||||||
ihdr.writeUInt8(0, 19) // Filter
|
|
||||||
ihdr.writeUInt8(0, 20) // Interlace
|
|
||||||
ihdr.writeUInt32BE(0, 21) // CRC (simplified)
|
|
||||||
|
|
||||||
// IEND chunk
|
|
||||||
const iend = Buffer.from([0, 0, 0, 0, 73, 69, 78, 68, 174, 66, 96, 130])
|
|
||||||
|
|
||||||
return Buffer.concat([signature, ihdr, iend])
|
|
||||||
}
|
|
||||||
|
|
||||||
beforeEach(() => {
|
|
||||||
handler = new ImageHandler()
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('Handler Detection', () => {
|
|
||||||
it('should identify as image format handler', () => {
|
|
||||||
expect(handler.format).toBe('image')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should handle JPEG files by filename', () => {
|
|
||||||
expect(handler.canHandle({ filename: 'photo.jpg' })).toBe(true)
|
|
||||||
expect(handler.canHandle({ filename: 'photo.jpeg' })).toBe(true)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should handle PNG files by filename', () => {
|
|
||||||
expect(handler.canHandle({ filename: 'logo.png' })).toBe(true)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should handle WebP files by filename', () => {
|
|
||||||
expect(handler.canHandle({ filename: 'image.webp' })).toBe(true)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should handle other image formats', () => {
|
|
||||||
expect(handler.canHandle({ filename: 'photo.gif' })).toBe(true)
|
|
||||||
expect(handler.canHandle({ filename: 'photo.tiff' })).toBe(true)
|
|
||||||
expect(handler.canHandle({ filename: 'photo.bmp' })).toBe(true)
|
|
||||||
expect(handler.canHandle({ filename: 'icon.svg' })).toBe(true)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should handle images by extension', () => {
|
|
||||||
expect(handler.canHandle({ ext: '.jpg' })).toBe(true)
|
|
||||||
expect(handler.canHandle({ ext: 'png' })).toBe(true)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should reject non-image files', () => {
|
|
||||||
expect(handler.canHandle({ filename: 'document.pdf' })).toBe(false)
|
|
||||||
expect(handler.canHandle({ filename: 'data.csv' })).toBe(false)
|
|
||||||
expect(handler.canHandle({ filename: 'text.txt' })).toBe(false)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should detect JPEG by magic bytes', () => {
|
|
||||||
const jpegBuffer = createMinimalJPEG(100, 100)
|
|
||||||
expect(handler.canHandle(jpegBuffer)).toBe(true)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should detect PNG by magic bytes', () => {
|
|
||||||
const pngBuffer = createMinimalPNG(100, 100)
|
|
||||||
expect(handler.canHandle(pngBuffer)).toBe(true)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('Image Metadata Extraction', () => {
|
|
||||||
it('should extract JPEG metadata', async () => {
|
|
||||||
const imageBuffer = createMinimalJPEG(800, 600)
|
|
||||||
|
|
||||||
const result = await handler.process(imageBuffer, {
|
|
||||||
extractEXIF: false
|
|
||||||
})
|
|
||||||
|
|
||||||
expect(result.format).toBe('image')
|
|
||||||
expect(result.data).toHaveLength(1)
|
|
||||||
|
|
||||||
const imageData = result.data[0]
|
|
||||||
expect(imageData.type).toBe('media')
|
|
||||||
expect(imageData.metadata).toBeDefined()
|
|
||||||
expect(imageData.metadata.subtype).toBe('image')
|
|
||||||
expect(imageData.metadata.width).toBe(800)
|
|
||||||
expect(imageData.metadata.height).toBe(600)
|
|
||||||
expect(imageData.metadata.format).toBe('jpg') // probe-image-size uses 'jpg' not 'jpeg'
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should extract PNG metadata', async () => {
|
|
||||||
const imageBuffer = createMinimalPNG(400, 300)
|
|
||||||
|
|
||||||
const result = await handler.process(imageBuffer, {
|
|
||||||
extractEXIF: false
|
|
||||||
})
|
|
||||||
|
|
||||||
const imageData = result.data[0]
|
|
||||||
expect(imageData.metadata.width).toBe(400)
|
|
||||||
expect(imageData.metadata.height).toBe(300)
|
|
||||||
expect(imageData.metadata.format).toBe('png')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should include image size in bytes', async () => {
|
|
||||||
const imageBuffer = createMinimalJPEG(200, 200)
|
|
||||||
|
|
||||||
const result = await handler.process(imageBuffer, {
|
|
||||||
extractEXIF: false
|
|
||||||
})
|
|
||||||
|
|
||||||
const imageData = result.data[0]
|
|
||||||
expect(imageData.metadata.size).toBe(imageBuffer.length)
|
|
||||||
expect(imageData.metadata.size).toBeGreaterThan(0)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should include MIME type when available', async () => {
|
|
||||||
const imageBuffer = createMinimalJPEG(100, 100)
|
|
||||||
|
|
||||||
const result = await handler.process(imageBuffer, {
|
|
||||||
extractEXIF: false
|
|
||||||
})
|
|
||||||
|
|
||||||
const imageData = result.data[0]
|
|
||||||
expect(imageData.metadata.mimeType).toBeDefined()
|
|
||||||
expect(imageData.metadata.mimeType).toContain('image/')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should include processing time in metadata', async () => {
|
|
||||||
const imageBuffer = createMinimalJPEG(800, 600)
|
|
||||||
|
|
||||||
const result = await handler.process(imageBuffer)
|
|
||||||
|
|
||||||
expect(result.metadata.processingTime).toBeGreaterThanOrEqual(0)
|
|
||||||
expect(result.metadata.processingTime).toBeLessThan(5000)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should include image metadata in result metadata', async () => {
|
|
||||||
const imageBuffer = createMinimalJPEG(800, 600)
|
|
||||||
|
|
||||||
const result = await handler.process(imageBuffer)
|
|
||||||
|
|
||||||
expect(result.metadata.imageMetadata).toBeDefined()
|
|
||||||
expect(result.metadata.imageMetadata.width).toBe(800)
|
|
||||||
expect(result.metadata.imageMetadata.height).toBe(600)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('EXIF Data Extraction', () => {
|
|
||||||
it('should handle images without EXIF data', async () => {
|
|
||||||
const imageBuffer = createMinimalJPEG(800, 600)
|
|
||||||
|
|
||||||
const result = await handler.process(imageBuffer, {
|
|
||||||
extractEXIF: true
|
|
||||||
})
|
|
||||||
|
|
||||||
const imageData = result.data[0]
|
|
||||||
// Should not crash, EXIF will be undefined
|
|
||||||
expect(imageData.metadata.exif).toBeUndefined()
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should extract EXIF by default', async () => {
|
|
||||||
const imageBuffer = createMinimalJPEG(800, 600)
|
|
||||||
|
|
||||||
// Default: EXIF extraction enabled
|
|
||||||
const result = await handler.process(imageBuffer)
|
|
||||||
|
|
||||||
// Will be undefined for test images, but should not crash
|
|
||||||
expect(result.metadata.exifData).toBeUndefined()
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should allow disabling EXIF extraction', async () => {
|
|
||||||
const imageBuffer = createMinimalJPEG(800, 600)
|
|
||||||
|
|
||||||
const result = await handler.process(imageBuffer, {
|
|
||||||
extractEXIF: false
|
|
||||||
})
|
|
||||||
|
|
||||||
const imageData = result.data[0]
|
|
||||||
expect(imageData.metadata.exif).toBeUndefined()
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('Error Handling', () => {
|
|
||||||
it('should handle invalid image data gracefully', async () => {
|
|
||||||
const invalidBuffer = Buffer.from('not an image', 'utf-8')
|
|
||||||
|
|
||||||
// Should still process but with fallback values
|
|
||||||
const result = await handler.process(invalidBuffer, { extractEXIF: false })
|
|
||||||
|
|
||||||
// Fallback detection should provide basic info
|
|
||||||
expect(result.data[0].metadata.format).toBeDefined()
|
|
||||||
expect(result.data[0].metadata.size).toBe(invalidBuffer.length)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should handle empty buffer', async () => {
|
|
||||||
const emptyBuffer = Buffer.alloc(0)
|
|
||||||
|
|
||||||
// Should not crash, but will have unknown format
|
|
||||||
const result = await handler.process(emptyBuffer, { extractEXIF: false })
|
|
||||||
expect(result.data[0].metadata.format).toBe('unknown')
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('Format Support', () => {
|
|
||||||
it('should process JPEG images', async () => {
|
|
||||||
const imageBuffer = createMinimalJPEG(200, 200)
|
|
||||||
|
|
||||||
const result = await handler.process(imageBuffer, {
|
|
||||||
extractEXIF: false
|
|
||||||
})
|
|
||||||
|
|
||||||
expect(result.data[0].metadata.format).toBe('jpg') // probe-image-size uses 'jpg' not 'jpeg'
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should process PNG images', async () => {
|
|
||||||
const imageBuffer = createMinimalPNG(200, 200)
|
|
||||||
|
|
||||||
const result = await handler.process(imageBuffer, {
|
|
||||||
extractEXIF: false
|
|
||||||
})
|
|
||||||
|
|
||||||
expect(result.data[0].metadata.format).toBe('png')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should handle various image dimensions', async () => {
|
|
||||||
const sizes = [
|
|
||||||
[100, 100],
|
|
||||||
[1920, 1080],
|
|
||||||
[400, 300],
|
|
||||||
[1000, 500]
|
|
||||||
]
|
|
||||||
|
|
||||||
for (const [width, height] of sizes) {
|
|
||||||
const imageBuffer = createMinimalJPEG(width, height)
|
|
||||||
const result = await handler.process(imageBuffer, { extractEXIF: false })
|
|
||||||
|
|
||||||
expect(result.data[0].metadata.width).toBe(width)
|
|
||||||
expect(result.data[0].metadata.height).toBe(height)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('Integration with BaseFormatHandler', () => {
|
|
||||||
it('should inherit MIME type detection from BaseFormatHandler', () => {
|
|
||||||
// getMimeType is protected, but we can verify behavior
|
|
||||||
expect(handler.canHandle({ filename: 'test.jpg' })).toBe(true)
|
|
||||||
expect(handler.canHandle({ filename: 'test.png' })).toBe(true)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should provide structured ProcessedData', async () => {
|
|
||||||
const imageBuffer = createMinimalJPEG(200, 200)
|
|
||||||
|
|
||||||
const result = await handler.process(imageBuffer, {
|
|
||||||
filename: 'test-image.jpg',
|
|
||||||
extractEXIF: false
|
|
||||||
})
|
|
||||||
|
|
||||||
// Verify ProcessedData structure
|
|
||||||
expect(result.format).toBe('image')
|
|
||||||
expect(result.data).toBeInstanceOf(Array)
|
|
||||||
expect(result.metadata).toBeDefined()
|
|
||||||
expect(result.filename).toBe('test-image.jpg')
|
|
||||||
|
|
||||||
// Verify data structure
|
|
||||||
const entity = result.data[0]
|
|
||||||
expect(entity.name).toBe('test-image')
|
|
||||||
expect(entity.type).toBe('media')
|
|
||||||
expect(entity.metadata).toBeDefined()
|
|
||||||
})
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
@ -15,13 +15,7 @@ describe('Brainy 3.0 Core (Unit Tests)', () => {
|
||||||
beforeEach(async () => {
|
beforeEach(async () => {
|
||||||
// Create instance with real embeddings for production-ready tests
|
// Create instance with real embeddings for production-ready tests
|
||||||
brain = new Brainy({
|
brain = new Brainy({
|
||||||
storage: { type: 'memory' },
|
storage: { type: 'memory' }
|
||||||
augmentations: {
|
|
||||||
cache: false,
|
|
||||||
metrics: false,
|
|
||||||
display: false,
|
|
||||||
monitoring: false
|
|
||||||
}
|
|
||||||
})
|
})
|
||||||
|
|
||||||
await brain.init()
|
await brain.init()
|
||||||
|
|
@ -196,7 +190,7 @@ describe('Brainy 3.0 Core (Unit Tests)', () => {
|
||||||
})
|
})
|
||||||
|
|
||||||
describe('Statistics and Metadata', () => {
|
describe('Statistics and Metadata', () => {
|
||||||
it('should track statistics through augmentations', async () => {
|
it('should track statistics', async () => {
|
||||||
await brain.add({
|
await brain.add({
|
||||||
data: { name: 'Test1' },
|
data: { name: 'Test1' },
|
||||||
type: NounType.Concept
|
type: NounType.Concept
|
||||||
|
|
|
||||||
|
|
@ -482,24 +482,6 @@ describe('Brainy.add()', () => {
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
describe('augmentations', () => {
|
|
||||||
it('should apply augmentations during add', async () => {
|
|
||||||
// This would test augmentation pipeline if configured
|
|
||||||
// For now, just verify the entity is added correctly
|
|
||||||
const params = createAddParams({
|
|
||||||
data: 'Augmentation test',
|
|
||||||
type: 'thing',
|
|
||||||
metadata: { augment: true }
|
|
||||||
})
|
|
||||||
|
|
||||||
const id = await brain.add(params)
|
|
||||||
const entity = await brain.get(id)
|
|
||||||
|
|
||||||
expect(entity).not.toBeNull()
|
|
||||||
expect(entity!.metadata.augment).toBe(true)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('caching behavior', () => {
|
describe('caching behavior', () => {
|
||||||
it('should retrieve consistent entities', async () => {
|
it('should retrieve consistent entities', async () => {
|
||||||
// Arrange (v5.1.0: use valid UUID format)
|
// Arrange (v5.1.0: use valid UUID format)
|
||||||
|
|
|
||||||
|
|
@ -18,13 +18,7 @@ describe('add() Performance Regression Tests', () => {
|
||||||
beforeEach(async () => {
|
beforeEach(async () => {
|
||||||
brain = new Brainy({
|
brain = new Brainy({
|
||||||
storage: { type: 'memory' },
|
storage: { type: 'memory' },
|
||||||
silent: true,
|
silent: true
|
||||||
augmentations: {
|
|
||||||
cache: false,
|
|
||||||
metrics: false,
|
|
||||||
display: false,
|
|
||||||
monitoring: false
|
|
||||||
}
|
|
||||||
})
|
})
|
||||||
await brain.init()
|
await brain.init()
|
||||||
})
|
})
|
||||||
|
|
|
||||||
|
|
@ -214,7 +214,6 @@ describe('Brainy plugin integration', () => {
|
||||||
const brain = new Brainy({
|
const brain = new Brainy({
|
||||||
storage: { type: 'memory' },
|
storage: { type: 'memory' },
|
||||||
silent: true,
|
silent: true,
|
||||||
augmentations: { cache: false, metrics: false, display: false, monitoring: false }
|
|
||||||
})
|
})
|
||||||
brain.use(mockPlugin)
|
brain.use(mockPlugin)
|
||||||
await brain.init()
|
await brain.init()
|
||||||
|
|
@ -244,7 +243,6 @@ describe('Brainy plugin integration', () => {
|
||||||
const brain = new Brainy({
|
const brain = new Brainy({
|
||||||
storage: { type: 'memory' },
|
storage: { type: 'memory' },
|
||||||
silent: true,
|
silent: true,
|
||||||
augmentations: { cache: false, metrics: false, display: false, monitoring: false }
|
|
||||||
})
|
})
|
||||||
brain.use(mockPlugin)
|
brain.use(mockPlugin)
|
||||||
await brain.init()
|
await brain.init()
|
||||||
|
|
@ -263,7 +261,6 @@ describe('Brainy plugin integration', () => {
|
||||||
const brain = new Brainy({
|
const brain = new Brainy({
|
||||||
storage: { type: 'memory' },
|
storage: { type: 'memory' },
|
||||||
silent: true,
|
silent: true,
|
||||||
augmentations: { cache: false, metrics: false, display: false, monitoring: false }
|
|
||||||
})
|
})
|
||||||
brain.use(mockPlugin)
|
brain.use(mockPlugin)
|
||||||
await brain.init()
|
await brain.init()
|
||||||
|
|
@ -293,7 +290,6 @@ describe('Brainy plugin integration', () => {
|
||||||
const brain = new Brainy({
|
const brain = new Brainy({
|
||||||
storage: { type: 'memory' },
|
storage: { type: 'memory' },
|
||||||
silent: true,
|
silent: true,
|
||||||
augmentations: { cache: false, metrics: false, display: false, monitoring: false }
|
|
||||||
})
|
})
|
||||||
brain.use(mockPlugin)
|
brain.use(mockPlugin)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,219 +0,0 @@
|
||||||
/**
|
|
||||||
* Tests for Intelligent Type Matching with embeddings
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { describe, it, expect, beforeAll, afterAll } from 'vitest'
|
|
||||||
import { IntelligentTypeMatcher } from '../../src/augmentations/typeMatching/intelligentTypeMatcher.js'
|
|
||||||
import { NounType, VerbType } from '../../src/types/graphTypes.js'
|
|
||||||
|
|
||||||
describe('Intelligent Type Matching', () => {
|
|
||||||
let matcher: IntelligentTypeMatcher
|
|
||||||
|
|
||||||
beforeAll(async () => {
|
|
||||||
matcher = new IntelligentTypeMatcher()
|
|
||||||
await matcher.init()
|
|
||||||
})
|
|
||||||
|
|
||||||
afterAll(async () => {
|
|
||||||
await matcher.dispose()
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('Noun Type Detection', () => {
|
|
||||||
it('should detect Person type from user data', async () => {
|
|
||||||
const result = await matcher.matchNounType({
|
|
||||||
name: 'John Doe',
|
|
||||||
email: 'john@example.com',
|
|
||||||
age: 30
|
|
||||||
})
|
|
||||||
|
|
||||||
expect(result.type).toBeDefined()
|
|
||||||
expect(result.confidence).toBeGreaterThan(0)
|
|
||||||
expect(result.alternatives).toBeDefined()
|
|
||||||
expect(result.alternatives.length).toBeGreaterThanOrEqual(0)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should detect Organization type from company data', async () => {
|
|
||||||
const result = await matcher.matchNounType({
|
|
||||||
companyName: 'Acme Corp',
|
|
||||||
employees: 500,
|
|
||||||
industry: 'Technology'
|
|
||||||
})
|
|
||||||
|
|
||||||
// With real type embeddings (v3.33.0+), type detection uses actual semantic similarity
|
|
||||||
// Results depend on the embedding quality and field patterns
|
|
||||||
expect(result.type).toBeDefined()
|
|
||||||
expect(result.confidence).toBeGreaterThan(0.1)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should detect Location type from geographic data', async () => {
|
|
||||||
const result = await matcher.matchNounType({
|
|
||||||
latitude: 37.7749,
|
|
||||||
longitude: -122.4194,
|
|
||||||
city: 'San Francisco'
|
|
||||||
})
|
|
||||||
|
|
||||||
// With real type embeddings (v3.33.0+), geographic data detection is semantic
|
|
||||||
expect(result.type).toBeDefined()
|
|
||||||
expect(result.confidence).toBeGreaterThan(0.1)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should detect Document type from text content', async () => {
|
|
||||||
const result = await matcher.matchNounType({
|
|
||||||
title: 'Research Paper',
|
|
||||||
content: 'Abstract: This paper discusses...',
|
|
||||||
author: 'Dr. Smith',
|
|
||||||
pages: 20
|
|
||||||
})
|
|
||||||
|
|
||||||
// With real type embeddings (v3.33.0+), could match various types based on semantic similarity
|
|
||||||
expect(result.type).toBeDefined()
|
|
||||||
expect(result.confidence).toBeGreaterThan(0.0)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should detect Product type from commercial data', async () => {
|
|
||||||
const result = await matcher.matchNounType({
|
|
||||||
price: 99.99,
|
|
||||||
sku: 'PROD-123',
|
|
||||||
inventory: 50,
|
|
||||||
productId: 'abc-123'
|
|
||||||
})
|
|
||||||
|
|
||||||
// With real type embeddings (v3.33.0+), commercial data uses semantic matching
|
|
||||||
expect(result.type).toBeDefined()
|
|
||||||
expect(result.confidence).toBeGreaterThan(0.1)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should detect Event type from temporal data', async () => {
|
|
||||||
const result = await matcher.matchNounType({
|
|
||||||
startTime: '2024-01-01',
|
|
||||||
endTime: '2024-01-02',
|
|
||||||
attendees: 100,
|
|
||||||
eventType: 'conference'
|
|
||||||
})
|
|
||||||
|
|
||||||
// With real type embeddings (v3.33.0+), temporal data uses semantic matching
|
|
||||||
expect(result.type).toBeDefined()
|
|
||||||
expect(result.confidence).toBeGreaterThan(0.1)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should handle ambiguous data with alternatives', async () => {
|
|
||||||
const result = await matcher.matchNounType({
|
|
||||||
value: 100,
|
|
||||||
type: 'unknown',
|
|
||||||
data: 'mixed'
|
|
||||||
})
|
|
||||||
|
|
||||||
expect(result.type).toBeDefined()
|
|
||||||
expect(result.alternatives).toBeDefined()
|
|
||||||
expect(result.alternatives.length).toBeGreaterThan(0)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('Verb Type Detection', () => {
|
|
||||||
it('should detect MemberOf relationship', async () => {
|
|
||||||
const result = await matcher.matchVerbType(
|
|
||||||
{ id: 'user1', type: 'person' },
|
|
||||||
{ id: 'org1', type: 'organization' },
|
|
||||||
'memberOf'
|
|
||||||
)
|
|
||||||
|
|
||||||
// With mocked embeddings, type detection is non-deterministic
|
|
||||||
expect(result.type).toBeDefined()
|
|
||||||
expect(Object.values(VerbType)).toContain(result.type)
|
|
||||||
expect(result.confidence).toBeGreaterThan(0)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should detect CreatedBy relationship', async () => {
|
|
||||||
const result = await matcher.matchVerbType(
|
|
||||||
{ id: 'doc1', type: 'document' },
|
|
||||||
{ id: 'author1', type: 'person' },
|
|
||||||
'created by'
|
|
||||||
)
|
|
||||||
|
|
||||||
// With mocked embeddings, type detection is non-deterministic
|
|
||||||
expect(result.type).toBeDefined()
|
|
||||||
expect(Object.values(VerbType)).toContain(result.type)
|
|
||||||
expect(result.confidence).toBeGreaterThan(0)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should detect Contains relationship', async () => {
|
|
||||||
const result = await matcher.matchVerbType(
|
|
||||||
{ id: 'folder1', type: 'collection' },
|
|
||||||
{ id: 'file1', type: 'file' },
|
|
||||||
'contains'
|
|
||||||
)
|
|
||||||
|
|
||||||
// With mocked embeddings, type detection is non-deterministic
|
|
||||||
expect(result.type).toBeDefined()
|
|
||||||
expect(Object.values(VerbType)).toContain(result.type)
|
|
||||||
expect(result.confidence).toBeGreaterThan(0)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should handle unknown relationships with defaults', async () => {
|
|
||||||
const result = await matcher.matchVerbType(
|
|
||||||
{ id: 'a' },
|
|
||||||
{ id: 'b' },
|
|
||||||
'somehow connected'
|
|
||||||
)
|
|
||||||
|
|
||||||
expect(result.type).toBeDefined()
|
|
||||||
expect(Object.values(VerbType)).toContain(result.type)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('Type Coverage', () => {
|
|
||||||
it('should have embeddings for all 42 noun types', async () => {
|
|
||||||
const nounTypes = Object.values(NounType)
|
|
||||||
expect(nounTypes.length).toBe(42)
|
|
||||||
|
|
||||||
// Test that each type can be matched
|
|
||||||
for (const nounType of nounTypes) {
|
|
||||||
const result = await matcher.matchNounType({
|
|
||||||
type: nounType,
|
|
||||||
test: 'coverage check'
|
|
||||||
})
|
|
||||||
expect(result.type).toBeDefined()
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should have embeddings for all 127 verb types', async () => {
|
|
||||||
const verbTypes = Object.values(VerbType)
|
|
||||||
expect(verbTypes.length).toBe(127)
|
|
||||||
|
|
||||||
// Test that each type can be matched
|
|
||||||
for (const verbType of verbTypes) {
|
|
||||||
const result = await matcher.matchVerbType(
|
|
||||||
{ id: 'source' },
|
|
||||||
{ id: 'target' },
|
|
||||||
verbType
|
|
||||||
)
|
|
||||||
expect(result.type).toBeDefined()
|
|
||||||
}
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('Cache Performance', () => {
|
|
||||||
it('should cache repeated type matches', async () => {
|
|
||||||
const testData = { name: 'Cache Test', value: 123 }
|
|
||||||
|
|
||||||
const result1 = await matcher.matchNounType(testData)
|
|
||||||
const result2 = await matcher.matchNounType(testData)
|
|
||||||
|
|
||||||
expect(result1.type).toBe(result2.type)
|
|
||||||
expect(result1.confidence).toBe(result2.confidence)
|
|
||||||
// Cache should return consistent results
|
|
||||||
expect(result2.type).toBeDefined()
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should clear cache when requested', async () => {
|
|
||||||
const testData = { name: 'Clear Cache Test' }
|
|
||||||
|
|
||||||
await matcher.matchNounType(testData) // Populate cache
|
|
||||||
matcher.clearCache()
|
|
||||||
|
|
||||||
// Should still work after cache clear
|
|
||||||
const result = await matcher.matchNounType(testData)
|
|
||||||
expect(result.type).toBeDefined()
|
|
||||||
})
|
|
||||||
})
|
|
||||||
})
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue