soulcraft/brainy
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

refactor: remove augmentation system and semantic type matching

Browse source 
Remove the entire augmentation pipeline infrastructure (52 files,
~15,000 lines) and the semantic type matching system. These were
unused middleware layers adding complexity without value.

What was removed:
- src/augmentations/ directory (all augmentation implementations)
- src/augmentationManager.ts (pipeline orchestrator)
- src/types/augmentations.ts, src/types/pipelineTypes.ts
- src/shared/default-augmentations.ts
- Semantic type suggestion (BrainyTypes.suggestNoun/suggestVerb)
- src/utils/typeMatching/ (embedding-based type matcher)

What was preserved by relocating:
- Import handlers (CSV, PDF, Excel) -> src/importers/handlers/
- NeuralImportAugmentation -> src/cortex/neuralImportAugmentation.ts
- Type matching utilities -> heuristic inference in consumers

What was simplified:
- brainy.ts: operations call storage directly (no execute() wrapper)
- IntegrationBase: standalone class (no BaseAugmentation parent)
- BrainyTypes: validation-only (nouns, verbs, isValid*, get*)
- Pipeline: direct execution (no augmentation interception)
- index.ts: removed TypeSuggestion, suggestType exports
- package.json: removed stale types/augmentations export

Build passes, 1176 tests pass, 0 failures.
This commit is contained in:
David Snelling 2026-02-01 10:48:56 -08:00
parent ac7a1f772c
commit d1db3510be
97 changed files with 349 additions and 19705 deletions
Download patch file Download diff file Expand all files Collapse all files

105
examples/api-server-example.ts
View file

@ -1,105 +0,0 @@
/**
* API Server Example
*
* This example shows how to expose Brainy through REST, WebSocket, and MCP APIs
* using the APIServerAugmentation.
*
* Zero-config philosophy: Just create and register the augmentation!
*/
import { Brainy } from '../src/index.js'
import { APIServerAugmentation } from '../src/augmentations/apiServerAugmentation.js'
async function main() {
// 1. Create Brainy with zero config
const brain = new Brainy()
await brain.init()
// 2. Add some sample data
await brain.add({ data: "The quick brown fox", type: 'content', metadata: { type: "sentence", category: "animals" } })
await brain.add({ data: "Machine learning models", type: 'content', metadata: { type: "tech", category: "AI" } })
await brain.add({ data: "Natural language processing", type: 'content', metadata: { type: "tech", category: "NLP" } })
// 3. Create and register the API Server augmentation
const apiServer = new APIServerAugmentation({
port: 3000,
host: 'localhost'
})
// Register the augmentation with Brainy
brain.augmentations.register(apiServer)
// Initialize augmentations with Brainy context
await brain.augmentations.initialize({
brain,
log: (msg: string, level?: string) => console.log(`[${level || 'info'}] ${msg}`),
config: {}
})
console.log('🚀 Brainy API Server is running!')
console.log('📡 REST API: http://localhost:3000')
console.log('🔌 WebSocket: ws://localhost:3000/ws')
console.log('🧠 MCP: http://localhost:3000/api/mcp')
console.log('')
console.log('Try these endpoints:')
console.log(' GET http://localhost:3000/health')
console.log(' POST http://localhost:3000/api/search')
console.log(' Body: { "query": "fox", "limit": 10 }')
console.log(' POST http://localhost:3000/api/add')
console.log(' Body: { "content": "New data", "metadata": {} }')
console.log('')
console.log('Press Ctrl+C to stop the server')
}
// Run the example
main().catch(console.error)
/**
* Example REST API calls:
*
* # Health check
* curl http://localhost:3000/health
*
* # Search
* curl -X POST http://localhost:3000/api/search \
* -H "Content-Type: application/json" \
* -d '{"query": "fox", "limit": 5}'
*
* # Add data
* curl -X POST http://localhost:3000/api/add \
* -H "Content-Type: application/json" \
* -d '{"content": "The cat sat on the mat", "metadata": {"type": "sentence"}}'
*
* # Get by ID
* curl http://localhost:3000/api/get/[id]
*
* # Statistics
* curl http://localhost:3000/api/stats
*/
/**
* Example WebSocket client (browser):
*
* const ws = new WebSocket('ws://localhost:3000/ws')
*
* ws.onopen = () => {
* // Subscribe to all operations
* ws.send(JSON.stringify({
* type: 'subscribe',
* operations: ['all']
* }))
*
* // Perform a search
* ws.send(JSON.stringify({
* type: 'search',
* query: 'fox',
* limit: 5,
* requestId: '123'
* }))
* }
*
* ws.onmessage = (event) => {
* const msg = JSON.parse(event.data)
* console.log('Received:', msg)
* }
*/

49
examples/augmentations-zero-config.ts
View file

@ -1,49 +0,0 @@
/**
* Zero-Config Augmentations Example
*
* Brainy's philosophy: Everything just works with zero configuration.
* Augmentations extend Brainy without any complex setup.
*/
import { Brainy } from '../src/index.js'
import { EntityRegistryAugmentation } from '../src/augmentations/entityRegistryAugmentation.js'
import { BatchProcessingAugmentation } from '../src/augmentations/batchProcessingAugmentation.js'
import { APIServerAugmentation } from '../src/augmentations/apiServerAugmentation.js'
async function main() {
// Create Brainy - zero config!
const brain = new Brainy()
// Register augmentations - they just work!
brain.augmentations.register(new EntityRegistryAugmentation()) // Deduplication
brain.augmentations.register(new BatchProcessingAugmentation()) // Performance
brain.augmentations.register(new APIServerAugmentation()) // API exposure
// Initialize Brainy (augmentations auto-initialize)
await brain.init()
// Use Brainy normally - augmentations work transparently
await brain.add({ data: "Hello world", type: 'content', metadata: { type: "greeting" } })
// Batch operations automatically optimized
await brain.addBatch([
{ content: "Item 1", metadata: { id: 1 } },
{ content: "Item 2", metadata: { id: 2 } },
{ content: "Item 3", metadata: { id: 3 } }
])
// Search works with all augmentations active
const results = await brain.search("hello")
console.log('Search results:', results)
// API server is running at http://localhost:3000
console.log('✨ Brainy is running with:')
console.log(' - Write-ahead logging (durability)')
console.log(' - Entity deduplication (performance)')
console.log(' - Batch optimization (speed)')
console.log(' - REST/WebSocket/MCP APIs (connectivity)')
console.log('')
console.log('Zero configuration required!')
}
main().catch(console.error)

5
examples/tests/instant-validation.js
View file

@ -50,10 +50,7 @@ try {
console.log('\n📊 API Architecture Validation:')
// Test 5: Augmentation system structure
test('Augmentations system exists', brain.augmentations !== undefined)
// Test 6: Core properties exist
// Test 5: Core properties exist
test('Index system exists', brain.index !== undefined)
test('Storage system exists', brain.storage !== undefined)

89
examples/tests/test-storage-augmentation.js
View file

@ -1,89 +0,0 @@
/**
* Test script to verify storage augmentation system works
*/
import { Brainy } from '../../dist/index.js'
import {
MemoryStorageAugmentation,
FileSystemStorageAugmentation
} from './dist/augmentations/storageAugmentations.js'
console.log('🧪 Testing Storage Augmentation System')
console.log('=' .repeat(50))
async function test1_ZeroConfig() {
console.log('\n1. Zero-Config Test')
const brain = new Brainy()
await brain.init()
await brain.add('test', { content: 'Zero-config test' })
const results = await brain.search('test', { limit: 1 })
console.log('✅ Zero-config works:', results.length > 0)
await brain.destroy()
}
async function test2_ConfigBased() {
console.log('\n2. Config-Based Storage Test')
const brain = new Brainy({
storage: {
forceMemoryStorage: true
}
})
await brain.init()
await brain.add('config test', { content: 'Config-based test' })
const results = await brain.search('config', { limit: 1 })
console.log('✅ Config-based works:', results.length > 0)
await brain.destroy()
}
async function test3_AugmentationOverride() {
console.log('\n3. Augmentation Override Test')
const brain = new Brainy()
// Register storage augmentation BEFORE init
brain.augmentations.register(new MemoryStorageAugmentation('test-memory'))
await brain.init()
await brain.add('augmentation test', { content: 'Augmentation override test' })
const results = await brain.search('augmentation', { limit: 1 })
console.log('✅ Augmentation override works:', results.length > 0)
await brain.destroy()
}
async function test4_BackwardCompatibility() {
console.log('\n4. Backward Compatibility Test')
// Old style with rootDirectory config
const brain = new Brainy({
storage: {
rootDirectory: './test-data',
forceFileSystemStorage: true
}
})
await brain.init()
console.log('✅ Backward compatible config works')
await brain.destroy()
}
async function runAllTests() {
try {
await test1_ZeroConfig()
await test2_ConfigBased()
await test3_AugmentationOverride()
await test4_BackwardCompatibility()
console.log('\n' + '='.repeat(50))
console.log('✅ ALL STORAGE AUGMENTATION TESTS PASSED!')
} catch (error) {
console.error('❌ Test failed:', error)
process.exit(1)
}
}
runAllTests()

4
package.json
View file

@ -28,10 +28,6 @@
"import": "./dist/types/graphTypes.js",
"types": "./dist/types/graphTypes.d.ts"
},
"./types/augmentations": {
"import": "./dist/types/augmentations.js",
"types": "./dist/types/augmentations.d.ts"
},
"./utils/textEncoding": {
"import": "./dist/utils/textEncoding.js",
"types": "./dist/utils/textEncoding.d.ts"

131
src/api/UniversalImportAPI.ts
View file

@ -16,8 +16,7 @@ import { NounType, VerbType } from '../types/graphTypes.js'
import { Vector } from '../coreTypes.js'
import type { Brainy } from '../brainy.js'
import type { Entity, Relation } from '../types/brainy.types.js'
import { BrainyTypes, getBrainyTypes } from '../augmentations/typeMatching/brainyTypes.js'
import { NeuralImportAugmentation } from '../augmentations/neuralImport.js'
import { NeuralImportAugmentation } from '../cortex/neuralImportAugmentation.js'
import { mimeDetector } from '../vfs/MimeTypeDetector.js'
export interface ImportSource {
@ -65,7 +64,6 @@ export interface NeuralImportProgress {
export class UniversalImportAPI {
private brain: Brainy<any>
private typeMatcher!: BrainyTypes
private neuralImport: NeuralImportAugmentation
private embedCache = new Map<string, Vector>()
@ -82,7 +80,6 @@ export class UniversalImportAPI {
* Initialize the neural import system
*/
async init(): Promise<void> {
this.typeMatcher = await getBrainyTypes()
// Neural import initializes itself
}
@ -370,21 +367,18 @@ export class UniversalImportAPI {
// Generate embedding for the item
const embedding = await this.generateEmbedding(item)
// Neural type matching - MANDATORY
const nounMatch = await this.typeMatcher.matchNounType(item)
// Never reject based on confidence - we ALWAYS accept the best match
// Determine noun type from data
const nounType = this.inferNounType(item)
const entityId = this.generateId(item)
entities.set(entityId, {
id: entityId,
type: nounMatch.type as NounType, // Always use the neural match
type: nounType,
data: item,
vector: embedding,
confidence: nounMatch.confidence,
confidence: 1.0,
metadata: {
...item,
_neuralMatch: nounMatch,
_importedAt: Date.now()
}
})
@ -465,28 +459,19 @@ export class UniversalImportAPI {
for (const [key, value] of Object.entries(item)) {
// Check if this looks like a reference
if (this.looksLikeReference(key, value)) {
// Find or predict target entity
const targetId = String(value)
const verbType = this.inferVerbType(key)
// Neural verb type matching
const verbMatch = await this.typeMatcher.matchVerbType(
item, // source object
{ id: targetId }, // target (we may not have full data)
key // field name as context
)
// Always create relationship with neural match
const relationId = `${sourceId}_${verbMatch.type}_${targetId}`
const relationId = `${sourceId}_${verbType}_${targetId}`
relationships.set(relationId, {
id: relationId,
from: sourceId,
to: targetId,
type: verbMatch.type as VerbType,
weight: verbMatch.confidence, // Use confidence as weight
confidence: verbMatch.confidence,
type: verbType,
weight: 1.0,
confidence: 1.0,
metadata: {
field: key,
_neuralMatch: verbMatch,
_importedAt: Date.now()
}
})
@ -494,27 +479,22 @@ export class UniversalImportAPI {
// Handle arrays of references
if (Array.isArray(value)) {
for (const item of value) {
if (this.looksLikeReference(key, item)) {
const targetId = String(item)
const verbMatch = await this.typeMatcher.matchVerbType(
item,
{ id: targetId },
key
)
for (const arrayItem of value) {
if (this.looksLikeReference(key, arrayItem)) {
const targetId = String(arrayItem)
const verbType = this.inferVerbType(key)
const relationId = `${sourceId}_${verbMatch.type}_${targetId}`
const relationId = `${sourceId}_${verbType}_${targetId}`
relationships.set(relationId, {
id: relationId,
from: sourceId,
to: targetId,
type: verbMatch.type as VerbType,
weight: verbMatch.confidence,
confidence: verbMatch.confidence,
type: verbType,
weight: 1.0,
confidence: 1.0,
metadata: {
field: key,
array: true,
_neuralMatch: verbMatch,
_importedAt: Date.now()
}
})
@ -553,6 +533,81 @@ export class UniversalImportAPI {
return fieldLooksLikeRef && valueLooksLikeId
}
/**
* Infer noun type from object structure using field heuristics
*/
private inferNounType(obj: any): NounType {
if (typeof obj !== 'object' || obj === null) return NounType.Thing
// Check for explicit type field
if (obj.type && typeof obj.type === 'string') {
const normalized = obj.type.charAt(0).toUpperCase() + obj.type.slice(1)
if (Object.values(NounType).includes(normalized as NounType)) {
return normalized as NounType
}
}
// Person heuristics
if (obj.email || obj.firstName || obj.lastName || obj.username || obj.age) {
return NounType.Person
}
// Organization heuristics
if (obj.companyName || obj.organizationId || obj.employees || obj.industry) {
return NounType.Organization
}
// Location heuristics
if (obj.latitude || obj.longitude || obj.address || obj.city || obj.country) {
return NounType.Location
}
// Document heuristics
if ((obj.content && (obj.title || obj.author)) || obj.documentType || obj.pages) {
return NounType.Document
}
// Event heuristics
if (obj.startTime || obj.endTime || obj.date || obj.eventType || obj.attendees) {
return NounType.Event
}
// Product heuristics
if (obj.price || obj.sku || obj.inventory || obj.productId) {
return NounType.Product
}
// Task heuristics
if ((obj.status && (obj.assignee || obj.dueDate)) || obj.priority || obj.completed !== undefined) {
return NounType.Task
}
// Dataset heuristics
if (Array.isArray(obj.data) || obj.rows || obj.columns || obj.schema) {
return NounType.Dataset
}
return NounType.Thing
}
/**
* Infer verb type from field name using common patterns
*/
private inferVerbType(fieldName: string): VerbType {
const field = fieldName.toLowerCase()
if (field.includes('parent') || field.includes('child') || field.includes('contain')) {
return VerbType.Contains
}
if (field.includes('owner') || field.includes('created') || field.includes('author')) {
return VerbType.Creates
}
if (field.includes('member') || field.includes('belong')) {
return VerbType.MemberOf
}
if (field.includes('depend') || field.includes('require')) {
return VerbType.DependsOn
}
if (field.includes('ref') || field.includes('link') || field.includes('source')) {
return VerbType.References
}
return VerbType.RelatedTo
}
/**
* Store processed data in brain
*/

137
src/augmentationManager.ts
View file

@ -1,137 +0,0 @@
/**
* Type-safe augmentation management system for Brainy
* Provides a clean API for managing augmentations without string literals
*/
import { IAugmentation, AugmentationType } from './types/augmentations.js'
import { augmentationPipeline } from './pipeline.js'
export interface AugmentationInfo {
name: string
type: string
enabled: boolean
description: string
}
/**
* Type-safe augmentation manager
* Accessed via brain.augmentations for all management operations
*/
export class AugmentationManager {
private pipeline = augmentationPipeline
/**
* List all registered augmentations with their status
* @returns Array of augmentation information
*/
list(): AugmentationInfo[] {
// Deprecated: use brain.augmentations instead
return []
}
/**
* Get information about a specific augmentation
* @param name The augmentation name
* @returns Augmentation info or undefined if not found
*/
get(name: string): AugmentationInfo | undefined {
const all = this.list()
return all.find(a => a.name === name)
}
/**
* Check if an augmentation is enabled
* @param name The augmentation name
* @returns True if enabled, false otherwise
*/
isEnabled(name: string): boolean {
const aug = this.get(name)
return aug?.enabled ?? false
}
/**
* Enable a specific augmentation
* @param name The augmentation name
* @returns True if successfully enabled
*/
enable(name: string): boolean {
// Deprecated: use brain.augmentations instead
return false
}
/**
* Disable a specific augmentation
* @param name The augmentation name
* @returns True if successfully disabled
*/
disable(name: string): boolean {
// Deprecated: use brain.augmentations instead
return false
}
/**
* Remove an augmentation from the pipeline
* @param name The augmentation name
* @returns True if successfully removed
*/
remove(name: string): boolean {
// Deprecated: use brain.augmentations instead
return true
}
/**
* Enable all augmentations of a specific type
* @param type The augmentation type
* @returns Number of augmentations enabled
*/
enableType(type: AugmentationType): number {
// Deprecated: use brain.augmentations instead
return 0
}
/**
* Disable all augmentations of a specific type
* @param type The augmentation type
* @returns Number of augmentations disabled
*/
disableType(type: AugmentationType): number {
// Deprecated: use brain.augmentations instead
return 0
}
/**
* Get all augmentations of a specific type
* @param type The augmentation type
* @returns Array of augmentations of that type
*/
listByType(type: AugmentationType): AugmentationInfo[] {
return this.list().filter(a => a.type === type)
}
/**
* Get all enabled augmentations
* @returns Array of enabled augmentations
*/
listEnabled(): AugmentationInfo[] {
return this.list().filter(a => a.enabled)
}
/**
* Get all disabled augmentations
* @returns Array of disabled augmentations
*/
listDisabled(): AugmentationInfo[] {
return this.list().filter(a => !a.enabled)
}
/**
* Register a new augmentation (internal use)
* @param augmentation The augmentation to register
*/
register(augmentation: IAugmentation): void {
// Deprecated: use brain.augmentations instead
}
}
// Export types for external use
export { AugmentationType } from './types/augmentations.js'

410
src/augmentations/AugmentationMetadataContract.ts
View file

@ -1,410 +0,0 @@
/**
* Augmentation Metadata Contract System
*
* Prevents accidental metadata corruption while allowing intentional enrichment
* Each augmentation declares its metadata intentions upfront
*/
export interface AugmentationMetadataContract {
// Augmentation identity
name: string
version: string
// What fields this augmentation READS
reads?: {
userFields?: string[] // e.g., ['title', 'description']
internalFields?: string[] // e.g., ['_brainy.deleted']
augmentationFields?: string[] // e.g., ['_augmentations.otherAug.score']
}
// What fields this augmentation WRITES
writes?: {
// User metadata it enriches (MUST be declared!)
userFields?: Array<{
field: string
type: 'create' | 'update' | 'merge' | 'delete'
description: string
example?: any
}>
// Its own augmentation namespace
augmentationFields?: Array<{
field: string
description: string
}>
// Internal fields (requires special permission)
internalFields?: Array<{
field: string
permission: 'granted' | 'requested'
reason: string
}>
}
// Conflict resolution strategy
conflictResolution?: {
strategy: 'error' | 'warn' | 'merge' | 'skip' | 'override'
priority?: number // Higher priority wins in conflicts
}
// Safety guarantees
guarantees?: {
preservesExisting?: boolean // Won't delete existing fields
reversible?: boolean // Can undo changes
idempotent?: boolean // Safe to run multiple times
validatesTypes?: boolean // Checks field types before modifying
}
}
/**
* Runtime metadata safety enforcer
*/
export class MetadataSafetyEnforcer {
private contracts: Map<string, AugmentationMetadataContract> = new Map()
private modifications: Map<string, Set<string>> = new Map() // field -> augmentations that modify it
/**
* Register an augmentation's contract
*/
registerContract(contract: AugmentationMetadataContract): void {
this.contracts.set(contract.name, contract)
// Track which augmentations modify which fields
if (contract.writes?.userFields) {
for (const fieldDef of contract.writes.userFields) {
if (!this.modifications.has(fieldDef.field)) {
this.modifications.set(fieldDef.field, new Set())
}
this.modifications.get(fieldDef.field)!.add(contract.name)
}
}
}
/**
* Check if an augmentation can modify a field
*/
canModifyField(augName: string, field: string, value: any): {
allowed: boolean
reason?: string
warnings?: string[]
} {
const contract = this.contracts.get(augName)
if (!contract) {
return {
allowed: false,
reason: `Augmentation '${augName}' has no registered contract`
}
}
// Check if field is in user namespace
if (!field.startsWith('_brainy.') && !field.startsWith('_augmentations.')) {
// It's a user field
const declaredField = contract.writes?.userFields?.find(f => f.field === field)
if (!declaredField) {
return {
allowed: false,
reason: `Augmentation '${augName}' did not declare intent to modify '${field}'`
}
}
// Check for conflicts
const modifiers = this.modifications.get(field)
if (modifiers && modifiers.size > 1) {
const others = Array.from(modifiers).filter(a => a !== augName)
return {
allowed: true, // Still allowed but with warning
warnings: [`Field '${field}' is also modified by: ${others.join(', ')}`]
}
}
return { allowed: true }
}
// Check internal fields
if (field.startsWith('_brainy.')) {
const internalField = contract.writes?.internalFields?.find(f =>
field === `_brainy.${f.field}`
)
if (!internalField) {
return {
allowed: false,
reason: `Augmentation '${augName}' cannot modify internal field '${field}'`
}
}
if (internalField.permission !== 'granted') {
return {
allowed: false,
reason: `Permission not granted for internal field '${field}'`
}
}
return { allowed: true }
}
// Check augmentation namespace
if (field.startsWith('_augmentations.')) {
const parts = field.split('.')
const targetAug = parts[1]
// Can only modify own namespace
if (targetAug !== augName) {
return {
allowed: false,
reason: `Cannot modify another augmentation's namespace: ${targetAug}`
}
}
return { allowed: true }
}
return { allowed: true }
}
/**
* Create safe metadata proxy for an augmentation
*/
createSafeProxy(metadata: any, augName: string): any {
const self = this
return new Proxy(metadata, {
set(target, prop, value) {
const field = String(prop)
// Check permission
const permission = self.canModifyField(augName, field, value)
if (!permission.allowed) {
throw new Error(`[${augName}] ${permission.reason}`)
}
if (permission.warnings) {
console.warn(`[${augName}] Warning:`, ...permission.warnings)
}
// Track modification for audit
if (!target._audit) {
target._audit = []
}
target._audit.push({
augmentation: augName,
field,
oldValue: target[prop],
newValue: value,
timestamp: Date.now()
})
target[prop] = value
return true
},
deleteProperty(target, prop) {
const field = String(prop)
const permission = self.canModifyField(augName, field, undefined)
if (!permission.allowed) {
throw new Error(`[${augName}] Cannot delete field: ${permission.reason}`)
}
delete target[prop]
return true
}
})
}
}
/**
* Example augmentation contracts
*/
export const EXAMPLE_CONTRACTS: Record<string, AugmentationMetadataContract> = {
// Enrichment augmentation that adds categories
categoryEnricher: {
name: 'categoryEnricher',
version: '1.0.0',
reads: {
userFields: ['title', 'description', 'content']
},
writes: {
userFields: [
{
field: 'category',
type: 'create',
description: 'Auto-detected category',
example: 'technology'
},
{
field: 'subcategories',
type: 'create',
description: 'List of relevant subcategories',
example: ['web', 'framework']
}
],
augmentationFields: [
{
field: 'confidence',
description: 'Confidence score of categorization'
}
]
},
guarantees: {
preservesExisting: true,
idempotent: true
}
},
// Translation augmentation
translator: {
name: 'translator',
version: '1.0.0',
reads: {
userFields: ['title', 'description']
},
writes: {
userFields: [
{
field: 'translations',
type: 'merge',
description: 'Translations in multiple languages',
example: { es: 'Título', fr: 'Titre' }
},
{
field: 'detectedLanguage',
type: 'create',
description: 'Detected source language',
example: 'en'
}
]
},
conflictResolution: {
strategy: 'merge',
priority: 10
}
},
// Sentiment analyzer
sentimentAnalyzer: {
name: 'sentimentAnalyzer',
version: '1.0.0',
reads: {
userFields: ['content', 'description', 'reviews']
},
writes: {
userFields: [
{
field: 'sentiment',
type: 'update',
description: 'Overall sentiment score',
example: { score: 0.8, label: 'positive' }
}
],
augmentationFields: [
{
field: 'analysis',
description: 'Detailed sentiment breakdown'
}
]
},
guarantees: {
reversible: true,
validatesTypes: true
}
},
// System augmentation with internal access
garbageCollector: {
name: 'garbageCollector',
version: '1.0.0',
reads: {
internalFields: ['_brainy.createdAt', '_brainy.lastAccessed']
},
writes: {
internalFields: [
{
field: 'deleted',
permission: 'granted',
reason: 'Soft delete expired items'
},
{
field: 'archived',
permission: 'granted',
reason: 'Archive old items'
}
]
}
}
}
/**
* Augmentation base class with safety
*/
export abstract class SafeAugmentation {
protected enforcer: MetadataSafetyEnforcer
protected contract: AugmentationMetadataContract
constructor(contract: AugmentationMetadataContract) {
this.contract = contract
this.enforcer = new MetadataSafetyEnforcer()
this.enforcer.registerContract(contract)
}
/**
* Get safe metadata proxy
*/
protected getSafeMetadata(metadata: any): any {
return this.enforcer.createSafeProxy(metadata, this.contract.name)
}
/**
* Abstract method to implement augmentation logic
*/
abstract execute(metadata: any): Promise<any>
}
/**
* Example: Category enricher implementation
*/
export class CategoryEnricherAugmentation extends SafeAugmentation {
constructor() {
super(EXAMPLE_CONTRACTS.categoryEnricher)
}
async execute(metadata: any): Promise<any> {
const safe = this.getSafeMetadata(metadata)
// Read declared fields
const title = safe.title
const description = safe.description
// Analyze and categorize
const category = this.detectCategory(title, description)
const subcategories = this.detectSubcategories(title, description)
// Write to declared fields (will be checked by proxy)
safe.category = category // ✅ Allowed - declared in contract
safe.subcategories = subcategories // ✅ Allowed
// Try to write undeclared field
// safe.randomField = 'test' // ❌ Would throw error!
// Write to our augmentation namespace
if (!safe._augmentations) safe._augmentations = {}
if (!safe._augmentations.categoryEnricher) {
safe._augmentations.categoryEnricher = {}
}
safe._augmentations.categoryEnricher.confidence = 0.95 // ✅ Allowed
return safe
}
private detectCategory(title: string, description: string): string {
// Simplified logic
return 'technology'
}
private detectSubcategories(title: string, description: string): string[] {
return ['web', 'framework']
}
}

292
src/augmentations/README.md
View file

@ -1,292 +0,0 @@
<div align="center">
<img src="../../brainy.png" alt="Brainy Logo" width="200"/>
# Brainy Augmentations
</div>
This directory contains the augmentation implementations for Brainy. Augmentations are pluggable components that extend
Brainy's functionality in various ways.
## Available Augmentations
### Core Augmentations
#### IntelligentImportAugmentation
Automatically detects and processes CSV, Excel, and PDF files with intelligent extraction. This augmentation is enabled by default and provides:
- **CSV Support**: Auto-detection of encoding, delimiters, and field types
- **Excel Support**: Multi-sheet extraction with metadata preservation
- **PDF Support**: Text extraction, table detection, and metadata extraction
- **Type Inference**: Automatically infers data types (string, number, boolean, date)
- **Neural Integration**: Seamlessly integrates with entity extraction and relationship detection
```javascript
import { Brainy } from '@soulcraft/brainy'
const brain = new Brainy({
intelligentImport: {
enableCSV: true,
enableExcel: true,
enablePDF: true,
maxFileSize: 100 * 1024 * 1024 // 100MB
}
})
await brain.init()
// Import CSV with auto-detection
await brain.import('customers.csv')
// Import Excel with specific sheets
await brain.import('sales-data.xlsx', {
excelSheets: ['Q1', 'Q2']
})
// Import PDF with table extraction
await brain.import('report.pdf', {
pdfExtractTables: true
})
```
**See:** [Import Anything Guide](../../docs/guides/import-anything.md) | [Example](../../examples/import-excel-pdf-csv.ts)
### Conduit Augmentations
Conduit augmentations provide data synchronization between Brainy instances.
#### WebSocketConduitAugmentation
A conduit augmentation that syncs Brainy instances using WebSockets. This is used for syncing between browsers and
servers, or between servers.
```javascript
import { createConduitAugmentation, augmentationPipeline } from '@soulcraft/brainy'
// Create a WebSocket conduit augmentation
const wsConduit = await createConduitAugmentation('websocket', 'my-websocket-sync')
// Register the augmentation with the pipeline
augmentationPipeline.register(wsConduit)
// Connect to another Brainy instance
const connectionResult = await wsConduit.establishConnection(
'wss://your-websocket-server.com/brainy-sync',
{ protocols: 'brainy-sync' }
)
```
#### WebRTCConduitAugmentation
A conduit augmentation that syncs Brainy instances using WebRTC. This is used for direct peer-to-peer syncing between
browsers.
```javascript
import { createConduitAugmentation, augmentationPipeline } from '@soulcraft/brainy'
// Create a WebRTC conduit augmentation
const webrtcConduit = await createConduitAugmentation('webrtc', 'my-webrtc-sync')
// Register the augmentation with the pipeline
augmentationPipeline.register(webrtcConduit)
// Connect to a peer
const connectionResult = await webrtcConduit.establishConnection(
'peer-id-to-connect-to',
{
signalServerUrl: 'wss://your-signal-server.com',
localPeerId: 'my-peer-id',
iceServers: [{ urls: 'stun:stun.l.google.com:19302' }]
}
)
```
#### ServerSearchConduitAugmentation
A specialized conduit augmentation that provides functionality for searching a server-hosted Brainy instance and storing
results locally. This allows you to:
- Search a server-hosted Brainy instance from a browser
- Store the search results in a local Brainy instance
- Perform further searches against the local instance without needing to query the server again
- Add data to both local and server instances
```javascript
import {
ServerSearchConduitAugmentation,
createServerSearchAugmentations,
augmentationPipeline
} from '@soulcraft/brainy'
// Using the factory function (recommended)
const { conduit, activation, connection } = await createServerSearchAugmentations(
'wss://your-brainy-server.com/ws',
{ protocols: 'brainy-sync' }
)
// Register the augmentations with the pipeline
augmentationPipeline.register(conduit)
augmentationPipeline.register(activation)
// Search the server and store results locally
const serverSearchResult = await conduit.searchServer(
connection.connectionId,
'your search query',
5 // limit
)
// Search the local instance
const localSearchResult = await conduit.searchLocal('your search query', 5)
// Perform a combined search (local first, then server if needed)
const combinedSearchResult = await conduit.searchCombined(
connection.connectionId,
'your search query',
5
)
// Add data to both local and server
const addResult = await conduit.addToBoth(
connection.connectionId,
'Text to add',
{ /* metadata */ }
)
```
### Activation Augmentations
Activation augmentations dictate how Brainy initiates actions, responses, or data manipulations.
#### ServerSearchActivationAugmentation
An activation augmentation that provides actions for server search functionality. This works in conjunction with the
ServerSearchConduitAugmentation to provide a complete solution for browser-server search.
```javascript
import {
ServerSearchActivationAugmentation,
createServerSearchAugmentations,
augmentationPipeline
} from '@soulcraft/brainy'
// Using the factory function (recommended)
const { conduit, activation, connection } = await createServerSearchAugmentations(
'wss://your-brainy-server.com/ws',
{ protocols: 'brainy-sync' }
)
// Register the augmentations with the pipeline
augmentationPipeline.register(conduit)
augmentationPipeline.register(activation)
// Use the activation augmentation to search the server
const serverSearchAction = activation.triggerAction('searchServer', {
connectionId: connection.connectionId,
query: 'your search query',
limit: 5
})
if (serverSearchAction.success) {
// The data property contains a promise that will resolve to the search results
const serverSearchResult = await serverSearchAction.data
console.log('Server search results:', serverSearchResult)
}
// Other available actions:
// - 'connectToServer': Connect to a server
// - 'searchLocal': Search the local instance
// - 'searchCombined': Search both local and server
// - 'addToBoth': Add data to both local and server
```
## Using the Augmentation Pipeline
The augmentation pipeline provides a way to execute augmentations based on their type.
```javascript
import { augmentationPipeline } from '@soulcraft/brainy'
// Execute a conduit augmentation
const conduitResults = await augmentationPipeline.executeConduitPipeline(
'methodName',
[arg1, arg2, ...],
{ /* options */ }
)
// Execute an activation augmentation
const activationResults = await augmentationPipeline.executeActivationPipeline(
'methodName',
[arg1, arg2, ...],
{ /* options */ }
)
```
## Creating Custom Augmentations
To create a custom augmentation, implement one of the augmentation interfaces:
- `ISenseAugmentation`: For processing raw data
- `IConduitAugmentation`: For data synchronization
- `ICognitionAugmentation`: For reasoning and inference
- `IMemoryAugmentation`: For data storage
- `IPerceptionAugmentation`: For data interpretation and visualization
- `IDialogAugmentation`: For natural language processing
- `IActivationAugmentation`: For triggering actions
Example:
```javascript
import { AugmentationType, IActivationAugmentation } from '@soulcraft/brainy'
class MyCustomActivation implements IActivationAugmentation {
readonly
name = 'my-custom-activation'
readonly
description = 'My custom activation augmentation'
enabled = true
getType(): AugmentationType {
return AugmentationType.ACTIVATION
}
async initialize(): Promise<void> {
// Initialization code
}
async shutDown(): Promise<void> {
// Cleanup code
}
async getStatus(): Promise<'active' | 'inactive' | 'error'> {
return 'active'
}
triggerAction(actionName: string, parameters
?:
Record<string, unknown>
):
AugmentationResponse<unknown> {
// Implementation
}
generateOutput(knowledgeId: string, format: string): AugmentationResponse<string | Record<string, unknown
>> {
// Implementation
}
interactExternal(systemId
:
string, payload
:
Record < string, unknown >
):
AugmentationResponse < unknown > {
// Implementation
}
}
```

915
src/augmentations/apiServerAugmentation.ts
View file

@ -1,915 +0,0 @@
/**
* API Server Augmentation - Universal API Exposure
*
* 🌐 Exposes Brainy through REST, WebSocket, and MCP
* 🔌 Works in Node.js, Deno, and Service Workers
* 🚀 Single augmentation for all API needs
*
* This unifies and replaces:
* - BrainyMCPBroadcast (Node-specific server)
* - WebSocketConduitAugmentation (client connections)
* - Future REST API implementations
*/
import { BaseAugmentation, AugmentationContext } from './brainyAugmentation.js'
import { BrainyMCPService } from '../mcp/brainyMCPService.js'
import { v4 as uuidv4 } from '../universal/uuid.js'
import { isNode, isBrowser } from '../utils/environment.js'
export interface APIServerConfig {
enabled?: boolean
port?: number
mcpPort?: number
wsPort?: number
host?: string
cors?: {
origin?: string | string[]
credentials?: boolean
}
auth?: {
required?: boolean
apiKeys?: string[]
bearerTokens?: string[]
}
rateLimit?: {
windowMs?: number
max?: number
}
ssl?: {
cert?: string
key?: string
}
}
interface ConnectedClient {
id: string
type: 'rest' | 'websocket' | 'mcp'
socket?: any
subscriptions?: string[]
metadata?: Record<string, any>
lastSeen: number
}
/**
* Unified API Server Augmentation
* Exposes Brainy through multiple protocols
*/
export class APIServerAugmentation extends BaseAugmentation {
readonly name = 'api-server'
readonly timing = 'after' as const
readonly metadata = 'readonly' as const // API server reads metadata to serve data
readonly operations = ['all'] as ('all')[]
readonly priority = 5 // Low priority, runs after other augmentations
protected config: APIServerConfig
private mcpService?: BrainyMCPService
private httpServer?: any
private wsServer?: any
private clients = new Map<string, ConnectedClient>()
private operationHistory: any[] = []
private maxHistorySize = 1000
private rateLimitStore = new Map<string, number[]>()
constructor(config: APIServerConfig = {}) {
super()
this.config = {
enabled: true,
port: 3000,
host: '0.0.0.0',
cors: { origin: '*', credentials: true },
auth: { required: false },
rateLimit: { windowMs: 60000, max: 100 },
...config
}
}
protected async onInitialize(): Promise<void> {
if (!this.config.enabled) {
this.log('API Server disabled in config')
return
}
// Initialize MCP service
this.mcpService = new BrainyMCPService(this.context!.brain, {
enableAuth: this.config.auth?.required
})
// Start appropriate server based on environment
if (isNode()) {
await this.startNodeServer()
} else if (typeof (globalThis as any).Deno !== 'undefined') {
await this.startDenoServer()
} else if (isBrowser() && 'serviceWorker' in navigator) {
await this.startServiceWorker()
} else {
this.log('No suitable server environment detected', 'warn')
}
}
/**
* Start Node.js server with Express
*/
private async startNodeServer(): Promise<void> {
try {
// Dynamic imports for Node.js dependencies
const express = await import('express').catch(() => null)
const cors = await import('cors').catch(() => null)
const ws = await import('ws').catch(() => null)
const { createServer } = await import('node:http')
if (!express || !cors || !ws) {
this.log('Express, cors, or ws not available. Install with: npm install express cors ws', 'error')
return
}
const WebSocketServer = (ws as any)?.WebSocketServer || (ws as any)?.default?.WebSocketServer || (ws as any)?.Server
const app = express.default()
// Middleware
app.use(cors.default(this.config.cors))
app.use((express.default || express).json({ limit: '50mb' }))
app.use(this.authMiddleware.bind(this))
app.use(this.rateLimitMiddleware.bind(this))
// REST API Routes
this.setupRESTRoutes(app)
// Create HTTP server
this.httpServer = createServer(app)
// WebSocket server
this.wsServer = new WebSocketServer({
server: this.httpServer,
path: '/ws'
})
this.setupWebSocketServer()
// Start listening
await new Promise<void>((resolve, reject) => {
this.httpServer.listen(this.config.port, this.config.host, () => {
this.log(`🌐 API Server listening on http://${this.config.host}:${this.config.port}`)
this.log(`🔌 WebSocket: ws://${this.config.host}:${this.config.port}/ws`)
this.log(`🧠 MCP endpoint: http://${this.config.host}:${this.config.port}/api/mcp`)
resolve()
}).on('error', reject)
})
// Heartbeat interval
setInterval(() => this.sendHeartbeats(), 30000)
} catch (error) {
this.log(`Failed to start Node.js server: ${error}`, 'error')
throw error
}
}
/**
* Setup REST API routes
*/
private setupRESTRoutes(app: any): void {
// Health check
app.get('/health', (_req: any, res: any) => {
res.json({
status: 'healthy',
version: '2.0.0',
clients: this.clients.size,
uptime: process.uptime ? process.uptime() : 0
})
})
// Search endpoint
app.post('/api/search', async (req: any, res: any) => {
try {
const { query, limit = 10, options = {} } = req.body
const results = await this.context!.brain.search(query, limit, options)
res.json({ success: true, results })
} catch (error: any) {
res.status(500).json({ success: false, error: error.message })
}
})
// Add data endpoint
app.post('/api/add', async (req: any, res: any) => {
try {
const { content, metadata } = req.body
const id = await this.context!.brain.add({ data: content, type: 'content', metadata })
res.json({ success: true, id })
} catch (error: any) {
res.status(500).json({ success: false, error: error.message })
}
})
// Get by ID endpoint
app.get('/api/get/:id', async (req: any, res: any) => {
try {
const data = await this.context!.brain.get(req.params.id)
if (data) {
res.json({ success: true, data })
} else {
res.status(404).json({ success: false, error: 'Not found' })
}
} catch (error: any) {
res.status(500).json({ success: false, error: error.message })
}
})
// Delete endpoint
app.delete('/api/delete/:id', async (req: any, res: any) => {
try {
await this.context!.brain.delete(req.params.id)
res.json({ success: true })
} catch (error: any) {
res.status(500).json({ success: false, error: error.message })
}
})
// Relate endpoint
app.post('/api/relate', async (req: any, res: any) => {
try {
const { source, target, verb, metadata } = req.body
await this.context!.brain.relate(source, target, verb, metadata)
res.json({ success: true })
} catch (error: any) {
res.status(500).json({ success: false, error: error.message })
}
})
// Find endpoint (complex queries)
app.post('/api/find', async (req: any, res: any) => {
try {
const results = await this.context!.brain.find(req.body)
res.json({ success: true, results })
} catch (error: any) {
res.status(500).json({ success: false, error: error.message })
}
})
// Cluster endpoint
app.post('/api/cluster', async (req: any, res: any) => {
try {
const { algorithm = 'kmeans', options = {} } = req.body
const clusters = await this.context!.brain.cluster(algorithm, options)
res.json({ success: true, clusters })
} catch (error: any) {
res.status(500).json({ success: false, error: error.message })
}
})
// MCP endpoint
app.post('/api/mcp', async (req: any, res: any) => {
try {
const response = await this.mcpService!.handleRequest(req.body)
res.json(response)
} catch (error: any) {
res.status(500).json({ success: false, error: error.message })
}
})
// Statistics endpoint
app.get('/api/stats', async (_req: any, res: any) => {
try {
const stats = this.context!.brain.getStats()
res.json({ success: true, stats })
} catch (error: any) {
res.status(500).json({ success: false, error: error.message })
}
})
// Operation history endpoint
app.get('/api/history', (_req: any, res: any) => {
res.json({
success: true,
history: this.operationHistory.slice(-100)
})
})
}
/**
* Setup WebSocket server
*/
private setupWebSocketServer(): void {
if (!this.wsServer) return
this.wsServer.on('connection', (socket: any, request: any) => {
const clientId = uuidv4()
const client: ConnectedClient = {
id: clientId,
type: 'websocket',
socket,
subscriptions: [],
lastSeen: Date.now()
}
this.clients.set(clientId, client)
// Send welcome message
socket.send(JSON.stringify({
type: 'welcome',
clientId,
message: 'Connected to Brainy API Server',
capabilities: ['search', 'add', 'delete', 'relate', 'subscribe', 'mcp']
}))
// Handle messages
socket.on('message', async (message: string) => {
try {
const msg = JSON.parse(message)
await this.handleWebSocketMessage(msg, client)
} catch (error: any) {
socket.send(JSON.stringify({
type: 'error',
error: error.message
}))
}
})
// Handle disconnect
socket.on('close', () => {
this.clients.delete(clientId)
this.log(`Client ${clientId} disconnected`)
})
// Handle errors
socket.on('error', (error: any) => {
this.log(`WebSocket error for client ${clientId}: ${error}`, 'error')
})
})
}
/**
* Handle WebSocket message
*/
private async handleWebSocketMessage(msg: any, client: ConnectedClient): Promise<void> {
const { socket } = client
switch (msg.type) {
case 'subscribe':
// Subscribe to operation types
client.subscriptions = msg.operations || ['all']
socket.send(JSON.stringify({
type: 'subscribed',
operations: client.subscriptions
}))
break
case 'search':
const searchResults = await this.context!.brain.search(
msg.query,
msg.limit || 10,
msg.options || {}
)
socket.send(JSON.stringify({
type: 'searchResults',
requestId: msg.requestId,
results: searchResults
}))
break
case 'add':
const id = await this.context!.brain.add({ data: msg.content, type: 'content', metadata: msg.metadata })
socket.send(JSON.stringify({
type: 'addResult',
requestId: msg.requestId,
id
}))
break
case 'mcp':
const mcpResponse = await this.mcpService!.handleRequest(msg.request)
socket.send(JSON.stringify({
type: 'mcpResponse',
requestId: msg.requestId,
response: mcpResponse
}))
break
case 'heartbeat':
client.lastSeen = Date.now()
socket.send(JSON.stringify({
type: 'heartbeat',
timestamp: Date.now()
}))
break
default:
socket.send(JSON.stringify({
type: 'error',
error: `Unknown message type: ${msg.type}`
}))
}
}
/**
* Execute augmentation - broadcast operations to clients
*/
async execute<T = any>(
operation: string,
params: any,
next: () => Promise<T>
): Promise<T> {
const startTime = Date.now()
const result = await next()
const duration = Date.now() - startTime
// Record operation in history
const historyEntry = {
operation,
params: this.sanitizeParams(params),
timestamp: Date.now(),
duration
}
this.operationHistory.push(historyEntry)
if (this.operationHistory.length > this.maxHistorySize) {
this.operationHistory.shift()
}
// Broadcast to subscribed WebSocket clients
const message = JSON.stringify({
type: 'operation',
operation,
params: historyEntry.params,
timestamp: historyEntry.timestamp,
duration
})
for (const client of this.clients.values()) {
if (client.type === 'websocket' && client.socket) {
if (client.subscriptions?.includes('all') ||
client.subscriptions?.includes(operation)) {
try {
client.socket.send(message)
} catch (error) {
// Client might be disconnected
this.clients.delete(client.id)
}
}
}
}
return result
}
/**
* Auth middleware for Express
*/
private authMiddleware(req: any, res: any, next: any): void {
if (!this.config.auth?.required) {
return next()
}
const apiKey = req.headers['x-api-key']
const bearerToken = req.headers.authorization?.replace('Bearer ', '')
if (apiKey && this.config.auth.apiKeys?.includes(apiKey)) {
return next()
}
if (bearerToken && this.config.auth.bearerTokens?.includes(bearerToken)) {
return next()
}
res.status(401).json({ error: 'Unauthorized' })
}
/**
* Rate limiting middleware
*/
private rateLimitMiddleware(req: any, res: any, next: any): void {
// Simple in-memory rate limiting
// In production, use redis or proper rate limiting library
const ip = req.ip || req.connection.remoteAddress
const now = Date.now()
const windowMs = this.config.rateLimit?.windowMs || 60000
const max = this.config.rateLimit?.max || 100
// Clean old entries
for (const [key, client] of this.clients.entries()) {
if (now - client.lastSeen > windowMs) {
this.clients.delete(key)
}
}
// For now, just pass through
// Real implementation would track requests per IP
next()
}
/**
* Sanitize parameters before broadcasting
*/
private sanitizeParams(params: any): any {
if (!params) return params
const sanitized = { ...params }
// Remove sensitive fields
delete sanitized.password
delete sanitized.apiKey
delete sanitized.token
delete sanitized.secret
// Truncate large data
if (sanitized.content && sanitized.content.length > 1000) {
sanitized.content = sanitized.content.substring(0, 1000) + '...'
}
return sanitized
}
/**
* Send heartbeats to all connected clients
*/
private sendHeartbeats(): void {
const now = Date.now()
for (const [id, client] of this.clients.entries()) {
if (client.type === 'websocket' && client.socket) {
// Remove inactive clients
if (now - client.lastSeen > 60000) {
this.clients.delete(id)
continue
}
// Send heartbeat
try {
client.socket.send(JSON.stringify({
type: 'heartbeat',
timestamp: now
}))
} catch {
// Client disconnected
this.clients.delete(id)
}
}
}
}
/**
* Start Deno server
*/
private async startDenoServer(): Promise<void> {
try {
// Check if Deno.serve is available (Deno 1.35+)
const DenoGlobal = (globalThis as any).Deno
if (DenoGlobal && 'serve' in DenoGlobal) {
const handler = this.createUniversalHandler()
this.httpServer = DenoGlobal.serve({
port: this.config.port,
hostname: this.config.host || '0.0.0.0',
handler: handler
})
this.log(`Deno server started on ${this.config.host || '0.0.0.0'}:${this.config.port}`)
// Setup WebSocket handling for Deno
this.setupUniversalWebSocket()
} else {
throw new Error('Deno.serve not available - requires Deno 1.35+')
}
} catch (error) {
this.log(`Failed to start Deno server: ${(error as Error).message}`, 'error')
throw error
}
}
/**
* Start Service Worker (for browser)
*/
private async startServiceWorker(): Promise<void> {
try {
if (typeof self !== 'undefined' && 'addEventListener' in self) {
// Service Worker environment - intercept fetch events
const handler = this.createUniversalHandler()
self.addEventListener('fetch', async (event: any) => {
const url = new URL(event.request.url)
// Only handle API requests
if (url.pathname.startsWith('/api/') || url.pathname.startsWith('/ws') || url.pathname.startsWith('/mcp/')) {
event.respondWith(handler(event.request))
}
})
this.log('Service Worker API server registered for /api/, /ws, and /mcp paths')
// Setup message handling for WebSocket-like communication
this.setupServiceWorkerMessaging()
} else if (typeof navigator !== 'undefined' && 'serviceWorker' in navigator) {
// Browser main thread - service worker registration should be handled by the application
this.log('Service Worker environment detected. Registration should be handled by your application.', 'info')
// Return early - the app will handle service worker registration
return
} else {
this.log('Service Worker environment not available', 'warn')
return
}
} catch (error) {
this.log(`Failed to start Service Worker server: ${(error as Error).message}`, 'error')
throw error
}
}
/**
* Create universal handler using Web Standards (works in Node, Deno, Service Workers)
*/
private createUniversalHandler(): (request: Request) => Promise<Response> {
return async (request: Request): Promise<Response> => {
try {
const url = new URL(request.url)
const method = request.method.toUpperCase()
const path = url.pathname
// Add CORS headers
const corsOrigin = Array.isArray(this.config.cors?.origin)
? this.config.cors.origin[0]
: this.config.cors?.origin || '*'
const headers = new Headers({
'Access-Control-Allow-Origin': corsOrigin,
'Access-Control-Allow-Methods': 'GET, POST, PUT, DELETE, OPTIONS',
'Access-Control-Allow-Headers': 'Content-Type, Authorization',
'Content-Type': 'application/json'
})
// Handle preflight requests
if (method === 'OPTIONS') {
return new Response(null, { status: 200, headers })
}
// Authentication
if (!this.authenticateRequest(request)) {
return new Response(JSON.stringify({ error: 'Unauthorized' }), {
status: 401,
headers
})
}
// Rate limiting
if (!this.checkRateLimit(request)) {
return new Response(JSON.stringify({ error: 'Rate limit exceeded' }), {
status: 429,
headers
})
}
// Route handling
if (path.startsWith('/api/brainy/')) {
return this.handleBrainyAPI(request, path.replace('/api/brainy/', ''), headers)
} else if (path.startsWith('/mcp/')) {
return this.handleMCPAPI(request, path.replace('/mcp/', ''), headers)
} else if (path === '/health') {
return new Response(JSON.stringify({ status: 'ok', timestamp: Date.now() }), {
status: 200,
headers
})
}
return new Response(JSON.stringify({ error: 'Not found' }), {
status: 404,
headers
})
} catch (error) {
return new Response(JSON.stringify({
error: 'Internal server error',
message: (error as Error).message
}), {
status: 500,
headers: new Headers({ 'Content-Type': 'application/json' })
})
}
}
}
/**
* Handle Brainy API requests using universal Request/Response
*/
private async handleBrainyAPI(request: Request, path: string, headers: Headers): Promise<Response> {
const method = request.method.toUpperCase()
const body = method !== 'GET' ? await request.json().catch(() => ({})) : {}
try {
let result: any
switch (`${method} ${path}`) {
case 'POST add':
result = { id: await this.context!.brain.add(body) }
break
case 'GET get':
const id = new URL(request.url).searchParams.get('id')
result = await this.context!.brain.get(id)
break
case 'PUT update':
await this.context!.brain.update(body)
result = { success: true }
break
case 'DELETE delete':
const deleteId = new URL(request.url).searchParams.get('id')
await this.context!.brain.delete(deleteId)
result = { success: true }
break
case 'POST find':
result = await this.context!.brain.find(body)
break
case 'POST relate':
result = { id: await this.context!.brain.relate(body) }
break
case 'GET insights':
result = await this.context!.brain.insights()
break
default:
return new Response(JSON.stringify({ error: `Unknown endpoint: ${method} ${path}` }), {
status: 404,
headers
})
}
return new Response(JSON.stringify(result), { status: 200, headers })
} catch (error) {
return new Response(JSON.stringify({
error: (error as Error).message
}), { status: 400, headers })
}
}
/**
* Handle MCP API requests
*/
private async handleMCPAPI(request: Request, path: string, headers: Headers): Promise<Response> {
try {
if (!this.mcpService) {
return new Response(JSON.stringify({ error: 'MCP service not available' }), {
status: 503,
headers
})
}
const body = await request.json().catch(() => ({}))
// Convert to MCP request format
const mcpRequest = {
type: path.includes('data') ? 'data_access' : 'tool_execution',
...body
}
const result = await this.mcpService.handleRequest(mcpRequest as any)
return new Response(JSON.stringify(result), { status: 200, headers })
} catch (error) {
return new Response(JSON.stringify({
error: (error as Error).message
}), { status: 400, headers })
}
}
/**
* Universal WebSocket setup (works in Node, Deno)
*/
private setupUniversalWebSocket(): void {
// WebSocket handling varies by platform but uses same interface
this.log('WebSocket support enabled for real-time updates')
}
/**
* Service Worker messaging for WebSocket-like communication
*/
private setupServiceWorkerMessaging(): void {
if (typeof self !== 'undefined') {
self.addEventListener('message', async (event: any) => {
if (event.data.type === 'brainy-api') {
try {
const response = await this.handleBrainyAPI(
new Request('http://localhost/api/brainy/' + event.data.endpoint, {
method: event.data.method || 'POST',
body: JSON.stringify(event.data.data)
}),
event.data.endpoint,
new Headers({ 'Content-Type': 'application/json' })
)
const result = await response.json()
event.ports[0]?.postMessage({
id: event.data.id,
success: response.ok,
data: result
})
} catch (error) {
event.ports[0]?.postMessage({
id: event.data.id,
success: false,
error: (error as Error).message
})
}
}
})
}
}
/**
* Universal authentication using Web Standards
*/
private authenticateRequest(request: Request): boolean {
if (!this.config.auth?.required) return true
const authHeader = request.headers.get('authorization')
if (!authHeader) return false
if (this.config.auth.apiKeys?.length) {
const apiKey = authHeader.replace('Bearer ', '')
return this.config.auth.apiKeys.includes(apiKey)
}
return true
}
private checkRateLimit(request: Request): boolean {
if (!this.config.rateLimit) return true
// Get client identifier from headers or use a default
const clientId = request.headers.get('x-forwarded-for') ||
request.headers.get('x-real-ip') ||
request.headers.get('cf-connecting-ip') || // Cloudflare
request.headers.get('x-vercel-forwarded-for') || // Vercel
'unknown'
const now = Date.now()
const windowMs = this.config.rateLimit.windowMs || 60000
const maxRequests = this.config.rateLimit.max || 100
const windowStart = now - windowMs
// Get or create request timestamps for this client
let timestamps = this.rateLimitStore.get(clientId) || []
// Remove old timestamps outside the window
timestamps = timestamps.filter(t => t > windowStart)
// Check if limit exceeded
if (timestamps.length >= maxRequests) {
this.log(`Rate limit exceeded for client ${clientId}: ${timestamps.length}/${maxRequests} requests`, 'warn')
return false
}
// Add current request timestamp
timestamps.push(now)
this.rateLimitStore.set(clientId, timestamps)
// Periodic cleanup of old entries to prevent memory leak
if (this.rateLimitStore.size > 1000) {
for (const [id, times] of this.rateLimitStore.entries()) {
const validTimes = times.filter(t => t > windowStart)
if (validTimes.length === 0) {
this.rateLimitStore.delete(id)
} else {
this.rateLimitStore.set(id, validTimes)
}
}
}
return true
}
/**
* Shutdown the server
*/
protected async onShutdown(): Promise<void> {
// Close all WebSocket connections
for (const client of this.clients.values()) {
if (client.socket) {
try {
client.socket.close()
} catch (error) {
// Socket already closed or errored
console.debug('Error closing WebSocket:', error)
}
}
}
this.clients.clear()
// Close servers
if (this.wsServer) {
this.wsServer.close()
}
if (this.httpServer) {
await new Promise<void>(resolve => {
this.httpServer.close(() => resolve())
})
}
this.log('API Server shut down')
}
}
/**
* Helper function to create and configure API server
*/
export function createAPIServer(config?: APIServerConfig): APIServerAugmentation {
return new APIServerAugmentation(config)
}

452
src/augmentations/auditLogAugmentation.ts
View file

@ -1,452 +0,0 @@
/**
* Audit Logging Augmentation
* Provides comprehensive audit trail for all Brainy operations
*/
import { BaseAugmentation } from './brainyAugmentation.js'
import { AugmentationManifest } from './manifest.js'
import { createHash } from '../universal/crypto.js'
export interface AuditLogConfig {
enabled?: boolean
logLevel?: 'minimal' | 'standard' | 'detailed'
includeData?: boolean
includeMetadata?: boolean
retention?: number // Days to keep logs
storage?: 'memory' | 'file' | 'database'
filePath?: string
maxMemoryLogs?: number
}
export interface AuditLogEntry {
id: string
timestamp: number
operation: string
params: any
result?: any
error?: any
duration: number
userId?: string
sessionId?: string
metadata?: Record<string, any>
}
/**
* Audit Log Augmentation
*/
export class AuditLogAugmentation extends BaseAugmentation {
readonly name = 'auditLogger'
readonly timing = 'around' as const
readonly metadata = 'readonly' as const // Read metadata for context
operations = ['all'] as any // Audit all operations
readonly priority = 90 // Low priority, runs last
// Augmentation metadata
readonly category = 'core' as const
readonly description = 'Comprehensive audit logging for compliance and debugging'
private logs: AuditLogEntry[] = []
private sessionId: string
constructor(config: AuditLogConfig = {}) {
super(config)
// Merge with defaults
this.config = {
enabled: config.enabled ?? true,
logLevel: config.logLevel ?? 'standard',
includeData: config.includeData ?? false,
includeMetadata: config.includeMetadata ?? true,
retention: config.retention ?? 90, // 90 days default
storage: config.storage ?? 'memory',
filePath: config.filePath,
maxMemoryLogs: config.maxMemoryLogs ?? 10000
}
// Generate session ID
this.sessionId = this.generateId()
}
getManifest(): AugmentationManifest {
return {
id: 'audit-logger',
name: 'Audit Logger',
version: '1.0.0',
description: 'Comprehensive audit trail for all operations',
longDescription: 'Records detailed audit logs of all Brainy operations for compliance, debugging, and analytics purposes.',
category: 'analytics',
configSchema: {
type: 'object',
properties: {
enabled: {
type: 'boolean',
default: true,
description: 'Enable audit logging'
},
logLevel: {
type: 'string',
enum: ['minimal', 'standard', 'detailed'],
default: 'standard',
description: 'Level of detail to log'
},
includeData: {
type: 'boolean',
default: false,
description: 'Include actual data in logs (privacy concern)'
},
includeMetadata: {
type: 'boolean',
default: true,
description: 'Include metadata in logs'
},
retention: {
type: 'number',
default: 90,
minimum: 1,
maximum: 365,
description: 'Days to retain logs'
},
storage: {
type: 'string',
enum: ['memory', 'file', 'database'],
default: 'memory',
description: 'Where to store audit logs'
},
maxMemoryLogs: {
type: 'number',
default: 10000,
description: 'Maximum logs to keep in memory'
}
}
},
configDefaults: {
enabled: true,
logLevel: 'standard',
includeData: false,
includeMetadata: true,
retention: 90,
storage: 'memory',
maxMemoryLogs: 10000
},
minBrainyVersion: '3.0.0',
keywords: ['audit', 'logging', 'compliance', 'analytics'],
documentation: 'https://docs.brainy.dev/augmentations/audit-log',
status: 'stable',
performance: {
memoryUsage: 'medium',
cpuUsage: 'low',
networkUsage: 'none'
},
features: ['operation-logging', 'configurable-detail', 'retention-management'],
enhancedOperations: ['all'],
metrics: [
{
name: 'audit_logs_created',
type: 'counter',
description: 'Total audit logs created'
},
{
name: 'audit_log_size',
type: 'gauge',
description: 'Current audit log size'
}
]
}
}
protected async onInitialize(): Promise<void> {
if (!this.config.enabled) {
this.log('Audit logger disabled by configuration')
return
}
this.log(`Audit logger initialized (level: ${this.config.logLevel}, storage: ${this.config.storage})`)
// Start retention cleanup if using memory storage
if (this.config.storage === 'memory') {
setInterval(() => {
this.cleanupOldLogs()
}, 3600000) // Every hour
}
}
protected async onShutdown(): Promise<void> {
// Save any pending logs if using file storage
if (this.config.storage === 'file' && this.logs.length > 0) {
await this.flushLogs()
}
this.log('Audit logger shut down')
}
/**
* Execute augmentation - log operations
*/
async execute<T = any>(
operation: string,
params: any,
next: () => Promise<T>
): Promise<T> {
// If audit logging is disabled, just pass through
if (!this.config.enabled) {
return next()
}
const startTime = Date.now()
const logEntry: Partial<AuditLogEntry> = {
id: this.generateId(),
timestamp: startTime,
operation,
sessionId: this.sessionId
}
// Add params based on log level
if (this.config.logLevel !== 'minimal') {
logEntry.params = this.sanitizeParams(params)
}
try {
const result = await next()
// Log successful operation
logEntry.duration = Date.now() - startTime
// Add result based on log level and config
if (this.config.logLevel === 'detailed' && this.config.includeData) {
logEntry.result = this.sanitizeResult(result)
}
await this.writeLog(logEntry as AuditLogEntry)
return result
} catch (error) {
// Log failed operation
logEntry.duration = Date.now() - startTime
logEntry.error = this.sanitizeError(error)
await this.writeLog(logEntry as AuditLogEntry)
throw error
}
}
/**
* Sanitize parameters to remove sensitive data
*/
private sanitizeParams(params: any): any {
if (!params) return params
// Don't include actual data unless configured
if (!this.config.includeData && params.data) {
return {
...params,
data: '[REDACTED]'
}
}
// Redact common sensitive fields
const sanitized = { ...params }
const sensitiveFields = ['password', 'token', 'apiKey', 'secret']
for (const field of sensitiveFields) {
if (sanitized[field]) {
sanitized[field] = '[REDACTED]'
}
}
return sanitized
}
/**
* Sanitize result data
*/
private sanitizeResult(result: any): any {
if (!result) return result
// For arrays, just log count
if (Array.isArray(result)) {
return { count: result.length, type: 'array' }
}
// For objects, remove sensitive fields
if (typeof result === 'object') {
const sanitized: any = {}
for (const key in result) {
if (!['password', 'token', 'apiKey', 'secret'].includes(key)) {
sanitized[key] = result[key]
}
}
return sanitized
}
return result
}
/**
* Sanitize error information
*/
private sanitizeError(error: any): any {
if (!error) return error
return {
message: error.message || 'Unknown error',
code: error.code,
statusCode: error.statusCode,
stack: this.config.logLevel === 'detailed' ? error.stack : undefined
}
}
/**
* Write log entry
*/
private async writeLog(entry: AuditLogEntry): Promise<void> {
switch (this.config.storage) {
case 'memory':
this.logs.push(entry)
// Enforce max memory logs
if (this.logs.length > this.config.maxMemoryLogs!) {
this.logs.shift() // Remove oldest
}
break
case 'file':
// In production, would write to file
// For now, just add to memory
this.logs.push(entry)
break
case 'database':
// In production, would write to database
// For now, just add to memory
this.logs.push(entry)
break
}
}
/**
* Flush logs to persistent storage
*/
private async flushLogs(): Promise<void> {
// In production, would write to file/database
// For now, just clear old logs
if (this.logs.length > this.config.maxMemoryLogs!) {
this.logs = this.logs.slice(-this.config.maxMemoryLogs!)
}
}
/**
* Clean up old logs based on retention
*/
private cleanupOldLogs(): void {
const cutoffTime = Date.now() - (this.config.retention! * 24 * 60 * 60 * 1000)
this.logs = this.logs.filter(log => log.timestamp >= cutoffTime)
}
/**
* Generate unique ID
*/
private generateId(): string {
return createHash('sha256')
.update(`${Date.now()}-${Math.random()}`)
.digest('hex')
.substring(0, 16)
}
/**
* Query audit logs
*/
queryLogs(filter?: {
operation?: string
startTime?: number
endTime?: number
sessionId?: string
hasError?: boolean
}): AuditLogEntry[] {
let results = [...this.logs]
if (filter) {
if (filter.operation) {
results = results.filter(log => log.operation === filter.operation)
}
if (filter.startTime) {
results = results.filter(log => log.timestamp >= filter.startTime!)
}
if (filter.endTime) {
results = results.filter(log => log.timestamp <= filter.endTime!)
}
if (filter.sessionId) {
results = results.filter(log => log.sessionId === filter.sessionId)
}
if (filter.hasError !== undefined) {
results = results.filter(log => (log.error !== undefined) === filter.hasError)
}
}
return results
}
/**
* Get audit statistics
*/
getStats(): {
totalLogs: number
operations: Record<string, number>
averageDuration: number
errorRate: number
} {
const stats: any = {
totalLogs: this.logs.length,
operations: {},
averageDuration: 0,
errorRate: 0
}
let totalDuration = 0
let errorCount = 0
for (const log of this.logs) {
// Count by operation
stats.operations[log.operation] = (stats.operations[log.operation] || 0) + 1
// Sum duration
totalDuration += log.duration
// Count errors
if (log.error) errorCount++
}
if (this.logs.length > 0) {
stats.averageDuration = totalDuration / this.logs.length
stats.errorRate = errorCount / this.logs.length
}
return stats
}
/**
* Export logs for analysis
*/
exportLogs(): AuditLogEntry[] {
return [...this.logs]
}
/**
* Clear all logs
*/
clearLogs(): void {
this.logs = []
}
}
/**
* Create audit log augmentation
*/
export function createAuditLogAugmentation(config?: AuditLogConfig): AuditLogAugmentation {
return new AuditLogAugmentation(config)
}

803
src/augmentations/batchProcessingAugmentation.ts
View file

@ -1,803 +0,0 @@
/**
* Batch Processing Augmentation
*
* Critical for enterprise-scale performance: 500,000+ operations/second
* Automatically batches operations for maximum throughput
* Handles streaming data, bulk imports, and high-frequency operations
*
* Performance Impact: 10-50x improvement for bulk operations
*/
import { BaseAugmentation, AugmentationContext } from './brainyAugmentation.js'
import { AugmentationManifest } from './manifest.js'
interface BatchConfig {
enabled?: boolean
adaptiveMode?: boolean // Zero-config: automatically decide when to batch
immediateThreshold?: number // Operations <= this count are immediate
batchThreshold?: number // Start batching when >= this many operations queued
maxBatchSize?: number // Maximum items per batch
maxWaitTime?: number // Maximum wait time before flushing (ms)
adaptiveBatching?: boolean // Dynamically adjust batch size based on performance
priorityLanes?: number // Number of priority processing lanes
memoryLimit?: number // Maximum memory for batching (bytes)
}
interface BatchedOperation<T = any> {
id: string
operation: string
params: any
executor: () => Promise<T> // The actual function to execute
resolver: (value: T) => void
rejector: (error: Error) => void
timestamp: number
priority: number
size: number // Estimated memory size
}
interface BatchMetrics {
totalOperations: number
batchesProcessed: number
averageBatchSize: number
averageLatency: number
throughputPerSecond: number
memoryUsage: number
adaptiveAdjustments: number
}
export class BatchProcessingAugmentation extends BaseAugmentation {
readonly metadata = 'readonly' as const // Reads metadata for batching decisions
name = 'BatchProcessing'
timing = 'around' as const
operations = ['add', 'addNoun', 'addVerb', 'saveNoun', 'saveVerb', 'storage'] as ('add' | 'addNoun' | 'addVerb' | 'saveNoun' | 'saveVerb' | 'storage')[]
priority = 80 // High priority for performance
protected config: Required<BatchConfig> = {
enabled: true,
adaptiveMode: true,
immediateThreshold: 1,
batchThreshold: 5,
maxBatchSize: 100,
maxWaitTime: 1000,
adaptiveBatching: true,
priorityLanes: 2,
memoryLimit: 100 * 1024 * 1024 // 100MB
}
private batches: Map<string, BatchedOperation<any>[]> = new Map()
private flushTimers: Map<string, NodeJS.Timeout> = new Map()
private metrics: BatchMetrics = {
totalOperations: 0,
batchesProcessed: 0,
averageBatchSize: 0,
averageLatency: 0,
throughputPerSecond: 0,
memoryUsage: 0,
adaptiveAdjustments: 0
}
private currentMemoryUsage = 0
private performanceHistory: number[] = []
constructor(config?: BatchConfig) {
super(config)
}
getManifest(): AugmentationManifest {
return {
id: 'batch-processing',
name: 'Batch Processing',
version: '2.0.0',
description: 'High-performance batching for bulk operations',
longDescription: 'Automatically batches operations for maximum throughput. Essential for enterprise-scale workloads, achieving 500,000+ operations/second. Provides 10-50x performance improvement for bulk operations.',
category: 'performance',
configSchema: {
type: 'object',
properties: {
enabled: {
type: 'boolean',
default: true,
description: 'Enable batch processing'
},
adaptiveMode: {
type: 'boolean',
default: true,
description: 'Automatically decide when to batch operations'
},
immediateThreshold: {
type: 'number',
default: 1,
minimum: 1,
maximum: 10,
description: 'Operations count below which to execute immediately'
},
batchThreshold: {
type: 'number',
default: 5,
minimum: 2,
maximum: 100,
description: 'Queue size at which to start batching'
},
maxBatchSize: {
type: 'number',
default: 1000,
minimum: 10,
maximum: 10000,
description: 'Maximum items per batch'
},
maxWaitTime: {
type: 'number',
default: 100,
minimum: 1,
maximum: 5000,
description: 'Maximum wait time before flushing batch (ms)'
},
adaptiveBatching: {
type: 'boolean',
default: true,
description: 'Dynamically adjust batch size based on performance'
},
priorityLanes: {
type: 'number',
default: 3,
minimum: 1,
maximum: 10,
description: 'Number of priority processing lanes'
},
memoryLimit: {
type: 'number',
default: 104857600, // 100MB
minimum: 10485760, // 10MB
maximum: 1073741824, // 1GB
description: 'Maximum memory for batching in bytes'
}
},
additionalProperties: false
},
configDefaults: {
enabled: true,
adaptiveMode: true,
immediateThreshold: 1,
batchThreshold: 5,
maxBatchSize: 1000,
maxWaitTime: 100,
adaptiveBatching: true,
priorityLanes: 3,
memoryLimit: 104857600
},
minBrainyVersion: '2.0.0',
keywords: ['batch', 'performance', 'bulk', 'streaming', 'throughput'],
documentation: 'https://docs.brainy.dev/augmentations/batch-processing',
status: 'stable',
performance: {
memoryUsage: 'high',
cpuUsage: 'medium',
networkUsage: 'none'
},
features: ['auto-batching', 'adaptive-sizing', 'priority-lanes', 'streaming-support'],
enhancedOperations: ['add', 'addNoun', 'addVerb', 'saveNoun', 'saveVerb'],
ui: {
icon: '📦',
color: '#9C27B0'
}
}
}
protected async onInitialize(): Promise<void> {
if (this.config.enabled) {
this.startMetricsCollection()
this.log(`Batch processing initialized: ${this.config.maxBatchSize} batch size, ${this.config.maxWaitTime}ms max wait`)
if (this.config.adaptiveBatching) {
this.log('Adaptive batching enabled - will optimize batch size dynamically')
}
} else {
this.log('Batch processing disabled')
}
}
shouldExecute(operation: string, params: any): boolean {
if (!this.config.enabled) return false
// Skip batching for single operations or already-batched operations
if (params?.batch === false || params?.streaming === false) return false
// Enable for high-volume operations
return operation.includes('add') ||
operation.includes('save') ||
operation.includes('storage')
}
async execute<T = any>(
operation: string,
params: any,
next: () => Promise<T>
): Promise<T> {
if (!this.shouldExecute(operation, params)) {
return next()
}
// Check if this should be batched based on system load
if (this.shouldBatch(operation, params)) {
return this.addToBatch(operation, params, next)
}
// Execute immediately for low-latency requirements
return next()
}
private shouldBatch(operation: string, params: any): boolean {
// ZERO-CONFIG INTELLIGENT ADAPTATION:
if (this.config.adaptiveMode) {
// CRITICAL WORKFLOW DETECTION: Never batch operations that break critical patterns
// 1. ENTITY REGISTRY PATTERN: Never batch when immediate lookup is expected
if (this.isEntityRegistryWorkflow(operation, params)) {
return false // Must be immediate for registry lookups to work
}
// 2. DEPENDENCY CHAIN PATTERN: Never batch when next operation depends on this one
if (this.isDependencyChainStart(operation, params)) {
return false // Must be immediate for noun → verb workflows
}
// Count pending operations in the current operation's batch (needed for write-only mode)
const batchKey = this.getBatchKey(operation, params)
const currentBatch = this.batches.get(batchKey) || []
const pendingCount = currentBatch.length
// 3. WRITE-ONLY MODE: Special handling for high-speed streaming
if (this.isWriteOnlyMode(params)) {
// In write-only mode, batch aggressively but ensure entity registry updates immediately
if (this.hasEntityRegistryMetadata(params)) {
return false // Entity registry updates must be immediate even in write-only mode
}
return pendingCount >= 3 // Lower threshold for write-only mode batching
}
// Apply intelligent thresholds:
// 4. Single operations are immediate (responsive user experience)
if (pendingCount < this.config.immediateThreshold!) {
return false // Execute immediately
}
// 5. Start batching when multiple operations are queued
if (pendingCount >= this.config.batchThreshold!) {
return true // Batch for efficiency
}
// 6. For in-between cases, use smart heuristics
const currentLoad = this.getCurrentLoad()
if (currentLoad > 0.5) return true // Higher load = more batching
// 7. Batch operations that naturally benefit from grouping
if (operation.includes('save') || operation.includes('add')) {
return pendingCount > 1 // Batch if others are already waiting
}
return false // Default to immediate for best responsiveness
}
// TRADITIONAL MODE: (for explicit configuration scenarios)
// Always batch if explicitly requested
if (params?.batch === true || params?.streaming === true) return true
// Batch based on current system load
const currentLoad = this.getCurrentLoad()
if (currentLoad > 0.7) return true // High load - batch everything
// Batch operations that benefit from grouping
return operation.includes('save') ||
operation.includes('add') ||
operation.includes('update')
}
/**
* SMART WORKFLOW DETECTION METHODS
* These methods detect critical patterns that must not be batched
*/
private isEntityRegistryWorkflow(operation: string, params: any): boolean {
// Detect operations that will likely be followed by immediate entity registry lookups
if (operation === 'addNoun' || operation === 'add') {
// Check if metadata contains external identifiers (DID, handle, etc.)
const metadata = params?.metadata || params?.data || {}
return !!(
metadata.did || // Bluesky DID
metadata.handle || // Social media handle
metadata.uri || // Resource URI
metadata.external_id || // External system ID
metadata.user_id || // User ID
metadata.profile_id || // Profile ID
metadata.account_id // Account ID
)
}
return false
}
private isDependencyChainStart(operation: string, params: any): boolean {
// Detect operations that are likely to be followed by dependent operations
if (operation === 'addNoun' || operation === 'add') {
// In interactive workflows, noun creation is often followed by verb creation
// Use heuristics to detect this pattern
const context = this.getOperationContext()
// If we've seen recent addVerb operations, this noun might be for a relationship
if (context.recentVerbOperations > 0) {
return true
}
// If this is part of a rapid sequence of operations, it might be a dependency chain
if (context.operationsInLastSecond > 3) {
return true
}
}
return false
}
private isWriteOnlyMode(params: any): boolean {
// Detect write-only mode from context or parameters
return !!(
params?.writeOnlyMode ||
params?.streaming ||
params?.highThroughput ||
this.context?.brain?.writeOnly
)
}
private hasEntityRegistryMetadata(params: any): boolean {
// Check if this operation has metadata that needs immediate entity registry updates
const metadata = params?.metadata || params?.data || {}
return !!(
metadata.did ||
metadata.handle ||
metadata.uri ||
metadata.external_id ||
// Also check for auto-registration hints
params?.autoCreateMissingNouns ||
params?.entityRegistry
)
}
private getOperationContext(): { recentVerbOperations: number; operationsInLastSecond: number } {
const now = Date.now()
const oneSecondAgo = now - 1000
let recentVerbOperations = 0
let operationsInLastSecond = 0
// Analyze recent operations across all batches
for (const batch of this.batches.values()) {
for (const op of batch) {
if (op.timestamp > oneSecondAgo) {
operationsInLastSecond++
if (op.operation.includes('Verb') || op.operation.includes('verb')) {
recentVerbOperations++
}
}
}
}
return { recentVerbOperations, operationsInLastSecond }
}
private getCurrentLoad(): number {
// Simple load calculation based on pending operations
let totalPending = 0
for (const batch of this.batches.values()) {
totalPending += batch.length
}
return Math.min(totalPending / 10000, 1.0) // Normalize to 0-1
}
private async addToBatch<T>(
operation: string,
params: any,
executor: () => Promise<T>
): Promise<T> {
return new Promise((resolve, reject) => {
const priority = this.getOperationPriority(operation, params)
const batchKey = this.getBatchKey(operation, priority)
const operationSize = this.estimateOperationSize(params)
// Check memory limit
if (this.currentMemoryUsage + operationSize > this.config.memoryLimit) {
// Memory limit reached - flush oldest batch
this.flushOldestBatch()
}
const batchedOp: BatchedOperation<T> = {
id: `op_${Date.now()}_${Math.random()}`,
operation,
params,
executor, // Store the actual executor
resolver: resolve,
rejector: reject,
timestamp: Date.now(),
priority,
size: operationSize
}
// Add to appropriate batch
if (!this.batches.has(batchKey)) {
this.batches.set(batchKey, [])
}
const batch = this.batches.get(batchKey)!
batch.push(batchedOp)
this.currentMemoryUsage += operationSize
this.metrics.totalOperations++
// Check if batch should be flushed immediately
if (this.shouldFlushBatch(batch, batchKey)) {
this.flushBatch(batchKey)
} else if (!this.flushTimers.has(batchKey)) {
// Set flush timer if not already set
this.setFlushTimer(batchKey)
}
})
}
private getOperationPriority(operation: string, params: any): number {
// Explicit priority
if (params?.priority !== undefined) return params.priority
// Operation-based priority
if (operation.includes('delete')) return 10 // Highest
if (operation.includes('update')) return 8
if (operation.includes('save')) return 6
if (operation.includes('add')) return 4
return 1 // Lowest
}
private getBatchKey(operation: string, priority: number): string {
// Group by operation type and priority for optimal batching
const opType = this.getOperationType(operation)
const priorityLane = Math.min(priority, this.config.priorityLanes - 1)
return `${opType}_p${priorityLane}`
}
private getOperationType(operation: string): string {
if (operation.includes('add')) return 'add'
if (operation.includes('save')) return 'save'
if (operation.includes('update')) return 'update'
if (operation.includes('delete')) return 'delete'
return 'other'
}
private estimateOperationSize(params: any): number {
// Rough estimation of memory usage
if (!params) return 100
let size = 0
if (params.vector && Array.isArray(params.vector)) {
size += params.vector.length * 8 // 8 bytes per float64
}
if (params.data) {
size += JSON.stringify(params.data).length * 2 // Rough UTF-16 estimate
}
if (params.metadata) {
size += JSON.stringify(params.metadata).length * 2
}
return Math.max(size, 100) // Minimum 100 bytes
}
private shouldFlushBatch(batch: BatchedOperation[], batchKey: string): boolean {
// Flush if batch is full
if (batch.length >= this.config.maxBatchSize) return true
// Flush if memory limit approaching
if (this.currentMemoryUsage > this.config.memoryLimit * 0.9) return true
// Flush high-priority batches more aggressively
const priority = this.extractPriorityFromKey(batchKey)
if (priority >= 8 && batch.length >= 100) return true
if (priority >= 6 && batch.length >= 500) return true
return false
}
private extractPriorityFromKey(batchKey: string): number {
const match = batchKey.match(/_p(\d+)$/)
return match ? parseInt(match[1]) : 0
}
private setFlushTimer(batchKey: string): void {
const priority = this.extractPriorityFromKey(batchKey)
const waitTime = this.getAdaptiveWaitTime(priority)
const timer = setTimeout(() => {
this.flushBatch(batchKey)
}, waitTime)
this.flushTimers.set(batchKey, timer)
}
private getAdaptiveWaitTime(priority: number): number {
if (!this.config.adaptiveBatching) {
return this.config.maxWaitTime
}
// Adaptive wait time based on performance and priority
const baseWaitTime = this.config.maxWaitTime
const performanceMultiplier = this.getPerformanceMultiplier()
const priorityMultiplier = priority >= 8 ? 0.5 : priority >= 6 ? 0.7 : 1.0
return Math.max(baseWaitTime * performanceMultiplier * priorityMultiplier, 10)
}
private getPerformanceMultiplier(): number {
if (this.performanceHistory.length < 10) return 1.0
// Calculate average latency trend
const recent = this.performanceHistory.slice(-10)
const average = recent.reduce((a, b) => a + b, 0) / recent.length
// If performance is degrading, reduce wait time
if (average > this.metrics.averageLatency * 1.2) return 0.7
if (average < this.metrics.averageLatency * 0.8) return 1.3
return 1.0
}
private async flushBatch(batchKey: string): Promise<void> {
const batch = this.batches.get(batchKey)
if (!batch || batch.length === 0) return
// Clear timer
const timer = this.flushTimers.get(batchKey)
if (timer) {
clearTimeout(timer)
this.flushTimers.delete(batchKey)
}
// Remove batch from queue
this.batches.delete(batchKey)
const startTime = Date.now()
try {
await this.processBatch(batch)
// Update metrics
const latency = Date.now() - startTime
this.updateMetrics(batch.length, latency)
// Adaptive adjustment
if (this.config.adaptiveBatching) {
this.adjustBatchSize(latency, batch.length)
}
} catch (error) {
this.log(`Batch processing failed for ${batchKey}: ${error}`, 'error')
// Reject all operations in batch
batch.forEach(op => {
op.rejector(error as Error)
this.currentMemoryUsage -= op.size
})
}
}
private async processBatch(batch: BatchedOperation<any>[]): Promise<void> {
// Group by operation type for efficient processing
const operationGroups = new Map<string, BatchedOperation<any>[]>()
for (const op of batch) {
const opType = this.getOperationType(op.operation)
if (!operationGroups.has(opType)) {
operationGroups.set(opType, [])
}
operationGroups.get(opType)!.push(op)
}
// Process each operation type
for (const [opType, operations] of operationGroups) {
await this.processBatchByType(opType, operations)
}
}
private async processBatchByType(opType: string, operations: BatchedOperation<any>[]): Promise<void> {
// Execute batch operation based on type
try {
if (opType === 'add' || opType === 'save') {
await this.processBatchSave(operations)
} else if (opType === 'update') {
await this.processBatchUpdate(operations)
} else if (opType === 'delete') {
await this.processBatchDelete(operations)
} else {
// Fallback: execute individually
await this.processIndividually(operations)
}
} catch (error) {
throw error
}
}
private async processBatchSave(operations: BatchedOperation<any>[]): Promise<void> {
// Try to use storage's bulk save if available
const storage = this.context?.storage
if (storage && typeof storage.saveBatch === 'function') {
// Use bulk save operation
const items = operations.map(op => ({
...op.params,
_batchId: op.id
}))
try {
const results = await storage.saveBatch(items)
// Resolve all operations with actual results
operations.forEach((op, index) => {
op.resolver(results[index] || op.params.id)
this.currentMemoryUsage -= op.size
})
} catch (error) {
// Reject all operations on batch error
operations.forEach(op => {
op.rejector(error as Error)
this.currentMemoryUsage -= op.size
})
throw error
}
} else {
// Execute using stored executors with concurrency control
await this.processWithConcurrency(operations, 10)
}
}
private async processBatchUpdate(operations: BatchedOperation<any>[]): Promise<void> {
await this.processWithConcurrency(operations, 5) // Lower concurrency for updates
}
private async processBatchDelete(operations: BatchedOperation<any>[]): Promise<void> {
await this.processWithConcurrency(operations, 5) // Lower concurrency for deletes
}
private async processIndividually(operations: BatchedOperation<any>[]): Promise<void> {
await this.processWithConcurrency(operations, 3) // Conservative concurrency
}
private async processWithConcurrency(operations: BatchedOperation<any>[], concurrency: number): Promise<void> {
const promises: Promise<void>[] = []
for (let i = 0; i < operations.length; i += concurrency) {
const chunk = operations.slice(i, i + concurrency)
const chunkPromise = Promise.all(
chunk.map(async (op) => {
try {
// Execute using the stored executor function - REAL EXECUTION!
const result = await op.executor()
op.resolver(result)
this.currentMemoryUsage -= op.size
} catch (error) {
op.rejector(error as Error)
this.currentMemoryUsage -= op.size
}
})
).then(() => {}) // Convert to void promise
promises.push(chunkPromise)
}
await Promise.all(promises)
}
// REMOVED executeOperation - no longer needed since we use stored executors
private flushOldestBatch(): void {
if (this.batches.size === 0) return
// Find oldest batch
let oldestKey = ''
let oldestTime = Infinity
for (const [key, batch] of this.batches) {
if (batch.length > 0) {
const batchAge = Math.min(...batch.map(op => op.timestamp))
if (batchAge < oldestTime) {
oldestTime = batchAge
oldestKey = key
}
}
}
if (oldestKey) {
this.flushBatch(oldestKey)
}
}
private updateMetrics(batchSize: number, latency: number): void {
this.metrics.batchesProcessed++
this.metrics.averageBatchSize =
(this.metrics.averageBatchSize * (this.metrics.batchesProcessed - 1) + batchSize) /
this.metrics.batchesProcessed
// Update latency with exponential moving average
this.metrics.averageLatency = this.metrics.averageLatency * 0.9 + latency * 0.1
// Add to performance history
this.performanceHistory.push(latency)
if (this.performanceHistory.length > 100) {
this.performanceHistory.shift()
}
}
private adjustBatchSize(latency: number, batchSize: number): void {
const targetLatency = this.config.maxWaitTime * 5 // Target: 5x wait time
if (latency > targetLatency && batchSize > 100) {
// Reduce batch size if latency too high
this.config.maxBatchSize = Math.max(this.config.maxBatchSize * 0.9, 100)
this.metrics.adaptiveAdjustments++
} else if (latency < targetLatency * 0.5 && batchSize === this.config.maxBatchSize) {
// Increase batch size if latency very low
this.config.maxBatchSize = Math.min(this.config.maxBatchSize * 1.1, 10000)
this.metrics.adaptiveAdjustments++
}
}
private startMetricsCollection(): void {
setInterval(() => {
// Calculate throughput
this.metrics.throughputPerSecond = this.metrics.totalOperations
this.metrics.totalOperations = 0 // Reset for next measurement
// Update memory usage
this.metrics.memoryUsage = this.currentMemoryUsage
}, 1000)
}
/**
* Get batch processing statistics
*/
getStats(): BatchMetrics & {
pendingBatches: number
pendingOperations: number
currentBatchSize: number
memoryUtilization: string
} {
let pendingOperations = 0
for (const batch of this.batches.values()) {
pendingOperations += batch.length
}
return {
...this.metrics,
pendingBatches: this.batches.size,
pendingOperations,
currentBatchSize: this.config.maxBatchSize,
memoryUtilization: `${Math.round((this.currentMemoryUsage / this.config.memoryLimit) * 100)}%`
}
}
/**
* Force flush all pending batches
*/
async flushAll(): Promise<void> {
const batchKeys = Array.from(this.batches.keys())
await Promise.all(batchKeys.map(key => this.flushBatch(key)))
}
protected async onShutdown(): Promise<void> {
// Clear all timers
for (const timer of this.flushTimers.values()) {
clearTimeout(timer)
}
this.flushTimers.clear()
// Flush all pending batches
await this.flushAll()
const stats = this.getStats()
this.log(`Batch processing shutdown: ${this.metrics.batchesProcessed} batches processed, ${stats.memoryUtilization} peak memory usage`)
}
}

766
src/augmentations/brainyAugmentation.ts
View file

@ -1,766 +0,0 @@
/**
* Single BrainyAugmentation Interface
*
* This replaces the 7 complex interfaces with one elegant, purpose-driven design.
* Each augmentation knows its place and when to execute automatically.
*
* The Vision: Components that enhance Brainy's capabilities seamlessly
* - RequestDeduplicator: Prevents duplicate concurrent requests
* - ConnectionPool: Optimizes cloud storage throughput
* - IntelligentVerbScoring: Enhances relationship analysis
* - StreamingPipeline: Enables unlimited data processing
*/
import { AugmentationManifest } from './manifest.js'
import { AugmentationConfigResolver } from './configResolver.js'
/**
* Metadata access declaration for augmentations
*/
export interface MetadataAccess {
reads?: string[] | '*' // Fields to read, or '*' for all
writes?: string[] | '*' // Fields to write, or '*' for all
namespace?: string // Optional: custom namespace like '_myAug'
}
export interface BrainyAugmentation {
/**
* Unique identifier for the augmentation
*/
name: string
/**
* When this augmentation should execute
* - 'before': Execute before the main operation
* - 'after': Execute after the main operation
* - 'around': Wrap the main operation (like middleware)
* - 'replace': Replace the main operation entirely
*/
timing: 'before' | 'after' | 'around' | 'replace'
/**
* Metadata access contract - REQUIRED
* - 'none': No metadata access at all
* - 'readonly': Can read any metadata but cannot write
* - MetadataAccess: Specific fields to read/write
*/
metadata: 'none' | 'readonly' | MetadataAccess
/**
* Which operations this augmentation applies to
* Granular operation matching for precise augmentation targeting
*/
operations: (
// Data Operations
| 'add' | 'addNoun' | 'addVerb'
| 'saveNoun' | 'saveVerb' | 'updateMetadata'
| 'update' | 'delete' | 'deleteVerb' | 'clear' | 'get'
// Search Operations
| 'search' | 'searchText' | 'searchByNounTypes'
| 'find' | 'findSimilar' | 'searchWithCursor' | 'similar'
// Relationship Operations
| 'relate' | 'unrelate' | 'getConnections' | 'getRelations'
// Storage Operations
| 'storage' | 'backup' | 'restore'
// Meta
| 'all'
)[]
/**
* Priority for execution order (higher numbers execute first)
* - 100: Critical system operations (ConnectionPool)
* - 50: Performance optimizations (RequestDeduplicator, Caching)
* - 10: Enhancement features (IntelligentVerbScoring)
* - 1: Optional features (Logging, Analytics)
*/
priority: number
/**
* Initialize the augmentation
* Called once during Brainy initialization
*
* @param context - The Brainy instance and storage
*/
initialize(context: AugmentationContext): Promise<void>
/**
* Execute the augmentation
*
* @param operation - The operation being performed
* @param params - Parameters for the operation
* @param next - Function to call the next augmentation or main operation
* @returns Result of the operation
*/
execute<T = any>(
operation: string,
params: any,
next: () => Promise<T>
): Promise<T>
/**
* Optional: Check if this augmentation should run for the given operation
* Return false to skip execution
*/
shouldExecute?(operation: string, params: any): boolean
/**
* Optional: Cleanup when Brainy is destroyed
*/
shutdown?(): Promise<void>
/**
* Optional: Computed fields this augmentation provides
* Used for discovery, TypeScript support, and API documentation
*/
computedFields?: {
[namespace: string]: {
[field: string]: {
type: 'string' | 'number' | 'boolean' | 'object' | 'array'
description: string
confidence?: number
}
}
}
/**
* Optional: Compute fields for a result entity
* Called when user accesses getDisplay(), getSchema(), etc.
*
* @param result - The result entity (VectorDocument, GraphVerb, etc.)
* @param namespace - The namespace being requested ('display', 'schema', etc.)
* @returns Computed fields for the namespace
*/
computeFields?(result: any, namespace: string): Promise<Record<string, any>> | Record<string, any>
}
/**
* Context provided to augmentations
*/
export interface AugmentationContext {
/**
* The Brainy instance (for accessing methods and config)
*/
brain: any // Brainy - avoiding circular imports
/**
* The storage adapter
*/
storage: any // StorageAdapter
/**
* Configuration for this augmentation
*/
config: any
/**
* Logging function
*/
log: (message: string, level?: 'info' | 'warn' | 'error') => void
}
/**
* Base class for augmentations with common functionality
*
* This is the unified base class that combines the features of both
* BaseAugmentation and ConfigurableAugmentation. All augmentations
* should extend this class for consistent configuration support.
*/
export abstract class BaseAugmentation implements BrainyAugmentation {
abstract name: string
abstract timing: 'before' | 'after' | 'around' | 'replace'
abstract metadata: 'none' | 'readonly' | MetadataAccess
abstract operations: (
// Data Operations
| 'add' | 'addNoun' | 'addVerb'
| 'saveNoun' | 'saveVerb' | 'updateMetadata'
| 'update' | 'delete' | 'deleteVerb' | 'clear' | 'get'
// Search Operations
| 'search' | 'searchText' | 'searchByNounTypes'
| 'find' | 'findSimilar' | 'searchWithCursor' | 'similar'
// Relationship Operations
| 'relate' | 'unrelate' | 'getConnections' | 'getRelations'
// Storage Operations
| 'storage' | 'backup' | 'restore'
// Meta
| 'all'
)[]
abstract priority: number
// Metadata for augmentation listing and management
category: 'internal' | 'core' | 'premium' | 'community' | 'external' = 'core'
description: string = ''
enabled: boolean = true
protected context?: AugmentationContext
protected isInitialized = false
protected config: any = {}
private configResolver?: AugmentationConfigResolver
/**
* Constructor with optional configuration
* @param config Optional configuration to override defaults
*/
constructor(config?: any) {
// Only resolve configuration if getManifest is implemented
if (this.getManifest) {
this.config = this.resolveConfiguration(config)
} else if (config) {
// Legacy support: direct config assignment for augmentations without manifests
this.config = config
}
}
/**
* Get the augmentation manifest for discovery
* Override this to enable configuration support
* CRITICAL: This enables tools to discover parameters and configuration
*/
getManifest?(): AugmentationManifest
/**
* Get parameter schema for operations
* Enables tools to know what parameters each operation needs
*/
getParameterSchema?(operation: string): any
/**
* Get operation descriptions
* Enables tools to show what each operation does
*/
getOperationInfo?(): Record<string, {
description: string
parameters?: any
returns?: any
examples?: any[]
}>
/**
* Get current configuration
*/
getConfig(): any {
return { ...this.config }
}
/**
* Update configuration at runtime
* @param partial Partial configuration to merge
*/
async updateConfig(partial: any): Promise<void> {
if (!this.configResolver) {
// For legacy augmentations without manifest, just merge config
const oldConfig = this.config
this.config = { ...this.config, ...partial }
if (this.onConfigChange) {
await this.onConfigChange(this.config, oldConfig)
}
return
}
const oldConfig = this.config
try {
// Use resolver to update and validate
this.config = this.configResolver.updateRuntime(partial)
// Call config change handler if implemented
if (this.onConfigChange) {
await this.onConfigChange(this.config, oldConfig)
}
} catch (error) {
// Revert on error
this.config = oldConfig
throw error
}
}
/**
* Optional: Handle configuration changes
* Override this to react to runtime configuration updates
*/
protected onConfigChange?(newConfig: any, oldConfig: any): Promise<void>
/**
* Resolve configuration from all sources
* Priority: constructor > env > files > defaults
*/
private resolveConfiguration(constructorConfig?: any): any {
const manifest = this.getManifest!()
// Create config resolver
this.configResolver = new AugmentationConfigResolver({
augmentationId: manifest.id,
schema: manifest.configSchema,
defaults: manifest.configDefaults
})
// Resolve configuration from all sources
return this.configResolver.resolve(constructorConfig)
}
async initialize(context: AugmentationContext): Promise<void> {
this.context = context
this.isInitialized = true
await this.onInitialize()
}
/**
* Override this in subclasses for initialization logic
*/
protected async onInitialize(): Promise<void> {
// Default: no-op
}
abstract execute<T = any>(
operation: string,
params: any,
next: () => Promise<T>
): Promise<T>
shouldExecute(operation: string, params: any): boolean {
// Default: execute if operations match exactly or includes 'all'
return this.operations.includes('all' as any) ||
this.operations.includes(operation as any) ||
this.operations.some(op => operation.includes(op))
}
async shutdown(): Promise<void> {
await this.onShutdown()
this.isInitialized = false
}
/**
* Override this in subclasses for cleanup logic
*/
protected async onShutdown(): Promise<void> {
// Default: no-op
}
/**
* Optional computed fields declaration (override in subclasses)
*/
computedFields?: {
[namespace: string]: {
[field: string]: {
type: 'string' | 'number' | 'boolean' | 'object' | 'array'
description: string
confidence?: number
}
}
}
/**
* Optional computed fields implementation (override in subclasses)
* @param result The result entity
* @param namespace The requested namespace
* @returns Computed fields for the namespace
*/
computeFields?(result: any, namespace: string): Promise<Record<string, any>> | Record<string, any>
/**
* Log a message with the augmentation name
*/
protected log(message: string, level: 'info' | 'warn' | 'error' = 'info'): void {
if (this.context) {
this.context.log(`[${this.name}] ${message}`, level)
}
}
/**
* Get CommitLog for temporal features
*
* Provides access to commit history for time-travel queries, audit trails,
* and branch management. Available after initialize() is called.
*
* @returns CommitLog instance
* @throws Error if called before initialize() or if COW not enabled
*
* @example
* ```typescript
* protected async onInitialize() {
* const commitLog = this.getCommitLog()
* const history = await commitLog.getHistory('heads/main', { maxCount: 10 })
* }
* ```
*/
protected getCommitLog(): any {
if (!this.context) {
throw new Error(
`${this.name}: Cannot access CommitLog before initialize(). ` +
`CommitLog is only available after the augmentation has been initialized.`
)
}
const storage = this.context.storage as any
if (!storage.commitLog) {
throw new Error(
`${this.name}: CommitLog not available. ` +
`COW (Copy-on-Write) is not enabled on this storage adapter. ` +
`Requires BaseStorage with initializeCOW() called. ` +
`This is expected if using a non-COW storage adapter.`
)
}
return storage.commitLog
}
/**
* Get BlobStorage for content-addressable storage
*
* Provides access to the underlying blob storage system for storing
* and retrieving content-addressed data. Available after initialize() is called.
*
* @returns BlobStorage instance
* @throws Error if called before initialize() or if COW not enabled
*
* @example
* ```typescript
* protected async onInitialize() {
* const blobStorage = this.getBlobStorage()
* const hash = await blobStorage.writeBlob(Buffer.from('data'))
* const data = await blobStorage.readBlob(hash)
* }
* ```
*/
protected getBlobStorage(): any {
if (!this.context) {
throw new Error(
`${this.name}: Cannot access BlobStorage before initialize(). ` +
`BlobStorage is only available after the augmentation has been initialized.`
)
}
const storage = this.context.storage as any
if (!storage.blobStorage) {
throw new Error(
`${this.name}: BlobStorage not available. ` +
`COW (Copy-on-Write) is not enabled on this storage adapter. ` +
`Requires BaseStorage with initializeCOW() called. ` +
`This is expected if using a non-COW storage adapter.`
)
}
return storage.blobStorage
}
/**
* Get RefManager for branch/ref management
*
* Provides access to the reference manager for creating, updating,
* and managing Git-style branches and refs. Available after initialize() is called.
*
* @returns RefManager instance
* @throws Error if called before initialize() or if COW not enabled
*
* @example
* ```typescript
* protected async onInitialize() {
* const refManager = this.getRefManager()
* await refManager.setRef('heads/experiment', commitHash, {
* author: 'system',
* message: 'Create experiment branch'
* })
* }
* ```
*/
protected getRefManager(): any {
if (!this.context) {
throw new Error(
`${this.name}: Cannot access RefManager before initialize(). ` +
`RefManager is only available after the augmentation has been initialized.`
)
}
const storage = this.context.storage as any
if (!storage.refManager) {
throw new Error(
`${this.name}: RefManager not available. ` +
`COW (Copy-on-Write) is not enabled on this storage adapter. ` +
`Requires BaseStorage with initializeCOW() called. ` +
`This is expected if using a non-COW storage adapter.`
)
}
return storage.refManager
}
/**
* Get current branch name
*
* Convenience helper for getting the current branch from the Brainy instance.
* Available after initialize() is called.
*
* @returns Current branch name (e.g., 'main')
* @throws Error if called before initialize()
*
* @example
* ```typescript
* protected async onInitialize() {
* const branch = await this.getCurrentBranch()
* console.log(`Current branch: ${branch}`)
* }
* ```
*/
protected async getCurrentBranch(): Promise<string> {
if (!this.context) {
throw new Error(
`${this.name}: Cannot access Brainy instance before initialize(). ` +
`getCurrentBranch() is only available after the augmentation has been initialized.`
)
}
const brain = this.context.brain as any
if (typeof brain.getCurrentBranch !== 'function') {
throw new Error(
`${this.name}: getCurrentBranch() not available on Brainy instance. ` +
`This method requires Brainy with VFS support.`
)
}
return brain.getCurrentBranch()
}
}
/**
* Alias for backward compatibility
* ConfigurableAugmentation is now merged into BaseAugmentation
* @deprecated Use BaseAugmentation instead
*/
export const ConfigurableAugmentation = BaseAugmentation
/**
* Registry for managing augmentations
*/
export class AugmentationRegistry {
private augmentations: BrainyAugmentation[] = []
private context?: AugmentationContext
/**
* Register an augmentation
*/
register(augmentation: BrainyAugmentation): void {
this.augmentations.push(augmentation)
// Sort by priority (highest first)
this.augmentations.sort((a, b) => b.priority - a.priority)
}
/**
* Find augmentations by operation (before initialization)
* Used for two-phase initialization to find storage augmentations
*/
findByOperation(operation: string): BrainyAugmentation | null {
return this.augmentations.find(aug =>
aug.operations.includes(operation as any) ||
aug.operations.includes('all' as any)
) || null
}
/**
* Initialize all augmentations
*/
async initialize(context: AugmentationContext): Promise<void> {
this.context = context
for (const augmentation of this.augmentations) {
await augmentation.initialize(context)
}
context.log(`Initialized ${this.augmentations.length} augmentations`)
}
/**
* Initialize all augmentations (alias for consistency)
*/
async initializeAll(context: AugmentationContext): Promise<void> {
return this.initialize(context)
}
/**
* Execute augmentations for an operation
*/
async execute<T = any>(
operation: string,
params: any,
mainOperation: () => Promise<T>
): Promise<T> {
// Filter augmentations that should execute for this operation
const applicable = this.augmentations.filter(aug =>
aug.shouldExecute ? aug.shouldExecute(operation, params) :
aug.operations.includes('all' as any) ||
aug.operations.includes(operation as any) ||
aug.operations.some(op => operation.includes(op))
)
if (applicable.length === 0) {
// No augmentations, execute main operation directly
return mainOperation()
}
// Create a chain of augmentations
let index = 0
const executeNext = async (): Promise<T> => {
if (index >= applicable.length) {
// All augmentations processed, execute main operation
return mainOperation()
}
const augmentation = applicable[index++]
return augmentation.execute(operation, params, executeNext)
}
return executeNext()
}
/**
* Get all registered augmentations
*/
getAll(): BrainyAugmentation[] {
return [...this.augmentations]
}
/**
* Get augmentation info for listing
*/
getInfo(): Array<{
name: string
type: string
enabled: boolean
description: string
category: string
priority: number
}> {
return this.augmentations.map(aug => {
const baseAug = aug as any
return {
name: aug.name,
type: baseAug.category || 'core',
enabled: baseAug.enabled !== false,
description: baseAug.description || `${aug.name} augmentation`,
category: baseAug.category || 'core',
priority: aug.priority
}
})
}
/**
* Get augmentations by name
*/
get(name: string): BrainyAugmentation | undefined {
return this.augmentations.find(aug => aug.name === name)
}
/**
* Discover augmentation parameters and schemas
* Critical for tools like brain-cloud to generate UIs
*/
discover(name?: string): any {
if (name) {
const aug = this.get(name)
if (!aug) return null
const baseAug = aug as BaseAugmentation
return {
name: aug.name,
operations: aug.operations,
priority: aug.priority,
timing: aug.timing,
metadata: aug.metadata,
manifest: baseAug.getManifest ? baseAug.getManifest() : undefined,
parameters: baseAug.getParameterSchema ?
aug.operations.reduce((acc, op) => {
acc[op] = baseAug.getParameterSchema!(op as string)
return acc
}, {} as any) : undefined,
operationInfo: baseAug.getOperationInfo ? baseAug.getOperationInfo() : undefined,
config: baseAug.getConfig ? baseAug.getConfig() : undefined
}
}
// Return all augmentations discovery info
return this.augmentations.map(aug => this.discover(aug.name))
}
/**
* Get configuration schema for an augmentation
* Enables UI generation for configuration
*/
getConfigSchema(name: string): any {
const aug = this.get(name) as BaseAugmentation
if (!aug || !aug.getManifest) return null
const manifest = aug.getManifest()
return manifest?.configSchema
}
/**
* Configure an augmentation at runtime
*/
async configure(name: string, config: any): Promise<void> {
const aug = this.get(name) as BaseAugmentation
if (!aug || !aug.updateConfig) {
throw new Error(`Augmentation ${name} does not support configuration`)
}
await aug.updateConfig(config)
}
/**
* Get metrics for an augmentation
*/
metrics(name?: string): any {
if (name) {
const aug = this.get(name) as any
if (!aug || !aug.metrics) return null
return aug.metrics()
}
// Return all metrics
const allMetrics: any = {}
for (const aug of this.augmentations) {
const a = aug as any
if (a.metrics) {
allMetrics[aug.name] = a.metrics()
}
}
return allMetrics
}
/**
* Get health status
*/
health(): any {
const health: any = {
overall: 'healthy',
augmentations: {}
}
for (const aug of this.augmentations) {
const a = aug as any
health.augmentations[aug.name] = a.health ? a.health() : 'unknown'
}
return health
}
/**
* Shutdown all augmentations
*/
async shutdown(): Promise<void> {
for (const augmentation of this.augmentations) {
if (augmentation.shutdown) {
await augmentation.shutdown()
}
}
this.augmentations = []
}
}

391
src/augmentations/cacheAugmentation.ts
View file

@ -1,391 +0,0 @@
/**
* Cache Augmentation - Optional Search Result Caching
*
* Replaces the hardcoded SearchCache in Brainy with an optional augmentation.
* This reduces core size and allows custom cache implementations.
*
* Zero-config: Automatically enabled with sensible defaults
* Can be disabled or customized via augmentation registry
*/
import { BaseAugmentation, AugmentationContext } from './brainyAugmentation.js'
import { AugmentationManifest } from './manifest.js'
import { SearchCache } from '../utils/searchCache.js'
import type { GraphNoun } from '../types/graphTypes.js'
export interface CacheConfig {
maxSize?: number
ttl?: number
enabled?: boolean
invalidateOnWrite?: boolean
silent?: boolean // Silent mode support
}
/**
* CacheAugmentation - Makes search caching optional and pluggable
*
* Features:
* - Transparent search result caching
* - Automatic invalidation on data changes
* - Memory-aware cache management
* - Zero-config with smart defaults
*/
export class CacheAugmentation extends BaseAugmentation {
readonly name = 'cache'
readonly timing = 'around' as const
readonly metadata = 'none' as const // Cache doesn't access metadata
operations = ['search', 'find', 'similar', 'add', 'update', 'delete', 'clear', 'all'] as ('search' | 'find' | 'similar' | 'add' | 'update' | 'delete' | 'clear' | 'all')[]
readonly priority = 50 // Mid-priority, runs after data operations
// Augmentation metadata
readonly category = 'core' as const
readonly description = 'Transparent search result caching with automatic invalidation'
private searchCache: SearchCache<GraphNoun> | null = null
constructor(config?: CacheConfig) {
super(config)
}
getManifest(): AugmentationManifest {
return {
id: 'cache',
name: 'Cache',
version: '2.0.0',
description: 'Intelligent caching for search and query operations',
longDescription: 'Provides transparent caching for search results with automatic invalidation on data changes. Significantly improves performance for repeated queries while maintaining data consistency.',
category: 'performance',
configSchema: {
type: 'object',
properties: {
enabled: {
type: 'boolean',
default: true,
description: 'Enable or disable caching'
},
maxSize: {
type: 'number',
default: 1000,
minimum: 10,
maximum: 100000,
description: 'Maximum number of cached entries'
},
ttl: {
type: 'number',
default: 300000, // 5 minutes
minimum: 1000, // 1 second
maximum: 3600000, // 1 hour
description: 'Time to live for cache entries in milliseconds'
},
invalidateOnWrite: {
type: 'boolean',
default: true,
description: 'Automatically invalidate cache on data modifications'
},
silent: {
type: 'boolean',
default: false,
description: 'Suppress all console output'
}
},
additionalProperties: false
},
configDefaults: {
enabled: true,
maxSize: 1000,
ttl: 300000,
invalidateOnWrite: true
},
configExamples: [
{
name: 'High Performance',
description: 'Large cache with longer TTL for read-heavy workloads',
config: {
enabled: true,
maxSize: 10000,
ttl: 1800000, // 30 minutes
invalidateOnWrite: true
}
},
{
name: 'Conservative',
description: 'Small cache with short TTL for frequently changing data',
config: {
enabled: true,
maxSize: 100,
ttl: 60000, // 1 minute
invalidateOnWrite: true
}
}
],
minBrainyVersion: '2.0.0',
keywords: ['cache', 'performance', 'search', 'optimization'],
documentation: 'https://docs.brainy.dev/augmentations/cache',
status: 'stable',
performance: {
memoryUsage: 'medium',
cpuUsage: 'low',
networkUsage: 'none'
},
features: ['search-caching', 'auto-invalidation', 'ttl-support', 'memory-management'],
enhancedOperations: ['search', 'searchText', 'findSimilar'],
metrics: [
{
name: 'cache_hits',
type: 'counter',
description: 'Number of cache hits'
},
{
name: 'cache_misses',
type: 'counter',
description: 'Number of cache misses'
},
{
name: 'cache_size',
type: 'gauge',
description: 'Current cache size'
}
],
ui: {
icon: '⚡',
color: '#FFC107'
}
}
}
protected async onInitialize(): Promise<void> {
if (!this.config.enabled) {
this.log('Cache augmentation disabled by configuration')
return
}
// Initialize search cache with config
this.searchCache = new SearchCache<GraphNoun>({
maxSize: this.config.maxSize!,
maxAge: this.config.ttl!, // SearchCache uses maxAge, not ttl
enabled: true
})
this.log(`Cache augmentation initialized (maxSize: ${this.config.maxSize}, ttl: ${this.config.ttl}ms)`)
}
protected async onShutdown(): Promise<void> {
if (this.searchCache) {
this.searchCache.clear()
this.searchCache = null
}
this.log('Cache augmentation shut down')
}
/**
* Execute augmentation - wrap operations with caching logic
*/
async execute<T = any>(
operation: string,
params: any,
next: () => Promise<T>
): Promise<T> {
// If cache is disabled, just pass through
if (!this.searchCache || !this.config.enabled) {
return next()
}
switch (operation) {
case 'search':
return this.handleSearch(params, next)
case 'add':
case 'update':
case 'delete':
// Invalidate cache on data changes
// Cache the reference to avoid race condition during async operation
const cache = this.searchCache
if (this.config.invalidateOnWrite && cache) {
const result = await next()
// Use cached reference - searchCache might have been nulled during await
cache.invalidateOnDataChange(operation as any)
this.log(`Cache invalidated due to ${operation} operation`)
return result
}
return next()
case 'clear':
// Clear cache when all data is cleared
const result = await next()
if (this.searchCache) {
this.searchCache.clear()
this.log('Cache cleared due to clear operation')
}
return result
default:
return next()
}
}
/**
* Handle search operation with caching
*/
private async handleSearch<T>(params: any, next: () => Promise<T>): Promise<T> {
if (!this.searchCache) return next()
// Extract search parameters
const { query, k, options = {} } = params
// Skip cache if explicitly disabled or has complex filters
if (options.skipCache || options.metadata) {
return next()
}
// Generate cache key
const cacheKey = this.searchCache.getCacheKey(query, k, options)
// Check cache
const cachedResult = this.searchCache.get(cacheKey)
if (cachedResult) {
this.log('Cache hit for search query')
// Update metrics if available
if (this.context?.brain) {
const metrics = this.context.brain.augmentations?.get('metrics')
if (metrics) {
metrics.recordCacheHit?.()
}
}
return cachedResult as T
}
// Execute search
const result = await next()
// Cache the result
this.searchCache.set(cacheKey, result as any)
this.log('Search result cached')
// Update metrics if available
if (this.context?.brain) {
const metrics = this.context.brain.augmentations?.get('metrics')
if (metrics) {
metrics.recordCacheMiss?.()
}
}
return result
}
/**
* Get cache statistics
*/
getStats() {
if (!this.searchCache) {
return {
enabled: false,
hits: 0,
misses: 0,
size: 0,
memoryUsage: 0
}
}
const stats = this.searchCache.getStats()
return {
...stats,
memoryUsage: this.searchCache.getMemoryUsage()
}
}
/**
* Clear the cache manually
*/
clear() {
if (this.searchCache) {
this.searchCache.clear()
this.log('Cache manually cleared')
}
}
/**
* Handle runtime configuration changes
*/
protected async onConfigChange(newConfig: CacheConfig, oldConfig: CacheConfig): Promise<void> {
if (this.searchCache && newConfig.enabled) {
this.searchCache.updateConfig({
maxSize: newConfig.maxSize!,
maxAge: newConfig.ttl!, // SearchCache uses maxAge
enabled: newConfig.enabled
})
this.log('Cache configuration updated')
} else if (!newConfig.enabled && this.searchCache) {
this.searchCache.clear()
this.log('Cache disabled and cleared')
}
}
/**
* Clean up expired entries
*/
cleanupExpiredEntries(): number {
if (!this.searchCache) return 0
const cleaned = this.searchCache.cleanupExpiredEntries()
if (cleaned > 0) {
this.log(`Cleaned ${cleaned} expired cache entries`)
}
return cleaned
}
/**
* Invalidate cache when data changes
*/
invalidateOnDataChange(operation: 'add' | 'update' | 'delete') {
if (!this.searchCache) return
this.searchCache.invalidateOnDataChange(operation)
this.log(`Cache invalidated due to ${operation} operation`)
}
/**
* Get cache key for a query
*/
getCacheKey(query: any, options?: any): string {
if (!this.searchCache) return ''
return this.searchCache.getCacheKey(query, options)
}
/**
* Direct cache get
*/
get(key: string): any {
if (!this.searchCache) return null
return this.searchCache.get(key)
}
/**
* Direct cache set
*/
set(key: string, value: any): void {
if (!this.searchCache) return
this.searchCache.set(key, value)
}
/**
* Get the underlying SearchCache instance (for compatibility)
*/
getSearchCache() {
return this.searchCache
}
/**
* Get memory usage
*/
getMemoryUsage(): number {
if (!this.searchCache) return 0
return this.searchCache.getMemoryUsage()
}
}
/**
* Factory function for zero-config cache augmentation
*/
export function createCacheAugmentation(config?: CacheConfig): CacheAugmentation {
return new CacheAugmentation(config)
}

274
src/augmentations/conduitAugmentations.ts
View file

@ -1,274 +0,0 @@
/**
* Conduit Augmentations - Data Synchronization Bridges
*
* These augmentations connect and synchronize data between multiple Brainy instances.
* Now using the unified BrainyAugmentation interface.
*/
import { BaseAugmentation, BrainyAugmentation, AugmentationContext } from './brainyAugmentation.js'
import { v4 as uuidv4 } from '../universal/uuid.js'
export interface WebSocketConnection {
connectionId: string
url: string
readyState: number
socket?: any
}
/**
* Base class for conduit augmentations that sync between Brainy instances
* Converted to use the unified BrainyAugmentation interface
*/
abstract class BaseConduitAugmentation extends BaseAugmentation {
readonly timing = 'after' as const // Conduits run after operations to sync
readonly metadata = 'readonly' as const // Conduits read metadata to pass to external systems
readonly operations = ['addNoun', 'delete', 'addVerb'] as ('addNoun' | 'delete' | 'addVerb')[]
readonly priority = 20 // Medium-low priority
protected connections = new Map<string, any>()
protected async onShutdown(): Promise<void> {
// Close all connections
for (const [connectionId, connection] of this.connections.entries()) {
try {
if (connection.close) {
await connection.close()
}
} catch (error) {
this.log(`Failed to close connection ${connectionId}: ${error}`, 'error')
}
}
this.connections.clear()
}
abstract establishConnection(
targetSystemId: string,
config?: Record<string, unknown>
): Promise<WebSocketConnection | null>
}
/**
* WebSocket Conduit Augmentation
* Syncs data between Brainy instances using WebSockets
*/
export class WebSocketConduitAugmentation extends BaseConduitAugmentation {
readonly name = 'websocket-conduit'
private webSocketConnections = new Map<string, WebSocketConnection>()
private messageCallbacks = new Map<string, Set<(data: any) => void>>()
async execute<T = any>(
operation: string,
params: any,
next: () => Promise<T>
): Promise<T> {
// Execute the operation first
const result = await next()
// Then sync to connected instances
if (this.shouldSync(operation)) {
await this.syncOperation(operation, params, result)
}
return result
}
private shouldSync(operation: string): boolean {
return ['addNoun', 'deleteNoun', 'addVerb'].includes(operation)
}
private async syncOperation(operation: string, params: any, result: any): Promise<void> {
// Broadcast to all connected WebSocket instances
for (const [id, connection] of this.webSocketConnections) {
if (connection.socket && connection.readyState === 1) { // OPEN state
try {
const message = JSON.stringify({
type: 'sync',
operation,
params,
timestamp: Date.now()
})
if (typeof connection.socket.send === 'function') {
connection.socket.send(message)
}
} catch (error) {
this.log(`Failed to sync to ${id}: ${error}`, 'error')
}
}
}
}
async establishConnection(
url: string,
config?: Record<string, unknown>
): Promise<WebSocketConnection | null> {
try {
const connectionId = uuidv4()
const protocols = config?.protocols as string | string[] | undefined
// Create WebSocket based on environment
let socket: any
if (typeof WebSocket !== 'undefined') {
// Browser environment
socket = new WebSocket(url, protocols)
} else {
// Node.js environment - dynamic import
try {
const ws = await import('ws')
socket = new ws.WebSocket(url, protocols)
} catch {
this.log('WebSocket not available in this environment', 'error')
return null
}
}
// Setup event handlers
socket.onopen = () => {
this.log(`Connected to ${url}`)
}
socket.onmessage = (event: any) => {
this.handleMessage(connectionId, event.data)
}
socket.onerror = (error: any) => {
this.log(`WebSocket error: ${error}`, 'error')
}
socket.onclose = () => {
this.log(`Disconnected from ${url}`)
this.webSocketConnections.delete(connectionId)
}
// Wait for connection to open
await new Promise<void>((resolve, reject) => {
const timeout = setTimeout(() => {
reject(new Error('Connection timeout'))
}, 5000)
socket.onopen = () => {
clearTimeout(timeout)
resolve()
}
socket.onerror = (error: any) => {
clearTimeout(timeout)
reject(error)
}
})
const connection: WebSocketConnection = {
connectionId,
url,
readyState: socket.readyState,
socket
}
this.webSocketConnections.set(connectionId, connection)
this.connections.set(connectionId, connection)
return connection
} catch (error) {
this.log(`Failed to establish connection to ${url}: ${error}`, 'error')
return null
}
}
private handleMessage(connectionId: string, data: any): void {
try {
const message = typeof data === 'string' ? JSON.parse(data) : data
// Handle sync messages from remote instances
if (message.type === 'sync') {
// Apply the operation to our local instance
this.applySyncOperation(message).catch(error => {
this.log(`Failed to apply sync operation: ${error}`, 'error')
})
}
// Notify any registered callbacks
const callbacks = this.messageCallbacks.get(connectionId)
if (callbacks) {
callbacks.forEach(callback => callback(message))
}
} catch (error) {
this.log(`Failed to handle message: ${error}`, 'error')
}
}
private async applySyncOperation(message: any): Promise<void> {
// Apply the synced operation to our local Brainy instance
const { operation, params } = message
try {
switch (operation) {
case 'addNoun':
await this.context?.brain.add({ data: params.content, type: 'content', metadata: params.metadata })
break
case 'deleteNoun':
await this.context?.brain.deleteNoun(params.id)
break
case 'addVerb':
await this.context?.brain.relate({
from: params.source,
to: params.target,
type: params.verb,
metadata: params.metadata
})
break
}
} catch (error) {
this.log(`Failed to apply ${operation}: ${error}`, 'error')
}
}
/**
* Subscribe to messages from a specific connection
*/
onMessage(connectionId: string, callback: (data: any) => void): void {
if (!this.messageCallbacks.has(connectionId)) {
this.messageCallbacks.set(connectionId, new Set())
}
this.messageCallbacks.get(connectionId)!.add(callback)
}
/**
* Send a message to a specific connection
*/
sendMessage(connectionId: string, data: any): boolean {
const connection = this.webSocketConnections.get(connectionId)
if (connection?.socket && connection.readyState === 1) {
try {
const message = typeof data === 'string' ? data : JSON.stringify(data)
connection.socket.send(message)
return true
} catch (error) {
this.log(`Failed to send message: ${error}`, 'error')
}
}
return false
}
}
/**
* Example usage:
*
* // Server instance
* const serverBrain = new Brainy()
* serverBrain.augmentations.register(new APIServerAugmentation())
* await serverBrain.init()
*
* // Client instance
* const clientBrain = new Brainy()
* const conduit = new WebSocketConduitAugmentation()
* clientBrain.augmentations.register(conduit)
* await clientBrain.init()
*
* // Connect client to server
* await conduit.establishConnection('ws://localhost:3000/ws')
*
* // Now operations sync automatically!
* await clientBrain.add({ data: 'synced data', type: 'content', metadata: { source: 'client' } })
* // This will automatically sync to the server
*/

539
src/augmentations/configResolver.ts
View file

@ -1,539 +0,0 @@
/**
* Configuration Resolver for Augmentations
*
* Handles loading and resolving configuration from multiple sources:
* - Environment variables
* - Configuration files
* - Runtime updates
* - Default values from schema
*/
import { JSONSchema } from './manifest.js'
import { isNode } from '../utils/environment.js'
/**
* Configuration source priority (highest to lowest)
*/
export enum ConfigPriority {
RUNTIME = 4, // Runtime updates (highest priority)
CONSTRUCTOR = 3, // Constructor parameters
ENVIRONMENT = 2, // Environment variables
FILE = 1, // Configuration files
DEFAULT = 0 // Schema defaults (lowest priority)
}
/**
* Configuration source information
*/
export interface ConfigSource {
priority: ConfigPriority
source: string
config: any
}
/**
* Configuration resolution options
*/
export interface ConfigResolverOptions {
augmentationId: string
schema?: JSONSchema
defaults?: Record<string, any>
configPaths?: string[]
envPrefix?: string
allowUndefined?: boolean
}
/**
* Augmentation Configuration Resolver
*/
export class AugmentationConfigResolver {
private sources: ConfigSource[] = []
private resolved: any = {}
constructor(private options: ConfigResolverOptions) {
// Default config paths - include home directory paths only in Node.js
const defaultConfigPaths = [
'.brainyrc',
'.brainyrc.json',
'brainy.config.json'
]
// Add home directory paths in Node.js environments
if (isNode()) {
try {
const nodeRequire = typeof require !== 'undefined' ? require : null
if (nodeRequire) {
const path = nodeRequire('node:path')
const os = nodeRequire('node:os')
defaultConfigPaths.push(
path.join(os.homedir(), '.brainy', 'config.json'),
path.join(os.homedir(), '.brainyrc')
)
}
} catch {
// Silently continue without home directory paths
}
}
this.options = {
configPaths: defaultConfigPaths,
envPrefix: `BRAINY_AUG_${options.augmentationId.toUpperCase()}_`,
allowUndefined: true,
...options
}
}
/**
* Resolve configuration from all sources
* @param constructorConfig Optional constructor configuration
* @returns Resolved configuration
*/
resolve(constructorConfig?: any): any {
this.sources = []
// Load from all sources in priority order
this.loadDefaults()
this.loadFromFiles()
this.loadFromEnvironment()
if (constructorConfig) {
this.sources.push({
priority: ConfigPriority.CONSTRUCTOR,
source: 'constructor',
config: constructorConfig
})
}
// Merge configurations by priority
this.resolved = this.mergeConfigurations()
// Validate against schema if provided
if (this.options.schema) {
this.validateConfiguration(this.resolved)
}
return this.resolved
}
/**
* Load default values from schema and defaults
*/
private loadDefaults(): void {
let defaults: any = {}
// Load from provided defaults
if (this.options.defaults) {
defaults = { ...defaults, ...this.options.defaults }
}
// Load from schema defaults
if (this.options.schema?.properties) {
for (const [key, prop] of Object.entries(this.options.schema.properties)) {
if (prop.default !== undefined && defaults[key] === undefined) {
defaults[key] = prop.default
}
}
}
if (Object.keys(defaults).length > 0) {
this.sources.push({
priority: ConfigPriority.DEFAULT,
source: 'defaults',
config: defaults
})
}
}
/**
* Load configuration from files
*/
private loadFromFiles(): void {
// Skip in browser environment
if (!isNode()) {
return
}
try {
const nodeRequire = typeof require !== 'undefined' ? require : null
if (!nodeRequire) return
const fs = nodeRequire('node:fs')
for (const configPath of this.options.configPaths || []) {
try {
if (fs.existsSync(configPath)) {
const content = fs.readFileSync(configPath, 'utf8')
const config = this.parseConfigFile(content, configPath)
// Extract augmentation-specific configuration
const augConfig = this.extractAugmentationConfig(config)
if (augConfig && Object.keys(augConfig).length > 0) {
this.sources.push({
priority: ConfigPriority.FILE,
source: `file:${configPath}`,
config: augConfig
})
break // Use first found config file
}
}
} catch (error) {
// Silently ignore file errors
console.debug(`Failed to load config from ${configPath}:`, error)
}
}
} catch {
// Silently continue if require fails
}
}
/**
* Parse configuration file based on extension
*/
private parseConfigFile(content: string, filepath: string): any {
try {
// Try JSON first
return JSON.parse(content)
} catch {
// Try other formats in the future (YAML, TOML, etc.)
throw new Error(`Unable to parse config file: ${filepath}`)
}
}
/**
* Extract augmentation-specific configuration from a config object
*/
private extractAugmentationConfig(config: any): any {
const augId = this.options.augmentationId
// Check for augmentations section
if (config.augmentations && config.augmentations[augId]) {
return config.augmentations[augId]
}
// Check for direct augmentation config (prefixed keys)
const prefix = `${augId}.`
const augConfig: any = {}
for (const [key, value] of Object.entries(config)) {
if (key.startsWith(prefix)) {
const configKey = key.slice(prefix.length)
augConfig[configKey] = value
}
}
return Object.keys(augConfig).length > 0 ? augConfig : null
}
/**
* Load configuration from environment variables
*/
private loadFromEnvironment(): void {
// Skip in browser environment
if (!isNode() || !process.env) {
return
}
const prefix = this.options.envPrefix!
const envConfig: any = {}
for (const [key, value] of Object.entries(process.env)) {
if (key.startsWith(prefix)) {
const configKey = this.envKeyToConfigKey(key.slice(prefix.length))
envConfig[configKey] = this.parseEnvValue(value as string)
}
}
if (Object.keys(envConfig).length > 0) {
this.sources.push({
priority: ConfigPriority.ENVIRONMENT,
source: 'environment',
config: envConfig
})
}
}
/**
* Convert environment variable key to config key
* ENABLED -> enabled
* MAX_SIZE -> maxSize
*/
private envKeyToConfigKey(envKey: string): string {
return envKey
.toLowerCase()
.replace(/_([a-z])/g, (_, letter) => letter.toUpperCase())
}
/**
* Parse environment variable value
*/
private parseEnvValue(value: string): any {
// Handle empty strings
if (value === '') return value
// Try to parse as JSON
try {
return JSON.parse(value)
} catch {
// Check for boolean strings
if (value.toLowerCase() === 'true') return true
if (value.toLowerCase() === 'false') return false
// Check for number strings
const num = Number(value)
if (!isNaN(num) && value.trim() !== '') return num
// Return as string
return value
}
}
/**
* Merge configurations by priority
*/
private mergeConfigurations(): any {
// Sort by priority (lowest to highest)
const sorted = [...this.sources].sort((a, b) => a.priority - b.priority)
// Merge configurations
let merged = {}
for (const source of sorted) {
merged = this.deepMerge(merged, source.config)
}
return merged
}
/**
* Deep merge two objects
*/
private deepMerge(target: any, source: any): any {
const output = { ...target }
for (const key in source) {
if (source.hasOwnProperty(key)) {
if (source[key] && typeof source[key] === 'object' && !Array.isArray(source[key])) {
if (target[key] && typeof target[key] === 'object' && !Array.isArray(target[key])) {
output[key] = this.deepMerge(target[key], source[key])
} else {
output[key] = source[key]
}
} else {
output[key] = source[key]
}
}
}
return output
}
/**
* Validate configuration against schema
*/
private validateConfiguration(config: any): void {
if (!this.options.schema) return
const schema = this.options.schema
const errors: string[] = []
// Check required fields
if (schema.required) {
for (const field of schema.required) {
if (config[field] === undefined) {
errors.push(`Missing required field: ${field}`)
}
}
}
// Validate properties
if (schema.properties) {
for (const [key, propSchema] of Object.entries(schema.properties)) {
const value = config[key]
if (value !== undefined) {
this.validateProperty(key, value, propSchema, errors)
}
}
}
// Check for additional properties
if (schema.additionalProperties === false) {
const allowedKeys = Object.keys(schema.properties || {})
for (const key of Object.keys(config)) {
if (!allowedKeys.includes(key)) {
errors.push(`Unknown configuration property: ${key}`)
}
}
}
if (errors.length > 0) {
throw new Error(`Configuration validation failed for ${this.options.augmentationId}:\n${errors.join('\n')}`)
}
}
/**
* Validate a single property against its schema
*/
private validateProperty(key: string, value: any, schema: JSONSchema, errors: string[]): void {
// Type validation
if (schema.type) {
const actualType = Array.isArray(value) ? 'array' : typeof value
if (actualType !== schema.type) {
errors.push(`${key}: expected ${schema.type}, got ${actualType}`)
return
}
}
// Number validations
if (schema.type === 'number') {
if (schema.minimum !== undefined && value < schema.minimum) {
errors.push(`${key}: value ${value} is less than minimum ${schema.minimum}`)
}
if (schema.maximum !== undefined && value > schema.maximum) {
errors.push(`${key}: value ${value} is greater than maximum ${schema.maximum}`)
}
}
// String validations
if (schema.type === 'string') {
if (schema.minLength !== undefined && value.length < schema.minLength) {
errors.push(`${key}: length ${value.length} is less than minimum ${schema.minLength}`)
}
if (schema.maxLength !== undefined && value.length > schema.maxLength) {
errors.push(`${key}: length ${value.length} is greater than maximum ${schema.maxLength}`)
}
if (schema.pattern) {
const regex = new RegExp(schema.pattern)
if (!regex.test(value)) {
errors.push(`${key}: value does not match pattern ${schema.pattern}`)
}
}
}
// Enum validation
if (schema.enum && !schema.enum.includes(value)) {
errors.push(`${key}: value ${value} is not one of allowed values: ${schema.enum.join(', ')}`)
}
}
/**
* Get configuration sources for debugging
*/
getSources(): ConfigSource[] {
return [...this.sources]
}
/**
* Get resolved configuration
*/
getResolved(): any {
return { ...this.resolved }
}
/**
* Update configuration at runtime
*/
updateRuntime(config: any): any {
// Add or update runtime source
const runtimeIndex = this.sources.findIndex(s => s.priority === ConfigPriority.RUNTIME)
if (runtimeIndex >= 0) {
this.sources[runtimeIndex].config = {
...this.sources[runtimeIndex].config,
...config
}
} else {
this.sources.push({
priority: ConfigPriority.RUNTIME,
source: 'runtime',
config
})
}
// Re-merge configurations
this.resolved = this.mergeConfigurations()
// Validate
if (this.options.schema) {
this.validateConfiguration(this.resolved)
}
return this.resolved
}
/**
* Save configuration to file
* @param filepath Path to save configuration
* @param format Format to save as (json, etc.)
*/
async saveToFile(filepath?: string, format: 'json' = 'json'): Promise<void> {
// Skip in browser environment
if (!isNode()) {
throw new Error('Cannot save configuration files in browser environment')
}
const fs = await import('node:fs')
const path = await import('node:path')
const configPath = filepath || this.options.configPaths?.[0] || '.brainyrc'
const augId = this.options.augmentationId
// Load existing config if it exists
let fullConfig: any = {}
try {
if (fs.existsSync(configPath)) {
const content = fs.readFileSync(configPath, 'utf8')
fullConfig = JSON.parse(content)
}
} catch {
// Start with empty config
}
// Ensure augmentations section exists
if (!fullConfig.augmentations) {
fullConfig.augmentations = {}
}
// Update augmentation config
fullConfig.augmentations[augId] = this.resolved
// Save based on format
let content: string
if (format === 'json') {
content = JSON.stringify(fullConfig, null, 2)
} else {
throw new Error(`Unsupported format: ${format}`)
}
// Ensure directory exists
const dir = path.dirname(configPath)
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir, { recursive: true })
}
// Write file
fs.writeFileSync(configPath, content, 'utf8')
}
/**
* Get environment variable names for this augmentation
*/
getEnvironmentVariables(): Record<string, any> {
const schema = this.options.schema
const prefix = this.options.envPrefix!
const vars: Record<string, any> = {}
if (schema?.properties) {
for (const [key, prop] of Object.entries(schema.properties)) {
const envKey = prefix + key.replace(/([A-Z])/g, '_$1').toUpperCase()
vars[envKey] = {
description: prop.description,
type: prop.type,
default: prop.default,
currentValue: process.env?.[envKey]
}
}
}
return vars
}
}

445
src/augmentations/connectionPoolAugmentation.ts
View file

@ -1,445 +0,0 @@
/**
* Connection Pool Augmentation
*
* Provides 10-20x throughput improvement for cloud storage (S3, R2, GCS)
* Manages connection pooling, request queuing, and parallel processing
* Critical for enterprise-scale operations with millions of entries
*/
import { BaseAugmentation, AugmentationContext } from './brainyAugmentation.js'
interface ConnectionPoolConfig {
enabled?: boolean
maxConnections?: number // Maximum concurrent connections
minConnections?: number // Minimum idle connections
acquireTimeout?: number // Timeout for getting connection (ms)
idleTimeout?: number // Connection idle timeout (ms)
maxQueueSize?: number // Maximum queued requests
retryAttempts?: number // Retry failed requests
healthCheckInterval?: number // Connection health check (ms)
}
interface PooledConnection {
id: string
connection: any // Actual storage client (S3, R2, etc)
isIdle: boolean
lastUsed: number
healthScore: number
activeRequests: number
requestCount: number // Total requests handled
}
interface QueuedRequest<T = any> {
id: string
operation: string
params: any
executor: () => Promise<T>
resolver: (value: T) => void
rejector: (error: Error) => void
timestamp: number
priority: number
}
export class ConnectionPoolAugmentation extends BaseAugmentation {
name = 'ConnectionPool'
timing = 'around' as const
metadata = 'none' as const // Connection pooling doesn't access metadata
operations = ['storage'] as ('storage')[]
priority = 95 // Very high priority for storage operations
protected config: Required<ConnectionPoolConfig>
private connections: Map<string, PooledConnection> = new Map()
private requestQueue: QueuedRequest<any>[] = []
private healthCheckInterval?: NodeJS.Timeout
private storageType: string = 'unknown'
private stats = {
totalRequests: 0,
queuedRequests: 0,
activeConnections: 0,
totalConnections: 0,
averageLatency: 0,
throughputPerSecond: 0
}
constructor(config: ConnectionPoolConfig = {}) {
super()
this.config = {
enabled: config.enabled ?? true,
maxConnections: config.maxConnections ?? 50,
minConnections: config.minConnections ?? 5,
acquireTimeout: config.acquireTimeout ?? 30000, // 30s
idleTimeout: config.idleTimeout ?? 300000, // 5 minutes
maxQueueSize: config.maxQueueSize ?? 10000,
retryAttempts: config.retryAttempts ?? 3,
healthCheckInterval: config.healthCheckInterval ?? 60000 // 1 minute
}
}
protected async onInitialize(): Promise<void> {
if (!this.config.enabled) {
this.log('Connection pooling disabled')
return
}
// Detect storage type
this.storageType = this.detectStorageType()
if (this.isCloudStorage()) {
await this.initializeConnectionPool()
this.startHealthChecks()
this.startMetricsCollection()
this.log(`Connection pool initialized for ${this.storageType}: ${this.config.minConnections}-${this.config.maxConnections} connections`)
} else {
this.log(`Connection pooling skipped for ${this.storageType} (local storage)`)
}
}
shouldExecute(operation: string, params: any): boolean {
return this.config.enabled &&
this.isCloudStorage() &&
this.isStorageOperation(operation)
}
async execute<T = any>(
operation: string,
params: any,
next: () => Promise<T>
): Promise<T> {
if (!this.shouldExecute(operation, params)) {
return next()
}
const startTime = Date.now()
this.stats.totalRequests++
try {
// High priority for critical operations
const priority = this.getOperationPriority(operation)
// Execute with pooled connection
const result = await this.executeWithPool(operation, params, next, priority)
// Update metrics
const latency = Date.now() - startTime
this.updateLatencyMetrics(latency)
return result
} catch (error) {
this.log(`Connection pool error for ${operation}: ${error}`, 'error')
// Fallback to direct execution for reliability
return next()
}
}
private detectStorageType(): string {
const storage = this.context?.storage
if (!storage) return 'unknown'
const className = storage.constructor.name.toLowerCase()
if (className.includes('s3')) return 's3'
if (className.includes('r2')) return 'r2'
if (className.includes('gcs') || className.includes('google')) return 'gcs'
if (className.includes('azure')) return 'azure'
if (className.includes('filesystem')) return 'filesystem'
if (className.includes('memory')) return 'memory'
return 'unknown'
}
private isCloudStorage(): boolean {
return ['s3', 'r2', 'gcs', 'azure'].includes(this.storageType)
}
private isStorageOperation(operation: string): boolean {
return operation.includes('save') ||
operation.includes('get') ||
operation.includes('delete') ||
operation.includes('list') ||
operation.includes('backup') ||
operation.includes('restore')
}
private getOperationPriority(operation: string): number {
// Critical operations get highest priority
if (operation.includes('save') || operation.includes('update')) return 10
if (operation.includes('delete')) return 9
if (operation.includes('get')) return 7
if (operation.includes('list')) return 5
if (operation.includes('backup')) return 3
return 1
}
private async executeWithPool<T>(
operation: string,
params: any,
executor: () => Promise<T>,
priority: number
): Promise<T> {
// Check queue size
if (this.requestQueue.length >= this.config.maxQueueSize) {
throw new Error('Connection pool queue full - system overloaded')
}
// Try to get available connection immediately
const connection = await this.getOrCreateConnection()
if (connection && connection.isIdle) {
return this.executeWithConnection(connection, operation, executor)
}
// Queue the request with the actual executor
return this.queueRequest(operation, params, executor, priority)
}
private getAvailableConnection(): PooledConnection | null {
// Find idle connection with best health score
let bestConnection: PooledConnection | null = null
let bestScore = -1
for (const connection of this.connections.values()) {
if (connection.isIdle && connection.healthScore > bestScore) {
bestConnection = connection
bestScore = connection.healthScore
}
}
return bestConnection
}
private async executeWithConnection<T>(
connection: PooledConnection,
operation: string,
executor: () => Promise<T>
): Promise<T> {
// Mark connection as active
connection.isIdle = false
connection.activeRequests++
connection.lastUsed = Date.now()
this.stats.activeConnections++
try {
const result = await executor()
// Update connection health on success
connection.healthScore = Math.min(connection.healthScore + 1, 100)
return result
} catch (error) {
// Decrease health on failure
connection.healthScore = Math.max(connection.healthScore - 5, 0)
throw error
} finally {
// Release connection
connection.isIdle = true
connection.activeRequests--
this.stats.activeConnections--
// Process next queued request
this.processQueue()
}
}
private async queueRequest<T>(
operation: string,
params: any,
executor: () => Promise<T>,
priority: number
): Promise<T> {
return new Promise((resolve, reject) => {
const request: QueuedRequest<T> = {
id: `req_${Date.now()}_${Math.random()}`,
operation,
params,
executor, // Store the actual executor function
resolver: resolve,
rejector: reject,
timestamp: Date.now(),
priority
}
// Insert by priority (higher priority first)
const insertIndex = this.requestQueue.findIndex(r => r.priority < priority)
if (insertIndex === -1) {
this.requestQueue.push(request)
} else {
this.requestQueue.splice(insertIndex, 0, request)
}
this.stats.queuedRequests++
// Set timeout
setTimeout(() => {
const index = this.requestQueue.findIndex(r => r.id === request.id)
if (index !== -1) {
this.requestQueue.splice(index, 1)
this.stats.queuedRequests--
reject(new Error(`Connection pool timeout: ${this.config.acquireTimeout}ms`))
}
}, this.config.acquireTimeout)
})
}
private processQueue(): void {
if (this.requestQueue.length === 0) return
const connection = this.getAvailableConnection()
if (!connection) return
const request = this.requestQueue.shift()!
this.stats.queuedRequests--
// Execute queued request with the REAL executor
this.executeWithConnection(connection, request.operation, request.executor)
.then(request.resolver)
.catch(request.rejector)
}
private async initializeConnectionPool(): Promise<void> {
// Create minimum connections
for (let i = 0; i < this.config.minConnections; i++) {
await this.createConnection()
}
}
private async createConnection(): Promise<PooledConnection> {
const connectionId = `conn_${Date.now()}_${Math.random()}`
// Create actual connection based on storage type
const actualConnection = await this.createStorageConnection()
const connection: PooledConnection = {
id: connectionId,
connection: actualConnection,
isIdle: true,
lastUsed: Date.now(),
healthScore: 100,
activeRequests: 0,
requestCount: 0
}
this.connections.set(connectionId, connection)
this.stats.totalConnections++
return connection
}
private async createStorageConnection(): Promise<any> {
// For cloud storage, reuse the existing storage instance
// Connection pooling in this context means managing concurrent requests
// not creating multiple storage instances (which would be wasteful)
const storage = this.context?.storage
if (!storage) {
throw new Error('Storage not available for connection pooling')
}
// Return a connection wrapper that tracks usage
return {
storage,
created: Date.now(),
requestCount: 0
}
}
private async getOrCreateConnection(): Promise<PooledConnection | null> {
// Try to get an available connection
let connection = this.getAvailableConnection()
// If no connection available and under max, create new one
if (!connection && this.connections.size < this.config.maxConnections) {
connection = await this.createConnection()
}
return connection
}
private startHealthChecks(): void {
this.healthCheckInterval = setInterval(() => {
this.performHealthChecks()
}, this.config.healthCheckInterval)
}
private performHealthChecks(): void {
const now = Date.now()
const toRemove: string[] = []
for (const [id, connection] of this.connections) {
// Remove idle connections that are too old
if (connection.isIdle &&
now - connection.lastUsed > this.config.idleTimeout &&
this.connections.size > this.config.minConnections) {
toRemove.push(id)
}
// Remove unhealthy connections
if (connection.healthScore < 20) {
toRemove.push(id)
}
}
// Remove unhealthy/old connections
for (const id of toRemove) {
this.connections.delete(id)
this.stats.totalConnections--
}
// Ensure minimum connections
while (this.connections.size < this.config.minConnections) {
this.createConnection()
}
}
private startMetricsCollection(): void {
setInterval(() => {
this.updateThroughputMetrics()
}, 1000) // Update every second
}
private updateLatencyMetrics(latency: number): void {
// Simple moving average
this.stats.averageLatency = (this.stats.averageLatency * 0.9) + (latency * 0.1)
}
private updateThroughputMetrics(): void {
// Reset counter for next second
this.stats.throughputPerSecond = this.stats.totalRequests
// Reset total for next measurement (in practice, use sliding window)
}
/**
* Get connection pool statistics
*/
getStats(): typeof this.stats & {
queueSize: number
activeConnections: number
totalConnections: number
poolUtilization: string
storageType: string
} {
return {
...this.stats,
queueSize: this.requestQueue.length,
activeConnections: this.stats.activeConnections,
totalConnections: this.connections.size,
poolUtilization: `${Math.round((this.stats.activeConnections / this.connections.size) * 100)}%`,
storageType: this.storageType
}
}
protected async onShutdown(): Promise<void> {
if (this.healthCheckInterval) {
clearInterval(this.healthCheckInterval)
}
// Close all connections
this.connections.clear()
// Reject all queued requests
this.requestQueue.forEach(request => {
request.rejector(new Error('Connection pool shutting down'))
})
this.requestQueue = []
const stats = this.getStats()
this.log(`Connection pool shutdown: ${stats.totalRequests} requests processed, ${stats.poolUtilization} avg utilization`)
}
}

130
src/augmentations/defaultAugmentations.ts
View file

@ -1,130 +0,0 @@
/**
* Default Augmentations Registration
*
* Maintains zero-config philosophy by automatically registering
* core augmentations that were previously hardcoded in Brainy.
*
* These augmentations are optional but enabled by default for
* backward compatibility and optimal performance.
*/
import { Brainy } from '../brainy.js'
import { BaseAugmentation } from './brainyAugmentation.js'
import { CacheAugmentation } from './cacheAugmentation.js'
import { MetricsAugmentation } from './metricsAugmentation.js'
import { MonitoringAugmentation } from './monitoringAugmentation.js'
import { UniversalDisplayAugmentation } from './universalDisplayAugmentation.js'
import { IntelligentImportAugmentation } from './intelligentImport/index.js'
/**
* Create default augmentations for zero-config operation
* Returns an array of augmentations to be registered
*
* @param config - Configuration options
* @returns Array of augmentations to register
*/
export function createDefaultAugmentations(
config: {
cache?: boolean | Record<string, any>
metrics?: boolean | Record<string, any>
monitoring?: boolean | Record<string, any>
display?: boolean | Record<string, any>
intelligentImport?: boolean | Record<string, any>
} = {}
): BaseAugmentation[] {
const augmentations: BaseAugmentation[] = []
// Intelligent Import augmentation (CSV, Excel, PDF)
if (config.intelligentImport !== false) {
const importConfig = typeof config.intelligentImport === 'object' ? config.intelligentImport : {}
augmentations.push(new IntelligentImportAugmentation(importConfig))
}
// Cache augmentation (was SearchCache)
if (config.cache !== false) {
const cacheConfig = typeof config.cache === 'object' ? config.cache : {}
augmentations.push(new CacheAugmentation(cacheConfig))
}
// Note: Index augmentation removed - metadata indexing is now core functionality
// Metrics augmentation (was StatisticsCollector)
if (config.metrics !== false) {
const metricsConfig = typeof config.metrics === 'object' ? config.metrics : {}
augmentations.push(new MetricsAugmentation(metricsConfig))
}
// Display augmentation (AI-powered intelligent display fields)
if (config.display !== false) {
const displayConfig = typeof config.display === 'object' ? config.display : {}
augmentations.push(new UniversalDisplayAugmentation(displayConfig))
}
// Monitoring augmentation (was HealthMonitor)
// Only enable by default in distributed mode
const isDistributed = process.env.BRAINY_MODE === 'distributed' ||
process.env.BRAINY_DISTRIBUTED === 'true'
if (config.monitoring !== false && (config.monitoring || isDistributed)) {
const monitoringConfig = typeof config.monitoring === 'object' ? config.monitoring : {}
augmentations.push(new MonitoringAugmentation(monitoringConfig))
}
return augmentations
}
/**
* Get augmentation by name with type safety
*/
export function getAugmentation<T>(brain: Brainy, name: string): T | null {
// Access augmentations through a public method or property
const augmentations = (brain as any).augmentations
if (!augmentations) return null
const aug = augmentations.get(name)
return aug as T | null
}
/**
* Compatibility helpers for migrating from hardcoded features
*/
export const AugmentationHelpers = {
/**
* Get cache augmentation
*/
getCache(brain: Brainy): CacheAugmentation | null {
return getAugmentation<CacheAugmentation>(brain, 'cache')
},
/**
* Note: Index augmentation removed - metadata indexing is now core functionality
* Use brain.metadataIndex directly instead
*/
/**
* Get metrics augmentation
*/
getMetrics(brain: Brainy): MetricsAugmentation | null {
return getAugmentation<MetricsAugmentation>(brain, 'metrics')
},
/**
* Get monitoring augmentation
*/
getMonitoring(brain: Brainy): MonitoringAugmentation | null {
return getAugmentation<MonitoringAugmentation>(brain, 'monitoring')
},
/**
* Get display augmentation
*/
getDisplay(brain: Brainy): UniversalDisplayAugmentation | null {
return getAugmentation<UniversalDisplayAugmentation>(brain, 'display')
},
/**
* Get intelligent import augmentation
*/
getIntelligentImport(brain: Brainy): IntelligentImportAugmentation | null {
return getAugmentation<IntelligentImportAugmentation>(brain, 'intelligent-import')
}
}

560
src/augmentations/discovery.ts
View file

@ -1,560 +0,0 @@
/**
* Augmentation Discovery API
*
* Provides discovery and configuration capabilities for augmentations
* Enables tools like brain-cloud to dynamically discover, configure, and manage augmentations
*/
import { AugmentationRegistry } from './brainyAugmentation.js'
import { AugmentationManifest, JSONSchema } from './manifest.js'
import { AugmentationConfigResolver } from './configResolver.js'
/**
* Augmentation listing with manifest and status
*/
export interface AugmentationListing {
id: string
name: string
manifest: AugmentationManifest
status: {
enabled: boolean
initialized: boolean
category: string
priority: number
}
config?: {
current: any
schema?: JSONSchema
sources?: any[]
}
}
/**
* Configuration validation result
*/
export interface ConfigValidationResult {
valid: boolean
errors?: string[]
warnings?: string[]
suggestions?: string[]
}
/**
* Discovery API options
*/
export interface DiscoveryOptions {
includeConfig?: boolean
includeSchema?: boolean
includeSources?: boolean
category?: string
enabled?: boolean
}
/**
* Augmentation Discovery API
*
* Provides a unified interface for discovering and managing augmentations
*/
export class AugmentationDiscovery {
constructor(private registry: AugmentationRegistry) {}
/**
* Discover all registered augmentations with manifests
* @param options Discovery options
* @returns List of augmentation listings
*/
async discover(options: DiscoveryOptions = {}): Promise<AugmentationListing[]> {
const augmentations = this.registry.getAll()
const listings: AugmentationListing[] = []
for (const aug of augmentations) {
// Check if augmentation has manifest support
const hasManifest = 'getManifest' in aug && typeof aug.getManifest === 'function'
if (!hasManifest) {
// Skip augmentations without manifest support (legacy)
continue
}
try {
// Check if augmentation has manifest method
if (!('getManifest' in aug) || typeof aug.getManifest !== 'function') {
continue
}
const getManifestFn = aug.getManifest as Function
const manifest = getManifestFn()
// Apply filters
if (options.category && manifest.category !== options.category) {
continue
}
if (options.enabled !== undefined) {
const isEnabled = (aug as any).enabled !== false
if (isEnabled !== options.enabled) {
continue
}
}
// Build listing
const listing: AugmentationListing = {
id: manifest.id,
name: manifest.name,
manifest,
status: {
enabled: (aug as any).enabled !== false,
initialized: (aug as any).isInitialized || false,
category: (aug as any).category || manifest.category,
priority: aug.priority
}
}
// Include configuration if requested
if (options.includeConfig && 'getConfig' in aug) {
const getConfigFn = aug.getConfig as Function
listing.config = {
current: getConfigFn()
}
if (options.includeSchema) {
listing.config.schema = manifest.configSchema
}
}
listings.push(listing)
} catch (error) {
console.warn(`Failed to get manifest for augmentation ${aug.name}:`, error)
}
}
// Sort by priority (highest first) then by name
listings.sort((a, b) => {
const priorityDiff = b.status.priority - a.status.priority
if (priorityDiff !== 0) return priorityDiff
return a.name.localeCompare(b.name)
})
return listings
}
/**
* Get a specific augmentation's manifest
* @param augId Augmentation ID
* @returns Augmentation manifest or null if not found
*/
async getManifest(augId: string): Promise<AugmentationManifest | null> {
const aug = this.registry.get(augId)
if (!aug || !('getManifest' in aug)) {
return null
}
try {
const getManifestFn = aug.getManifest as Function
return getManifestFn()
} catch (error) {
console.error(`Failed to get manifest for ${augId}:`, error)
return null
}
}
/**
* Get configuration schema for an augmentation
* @param augId Augmentation ID
* @returns Configuration schema or null
*/
async getConfigSchema(augId: string): Promise<JSONSchema | null> {
const manifest = await this.getManifest(augId)
return manifest?.configSchema || null
}
/**
* Get current configuration for an augmentation
* @param augId Augmentation ID
* @returns Current configuration or null
*/
async getConfig(augId: string): Promise<any | null> {
const aug = this.registry.get(augId)
if (!aug || !('getConfig' in aug)) {
return null
}
try {
const getConfigFn = aug.getConfig as Function
return getConfigFn()
} catch (error) {
console.error(`Failed to get config for ${augId}:`, error)
return null
}
}
/**
* Update configuration for an augmentation
* @param augId Augmentation ID
* @param config New configuration
* @returns Updated configuration or null on failure
*/
async updateConfig(augId: string, config: any): Promise<any | null> {
const aug = this.registry.get(augId)
if (!aug || !('updateConfig' in aug) || !('getConfig' in aug)) {
throw new Error(`Augmentation ${augId} does not support configuration updates`)
}
try {
const updateConfigFn = aug.updateConfig as Function
await updateConfigFn(config)
const getConfigFn = aug.getConfig as Function
return getConfigFn()
} catch (error) {
throw new Error(`Failed to update config for ${augId}: ${error}`)
}
}
/**
* Validate configuration against schema
* @param augId Augmentation ID
* @param config Configuration to validate
* @returns Validation result
*/
async validateConfig(augId: string, config: any): Promise<ConfigValidationResult> {
const schema = await this.getConfigSchema(augId)
if (!schema) {
return {
valid: true,
warnings: ['No schema available for validation']
}
}
const errors: string[] = []
const warnings: string[] = []
const suggestions: string[] = []
// Check required fields
if (schema.required) {
for (const field of schema.required) {
if (config[field] === undefined) {
errors.push(`Missing required field: ${field}`)
}
}
}
// Validate properties
if (schema.properties) {
for (const [key, propSchema] of Object.entries(schema.properties)) {
const value = config[key]
if (value === undefined) {
// Check if there's a default
if (propSchema.default !== undefined) {
suggestions.push(`Field '${key}' not provided, will use default: ${JSON.stringify(propSchema.default)}`)
}
continue
}
// Type validation
if (propSchema.type) {
const actualType = Array.isArray(value) ? 'array' : typeof value
if (actualType !== propSchema.type) {
errors.push(`${key}: expected ${propSchema.type}, got ${actualType}`)
}
}
// Additional validations for specific types
this.validatePropertyValue(key, value, propSchema, errors, warnings)
}
}
// Check for unknown properties
if (schema.additionalProperties === false && schema.properties) {
const allowedKeys = Object.keys(schema.properties)
for (const key of Object.keys(config)) {
if (!allowedKeys.includes(key)) {
warnings.push(`Unknown property: ${key}`)
}
}
}
return {
valid: errors.length === 0,
errors: errors.length > 0 ? errors : undefined,
warnings: warnings.length > 0 ? warnings : undefined,
suggestions: suggestions.length > 0 ? suggestions : undefined
}
}
/**
* Validate a property value against its schema
*/
private validatePropertyValue(
key: string,
value: any,
schema: JSONSchema,
errors: string[],
warnings: string[]
): void {
// Number validations
if (schema.type === 'number') {
if (schema.minimum !== undefined && value < schema.minimum) {
errors.push(`${key}: value ${value} is less than minimum ${schema.minimum}`)
}
if (schema.maximum !== undefined && value > schema.maximum) {
errors.push(`${key}: value ${value} is greater than maximum ${schema.maximum}`)
}
}
// String validations
if (schema.type === 'string') {
if (schema.minLength !== undefined && value.length < schema.minLength) {
errors.push(`${key}: length ${value.length} is less than minimum ${schema.minLength}`)
}
if (schema.maxLength !== undefined && value.length > schema.maxLength) {
errors.push(`${key}: length ${value.length} is greater than maximum ${schema.maxLength}`)
}
if (schema.pattern) {
const regex = new RegExp(schema.pattern)
if (!regex.test(value)) {
errors.push(`${key}: value does not match pattern ${schema.pattern}`)
}
}
}
// Enum validation
if (schema.enum && !schema.enum.includes(value)) {
errors.push(`${key}: value '${value}' is not one of allowed values: ${schema.enum.join(', ')}`)
}
}
/**
* Get environment variables for an augmentation
* @param augId Augmentation ID
* @returns Map of environment variable names to descriptions
*/
async getEnvironmentVariables(augId: string): Promise<Record<string, any> | null> {
const manifest = await this.getManifest(augId)
if (!manifest?.configSchema?.properties) {
return null
}
const prefix = `BRAINY_AUG_${augId.toUpperCase()}_`
const vars: Record<string, any> = {}
for (const [key, prop] of Object.entries(manifest.configSchema.properties)) {
const envKey = prefix + key.replace(/([A-Z])/g, '_$1').toUpperCase()
vars[envKey] = {
configKey: key,
description: prop.description,
type: prop.type,
default: prop.default,
required: manifest.configSchema.required?.includes(key),
currentValue: typeof process !== 'undefined' ? process.env?.[envKey] : undefined
}
}
return vars
}
/**
* Get configuration examples for an augmentation
* @param augId Augmentation ID
* @returns Configuration examples or empty array
*/
async getConfigExamples(augId: string): Promise<any[]> {
const manifest = await this.getManifest(augId)
return manifest?.configExamples || []
}
/**
* Check if an augmentation supports configuration
* @param augId Augmentation ID
* @returns True if augmentation supports configuration
*/
async supportsConfiguration(augId: string): Promise<boolean> {
const aug = this.registry.get(augId)
return !!(aug && 'getConfig' in aug && 'updateConfig' in aug)
}
/**
* Get augmentations by category
* @param category Category to filter by
* @returns List of augmentations in the category
*/
async getByCategory(category: string): Promise<AugmentationListing[]> {
return this.discover({ category })
}
/**
* Get enabled augmentations
* @returns List of enabled augmentations
*/
async getEnabled(): Promise<AugmentationListing[]> {
return this.discover({ enabled: true })
}
/**
* Search augmentations by keyword
* @param query Search query
* @returns Matching augmentations
*/
async search(query: string): Promise<AugmentationListing[]> {
const all = await this.discover()
const queryLower = query.toLowerCase()
return all.filter(listing => {
const manifest = listing.manifest
// Search in various fields
const searchFields = [
manifest.name,
manifest.description,
manifest.longDescription,
...(manifest.keywords || []),
manifest.category
].filter(Boolean).map(s => s!.toLowerCase())
return searchFields.some(field => field.includes(queryLower))
})
}
/**
* Export configuration for all augmentations
* @returns Map of augmentation IDs to configurations
*/
async exportConfigurations(): Promise<Record<string, any>> {
const configs: Record<string, any> = {}
const listings = await this.discover({ includeConfig: true })
for (const listing of listings) {
if (listing.config?.current) {
configs[listing.id] = listing.config.current
}
}
return configs
}
/**
* Import configurations for multiple augmentations
* @param configs Map of augmentation IDs to configurations
* @returns Results of import operation
*/
async importConfigurations(configs: Record<string, any>): Promise<Record<string, { success: boolean; error?: string }>> {
const results: Record<string, { success: boolean; error?: string }> = {}
for (const [augId, config] of Object.entries(configs)) {
try {
// Validate before applying
const validation = await this.validateConfig(augId, config)
if (!validation.valid) {
results[augId] = {
success: false,
error: `Validation failed: ${validation.errors?.join(', ')}`
}
continue
}
// Apply configuration
await this.updateConfig(augId, config)
results[augId] = { success: true }
} catch (error) {
results[augId] = {
success: false,
error: error instanceof Error ? error.message : String(error)
}
}
}
return results
}
/**
* Generate configuration documentation
* @param augId Augmentation ID
* @returns Markdown documentation
*/
async generateConfigDocs(augId: string): Promise<string | null> {
const manifest = await this.getManifest(augId)
if (!manifest) return null
const schema = manifest.configSchema
const examples = manifest.configExamples || []
const envVars = await this.getEnvironmentVariables(augId)
let docs = `# ${manifest.name} Configuration\n\n`
docs += `${manifest.description}\n\n`
if (manifest.longDescription) {
docs += `## Overview\n\n${manifest.longDescription}\n\n`
}
// Configuration options
if (schema?.properties) {
docs += `## Configuration Options\n\n`
for (const [key, prop] of Object.entries(schema.properties)) {
const required = schema.required?.includes(key) ? ' *(required)*' : ''
docs += `### \`${key}\`${required}\n\n`
if (prop.description) {
docs += `${prop.description}\n\n`
}
docs += `- **Type**: ${prop.type}\n`
if (prop.default !== undefined) {
docs += `- **Default**: \`${JSON.stringify(prop.default)}\`\n`
}
if (prop.minimum !== undefined) {
docs += `- **Minimum**: ${prop.minimum}\n`
}
if (prop.maximum !== undefined) {
docs += `- **Maximum**: ${prop.maximum}\n`
}
if (prop.enum) {
docs += `- **Allowed values**: ${prop.enum.map(v => `\`${v}\``).join(', ')}\n`
}
docs += '\n'
}
}
// Environment variables
if (envVars && Object.keys(envVars).length > 0) {
docs += `## Environment Variables\n\n`
docs += `| Variable | Config Key | Type | Required | Default |\n`
docs += `|----------|------------|------|----------|----------|\n`
for (const [envKey, info] of Object.entries(envVars)) {
docs += `| \`${envKey}\` | ${info.configKey} | ${info.type} | ${info.required ? 'Yes' : 'No'} | ${info.default !== undefined ? `\`${info.default}\`` : '-'} |\n`
}
docs += '\n'
}
// Examples
if (examples.length > 0) {
docs += `## Examples\n\n`
for (const example of examples) {
docs += `### ${example.name}\n\n`
if (example.description) {
docs += `${example.description}\n\n`
}
docs += '```json\n'
docs += JSON.stringify(example.config, null, 2)
docs += '\n```\n\n'
}
}
return docs
}
}

357
src/augmentations/discovery/catalogDiscovery.ts
View file

@ -1,357 +0,0 @@
/**
* Brain-Cloud Catalog Discovery
*
* Discovers augmentations available in the brain-cloud catalog
* Handles free, premium, and community augmentations
*/
import { AugmentationManifest } from '../manifest.js'
export interface CatalogAugmentation {
id: string
name: string
description: string
longDescription?: string
category: string
status: 'available' | 'coming_soon' | 'deprecated'
tier: 'free' | 'premium' | 'enterprise'
price?: {
monthly?: number
yearly?: number
oneTime?: number
}
manifest?: AugmentationManifest
source: 'catalog'
cdnUrl?: string
npmPackage?: string
githubRepo?: string
author?: {
name: string
url?: string
}
metrics?: {
installations: number
rating: number
reviews: number
}
requirements?: {
minBrainyVersion?: string
maxBrainyVersion?: string
dependencies?: string[]
}
}
export interface CatalogOptions {
apiUrl?: string
apiKey?: string
cache?: boolean
cacheTimeout?: number
}
export interface CatalogFilters {
category?: string
tier?: 'free' | 'premium' | 'enterprise'
status?: 'available' | 'coming_soon' | 'deprecated'
search?: string
installed?: boolean
minRating?: number
}
/**
* Brain-Cloud Catalog Discovery
*/
export class CatalogDiscovery {
private apiUrl: string
private apiKey?: string
private cache: Map<string, { data: any; timestamp: number }> = new Map()
private cacheTimeout: number
constructor(options: CatalogOptions = {}) {
this.apiUrl = options.apiUrl || 'https://api.soulcraft.com/brain-cloud'
this.apiKey = options.apiKey
this.cacheTimeout = options.cacheTimeout || 5 * 60 * 1000 // 5 minutes
}
/**
* Discover augmentations from catalog
*/
async discover(filters: CatalogFilters = {}): Promise<CatalogAugmentation[]> {
const cacheKey = JSON.stringify(filters)
// Check cache
if (this.cache.has(cacheKey)) {
const cached = this.cache.get(cacheKey)!
if (Date.now() - cached.timestamp < this.cacheTimeout) {
return cached.data
}
}
// Build query parameters
const params = new URLSearchParams()
if (filters.category) params.append('category', filters.category)
if (filters.tier) params.append('tier', filters.tier)
if (filters.status) params.append('status', filters.status)
if (filters.search) params.append('q', filters.search)
if (filters.minRating) params.append('minRating', filters.minRating.toString())
// Fetch from API
const response = await fetch(`${this.apiUrl}/augmentations/discover?${params}`, {
headers: this.getHeaders()
})
if (!response.ok) {
throw new Error(`Failed to fetch catalog: ${response.statusText}`)
}
const data = await response.json()
const augmentations = this.transformCatalogData(data)
// Cache result
this.cache.set(cacheKey, {
data: augmentations,
timestamp: Date.now()
})
return augmentations
}
/**
* Get specific augmentation details
*/
async getAugmentation(id: string): Promise<CatalogAugmentation | null> {
const response = await fetch(`${this.apiUrl}/augmentations/${id}`, {
headers: this.getHeaders()
})
if (!response.ok) {
if (response.status === 404) return null
throw new Error(`Failed to fetch augmentation: ${response.statusText}`)
}
const data = await response.json()
return this.transformAugmentation(data)
}
/**
* Get augmentation manifest
*/
async getManifest(id: string): Promise<AugmentationManifest | null> {
const response = await fetch(`${this.apiUrl}/augmentations/${id}/manifest`, {
headers: this.getHeaders()
})
if (!response.ok) {
if (response.status === 404) return null
throw new Error(`Failed to fetch manifest: ${response.statusText}`)
}
return response.json()
}
/**
* Get CDN URL for dynamic loading
*/
async getCDNUrl(id: string): Promise<string | null> {
const aug = await this.getAugmentation(id)
return aug?.cdnUrl || null
}
/**
* Check if user has access to augmentation
*/
async checkAccess(id: string): Promise<{
hasAccess: boolean
requiresPurchase?: boolean
requiredTier?: string
}> {
if (!this.apiKey) {
// No API key, only free augmentations
const aug = await this.getAugmentation(id)
return {
hasAccess: aug?.tier === 'free',
requiresPurchase: aug?.tier !== 'free',
requiredTier: aug?.tier
}
}
const response = await fetch(`${this.apiUrl}/augmentations/${id}/access`, {
headers: this.getHeaders()
})
if (!response.ok) {
throw new Error(`Failed to check access: ${response.statusText}`)
}
return response.json()
}
/**
* Purchase/activate augmentation
*/
async purchase(id: string, licenseKey?: string): Promise<{
success: boolean
cdnUrl?: string
npmPackage?: string
licenseKey?: string
}> {
const body = licenseKey ? { licenseKey } : {}
const response = await fetch(`${this.apiUrl}/augmentations/${id}/purchase`, {
method: 'POST',
headers: {
...this.getHeaders(),
'Content-Type': 'application/json'
},
body: JSON.stringify(body)
})
if (!response.ok) {
throw new Error(`Failed to purchase: ${response.statusText}`)
}
return response.json()
}
/**
* Get user's purchased augmentations
*/
async getPurchased(): Promise<CatalogAugmentation[]> {
if (!this.apiKey) {
return []
}
const response = await fetch(`${this.apiUrl}/user/augmentations`, {
headers: this.getHeaders()
})
if (!response.ok) {
throw new Error(`Failed to fetch purchased: ${response.statusText}`)
}
const data = await response.json()
return this.transformCatalogData(data)
}
/**
* Get categories
*/
async getCategories(): Promise<Array<{
id: string
name: string
description: string
icon?: string
}>> {
const response = await fetch(`${this.apiUrl}/augmentations/categories`, {
headers: this.getHeaders()
})
if (!response.ok) {
throw new Error(`Failed to fetch categories: ${response.statusText}`)
}
return response.json()
}
/**
* Search augmentations
*/
async search(query: string): Promise<CatalogAugmentation[]> {
return this.discover({ search: query })
}
/**
* Get trending augmentations
*/
async getTrending(limit: number = 10): Promise<CatalogAugmentation[]> {
const response = await fetch(`${this.apiUrl}/augmentations/trending?limit=${limit}`, {
headers: this.getHeaders()
})
if (!response.ok) {
throw new Error(`Failed to fetch trending: ${response.statusText}`)
}
const data = await response.json()
return this.transformCatalogData(data)
}
/**
* Get recommended augmentations
*/
async getRecommended(): Promise<CatalogAugmentation[]> {
if (!this.apiKey) {
// Return popular free augmentations
return this.discover({ tier: 'free', minRating: 4 })
}
const response = await fetch(`${this.apiUrl}/augmentations/recommended`, {
headers: this.getHeaders()
})
if (!response.ok) {
throw new Error(`Failed to fetch recommended: ${response.statusText}`)
}
const data = await response.json()
return this.transformCatalogData(data)
}
/**
* Transform catalog data
*/
private transformCatalogData(data: any[]): CatalogAugmentation[] {
return data.map(item => this.transformAugmentation(item))
}
/**
* Transform single augmentation
*/
private transformAugmentation(item: any): CatalogAugmentation {
return {
id: item.id,
name: item.name,
description: item.description,
longDescription: item.longDescription,
category: item.category,
status: item.status || 'available',
tier: item.tier || 'free',
price: item.price,
manifest: item.manifest,
source: 'catalog',
cdnUrl: item.cdnUrl || `https://cdn.soulcraft.com/augmentations/${item.id}@latest`,
npmPackage: item.npmPackage,
githubRepo: item.githubRepo,
author: item.author,
metrics: item.metrics,
requirements: item.requirements
}
}
/**
* Get request headers
*/
private getHeaders(): HeadersInit {
const headers: HeadersInit = {}
if (this.apiKey) {
headers['Authorization'] = `Bearer ${this.apiKey}`
}
return headers
}
/**
* Clear cache
*/
clearCache(): void {
this.cache.clear()
}
/**
* Set API key
*/
setApiKey(apiKey: string): void {
this.apiKey = apiKey
this.clearCache() // Clear cache when API key changes
}
}

315
src/augmentations/discovery/localDiscovery.ts
View file

@ -1,315 +0,0 @@
/**
* Local Augmentation Discovery
*
* Discovers augmentations installed locally in node_modules
* and built-in augmentations that ship with Brainy
*
* NOTE: This is a Node.js-only feature that requires filesystem access
*/
import { AugmentationManifest } from '../manifest.js'
// Node.js modules - dynamically imported to avoid bundler issues
let fs: any = null
let path: any = null
// Load Node.js modules if available
if (typeof window === 'undefined') {
try {
fs = await import('node:fs')
path = await import('node:path')
} catch (e) {
// Will throw error in methods if not available
}
}
// Create compatibility layer for sync methods
const existsSync = fs?.existsSync || (() => { throw new Error('Filesystem not available') })
const readdirSync = fs?.readdirSync || (() => { throw new Error('Filesystem not available') })
const readFileSync = fs?.readFileSync || (() => { throw new Error('Filesystem not available') })
const join = path?.join || ((...args: string[]) => args.join('/'))
export interface LocalAugmentation {
id: string
name: string
source: 'builtin' | 'npm' | 'local'
path: string
manifest?: AugmentationManifest
package?: {
name: string
version: string
description?: string
}
}
/**
* Discovers augmentations installed locally
*/
export class LocalAugmentationDiscovery {
private builtInAugmentations: Map<string, LocalAugmentation> = new Map()
private installedAugmentations: Map<string, LocalAugmentation> = new Map()
constructor(private options: {
brainyPath?: string
projectPath?: string
scanNodeModules?: boolean
} = {}) {
this.options = {
brainyPath: this.options.brainyPath || this.findBrainyPath(),
projectPath: this.options.projectPath || process.cwd(),
scanNodeModules: this.options.scanNodeModules ?? true
}
// Register built-in augmentations
this.registerBuiltIn()
}
/**
* Register built-in augmentations that ship with Brainy
*/
private registerBuiltIn(): void {
const builtIn = [
{ id: 'cache', name: 'Cache', path: 'cacheAugmentation' },
{ id: 'batch', name: 'Batch Processing', path: 'batchProcessingAugmentation' },
{ id: 'entity-registry', name: 'Entity Registry', path: 'entityRegistryAugmentation' },
{ id: 'index', name: 'Index', path: 'indexAugmentation' },
{ id: 'metrics', name: 'Metrics', path: 'metricsAugmentation' },
{ id: 'monitoring', name: 'Monitoring', path: 'monitoringAugmentation' },
{ id: 'connection-pool', name: 'Connection Pool', path: 'connectionPoolAugmentation' },
{ id: 'request-deduplicator', name: 'Request Deduplicator', path: 'requestDeduplicatorAugmentation' },
{ id: 'api-server', name: 'API Server', path: 'apiServerAugmentation' },
{ id: 'neural-import', name: 'Neural Import', path: 'neuralImport' },
{ id: 'intelligent-verb-scoring', name: 'Intelligent Verb Scoring', path: 'intelligentVerbScoringAugmentation' },
{ id: 'universal-display', name: 'Universal Display', path: 'universalDisplayAugmentation' },
// Storage augmentations
{ id: 'memory-storage', name: 'Memory Storage', path: 'storageAugmentations', export: 'MemoryStorageAugmentation' },
{ id: 'filesystem-storage', name: 'FileSystem Storage', path: 'storageAugmentations', export: 'FileSystemStorageAugmentation' },
{ id: 'opfs-storage', name: 'OPFS Storage', path: 'storageAugmentations', export: 'OPFSStorageAugmentation' },
{ id: 's3-storage', name: 'S3 Storage', path: 'storageAugmentations', export: 'S3StorageAugmentation' },
]
for (const aug of builtIn) {
this.builtInAugmentations.set(aug.id, {
id: aug.id,
name: aug.name,
source: 'builtin',
path: `@soulcraft/brainy/augmentations/${aug.path}`,
package: {
name: '@soulcraft/brainy',
version: 'builtin',
description: `Built-in ${aug.name} augmentation`
}
})
}
}
/**
* Find Brainy installation path
*/
private findBrainyPath(): string {
// Try to find brainy in node_modules
const possiblePaths = [
join(process.cwd(), 'node_modules', '@soulcraft', 'brainy'),
join(process.cwd(), 'node_modules', 'brainy'),
join(process.cwd(), '..', 'node_modules', '@soulcraft', 'brainy'),
]
for (const path of possiblePaths) {
if (existsSync(path)) {
return path
}
}
// Fallback to current directory
return process.cwd()
}
/**
* Discover all augmentations
*/
async discoverAll(): Promise<LocalAugmentation[]> {
const augmentations: LocalAugmentation[] = []
// Add built-in augmentations
augmentations.push(...this.builtInAugmentations.values())
// Scan node_modules if enabled
if (this.options.scanNodeModules) {
const installed = await this.scanNodeModules()
augmentations.push(...installed)
}
// Scan local project
const local = await this.scanLocalProject()
augmentations.push(...local)
return augmentations
}
/**
* Scan node_modules for installed augmentations
*/
private async scanNodeModules(): Promise<LocalAugmentation[]> {
const augmentations: LocalAugmentation[] = []
const nodeModulesPath = join(this.options.projectPath!, 'node_modules')
if (!existsSync(nodeModulesPath)) {
return augmentations
}
// Scan @brainy/* packages
const brainyPath = join(nodeModulesPath, '@brainy')
if (existsSync(brainyPath)) {
const packages = readdirSync(brainyPath)
for (const pkg of packages) {
const augmentation = await this.loadPackageAugmentation(join(brainyPath, pkg))
if (augmentation) {
augmentations.push(augmentation)
}
}
}
// Scan packages with brainy-augmentation keyword
const packages = readdirSync(nodeModulesPath)
for (const pkg of packages) {
if (pkg.startsWith('@') || pkg.startsWith('.')) continue
const pkgPath = join(nodeModulesPath, pkg)
const packageJson = this.loadPackageJson(pkgPath)
if (packageJson?.keywords?.includes('brainy-augmentation')) {
const augmentation = await this.loadPackageAugmentation(pkgPath)
if (augmentation) {
augmentations.push(augmentation)
}
}
}
return augmentations
}
/**
* Scan local project for augmentations
*/
private async scanLocalProject(): Promise<LocalAugmentation[]> {
const augmentations: LocalAugmentation[] = []
// Check for augmentations directory
const augPath = join(this.options.projectPath!, 'augmentations')
if (existsSync(augPath)) {
const files = readdirSync(augPath)
for (const file of files) {
if (file.endsWith('.ts') || file.endsWith('.js')) {
const name = file.replace(/\.(ts|js)$/, '')
augmentations.push({
id: name,
name: this.humanizeName(name),
source: 'local',
path: join(augPath, file)
})
}
}
}
return augmentations
}
/**
* Load augmentation from package
*/
private async loadPackageAugmentation(pkgPath: string): Promise<LocalAugmentation | null> {
const packageJson = this.loadPackageJson(pkgPath)
if (!packageJson) return null
// Check if it's a brainy augmentation
const isBrainyAug = packageJson.keywords?.includes('brainy-augmentation') ||
packageJson.brainy?.type === 'augmentation'
if (!isBrainyAug) return null
const manifest = packageJson.brainy?.manifest || null
return {
id: packageJson.brainy?.id || packageJson.name.replace('@brainy/', ''),
name: packageJson.brainy?.name || packageJson.name,
source: 'npm',
path: pkgPath,
manifest,
package: {
name: packageJson.name,
version: packageJson.version,
description: packageJson.description
}
}
}
/**
* Load package.json
*/
private loadPackageJson(pkgPath: string): any {
const packageJsonPath = join(pkgPath, 'package.json')
if (!existsSync(packageJsonPath)) return null
try {
return JSON.parse(readFileSync(packageJsonPath, 'utf8'))
} catch {
return null
}
}
/**
* Convert name to human-readable format
*/
private humanizeName(name: string): string {
return name
.replace(/[-_]/g, ' ')
.replace(/augmentation/gi, '')
.replace(/\b\w/g, l => l.toUpperCase())
.trim()
}
/**
* Get built-in augmentations
*/
getBuiltIn(): LocalAugmentation[] {
return Array.from(this.builtInAugmentations.values())
}
/**
* Get installed augmentations
*/
getInstalled(): LocalAugmentation[] {
return Array.from(this.installedAugmentations.values())
}
/**
* Check if augmentation is installed
*/
isInstalled(id: string): boolean {
return this.builtInAugmentations.has(id) ||
this.installedAugmentations.has(id)
}
/**
* Get import path for augmentation
*/
getImportPath(id: string): string | null {
const aug = this.builtInAugmentations.get(id) ||
this.installedAugmentations.get(id)
return aug?.path || null
}
/**
* Load augmentation module dynamically
*/
async loadAugmentation(id: string): Promise<any> {
const path = this.getImportPath(id)
if (!path) {
throw new Error(`Augmentation ${id} not found`)
}
// Dynamic import
const module = await import(path)
return module.default || module
}
}

443
src/augmentations/discovery/runtimeLoader.ts
View file

@ -1,443 +0,0 @@
/**
* Runtime Augmentation Loader
*
* Dynamically loads and registers augmentations at runtime
* Supports CDN loading for browser environments and npm imports for Node.js
*/
import { BrainyAugmentation, AugmentationRegistry } from '../brainyAugmentation.js'
import { AugmentationManifest } from '../manifest.js'
export interface LoaderOptions {
cdnUrl?: string
allowUnsafe?: boolean
sandbox?: boolean
timeout?: number
cache?: boolean
}
export interface LoadedAugmentation {
id: string
instance: BrainyAugmentation
manifest: AugmentationManifest
source: 'cdn' | 'npm' | 'local'
loadTime: number
}
/**
* Runtime Augmentation Loader
*
* Enables dynamic loading of augmentations from various sources
*/
export class RuntimeAugmentationLoader {
private loaded: Map<string, LoadedAugmentation> = new Map()
private cdnCache: Map<string, any> = new Map()
private registry?: AugmentationRegistry
constructor(private options: LoaderOptions = {}) {
this.options = {
cdnUrl: options.cdnUrl || 'https://cdn.soulcraft.com/augmentations',
allowUnsafe: options.allowUnsafe || false,
sandbox: options.sandbox || true,
timeout: options.timeout || 30000,
cache: options.cache ?? true
}
}
/**
* Set the augmentation registry
*/
setRegistry(registry: AugmentationRegistry): void {
this.registry = registry
}
/**
* Load augmentation from CDN (browser)
*/
async loadFromCDN(
id: string,
version: string = 'latest',
config?: any
): Promise<LoadedAugmentation> {
// Check if already loaded
if (this.loaded.has(id)) {
return this.loaded.get(id)!
}
const url = `${this.options.cdnUrl}/${id}@${version}/index.js`
const startTime = Date.now()
try {
// Load module from CDN
const module = await this.loadCDNModule(url)
// Extract augmentation class
const AugmentationClass = module.default || module[Object.keys(module)[0]]
if (!AugmentationClass) {
throw new Error(`No augmentation class found in module ${id}`)
}
// Create instance
const instance = new AugmentationClass(config)
// Validate it's a proper augmentation
if (!this.isValidAugmentation(instance)) {
throw new Error(`Invalid augmentation: ${id}`)
}
// Get manifest
const manifest = instance.getManifest ? instance.getManifest() : {
id,
name: id,
version: version,
description: `Dynamically loaded ${id}`,
category: 'external'
}
const loaded: LoadedAugmentation = {
id,
instance,
manifest,
source: 'cdn',
loadTime: Date.now() - startTime
}
// Cache
this.loaded.set(id, loaded)
// Auto-register if registry is set
if (this.registry) {
this.registry.register(instance)
}
return loaded
} catch (error) {
throw new Error(`Failed to load augmentation ${id} from CDN: ${error}`)
}
}
/**
* Load augmentation from NPM (Node.js)
*/
async loadFromNPM(
packageName: string,
config?: any
): Promise<LoadedAugmentation> {
// Check if already loaded
const id = packageName.replace('@', '').replace('/', '-')
if (this.loaded.has(id)) {
return this.loaded.get(id)!
}
const startTime = Date.now()
try {
// Dynamic import
const module = await import(packageName)
// Extract augmentation class
const AugmentationClass = module.default || module[Object.keys(module)[0]]
if (!AugmentationClass) {
throw new Error(`No augmentation class found in package ${packageName}`)
}
// Create instance
const instance = new AugmentationClass(config)
// Validate
if (!this.isValidAugmentation(instance)) {
throw new Error(`Invalid augmentation in package: ${packageName}`)
}
// Get manifest
const manifest = instance.getManifest ? instance.getManifest() : {
id,
name: packageName,
version: 'unknown',
description: `Loaded from ${packageName}`,
category: 'external'
}
const loaded: LoadedAugmentation = {
id,
instance,
manifest,
source: 'npm',
loadTime: Date.now() - startTime
}
// Cache
this.loaded.set(id, loaded)
// Auto-register
if (this.registry) {
this.registry.register(instance)
}
return loaded
} catch (error) {
throw new Error(`Failed to load augmentation from NPM ${packageName}: ${error}`)
}
}
/**
* Load augmentation from local file
*/
async loadFromFile(
path: string,
config?: any
): Promise<LoadedAugmentation> {
const startTime = Date.now()
try {
// Dynamic import
const module = await import(path)
// Extract augmentation class
const AugmentationClass = module.default || module[Object.keys(module)[0]]
if (!AugmentationClass) {
throw new Error(`No augmentation class found in file ${path}`)
}
// Create instance
const instance = new AugmentationClass(config)
// Validate
if (!this.isValidAugmentation(instance)) {
throw new Error(`Invalid augmentation in file: ${path}`)
}
// Extract ID from path
const id = path.split('/').pop()?.replace(/\.(js|ts)$/, '') || 'unknown'
// Get manifest
const manifest = instance.getManifest ? instance.getManifest() : {
id,
name: id,
version: 'local',
description: `Loaded from ${path}`,
category: 'local'
}
const loaded: LoadedAugmentation = {
id,
instance,
manifest,
source: 'local',
loadTime: Date.now() - startTime
}
// Cache
this.loaded.set(id, loaded)
// Auto-register
if (this.registry) {
this.registry.register(instance)
}
return loaded
} catch (error) {
throw new Error(`Failed to load augmentation from file ${path}: ${error}`)
}
}
/**
* Load multiple augmentations
*/
async loadBatch(
augmentations: Array<{
source: 'cdn' | 'npm' | 'local'
id: string
version?: string
path?: string
config?: any
}>
): Promise<LoadedAugmentation[]> {
const results = await Promise.allSettled(
augmentations.map(aug => {
switch (aug.source) {
case 'cdn':
return this.loadFromCDN(aug.id, aug.version, aug.config)
case 'npm':
return this.loadFromNPM(aug.id, aug.config)
case 'local':
return this.loadFromFile(aug.path || aug.id, aug.config)
default:
return Promise.reject(new Error(`Unknown source: ${aug.source}`))
}
})
)
const loaded: LoadedAugmentation[] = []
const errors: string[] = []
for (const result of results) {
if (result.status === 'fulfilled') {
loaded.push(result.value)
} else {
errors.push(result.reason.message)
}
}
if (errors.length > 0) {
console.warn('Some augmentations failed to load:', errors)
}
return loaded
}
/**
* Unload augmentation
*/
unload(id: string): boolean {
const loaded = this.loaded.get(id)
if (!loaded) return false
// Shutdown if possible
if (loaded.instance.shutdown) {
loaded.instance.shutdown()
}
// Remove from registry if set
// Note: Registry doesn't have unregister yet, would need to add
// Remove from cache
this.loaded.delete(id)
return true
}
/**
* Get loaded augmentations
*/
getLoaded(): LoadedAugmentation[] {
return Array.from(this.loaded.values())
}
/**
* Check if augmentation is loaded
*/
isLoaded(id: string): boolean {
return this.loaded.has(id)
}
/**
* Get loaded augmentation
*/
getAugmentation(id: string): BrainyAugmentation | null {
return this.loaded.get(id)?.instance || null
}
/**
* Load CDN module (browser-specific)
*/
private async loadCDNModule(url: string): Promise<any> {
// Check cache
if (this.options.cache && this.cdnCache.has(url)) {
return this.cdnCache.get(url)
}
// In browser environment
if (typeof window !== 'undefined') {
return new Promise((resolve, reject) => {
const timeout = setTimeout(() => {
reject(new Error(`Timeout loading ${url}`))
}, this.options.timeout!)
// Create script element
const script = document.createElement('script')
script.type = 'module'
script.src = url
// Handle load
script.onload = async () => {
clearTimeout(timeout)
// The module should register itself on window
const moduleId = url.split('/').pop()?.split('@')[0]
if (moduleId && (window as any)[moduleId]) {
const module = (window as any)[moduleId]
// Cache
if (this.options.cache) {
this.cdnCache.set(url, module)
}
resolve(module)
} else {
reject(new Error(`Module not found on window: ${moduleId}`))
}
}
// Handle error
script.onerror = () => {
clearTimeout(timeout)
reject(new Error(`Failed to load script: ${url}`))
}
// Add to document
document.head.appendChild(script)
})
} else {
// In Node.js, use dynamic import
const module = await import(url)
// Cache
if (this.options.cache) {
this.cdnCache.set(url, module)
}
return module
}
}
/**
* Validate augmentation instance
*/
private isValidAugmentation(instance: any): boolean {
// Check required properties
return !!(
instance.name &&
instance.timing &&
instance.operations &&
instance.priority !== undefined &&
typeof instance.execute === 'function' &&
typeof instance.initialize === 'function'
)
}
/**
* Clear all caches
*/
clearCache(): void {
this.cdnCache.clear()
}
/**
* Get load statistics
*/
getStats(): {
loaded: number
totalLoadTime: number
averageLoadTime: number
sources: Record<string, number>
} {
const loaded = Array.from(this.loaded.values())
const totalLoadTime = loaded.reduce((sum, aug) => sum + aug.loadTime, 0)
const sources = loaded.reduce((acc, aug) => {
acc[aug.source] = (acc[aug.source] || 0) + 1
return acc
}, {} as Record<string, number>)
return {
loaded: loaded.length,
totalLoadTime,
averageLoadTime: loaded.length > 0 ? totalLoadTime / loaded.length : 0,
sources
}
}
}

375
src/augmentations/display/cache.ts
View file

@ -1,375 +0,0 @@
/**
* Universal Display Augmentation - Intelligent Caching System
*
* High-performance LRU cache with smart eviction and batch optimization
* Designed for minimal memory footprint and maximum hit ratio
*/
import type { DisplayCacheEntry, ComputedDisplayFields, DisplayAugmentationStats } from './types.js'
/**
* LRU (Least Recently Used) Cache for computed display fields
* Optimized for the display augmentation use case
*/
export class DisplayCache {
private cache = new Map<string, DisplayCacheEntry>()
private readonly maxSize: number
private stats = {
hits: 0,
misses: 0,
evictions: 0,
totalComputations: 0,
totalComputationTime: 0
}
constructor(maxSize: number = 1000) {
this.maxSize = maxSize
}
/**
* Get cached display fields with LRU update
* @param key Cache key
* @returns Cached fields or null if not found
*/
get(key: string): ComputedDisplayFields | null {
const entry = this.cache.get(key)
if (!entry) {
this.stats.misses++
return null
}
// Update LRU - move to end
this.cache.delete(key)
entry.lastAccessed = Date.now()
entry.accessCount++
this.cache.set(key, entry)
this.stats.hits++
return entry.fields
}
/**
* Store computed display fields in cache
* @param key Cache key
* @param fields Computed display fields
* @param computationTime Time taken to compute (for stats)
*/
set(key: string, fields: ComputedDisplayFields, computationTime?: number): void {
// Remove if already exists (for LRU update)
if (this.cache.has(key)) {
this.cache.delete(key)
}
// Create cache entry
const entry: DisplayCacheEntry = {
fields,
lastAccessed: Date.now(),
accessCount: 1
}
// Add to end (most recently used)
this.cache.set(key, entry)
// Update stats
this.stats.totalComputations++
if (computationTime) {
this.stats.totalComputationTime += computationTime
}
// Evict oldest if over capacity
if (this.cache.size > this.maxSize) {
this.evictOldest()
}
}
/**
* Check if a key exists in cache without affecting LRU order
* @param key Cache key
* @returns True if key exists
*/
has(key: string): boolean {
return this.cache.has(key)
}
/**
* Generate cache key from data
* @param id Entity ID (preferred)
* @param data Fallback data for key generation
* @param entityType Type of entity (noun/verb)
* @returns Cache key string
*/
generateKey(id?: string, data?: any, entityType: 'noun' | 'verb' = 'noun'): string {
// Use ID if available (most reliable)
if (id) {
return `${entityType}:${id}`
}
// Generate hash from data
if (data) {
const dataString = JSON.stringify(data, Object.keys(data).sort())
const hash = this.simpleHash(dataString)
return `${entityType}:hash:${hash}`
}
// Fallback to timestamp (not ideal but prevents crashes)
return `${entityType}:temp:${Date.now()}:${Math.random()}`
}
/**
* Clear all cached entries
*/
clear(): void {
this.cache.clear()
this.stats = {
hits: 0,
misses: 0,
evictions: 0,
totalComputations: 0,
totalComputationTime: 0
}
}
/**
* Get cache statistics
* @returns Cache performance statistics
*/
getStats(): DisplayAugmentationStats {
const hitRatio = this.stats.hits + this.stats.misses > 0
? this.stats.hits / (this.stats.hits + this.stats.misses)
: 0
const avgComputationTime = this.stats.totalComputations > 0
? this.stats.totalComputationTime / this.stats.totalComputations
: 0
// Analyze cached types for common types statistics
const typeCount = new Map<string, number>()
let fastestComputation = Infinity
let slowestComputation = 0
for (const entry of this.cache.values()) {
const type = entry.fields.type
typeCount.set(type, (typeCount.get(type) || 0) + 1)
}
const commonTypes = Array.from(typeCount.entries())
.sort(([,a], [,b]) => b - a)
.slice(0, 10)
.map(([type, count]) => ({
type,
count,
percentage: Math.round((count / this.cache.size) * 100)
}))
return {
totalComputations: this.stats.totalComputations,
cacheHitRatio: Math.round(hitRatio * 100) / 100,
averageComputationTime: Math.round(avgComputationTime * 100) / 100,
commonTypes,
performance: {
fastestComputation,
slowestComputation,
totalComputationTime: this.stats.totalComputationTime
}
}
}
/**
* Get current cache size
* @returns Number of cached entries
*/
size(): number {
return this.cache.size
}
/**
* Get cache capacity
* @returns Maximum cache size
*/
capacity(): number {
return this.maxSize
}
/**
* Evict least recently used entry
*/
private evictOldest(): void {
// First entry is oldest (LRU)
const firstKey = this.cache.keys().next().value
if (firstKey) {
this.cache.delete(firstKey)
this.stats.evictions++
}
}
/**
* Simple hash function for cache keys
* @param str String to hash
* @returns Simple hash number
*/
private simpleHash(str: string): number {
let hash = 0
for (let i = 0; i < str.length; i++) {
const char = str.charCodeAt(i)
hash = ((hash << 5) - hash) + char
hash = hash & hash // Convert to 32-bit integer
}
return Math.abs(hash)
}
/**
* Optimize cache by removing stale entries
* Called periodically to maintain cache health
*/
optimizeCache(): void {
const now = Date.now()
const maxAge = 24 * 60 * 60 * 1000 // 24 hours
const minAccessCount = 2 // Minimum access count to keep
const toDelete: string[] = []
for (const [key, entry] of this.cache.entries()) {
// Remove very old entries with low access count
if (now - entry.lastAccessed > maxAge && entry.accessCount < minAccessCount) {
toDelete.push(key)
}
}
// Remove stale entries
for (const key of toDelete) {
this.cache.delete(key)
this.stats.evictions++
}
}
/**
* Precompute display fields for a batch of entities
* @param entities Array of entities with their compute functions
* @returns Promise resolving when batch is complete
*/
async batchPrecompute<T>(
entities: Array<{
key: string
computeFn: () => Promise<ComputedDisplayFields>
}>
): Promise<void> {
const promises = entities.map(async ({ key, computeFn }) => {
if (!this.has(key)) {
const startTime = Date.now()
try {
const fields = await computeFn()
const computationTime = Date.now() - startTime
this.set(key, fields, computationTime)
} catch (error) {
console.warn(`Batch precompute failed for key ${key}:`, error)
}
}
})
await Promise.all(promises)
}
}
/**
* Request deduplicator for batch processing
* Prevents duplicate computations for the same data
*/
export class RequestDeduplicator {
private pendingRequests = new Map<string, Promise<ComputedDisplayFields>>()
private readonly batchSize: number
constructor(batchSize: number = 50) {
this.batchSize = batchSize
}
/**
* Deduplicate computation request
* @param key Unique key for the computation
* @param computeFn Function to compute the result
* @returns Promise that resolves to the computed fields
*/
async deduplicate(
key: string,
computeFn: () => Promise<ComputedDisplayFields>
): Promise<ComputedDisplayFields> {
// Return existing promise if already pending
if (this.pendingRequests.has(key)) {
return this.pendingRequests.get(key)!
}
// Create new computation promise
const promise = computeFn().finally(() => {
// Remove from pending when complete
this.pendingRequests.delete(key)
})
this.pendingRequests.set(key, promise)
return promise
}
/**
* Get number of pending requests
* @returns Number of pending computations
*/
getPendingCount(): number {
return this.pendingRequests.size
}
/**
* Clear all pending requests
*/
clear(): void {
this.pendingRequests.clear()
}
/**
* Shutdown the deduplicator
*/
shutdown(): void {
this.clear()
}
}
/**
* Global cache instance management
* Provides singleton access to display cache
*/
let globalDisplayCache: DisplayCache | null = null
/**
* Get global display cache instance
* @param maxSize Optional cache size (only used on first call)
* @returns Shared display cache instance
*/
export function getGlobalDisplayCache(maxSize?: number): DisplayCache {
if (!globalDisplayCache) {
globalDisplayCache = new DisplayCache(maxSize)
// Set up periodic optimization
setInterval(() => {
globalDisplayCache?.optimizeCache()
}, 60 * 60 * 1000) // Every hour
}
return globalDisplayCache
}
/**
* Clear global cache (for testing or memory management)
*/
export function clearGlobalDisplayCache(): void {
if (globalDisplayCache) {
globalDisplayCache.clear()
}
}
/**
* Shutdown global cache and cleanup
*/
export function shutdownGlobalDisplayCache(): void {
if (globalDisplayCache) {
globalDisplayCache.clear()
globalDisplayCache = null
}
}

429
src/augmentations/display/fieldPatterns.ts
View file

@ -1,429 +0,0 @@
/**
* Universal Display Augmentation - Smart Field Patterns
*
* Intelligent field detection patterns for mapping user data to display fields
* Uses semantic understanding and common naming conventions
*/
import type { FieldPattern, FieldComputationContext } from './types.js'
import { NounType, VerbType } from '../../types/graphTypes.js'
/**
* Universal field patterns that work across all data types
* Ordered by confidence level (highest first)
*/
export const UNIVERSAL_FIELD_PATTERNS: FieldPattern[] = [
// Title/Name Patterns (Highest Priority)
{
fields: ['name', 'title', 'displayName', 'label', 'heading'],
displayField: 'title',
confidence: 0.95
},
{
fields: ['firstName', 'lastName', 'fullName', 'realName'],
displayField: 'title',
confidence: 0.9,
applicableTypes: [NounType.Person],
transform: (value: any, context: FieldComputationContext) => {
const { metadata } = context
if (metadata.firstName && metadata.lastName) {
return `${metadata.firstName} ${metadata.lastName}`.trim()
}
return String(value || '')
}
},
{
fields: ['companyName', 'organizationName', 'orgName', 'businessName'],
displayField: 'title',
confidence: 0.9,
applicableTypes: [NounType.Organization]
},
{
fields: ['filename', 'fileName', 'documentTitle', 'docName'],
displayField: 'title',
confidence: 0.85,
applicableTypes: [NounType.Document, NounType.File, NounType.Media]
},
{
fields: ['projectName', 'projectTitle', 'initiative'],
displayField: 'title',
confidence: 0.9,
applicableTypes: [NounType.Project]
},
{
fields: ['taskName', 'taskTitle', 'action', 'todo'],
displayField: 'title',
confidence: 0.85,
applicableTypes: [NounType.Task]
},
{
fields: ['subject', 'topic', 'headline', 'caption'],
displayField: 'title',
confidence: 0.8
},
// Description Patterns (High Priority)
{
fields: ['description', 'summary', 'overview', 'details'],
displayField: 'description',
confidence: 0.9
},
{
fields: ['bio', 'biography', 'profile', 'about'],
displayField: 'description',
confidence: 0.85,
applicableTypes: [NounType.Person]
},
{
fields: ['content', 'text', 'body', 'message'],
displayField: 'description',
confidence: 0.8
},
{
fields: ['abstract', 'excerpt', 'snippet', 'preview'],
displayField: 'description',
confidence: 0.75
},
{
fields: ['notes', 'comments', 'remarks', 'observations'],
displayField: 'description',
confidence: 0.7
},
// Type Patterns (Medium Priority)
{
fields: ['type', 'category', 'classification', 'kind'],
displayField: 'type',
confidence: 0.9
},
{
fields: ['nounType', 'entityType', 'objectType'],
displayField: 'type',
confidence: 0.95
},
{
fields: ['role', 'position', 'jobTitle', 'occupation'],
displayField: 'type',
confidence: 0.8,
applicableTypes: [NounType.Person],
transform: (value: any) => String(value || 'Person')
},
{
fields: ['industry', 'sector', 'domain', 'field'],
displayField: 'type',
confidence: 0.7,
applicableTypes: [NounType.Organization]
},
// Tag Patterns (Medium Priority)
{
fields: ['tags', 'keywords', 'labels', 'categories'],
displayField: 'tags',
confidence: 0.85
},
{
fields: ['topics', 'subjects', 'themes'],
displayField: 'tags',
confidence: 0.8
}
]
/**
* Type-specific field patterns for enhanced detection
* Used when we know the specific type of the entity
*/
export const TYPE_SPECIFIC_PATTERNS: Record<string, FieldPattern[]> = {
[NounType.Person]: [
{
fields: ['email', 'emailAddress', 'contactEmail'],
displayField: 'description',
confidence: 0.7,
transform: (value: any, context: FieldComputationContext) => {
const { metadata } = context
const role = metadata.role || metadata.jobTitle || metadata.position
const company = metadata.company || metadata.organization || metadata.employer
const parts = []
if (role) parts.push(role)
if (company) parts.push(`at ${company}`)
if (parts.length === 0 && value) parts.push(`Contact: ${value}`)
return parts.join(' ') || 'Person'
}
},
{
fields: ['phone', 'phoneNumber', 'mobile', 'cell'],
displayField: 'tags',
confidence: 0.6
}
],
[NounType.Organization]: [
{
fields: ['website', 'url', 'homepage', 'domain'],
displayField: 'description',
confidence: 0.7,
transform: (value: any, context: FieldComputationContext) => {
const { metadata } = context
const industry = metadata.industry || metadata.sector
const location = metadata.location || metadata.city || metadata.country
const parts = []
if (industry) parts.push(industry)
parts.push('organization')
if (location) parts.push(`in ${location}`)
return parts.join(' ')
}
},
{
fields: ['employees', 'size', 'headcount'],
displayField: 'tags',
confidence: 0.6
}
],
[NounType.Project]: [
{
fields: ['status', 'phase', 'stage', 'state'],
displayField: 'description',
confidence: 0.8,
transform: (value: any, context: FieldComputationContext) => {
const { metadata } = context
const status = String(value || 'active').toLowerCase()
const budget = metadata.budget || metadata.cost
const lead = metadata.lead || metadata.manager || metadata.owner
const parts = []
parts.push(status.charAt(0).toUpperCase() + status.slice(1))
if (metadata.description) parts.push('project')
if (lead) parts.push(`led by ${lead}`)
if (budget) parts.push(`($${parseInt(String(budget)).toLocaleString()} budget)`)
return parts.join(' ')
}
}
],
[NounType.Document]: [
{
fields: ['author', 'creator', 'writer'],
displayField: 'description',
confidence: 0.7,
transform: (value: any, context: FieldComputationContext) => {
const { metadata } = context
const docType = metadata.type || metadata.category || 'document'
const date = metadata.date || metadata.created || metadata.published
const parts = []
if (docType) parts.push(docType)
if (value) parts.push(`by ${value}`)
if (date) {
const dateStr = new Date(date).toLocaleDateString()
parts.push(`(${dateStr})`)
}
return parts.join(' ')
}
}
],
[NounType.Task]: [
{
fields: ['priority', 'urgency', 'importance'],
displayField: 'tags',
confidence: 0.7
}
]
}
/**
* Get field patterns for a specific entity type
* @param entityType The type of entity (noun or verb)
* @param specificType Optional specific noun/verb type
* @returns Array of applicable field patterns
*/
export function getFieldPatterns(entityType: 'noun' | 'verb', specificType?: string): FieldPattern[] {
const patterns = [...UNIVERSAL_FIELD_PATTERNS]
if (entityType === 'noun' && specificType && TYPE_SPECIFIC_PATTERNS[specificType]) {
patterns.unshift(...TYPE_SPECIFIC_PATTERNS[specificType])
}
return patterns.sort((a, b) => b.confidence - a.confidence)
}
/**
* Priority fields for different entity types (for AI analysis)
* Used by the BrainyTypes and neural processing
*/
export const TYPE_PRIORITY_FIELDS: Record<string, string[]> = {
[NounType.Person]: [
'name', 'firstName', 'lastName', 'fullName', 'displayName',
'email', 'role', 'jobTitle', 'position', 'title',
'bio', 'description', 'about', 'profile',
'company', 'organization', 'employer'
],
[NounType.Organization]: [
'name', 'companyName', 'organizationName', 'title',
'industry', 'sector', 'domain', 'type',
'description', 'about', 'summary',
'location', 'city', 'country', 'headquarters',
'website', 'url'
],
[NounType.Project]: [
'name', 'projectName', 'title', 'projectTitle',
'description', 'summary', 'overview', 'goal',
'status', 'phase', 'stage', 'state',
'lead', 'manager', 'owner', 'team',
'budget', 'timeline', 'deadline'
],
[NounType.Document]: [
'title', 'filename', 'name', 'subject',
'content', 'text', 'body', 'summary',
'author', 'creator', 'writer',
'type', 'category', 'format',
'date', 'created', 'published'
],
[NounType.Task]: [
'title', 'name', 'taskName', 'action',
'description', 'details', 'notes',
'status', 'state', 'priority',
'assignee', 'owner', 'responsible',
'due', 'deadline', 'dueDate'
],
[NounType.Event]: [
'name', 'title', 'eventName',
'description', 'details', 'summary',
'startDate', 'endDate', 'date', 'time',
'location', 'venue', 'address',
'organizer', 'host', 'creator'
],
[NounType.Product]: [
'name', 'productName', 'title',
'description', 'summary', 'features',
'price', 'cost', 'value',
'category', 'type', 'brand',
'manufacturer', 'vendor'
]
}
/**
* Get priority fields for intelligent analysis
* @param entityType The type of entity
* @param specificType Optional specific type
* @returns Array of priority field names
*/
export function getPriorityFields(entityType: 'noun' | 'verb', specificType?: string): string[] {
if (entityType === 'noun' && specificType && TYPE_PRIORITY_FIELDS[specificType]) {
return TYPE_PRIORITY_FIELDS[specificType]
}
// Default priority fields for any entity
return [
'name', 'title', 'label', 'displayName',
'description', 'summary', 'about', 'details',
'type', 'category', 'kind', 'classification',
'tags', 'keywords', 'labels'
]
}
/**
* Smart field value extraction with type-aware processing
* @param data The data object to extract from
* @param pattern The field pattern to apply
* @param context The computation context
* @returns The extracted and processed field value
*/
export function extractFieldValue(
data: any,
pattern: FieldPattern,
context: FieldComputationContext
): any {
// Find the first matching field
let value: any = null
let matchedField: string | null = null
for (const field of pattern.fields) {
if (data[field] !== undefined && data[field] !== null && data[field] !== '') {
value = data[field]
matchedField = field
break
}
}
if (value === null) return null
// Apply transformation if provided
if (pattern.transform) {
try {
return pattern.transform(value, context)
} catch (error) {
console.warn(`Field transformation error for ${matchedField}:`, error)
return String(value)
}
}
// Default processing based on display field type
switch (pattern.displayField) {
case 'title':
case 'description':
case 'type':
return String(value)
case 'tags':
if (Array.isArray(value)) return value
if (typeof value === 'string') {
return value.split(/[,;]\s*|\s+/).filter(Boolean)
}
return [String(value)]
default:
return value
}
}
/**
* Calculate confidence score for field detection
* @param pattern The field pattern
* @param context The computation context
* @param value The extracted value
* @returns Confidence score (0-1)
*/
export function calculateFieldConfidence(
pattern: FieldPattern,
context: FieldComputationContext,
value: any
): number {
let confidence = pattern.confidence
// Boost confidence if type matches
if (pattern.applicableTypes && context.typeResult) {
if (pattern.applicableTypes.includes(context.typeResult.type)) {
confidence = Math.min(1.0, confidence + 0.1)
}
}
// Reduce confidence for empty or very short values
if (typeof value === 'string') {
if (value.length < 2) {
confidence *= 0.5
} else if (value.length < 5) {
confidence *= 0.8
}
}
// Reduce confidence for generic values
const genericValues = ['unknown', 'n/a', 'null', 'undefined', 'default']
if (typeof value === 'string' && genericValues.includes(value.toLowerCase())) {
confidence *= 0.3
}
return Math.max(0, Math.min(1, confidence))
}

76
src/augmentations/display/iconMappings.ts
View file

@ -1,76 +0,0 @@
/**
* Universal Display Augmentation - Clean Display
*
* Simple, clean display without icons - focusing on AI-powered
* titles, descriptions, and smart formatting that matches
* Soulcraft's minimal aesthetic
*/
import { NounType, VerbType } from '../../types/graphTypes.js'
/**
* No icon mappings - clean, minimal approach
* The real value is in AI-generated titles and enhanced descriptions,
* not visual clutter that doesn't align with professional aesthetics
*/
export const NOUN_TYPE_ICONS: Record<string, string> = {}
/**
* No icon mappings for verbs either - focus on clear relationship descriptions
* Human-readable relationship text is more valuable than symbolic representations
*/
export const VERB_TYPE_ICONS: Record<string, string> = {}
/**
* Get icon for a noun type (returns empty string for clean display)
* @param type The noun type
* @returns Empty string (no icons)
*/
export function getNounIcon(type: string): string {
return '' // Clean, no icons
}
/**
* Get icon for a verb type (returns empty string for clean display)
* @param type The verb type
* @returns Empty string (no icons)
*/
export function getVerbIcon(type: string): string {
return '' // Clean, no icons
}
/**
* Get coverage statistics (for backwards compatibility)
* @returns Coverage info showing clean approach
*/
export function getIconCoverage() {
return {
nounTypes: {
total: 'Clean display - no icons needed',
covered: 'Focus on AI-powered content'
},
verbTypes: {
total: 'Clean display - no icons needed',
covered: 'Focus on relationship descriptions'
}
}
}
/**
* Check if an icon exists for a type (always false for clean display)
* @param type The type to check
* @param entityType Whether it's a noun or verb
* @returns Always false (no icons)
*/
export function hasIcon(type: string, entityType: 'noun' | 'verb' = 'noun'): boolean {
return false // Clean approach - no icons
}
/**
* Get fallback icon (returns empty string for clean display)
* @param entityType The entity type
* @returns Empty string (no fallback icons)
*/
export function getFallbackIcon(entityType: 'noun' | 'verb' = 'noun'): string {
return '' // Clean, minimal display
}

539
src/augmentations/display/intelligentComputation.ts
View file

@ -1,539 +0,0 @@
/**
* Universal Display Augmentation - Intelligent Computation Engine
*
* Leverages existing Brainy AI infrastructure for intelligent field computation:
* - BrainyTypes for semantic type detection
* - Neural Import patterns for field analysis
* - JSON processing utilities for field extraction
* - Existing NounType/VerbType taxonomy (42+127 types)
*/
import type {
ComputedDisplayFields,
FieldComputationContext,
TypeMatchResult,
DisplayConfig
} from './types.js'
import type { VectorDocument, GraphVerb } from '../../coreTypes.js'
import { BrainyTypes, getBrainyTypes } from '../typeMatching/brainyTypes.js'
import { getNounIcon, getVerbIcon } from './iconMappings.js'
import {
getFieldPatterns,
getPriorityFields,
extractFieldValue,
calculateFieldConfidence
} from './fieldPatterns.js'
import { prepareJsonForVectorization, extractFieldFromJson } from '../../utils/jsonProcessing.js'
import { NounType, VerbType } from '../../types/graphTypes.js'
/**
* Intelligent field computation engine
* Coordinates AI-powered analysis with fallback heuristics
*/
export class IntelligentComputationEngine {
private typeMatcher: BrainyTypes | null = null
protected config: DisplayConfig
private initialized = false
constructor(config: DisplayConfig) {
this.config = config
}
/**
* Initialize the computation engine with AI components
*/
async initialize(): Promise<void> {
if (this.initialized) return
try {
// 🧠 LEVERAGE YOUR EXISTING AI INFRASTRUCTURE
this.typeMatcher = await getBrainyTypes()
if (this.typeMatcher) {
console.log('🎨 Display computation engine initialized with AI intelligence')
} else {
console.warn('🎨 Display computation engine running in basic mode (AI unavailable)')
}
} catch (error) {
console.warn('🎨 AI initialization failed, using heuristic fallback:', error)
}
this.initialized = true
}
/**
* Compute display fields for a noun using AI-first approach
* @param data The noun data/metadata
* @param id Optional noun ID
* @returns Computed display fields
*/
async computeNounDisplay(data: any, id?: string): Promise<ComputedDisplayFields> {
const startTime = Date.now()
try {
// 🟢 PRIMARY PATH: Use your existing AI intelligence
if (this.typeMatcher) {
return await this.computeWithAI(data, 'noun', { id })
}
// 🟡 FALLBACK PATH: Use heuristic patterns
return await this.computeWithHeuristics(data, 'noun', { id })
} catch (error) {
console.warn('Display computation failed, using minimal fallback:', error)
return this.createMinimalDisplay(data, 'noun')
} finally {
const computationTime = Date.now() - startTime
if (this.config.debugMode) {
console.log(`Display computation took ${computationTime}ms`)
}
}
}
/**
* Compute display fields for a verb using AI-first approach
* @param verb The verb/relationship data
* @returns Computed display fields
*/
async computeVerbDisplay(verb: GraphVerb): Promise<ComputedDisplayFields> {
const startTime = Date.now()
try {
// 🟢 PRIMARY PATH: Use your existing AI for verb analysis
if (this.typeMatcher) {
return await this.computeVerbWithAI(verb)
}
// 🟡 FALLBACK PATH: Use heuristic patterns for verbs
return await this.computeWithHeuristics(verb, 'verb')
} catch (error) {
console.warn('Verb display computation failed, using minimal fallback:', error)
return this.createMinimalDisplay(verb, 'verb')
} finally {
const computationTime = Date.now() - startTime
if (this.config.debugMode) {
console.log(`Verb display computation took ${computationTime}ms`)
}
}
}
/**
* AI-powered computation using your existing BrainyTypes
* @param data Entity data/metadata
* @param entityType Type of entity (noun/verb)
* @param options Additional options
* @returns AI-computed display fields
*/
private async computeWithAI(
data: any,
entityType: 'noun' | 'verb',
options: { id?: string } = {}
): Promise<ComputedDisplayFields> {
// 🧠 USE YOUR EXISTING TYPE DETECTION AI
const typeResult = await this.typeMatcher!.matchNounType(data)
// Create computation context
const context: FieldComputationContext = {
data,
metadata: data,
typeResult,
config: this.config,
entityType
}
// 🟢 INTELLIGENT FIELD EXTRACTION using your patterns + AI insights
const displayFields = {
title: await this.computeIntelligentTitle(context),
description: await this.computeIntelligentDescription(context),
type: typeResult.type,
tags: await this.computeIntelligentTags(context),
confidence: typeResult.confidence,
reasoning: this.config.debugMode ? typeResult.reasoning : undefined,
alternatives: this.config.debugMode ? typeResult.alternatives : undefined,
computedAt: Date.now(),
version: '1.0.0'
}
return displayFields
}
/**
* AI-powered verb computation using relationship analysis
* @param verb The verb/relationship
* @returns AI-computed display fields
*/
private async computeVerbWithAI(verb: GraphVerb): Promise<ComputedDisplayFields> {
// 🧠 USE YOUR EXISTING VERB TYPE DETECTION
const typeResult = await this.typeMatcher!.matchVerbType(verb, 0.7)
// Create verb computation context
const context: FieldComputationContext = {
data: verb,
metadata: verb.metadata || {},
typeResult,
config: this.config,
entityType: 'verb',
verbContext: {
sourceId: verb.sourceId,
targetId: verb.targetId,
verbType: verb.type
}
}
// 🟢 INTELLIGENT VERB DISPLAY COMPUTATION
const displayFields = {
title: await this.computeVerbTitle(context),
description: await this.computeVerbDescription(context),
type: typeResult.type,
tags: await this.computeVerbTags(context),
relationship: await this.computeHumanReadableRelationship(context),
confidence: typeResult.confidence,
reasoning: this.config.debugMode ? typeResult.reasoning : undefined,
alternatives: this.config.debugMode ? typeResult.alternatives : undefined,
computedAt: Date.now(),
version: '1.0.0'
}
return displayFields
}
/**
* Heuristic computation when AI is unavailable
* @param data Entity data
* @param entityType Type of entity
* @param options Additional options
* @returns Heuristically computed display fields
*/
private async computeWithHeuristics(
data: any,
entityType: 'noun' | 'verb',
options: { id?: string } = {}
): Promise<ComputedDisplayFields> {
// Use basic type detection
const detectedType = this.detectTypeHeuristically(data, entityType)
const typeResult: TypeMatchResult = {
type: detectedType,
confidence: 0.6, // Lower confidence for heuristics
reasoning: 'Heuristic detection (AI unavailable)',
alternatives: []
}
const context: FieldComputationContext = {
data,
metadata: data,
typeResult: typeResult,
config: this.config,
entityType
}
// Use pattern-based field extraction
const patterns = getFieldPatterns(entityType, detectedType)
return {
title: this.extractFieldWithPatterns(data, patterns, 'title') || 'Untitled',
description: this.extractFieldWithPatterns(data, patterns, 'description') || 'No description',
type: detectedType,
tags: this.extractFieldWithPatterns(data, patterns, 'tags') || [],
confidence: typeResult.confidence,
reasoning: this.config.debugMode ? typeResult.reasoning : undefined,
computedAt: Date.now(),
version: '1.0.0'
}
}
/**
* Compute intelligent title using AI insights and your field extraction
* @param context Computation context with AI results
* @returns Computed title
*/
private async computeIntelligentTitle(context: FieldComputationContext): Promise<string> {
const { data, typeResult } = context
// 🟢 USE TYPE-SPECIFIC LOGIC based on your NounType taxonomy
switch (typeResult?.type) {
case NounType.Person:
return this.computePersonTitle(data)
case NounType.Organization:
return this.computeOrganizationTitle(data)
case NounType.Project:
return this.computeProjectTitle(data)
case NounType.Document:
return this.computeDocumentTitle(data)
default:
// 🟢 LEVERAGE YOUR JSON PROCESSING for unknown types
return this.extractBestTitle(data, typeResult?.type)
}
}
/**
* Compute intelligent description using AI insights and context
* @param context Computation context
* @returns Enhanced description
*/
private async computeIntelligentDescription(context: FieldComputationContext): Promise<string> {
const { data, typeResult } = context
// 🟢 USE YOUR EXISTING JSON PROCESSING for vectorization-quality text
const priorityFields = getPriorityFields('noun', typeResult?.type)
const enhancedText = prepareJsonForVectorization(data, {
priorityFields,
includeFieldNames: false,
maxDepth: 2
})
// Create context-aware description based on type
return this.createContextAwareDescription(data, typeResult, enhancedText)
}
/**
* Compute intelligent tags using type analysis
* @param context Computation context
* @returns Generated tags array
*/
private async computeIntelligentTags(context: FieldComputationContext): Promise<string[]> {
const { data, typeResult } = context
const tags: string[] = []
// Add type-based tag
if (typeResult?.type) {
tags.push(typeResult.type.toLowerCase())
}
// Extract explicit tags from data
const explicitTags = this.extractExplicitTags(data)
tags.push(...explicitTags)
// Add semantic tags based on AI analysis
if (typeResult && this.typeMatcher) {
const semanticTags = this.generateSemanticTags(data, typeResult)
tags.push(...semanticTags)
}
// Remove duplicates and return
return [...new Set(tags.filter(Boolean))]
}
/**
* Compute verb title (relationship summary)
* @param context Verb computation context
* @returns Verb title
*/
private async computeVerbTitle(context: FieldComputationContext): Promise<string> {
const { verbContext, typeResult } = context
if (!verbContext) return 'Relationship'
const { sourceId, targetId } = verbContext
const relationshipType = typeResult?.type || 'RelatedTo'
// Try to get readable names for source and target
// This could be enhanced to actually resolve the entities
return `${sourceId} ${this.getReadableVerbPhrase(relationshipType)} ${targetId}`
}
/**
* Create minimal display for error cases
* @param data Entity data
* @param entityType Entity type
* @returns Minimal display fields
*/
private createMinimalDisplay(data: any, entityType: 'noun' | 'verb'): ComputedDisplayFields {
return {
title: data.name || data.title || data.id || 'Untitled',
description: data.description || data.summary || 'No description available',
type: entityType === 'noun' ? 'Item' : 'RelatedTo',
tags: [],
confidence: 0.1, // Very low confidence for fallback
computedAt: Date.now(),
version: '1.0.0'
}
}
// Helper methods for specific noun types
private computePersonTitle(data: any): string {
if (data.firstName && data.lastName) {
return `${data.firstName} ${data.lastName}`.trim()
}
return data.name || data.fullName || data.displayName || data.firstName || data.lastName || 'Person'
}
private computeOrganizationTitle(data: any): string {
return data.name || data.companyName || data.organizationName || data.title || 'Organization'
}
private computeProjectTitle(data: any): string {
return data.name || data.projectName || data.title || data.projectTitle || 'Project'
}
private computeDocumentTitle(data: any): string {
return data.title || data.filename || data.name || data.subject || 'Document'
}
private extractBestTitle(data: any, type?: string): string {
const titleFields = ['name', 'title', 'displayName', 'label', 'subject', 'heading']
for (const field of titleFields) {
if (data[field]) return String(data[field])
}
return data.id || Object.keys(data)[0] || 'Untitled'
}
private createContextAwareDescription(data: any, typeResult?: TypeMatchResult, enhancedText?: string): string {
// Start with basic description fields
const basicDesc = data.description || data.summary || data.about || data.details
if (basicDesc) return String(basicDesc)
// Use enhanced text from JSON processing
if (enhancedText && enhancedText.length > 10) {
return enhancedText.substring(0, 200) + (enhancedText.length > 200 ? '...' : '')
}
// Generate from available fields
const parts = []
if (data.role) parts.push(data.role)
if (data.company) parts.push(`at ${data.company}`)
if (data.location) parts.push(`in ${data.location}`)
return parts.length > 0 ? parts.join(' ') : 'No description available'
}
private extractExplicitTags(data: any): string[] {
const tagFields = ['tags', 'keywords', 'labels', 'categories', 'topics']
for (const field of tagFields) {
if (data[field]) {
if (Array.isArray(data[field])) {
return data[field].map(String).filter(Boolean)
}
if (typeof data[field] === 'string') {
return data[field].split(/[,;]\s*|\s+/).filter(Boolean)
}
}
}
return []
}
private generateSemanticTags(data: any, typeResult: TypeMatchResult): string[] {
const tags: string[] = []
// Add confidence-based tags
if (typeResult.confidence > 0.9) tags.push('verified')
else if (typeResult.confidence < 0.7) tags.push('uncertain')
// Add type-specific semantic tags
if (data.status) tags.push(String(data.status).toLowerCase())
if (data.priority) tags.push(String(data.priority).toLowerCase())
if (data.category) tags.push(String(data.category).toLowerCase())
return tags
}
private getReadableVerbPhrase(verbType: string): string {
const verbPhrases: Record<string, string> = {
[VerbType.WorksWith]: 'works with',
[VerbType.MemberOf]: 'is member of',
[VerbType.ReportsTo]: 'reports to',
[VerbType.Owns]: 'owns',
[VerbType.LocatedAt]: 'located at',
[VerbType.Likes]: 'likes',
[VerbType.Follows]: 'follows',
}
return verbPhrases[verbType] || 'related to'
}
private async computeVerbDescription(context: FieldComputationContext): Promise<string> {
const { data, verbContext, typeResult } = context
if (data.description) return String(data.description)
// Generate contextual description for relationship
if (verbContext && typeResult) {
const parts = []
const relationshipPhrase = this.getReadableVerbPhrase(typeResult.type)
if (data.role) parts.push(`Role: ${data.role}`)
if (data.startDate) parts.push(`Since: ${new Date(data.startDate).toLocaleDateString()}`)
if (data.department) parts.push(`Department: ${data.department}`)
return parts.length > 0
? `${relationshipPhrase} - ${parts.join(', ')}`
: `${relationshipPhrase} relationship`
}
return 'Relationship'
}
private async computeVerbTags(context: FieldComputationContext): Promise<string[]> {
const { data, typeResult } = context
const tags = ['relationship']
if (typeResult?.type) {
tags.push(typeResult.type.toLowerCase())
}
// Add relationship-specific tags
if (data.status) tags.push(String(data.status).toLowerCase())
if (data.type) tags.push(String(data.type).toLowerCase())
return [...new Set(tags)]
}
private async computeHumanReadableRelationship(context: FieldComputationContext): Promise<string> {
const { verbContext, typeResult } = context
if (!verbContext || !typeResult) return 'Related'
const { sourceId, targetId } = verbContext
const phrase = this.getReadableVerbPhrase(typeResult.type)
return `${sourceId} ${phrase} ${targetId}`
}
private detectTypeHeuristically(data: any, entityType: 'noun' | 'verb'): string {
if (entityType === 'verb') return VerbType.RelatedTo
// Basic heuristics for noun types
if (data.firstName || data.lastName || data.email) return NounType.Person
if (data.companyName || data.organization) return NounType.Organization
if (data.filename || data.fileType) return NounType.Document
if (data.projectName || data.initiative) return NounType.Project
if (data.taskName || data.todo) return NounType.Task
if (data.startDate || data.endDate) return NounType.Event
return 'Item' // Generic fallback
}
private extractFieldWithPatterns(data: any, patterns: any[], fieldType: string): any {
const relevantPatterns = patterns.filter(p => p.displayField === fieldType)
for (const pattern of relevantPatterns) {
for (const field of pattern.fields) {
if (data[field]) {
return pattern.transform ? pattern.transform(data[field], { data, config: this.config } as any) : data[field]
}
}
}
return null
}
/**
* Shutdown the computation engine
*/
async shutdown(): Promise<void> {
// Cleanup if needed
this.typeMatcher = null
this.initialized = false
}
}

256
src/augmentations/display/types.ts
View file

@ -1,256 +0,0 @@
/**
* Universal Display Augmentation - Type Definitions
*
* Clean TypeScript interfaces for the display augmentation system
*/
import type { VectorDocument, GraphVerb } from '../../coreTypes.js'
/**
* Configuration interface for the Universal Display Augmentation
*/
export interface DisplayConfig {
/** Enable/disable the augmentation */
enabled: boolean
/** LRU cache size for computed display fields */
cacheSize: number
/** Use lazy computation (recommended for performance) */
lazyComputation: boolean
/** Batch processing size for multiple requests */
batchSize: number
/** Minimum confidence threshold for AI type detection */
confidenceThreshold: number
/** Silent mode - suppress all console output */
silent?: boolean
// No icon configuration needed - clean, minimal approach
/** Custom field mappings (userField -> displayField) */
customFieldMappings: Record<string, string>
/** Type-specific priority fields for intelligent detection */
priorityFields: Record<string, string[]>
/** Enable debug mode with reasoning output */
debugMode: boolean
}
/**
* Computed display fields for any noun or verb
*/
export interface ComputedDisplayFields {
/** Primary display name (AI-detected best field combination) */
title: string
/** Enhanced description with context awareness */
description: string
/** Human-readable type name */
type: string
// No icon field - clean, minimal approach
/** Generated display tags for categorization */
tags: string[]
/** For verbs: human-readable relationship description */
relationship?: string
/** AI confidence score (0-1) */
confidence: number
/** Explanation of type detection reasoning (debug mode) */
reasoning?: string
/** Alternative type suggestions with confidence scores */
alternatives?: Array<{
type: string
confidence: number
}>
/** Timestamp when fields were computed */
computedAt: number
/** Version of augmentation that computed these fields */
version: string
}
/**
* Cache entry for computed display fields
*/
export interface DisplayCacheEntry {
fields: ComputedDisplayFields
lastAccessed: number
accessCount: number
}
/**
* Field computation context passed to computation functions
*/
export interface FieldComputationContext {
/** The original data object */
data: any
/** Metadata associated with the object */
metadata: any
/** Type detection result from AI */
typeResult?: TypeMatchResult
/** Display configuration */
config: DisplayConfig
/** Whether this is a noun or verb */
entityType: 'noun' | 'verb'
/** For verbs: source and target information */
verbContext?: {
sourceId: string
targetId: string
verbType?: string
}
}
/**
* Type matching result from BrainyTypes
*/
export interface TypeMatchResult {
type: string
confidence: number
reasoning: string
alternatives: Array<{
type: string
confidence: number
}>
}
/**
* Enhanced VectorDocument with display capabilities
*/
export interface EnhancedVectorDocument<T = any> extends VectorDocument<T> {
/**
* Get computed display field(s)
* @param field Optional specific field name
* @returns Single field value or all display fields
*/
getDisplay(): Promise<ComputedDisplayFields>
getDisplay(field: keyof ComputedDisplayFields): Promise<any>
/**
* Get available fields for a specific augmentation namespace
* @param namespace The augmentation namespace (e.g., 'display')
* @returns Array of available field names
*/
getAvailableFields(namespace: string): string[]
/**
* Get available augmentation namespaces
* @returns Array of available augmentation names
*/
getAvailableAugmentations(): string[]
/**
* Debug exploration of all computed fields
*/
explore(): Promise<void>
}
/**
* Enhanced GraphVerb with display capabilities
*/
export interface EnhancedGraphVerb extends GraphVerb {
/**
* Get computed display field(s) for relationships
* @param field Optional specific field name
* @returns Single field value or all display fields
*/
getDisplay(): Promise<ComputedDisplayFields>
getDisplay(field: keyof ComputedDisplayFields): Promise<any>
/**
* Get available fields for a specific augmentation namespace
* @param namespace The augmentation namespace (e.g., 'display')
* @returns Array of available field names
*/
getAvailableFields(namespace: string): string[]
}
/**
* Batch computation request for performance optimization
*/
export interface BatchComputationRequest {
id: string
data: any
metadata: any
entityType: 'noun' | 'verb'
verbContext?: {
sourceId: string
targetId: string
verbType?: string
}
}
/**
* Batch computation result
*/
export interface BatchComputationResult {
id: string
fields: ComputedDisplayFields
error?: string
}
/**
* Field pattern for intelligent field detection
*/
export interface FieldPattern {
/** Field names that match this pattern */
fields: string[]
/** Target display field name */
displayField: keyof ComputedDisplayFields
/** Confidence score for this pattern match */
confidence: number
/** Optional: specific noun/verb types this applies to */
applicableTypes?: string[]
/** Optional: transformation function for the field value */
transform?: (value: any, context: FieldComputationContext) => string
}
/**
* Statistics for the display augmentation
*/
export interface DisplayAugmentationStats {
/** Total number of computations performed */
totalComputations: number
/** Cache hit ratio */
cacheHitRatio: number
/** Average computation time in milliseconds */
averageComputationTime: number
/** Type detection accuracy (when ground truth available) */
typeDetectionAccuracy?: number
/** Most commonly detected types */
commonTypes: Array<{
type: string
count: number
percentage: number
}>
/** Performance metrics */
performance: {
fastestComputation: number
slowestComputation: number
totalComputationTime: number
}
}

518
src/augmentations/entityRegistryAugmentation.ts
View file

@ -1,518 +0,0 @@
/**
* Entity Registry Augmentation
* Fast external-ID to internal-UUID mapping for streaming data processing
* Works in write-only mode for high-performance deduplication
*/
import { BrainyAugmentation, BaseAugmentation, AugmentationContext } from './brainyAugmentation.js'
export interface EntityRegistryConfig {
/**
* Maximum number of entries to keep in memory cache
* Default: 100,000 entries
*/
maxCacheSize?: number
/**
* Time to live for cache entries in milliseconds
* Default: 300,000 (5 minutes)
*/
cacheTTL?: number
/**
* Fields to index for fast lookup
* Default: ['did', 'handle', 'uri', 'id', 'external_id']
*/
indexedFields?: string[]
/**
* Persistence strategy
* memory: Keep only in memory (fast, but lost on restart)
* storage: Persist to storage (survives restarts)
* hybrid: Memory + periodic storage sync
*/
persistence?: 'memory' | 'storage' | 'hybrid'
/**
* How often to sync memory cache to storage (hybrid mode)
* Default: 30000 (30 seconds)
*/
syncInterval?: number
}
export interface EntityMapping {
externalId: string
field: string
brainyId: string
nounType: string
lastAccessed: number
metadata?: any
}
/**
* High-performance entity registry for external ID to Brainy UUID mapping
* Optimized for streaming data scenarios like Bluesky firehose processing
*/
export class EntityRegistryAugmentation extends BaseAugmentation {
readonly metadata = 'readonly' as const // Reads metadata to register entities
readonly name = 'entity-registry'
readonly description = 'Fast external-ID to internal-UUID mapping for streaming data'
readonly timing: 'before' | 'after' | 'around' | 'replace' = 'before'
readonly operations = ['add', 'addNoun', 'addVerb'] as ('add' | 'addNoun' | 'addVerb')[]
readonly priority = 90 // High priority for entity registration
protected config: Required<EntityRegistryConfig>
private memoryIndex = new Map<string, EntityMapping>()
private fieldIndices = new Map<string, Map<string, string>>() // field -> value -> brainyId
private syncTimer?: NodeJS.Timeout
private brain?: any
private storage?: any
private cacheHits = 0
private cacheMisses = 0
constructor(config: EntityRegistryConfig = {}) {
super()
this.config = {
maxCacheSize: config.maxCacheSize ?? 100000,
cacheTTL: config.cacheTTL ?? 300000, // 5 minutes
indexedFields: config.indexedFields ?? ['did', 'handle', 'uri', 'id', 'external_id'],
persistence: config.persistence ?? 'hybrid',
syncInterval: config.syncInterval ?? 30000 // 30 seconds
}
// Initialize field indices
for (const field of this.config.indexedFields) {
this.fieldIndices.set(field, new Map())
}
}
async initialize(context: AugmentationContext): Promise<void> {
this.brain = context.brain
this.storage = context.storage
// Load existing mappings from storage
if (this.config.persistence === 'storage' || this.config.persistence === 'hybrid') {
await this.loadFromStorage()
}
// Start sync timer for hybrid mode
if (this.config.persistence === 'hybrid') {
this.syncTimer = setInterval(() => {
this.syncToStorage().catch(console.error)
}, this.config.syncInterval)
}
console.log(`🔍 EntityRegistry initialized: ${this.memoryIndex.size} cached mappings`)
}
async shutdown(): Promise<void> {
// Final sync before shutdown
if (this.config.persistence === 'storage' || this.config.persistence === 'hybrid') {
await this.syncToStorage()
}
if (this.syncTimer) {
clearInterval(this.syncTimer)
}
}
/**
* Execute the augmentation
*/
async execute<T = any>(
operation: string,
params: any,
next: () => Promise<T>
): Promise<T> {
console.log(`🔍 [EntityRegistry] execute called: operation=${operation}`)
// For add operations, check for duplicates first
if (operation === 'add' || operation === 'addNoun') {
const metadata = params.metadata || {}
// Check if entity already exists
for (const field of this.config.indexedFields) {
const value = this.extractFieldValue(metadata, field)
if (value) {
const existingId = await this.lookupEntity(field, value)
if (existingId) {
// Entity already exists, return the existing one
console.log(`🔍 Duplicate detected: ${field}:${value} → ${existingId}`)
return { id: existingId, duplicate: true } as any
}
}
}
}
// For addVerb operations, resolve external IDs to internal UUIDs
if (operation === 'addVerb') {
const sourceId = params.sourceId
const targetId = params.targetId
// Try to resolve source and target IDs if they look like external IDs
for (const field of this.config.indexedFields) {
// Check if sourceId matches an external ID pattern
if (typeof sourceId === 'string' && this.looksLikeExternalId(sourceId, field)) {
const resolvedSourceId = await this.lookupEntity(field, sourceId)
if (resolvedSourceId) {
console.log(`🔍 [EntityRegistry] Resolved source: ${sourceId} → ${resolvedSourceId}`)
params.sourceId = resolvedSourceId
}
}
// Check if targetId matches an external ID pattern
if (typeof targetId === 'string' && this.looksLikeExternalId(targetId, field)) {
const resolvedTargetId = await this.lookupEntity(field, targetId)
if (resolvedTargetId) {
console.log(`🔍 [EntityRegistry] Resolved target: ${targetId} → ${resolvedTargetId}`)
params.targetId = resolvedTargetId
}
}
}
}
// Proceed with the operation
const result = await next()
// Register the entity after successful add
if ((operation === 'add' || operation === 'addNoun' || operation === 'addVerb') && result) {
// Handle both formats: string UUID or object with id property
const brainyId = typeof result === 'string' ? result : (result as any).id
if (brainyId) {
const metadata = params.metadata || {}
const nounType = params.nounType || 'default'
console.log(`🔍 [EntityRegistry] Registering entity: ${brainyId}`)
await this.registerEntity(brainyId, metadata, nounType)
console.log(`✅ [EntityRegistry] Entity registered successfully`)
}
}
return result
}
/**
* Register a new entity mapping
*/
async registerEntity(brainyId: string, metadata: any, nounType: string): Promise<void> {
const now = Date.now()
// Extract indexed fields from metadata
for (const field of this.config.indexedFields) {
const value = this.extractFieldValue(metadata, field)
if (value) {
const key = `${field}:${value}`
// Add to memory index
const mapping: EntityMapping = {
externalId: value,
field,
brainyId,
nounType,
lastAccessed: now,
metadata
}
this.memoryIndex.set(key, mapping)
// Add to field-specific index
const fieldIndex = this.fieldIndices.get(field)
if (fieldIndex) {
fieldIndex.set(value, brainyId)
}
}
}
// Enforce cache size limit (LRU eviction)
await this.evictOldEntries()
}
/**
* Fast lookup: external ID → Brainy UUID
* Works in write-only mode without search indexes
*/
async lookupEntity(field: string, value: string): Promise<string | null> {
const key = `${field}:${value}`
const cached = this.memoryIndex.get(key)
if (cached) {
// Update last accessed time
cached.lastAccessed = Date.now()
this.cacheHits++
return cached.brainyId
}
this.cacheMisses++
// If not in cache and using storage persistence, try loading from storage
if (this.config.persistence === 'storage' || this.config.persistence === 'hybrid') {
const stored = await this.loadFromStorageByField(field, value)
if (stored) {
// Add to memory cache
this.memoryIndex.set(key, stored)
const fieldIndex = this.fieldIndices.get(field)
if (fieldIndex) {
fieldIndex.set(value, stored.brainyId)
}
return stored.brainyId
}
}
return null
}
/**
* Batch lookup for multiple external IDs
*/
async lookupBatch(lookups: Array<{ field: string; value: string }>): Promise<Map<string, string | null>> {
const results = new Map<string, string | null>()
const missingKeys: Array<{ field: string; value: string; key: string }> = []
// Check memory cache first
for (const lookup of lookups) {
const key = `${lookup.field}:${lookup.value}`
const cached = this.memoryIndex.get(key)
if (cached) {
cached.lastAccessed = Date.now()
results.set(key, cached.brainyId)
} else {
missingKeys.push({ ...lookup, key })
results.set(key, null)
}
}
// Batch load missing keys from storage
if (missingKeys.length > 0 && (this.config.persistence === 'storage' || this.config.persistence === 'hybrid')) {
const stored = await this.loadBatchFromStorage(missingKeys)
for (const [key, mapping] of stored) {
if (mapping) {
// Add to memory cache
this.memoryIndex.set(key, mapping)
const fieldIndex = this.fieldIndices.get(mapping.field)
if (fieldIndex) {
fieldIndex.set(mapping.externalId, mapping.brainyId)
}
results.set(key, mapping.brainyId)
}
}
}
return results
}
/**
* Check if entity exists (faster than lookupEntity for existence checks)
*/
async hasEntity(field: string, value: string): Promise<boolean> {
const fieldIndex = this.fieldIndices.get(field)
if (fieldIndex && fieldIndex.has(value)) {
return true
}
return (await this.lookupEntity(field, value)) !== null
}
/**
* Get all entities by field (e.g., all DIDs)
*/
async getEntitiesByField(field: string): Promise<string[]> {
const fieldIndex = this.fieldIndices.get(field)
return fieldIndex ? Array.from(fieldIndex.keys()) : []
}
/**
* Get registry statistics
*/
getStats(): {
totalMappings: number
fieldCounts: Record<string, number>
cacheHitRate: number
memoryUsage: number
} {
const fieldCounts: Record<string, number> = {}
for (const [field, index] of this.fieldIndices) {
fieldCounts[field] = index.size
}
return {
totalMappings: this.memoryIndex.size,
fieldCounts,
cacheHitRate: this.cacheHits > 0 ? this.cacheHits / (this.cacheHits + this.cacheMisses) : 0,
memoryUsage: this.estimateMemoryUsage()
}
}
/**
* Clear all cached mappings
*/
async clearCache(): Promise<void> {
this.memoryIndex.clear()
for (const fieldIndex of this.fieldIndices.values()) {
fieldIndex.clear()
}
}
// Private helper methods
/**
* Check if an ID looks like it could be an external ID for a specific field
*/
private looksLikeExternalId(id: string, field: string): boolean {
// Basic heuristics to detect external ID patterns
switch (field) {
case 'did':
return id.startsWith('did:')
case 'handle':
return id.includes('.') && (id.includes('bsky') || id.includes('social'))
case 'external_id':
return !id.match(/^[a-f0-9-]{36}$/i) // Not a UUID
case 'uri':
return id.startsWith('http') || id.startsWith('at://')
case 'id':
return !id.match(/^[a-f0-9-]{36}$/i) // Not a UUID
default:
// For custom fields, assume non-UUID strings might be external IDs
return typeof id === 'string' && id.length > 3 && !id.match(/^[a-f0-9-]{36}$/i)
}
}
private extractFieldValue(metadata: any, field: string): string | null {
if (!metadata) return null
// Support nested field access (e.g., "author.did")
const parts = field.split('.')
let value = metadata
for (const part of parts) {
value = value?.[part]
if (value === undefined || value === null) {
return null
}
}
return typeof value === 'string' ? value : String(value)
}
private async evictOldEntries(): Promise<void> {
if (this.memoryIndex.size <= this.config.maxCacheSize) {
return
}
// Sort by last accessed time and remove oldest entries
const entries = Array.from(this.memoryIndex.entries())
entries.sort(([, a], [, b]) => a.lastAccessed - b.lastAccessed)
const toRemove = entries.slice(0, entries.length - this.config.maxCacheSize)
for (const [key, mapping] of toRemove) {
this.memoryIndex.delete(key)
const fieldIndex = this.fieldIndices.get(mapping.field)
if (fieldIndex) {
fieldIndex.delete(mapping.externalId)
}
}
}
private async loadFromStorage(): Promise<void> {
if (!this.brain) return
try {
// Load registry data from a special storage location
const registryData = await this.brain.storage?.getMetadata('__entity_registry__')
if (registryData && registryData.mappings) {
for (const mapping of registryData.mappings) {
const key = `${mapping.field}:${mapping.externalId}`
this.memoryIndex.set(key, mapping)
const fieldIndex = this.fieldIndices.get(mapping.field)
if (fieldIndex) {
fieldIndex.set(mapping.externalId, mapping.brainyId)
}
}
}
} catch (error) {
console.warn('Failed to load entity registry from storage:', error)
}
}
private async syncToStorage(): Promise<void> {
if (!this.brain) return
try {
const mappings = Array.from(this.memoryIndex.values())
await this.brain.storage?.saveMetadata('__entity_registry__', {
version: 1,
lastSync: Date.now(),
mappings
})
} catch (error) {
console.warn('Failed to sync entity registry to storage:', error)
}
}
private async loadFromStorageByField(field: string, value: string): Promise<EntityMapping | null> {
// For now, this would require a full load. In production, you'd want
// a more sophisticated storage index system
return null
}
private async loadBatchFromStorage(keys: Array<{ field: string; value: string; key: string }>): Promise<Map<string, EntityMapping | null>> {
// For now, return empty. In production, implement batch storage lookup
return new Map()
}
private estimateMemoryUsage(): number {
// Rough estimate: 200 bytes per mapping on average
return this.memoryIndex.size * 200
}
}
// Hook into Brainy's add operations to automatically register entities
export class AutoRegisterEntitiesAugmentation extends BaseAugmentation {
readonly metadata = 'readonly' as const // Reads metadata for auto-registration
readonly name = 'auto-register-entities'
readonly description = 'Automatically register entities in the registry when added'
readonly timing: 'before' | 'after' | 'around' | 'replace' = 'after'
readonly operations = ['add', 'addNoun', 'addVerb'] as ('add' | 'addNoun' | 'addVerb')[]
readonly priority = 85 // After entity registry
private registry?: EntityRegistryAugmentation
private brain?: any
async initialize(context: AugmentationContext): Promise<void> {
this.brain = context.brain
// Find the entity registry augmentation from the registry
this.registry = this.brain?.augmentations?.augmentations?.find(
(aug: any) => aug instanceof EntityRegistryAugmentation
) as EntityRegistryAugmentation
}
async execute<T = any>(
operation: string,
params: any,
next: () => Promise<T>
): Promise<T> {
const result = await next()
// After successful add, register the entity
if ((operation === 'add' || operation === 'addNoun' || operation === 'addVerb') && result) {
if (this.registry) {
// Handle both formats: string UUID or object with id property
const brainyId = typeof result === 'string' ? result : (result as any).id
if (brainyId) {
const metadata = params.metadata || {}
const nounType = params.nounType || 'default'
await this.registry.registerEntity(brainyId, metadata, nounType)
}
}
}
return result
}
}

337
src/augmentations/indexAugmentation.ts
View file

@ -1,337 +0,0 @@
/**
* Index Augmentation - Optional Metadata Indexing
*
* Replaces the hardcoded MetadataIndex in Brainy with an optional augmentation.
* Provides O(1) metadata filtering and field lookups.
*
* Zero-config: Automatically enabled for better search performance
* Can be disabled or customized via augmentation registry
*/
import { BaseAugmentation, AugmentationContext } from './brainyAugmentation.js'
import { MetadataIndexManager } from '../utils/metadataIndex.js'
import type { BaseStorageAdapter as StorageAdapter } from '../storage/adapters/baseStorageAdapter.js'
export interface IndexConfig {
enabled?: boolean
maxFieldValues?: number
maxIndexSize?: number // Max number of entries per field value
autoRebuild?: boolean
rebuildThreshold?: number
flushInterval?: number
}
/**
* IndexAugmentation - Makes metadata indexing optional and pluggable
*
* Features:
* - O(1) metadata field lookups
* - Fast pre-filtering for searches
* - Automatic index maintenance
* - Zero-config with smart defaults
*/
export class IndexAugmentation extends BaseAugmentation {
readonly metadata = 'readonly' as const // Reads metadata to build indexes
readonly name = 'index'
readonly timing = 'after' as const
operations = ['add', 'update', 'updateMetadata', 'delete', 'clear', 'all'] as ('add' | 'update' | 'updateMetadata' | 'delete' | 'clear' | 'all')[]
readonly priority = 60 // Run after data operations
// Augmentation metadata
readonly category = 'core' as const
readonly description = 'Fast metadata field indexing for O(1) filtering and lookups'
private metadataIndex: MetadataIndexManager | null = null
protected config: IndexConfig
private flushTimer: NodeJS.Timeout | null = null
constructor(config: IndexConfig = {}) {
super()
this.config = {
enabled: true,
maxFieldValues: 1000,
autoRebuild: true,
rebuildThreshold: 0.3, // Rebuild if 30% inconsistent
flushInterval: 30000, // Flush every 30 seconds
...config
}
}
protected async onInitialize(): Promise<void> {
if (!this.config.enabled) {
this.log('Index augmentation disabled by configuration')
return
}
// Get storage from context
const storage = this.context?.storage
if (!storage) {
this.log('No storage available, index augmentation inactive', 'warn')
return
}
// Initialize metadata index
this.metadataIndex = new MetadataIndexManager(
storage as StorageAdapter,
{
maxIndexSize: this.config.maxIndexSize || 10000
}
)
// Check if we need to rebuild
if (this.config.autoRebuild) {
const stats = await this.metadataIndex.getStats()
if (stats.totalEntries === 0) {
// Check if storage has data but index is empty
try {
const storageStats = await storage.getStatistics?.()
if (storageStats && storageStats.totalNouns > 0) {
this.log('Rebuilding metadata index for existing data...')
await this.metadataIndex.rebuild()
const newStats = await this.metadataIndex.getStats()
this.log(`Index rebuilt: ${newStats.totalEntries} entries, ${newStats.fieldsIndexed.length} fields`)
}
} catch (e) {
this.log('Could not check storage statistics', 'info')
}
}
}
// Start flush timer
if (this.config.flushInterval && this.config.flushInterval > 0) {
this.startFlushTimer()
}
this.log('Index augmentation initialized')
}
protected async onShutdown(): Promise<void> {
// Stop flush timer
if (this.flushTimer) {
clearInterval(this.flushTimer)
this.flushTimer = null
}
// Flush index one last time
if (this.metadataIndex) {
try {
await this.metadataIndex.flush()
} catch (error) {
this.log('Error flushing index during shutdown', 'warn')
}
this.metadataIndex = null
}
this.log('Index augmentation shut down')
}
/**
* Execute augmentation - maintain index on data operations
*/
async execute<T = any>(
operation: string,
params: any,
next: () => Promise<T>
): Promise<T> {
// Execute the operation first
const result = await next()
// If index is disabled, just return
if (!this.metadataIndex || !this.config.enabled) {
return result
}
// Handle index updates after operation completes
switch (operation) {
case 'add':
await this.handleAdd(params)
break
case 'updateMetadata':
await this.handleUpdate(params)
break
case 'delete':
await this.handleDelete(params)
break
case 'clear':
await this.handleClear()
break
}
return result
}
/**
* Handle add operation - index new metadata
*/
private async handleAdd(params: any): Promise<void> {
if (!this.metadataIndex) return
const { id, metadata } = params
if (id && metadata) {
await this.metadataIndex.addToIndex(id, metadata)
this.log(`Indexed metadata for ${id}`, 'info')
}
}
/**
* Handle update operation - reindex metadata
*/
private async handleUpdate(params: any): Promise<void> {
if (!this.metadataIndex) return
const { id, oldMetadata, newMetadata } = params
// Remove old metadata
if (id && oldMetadata) {
await this.metadataIndex.removeFromIndex(id, oldMetadata)
}
// Add new metadata
if (id && newMetadata) {
await this.metadataIndex.addToIndex(id, newMetadata)
this.log(`Reindexed metadata for ${id}`, 'info')
}
}
/**
* Handle delete operation - remove from index
*/
private async handleDelete(params: any): Promise<void> {
if (!this.metadataIndex) return
const { id, metadata } = params
if (id && metadata) {
await this.metadataIndex.removeFromIndex(id, metadata)
this.log(`Removed ${id} from index`, 'info')
}
}
/**
* Handle clear operation - clear index
*/
private async handleClear(): Promise<void> {
if (!this.metadataIndex) return
// Clear the index when all data is cleared (rebuild effectively clears it)
await this.metadataIndex.rebuild()
this.log('Index cleared due to clear operation')
}
/**
* Start periodic flush timer
*/
private startFlushTimer(): void {
if (this.flushTimer) return
this.flushTimer = setInterval(async () => {
if (this.metadataIndex) {
try {
await this.metadataIndex.flush()
} catch (error) {
this.log('Error during periodic index flush', 'warn')
}
}
}, this.config.flushInterval!)
}
/**
* Get IDs that match metadata filter (for pre-filtering)
*/
async getIdsForFilter(filter: Record<string, any>): Promise<string[]> {
if (!this.metadataIndex) return []
return this.metadataIndex.getIdsForFilter(filter)
}
/**
* Get available values for a field
*/
async getFilterValues(field: string): Promise<any[]> {
if (!this.metadataIndex) return []
return this.metadataIndex.getFilterValues(field)
}
/**
* Get all indexed fields
*/
async getFilterFields(): Promise<string[]> {
if (!this.metadataIndex) return []
return this.metadataIndex.getFilterFields()
}
/**
* Get index statistics
*/
async getStats() {
if (!this.metadataIndex) {
return {
enabled: false,
totalEntries: 0,
fieldsIndexed: [],
memoryUsage: 0
}
}
const stats = await this.metadataIndex.getStats()
return {
enabled: true,
...stats
}
}
/**
* Rebuild the index from storage
*/
async rebuild(): Promise<void> {
if (!this.metadataIndex) {
throw new Error('Index augmentation is not initialized')
}
this.log('Rebuilding metadata index...')
await this.metadataIndex.rebuild()
const stats = await this.metadataIndex.getStats()
this.log(`Index rebuilt: ${stats.totalEntries} entries, ${stats.fieldsIndexed.length} fields`)
}
/**
* Flush index to storage
*/
async flush(): Promise<void> {
if (this.metadataIndex) {
await this.metadataIndex.flush()
this.log('Index flushed to storage', 'info')
}
}
/**
* Add entry to index (public method for direct access)
*/
async addToIndex(id: string, metadata: Record<string, any>): Promise<void> {
if (!this.metadataIndex) return
await this.metadataIndex.addToIndex(id, metadata)
}
/**
* Remove entry from index (public method for direct access)
*/
async removeFromIndex(id: string, metadata: Record<string, any>): Promise<void> {
if (!this.metadataIndex) return
await this.metadataIndex.removeFromIndex(id, metadata)
}
/**
* Get the underlying MetadataIndexManager instance
*/
getMetadataIndex() {
return this.metadataIndex
}
}
/**
* Factory function for zero-config index augmentation
*/
export function createIndexAugmentation(config?: IndexConfig): IndexAugmentation {
return new IndexAugmentation(config)
}

246
src/augmentations/intelligentImport/FormatHandlerRegistry.ts
View file

@ -1,246 +0,0 @@
/**
* Format Handler Registry
*
* Central registry for format handlers with:
* - MIME type-based routing
* - Lazy loading support
* - Pluggable handler registration
* - Handler lifecycle management
*
* NO MOCKS - Production implementation
*/
import { FormatHandler, HandlerRegistry as IHandlerRegistry } from './types.js'
import { mimeDetector } from '../../vfs/MimeTypeDetector.js'
export interface HandlerRegistration {
/** Handler name (e.g., 'csv', 'image', 'pdf') */
name: string
/** MIME types this handler supports */
mimeTypes: string[]
/** File extensions (fallback if MIME detection fails) */
extensions: string[]
/** Lazy loader function */
loader: () => Promise<FormatHandler>
/** Loaded handler instance (lazy-loaded) */
instance?: FormatHandler
}
/**
* FormatHandlerRegistry - Central handler management
*
* Implements the HandlerRegistry interface with:
* - MIME type-based routing
* - Lazy loading for performance
* - Multiple handlers per MIME type
* - Priority-based selection
*/
export class FormatHandlerRegistry implements IHandlerRegistry {
// Interface compatibility
handlers: Map<string, () => Promise<FormatHandler>> = new Map()
loaded: Map<string, FormatHandler> = new Map()
// Enhanced registry
private registrations: Map<string, HandlerRegistration> = new Map()
private mimeTypeIndex: Map<string, string[]> = new Map() // MIME type → handler names
private extensionIndex: Map<string, string[]> = new Map() // Extension → handler names
/**
* Register a format handler
*
* @param registration Handler registration details
*/
registerHandler(registration: HandlerRegistration): void {
const { name, mimeTypes, extensions, loader } = registration
// Store registration
this.registrations.set(name, registration)
// Index by MIME types
for (const mimeType of mimeTypes) {
const handlers = this.mimeTypeIndex.get(mimeType) || []
handlers.push(name)
this.mimeTypeIndex.set(mimeType, handlers)
}
// Index by extensions
for (const ext of extensions) {
const normalized = ext.toLowerCase().replace(/^\./, '')
const handlers = this.extensionIndex.get(normalized) || []
handlers.push(name)
this.extensionIndex.set(normalized, handlers)
}
// Interface compatibility
this.handlers.set(name, loader)
}
/**
* Register a handler (interface compatibility)
*/
register(extensions: string[], loader: () => Promise<FormatHandler>): void {
const name = extensions[0].replace(/^\./, '')
// Auto-detect MIME types from extensions for better routing
const mimeTypes: string[] = []
for (const ext of extensions) {
const mimeType = mimeDetector.detectMimeType(`file${ext}`)
if (mimeType && mimeType !== 'application/octet-stream' && !mimeTypes.includes(mimeType)) {
mimeTypes.push(mimeType)
}
}
this.registerHandler({
name,
mimeTypes,
extensions,
loader
})
}
/**
* Get handler by filename or extension
*
* Uses MIME detection first, falls back to extension matching
*
* @param filenameOrExt Filename or extension
* @returns Handler instance or null
*/
async getHandler(filenameOrExt: string): Promise<FormatHandler | null> {
// Try MIME type detection first
const mimeType = mimeDetector.detectMimeType(filenameOrExt)
const byMime = await this.getHandlerByMimeType(mimeType)
if (byMime) return byMime
// Fallback to extension matching
const ext = this.extractExtension(filenameOrExt)
if (ext) {
return this.getHandlerByExtension(ext)
}
return null
}
/**
* Get handler by MIME type
*
* @param mimeType MIME type string
* @returns Handler instance or null
*/
async getHandlerByMimeType(mimeType: string): Promise<FormatHandler | null> {
const handlerNames = this.mimeTypeIndex.get(mimeType)
if (!handlerNames || handlerNames.length === 0) {
return null
}
// Return first matching handler (could add priority later)
return this.loadHandler(handlerNames[0])
}
/**
* Get handler by file extension
*
* @param ext File extension (with or without dot)
* @returns Handler instance or null
*/
async getHandlerByExtension(ext: string): Promise<FormatHandler | null> {
const normalized = ext.toLowerCase().replace(/^\./, '')
const handlerNames = this.extensionIndex.get(normalized)
if (!handlerNames || handlerNames.length === 0) {
return null
}
// Return first matching handler
return this.loadHandler(handlerNames[0])
}
/**
* Get handler by name
*
* @param name Handler name
* @returns Handler instance or null
*/
async getHandlerByName(name: string): Promise<FormatHandler | null> {
return this.loadHandler(name)
}
/**
* Load handler (lazy loading)
*
* @param name Handler name
* @returns Loaded handler instance
*/
private async loadHandler(name: string): Promise<FormatHandler | null> {
const registration = this.registrations.get(name)
if (!registration) return null
// Return cached instance if available
if (registration.instance) {
return registration.instance
}
// Load handler
try {
const handler = await registration.loader()
registration.instance = handler
// Interface compatibility
this.loaded.set(name, handler)
return handler
} catch (error) {
console.error(`Failed to load handler ${name}:`, error)
return null
}
}
/**
* Get all registered handler names
*/
getRegisteredHandlers(): string[] {
return Array.from(this.registrations.keys())
}
/**
* Get handlers for a MIME type
*/
getHandlersForMimeType(mimeType: string): string[] {
return this.mimeTypeIndex.get(mimeType) || []
}
/**
* Check if handler is registered
*/
hasHandler(name: string): boolean {
return this.registrations.has(name)
}
/**
* Clear all handlers (for testing)
*/
clear(): void {
this.registrations.clear()
this.mimeTypeIndex.clear()
this.extensionIndex.clear()
this.handlers.clear()
this.loaded.clear()
}
/**
* Extract file extension from filename
*/
private extractExtension(filename: string): string | null {
const lastDot = filename.lastIndexOf('.')
if (lastDot === -1 || lastDot === 0) return null
return filename.substring(lastDot + 1).toLowerCase()
}
}
/**
* Global handler registry singleton
*/
export const globalHandlerRegistry = new FormatHandlerRegistry()

227
src/augmentations/intelligentImport/IntelligentImportAugmentation.ts
View file

@ -1,227 +0,0 @@
/**
* Intelligent Import Augmentation
*
* Automatically detects and processes CSV, Excel, and PDF files with:
* - Format detection and routing
* - Lazy-loaded handlers
* - Intelligent entity and relationship extraction
* - Integration with NeuralImport augmentation
*/
import { BaseAugmentation, AugmentationContext } from '../brainyAugmentation.js'
import { FormatHandler, IntelligentImportConfig, ProcessedData } from './types.js'
import { CSVHandler } from './handlers/csvHandler.js'
import { ExcelHandler } from './handlers/excelHandler.js'
import { PDFHandler } from './handlers/pdfHandler.js'
import { ImageHandler } from './handlers/imageHandler.js'
export class IntelligentImportAugmentation extends BaseAugmentation {
readonly name = 'intelligent-import'
readonly timing = 'before' as const
readonly metadata = {
reads: '*' as '*',
writes: ['_intelligentImport', '_processedFormat', '_extractedData'] as string[]
}
readonly operations = ['import', 'importFile', 'importFromFile', 'importFromURL', 'all'] as any[]
readonly priority = 75 // Before NeuralImport (80), after validation
protected config: IntelligentImportConfig
private handlers: Map<string, FormatHandler> = new Map()
private initialized = false
constructor(config: Partial<IntelligentImportConfig> = {}) {
super(config)
this.config = {
enableCSV: true,
enableExcel: true,
enablePDF: true,
enableImage: true, // Image handler enabled by default
maxFileSize: 100 * 1024 * 1024, // 100MB default
enableCache: true,
cacheTTL: 24 * 60 * 60 * 1000, // 24 hours
...config
}
}
protected async onInitialize(): Promise<void> {
// Initialize handlers based on config
if (this.config.enableCSV) {
this.handlers.set('csv', new CSVHandler())
}
if (this.config.enableExcel) {
this.handlers.set('excel', new ExcelHandler())
}
if (this.config.enablePDF) {
this.handlers.set('pdf', new PDFHandler())
}
if (this.config.enableImage) {
this.handlers.set('image', new ImageHandler())
}
this.initialized = true
this.log(`Initialized with ${this.handlers.size} format handlers (CSV: ${this.config.enableCSV}, Excel: ${this.config.enableExcel}, PDF: ${this.config.enablePDF}, Image: ${this.config.enableImage})`)
}
async execute<T = any>(
operation: string,
params: any,
next: () => Promise<T>
): Promise<T> {
// Only process import operations
if (!this.shouldProcess(operation, params)) {
return next()
}
try {
// Extract file data from params
const fileData = this.extractFileData(params)
if (!fileData) {
return next()
}
// Check file size limit
if (this.config.maxFileSize && fileData.data.length > this.config.maxFileSize) {
this.log(`File too large (${fileData.data.length} bytes), skipping intelligent import`, 'warn')
return next()
}
// Detect format and get appropriate handler
const handler = this.detectHandler(fileData.data, fileData.filename)
if (!handler) {
// Not a supported format, pass through
return next()
}
this.log(`Processing ${fileData.filename || 'file'} with ${handler.format} handler`)
// Process the file
const processed = await handler.process(fileData.data, {
filename: fileData.filename,
ext: fileData.ext,
...this.config.csvDefaults,
...this.config.excelDefaults,
...this.config.pdfDefaults,
...this.config.imageDefaults,
...params.options
})
// Enrich params with processed data
params._intelligentImport = true
params._processedFormat = processed.format
params._extractedData = processed.data
params._metadata = {
...params._metadata,
intelligentImport: processed.metadata
}
// If this is an import operation, transform params to include the structured data
if (processed.data.length > 0) {
// Store processed data for the neural import augmentation to use
params.data = processed.data
params.metadata = params._metadata
}
this.log(`Extracted ${processed.data.length} items from ${processed.format} file`)
return next()
} catch (error) {
this.log(`Intelligent import processing failed: ${error instanceof Error ? error.message : String(error)}`, 'warn')
// Fall through to normal import on error
return next()
}
}
/**
* Check if we should process this operation
*/
private shouldProcess(operation: string, params: any): boolean {
// Only process if we have handlers initialized
if (!this.initialized || this.handlers.size === 0) {
return false
}
// Check operation type
const validOps = ['import', 'importFile', 'importFromFile', 'importFromURL']
if (!validOps.some(op => operation.includes(op))) {
return false
}
// Must have some data
if (!params || (!params.source && !params.data && !params.filePath && !params.url)) {
return false
}
return true
}
/**
* Extract file data from various param formats
*/
private extractFileData(params: any): { data: Buffer, filename?: string, ext?: string } | null {
// From source parameter
if (params.source) {
if (Buffer.isBuffer(params.source)) {
return { data: params.source, filename: params.filename }
}
if (typeof params.source === 'string') {
return { data: Buffer.from(params.source), filename: params.filename }
}
}
// From data parameter
if (params.data) {
if (Buffer.isBuffer(params.data)) {
return { data: params.data, filename: params.filename }
}
if (typeof params.data === 'string') {
return { data: Buffer.from(params.data), filename: params.filename }
}
}
// From file path (would need to read - but that should be handled by UniversalImportAPI)
if (params.filePath && typeof params.filePath === 'string') {
const ext = params.filePath.split('.').pop()
return null // File reading handled elsewhere
}
return null
}
/**
* Detect which handler can process this file
*/
private detectHandler(data: Buffer, filename?: string): FormatHandler | null {
// Try each handler's canHandle method
for (const handler of this.handlers.values()) {
if (handler.canHandle(data) || (filename && handler.canHandle({ filename }))) {
return handler
}
}
return null
}
/**
* Get handler by format name
*/
getHandler(format: string): FormatHandler | undefined {
return this.handlers.get(format.toLowerCase())
}
/**
* Get all registered handlers
*/
getHandlers(): FormatHandler[] {
return Array.from(this.handlers.values())
}
/**
* Get supported formats
*/
getSupportedFormats(): string[] {
return Array.from(this.handlers.keys())
}
}

350
src/augmentations/intelligentImport/handlers/imageHandler.ts
View file

@ -1,350 +0,0 @@
/**
* Image Import Handler
*
* Handles image files with:
* - EXIF metadata extraction (camera, GPS, timestamps) via exifr
* - Image metadata (dimensions, format) via probe-image-size
* - Support for JPEG, PNG, WebP, GIF, TIFF, BMP, SVG
*
* NO NATIVE DEPENDENCIES - Pure JavaScript implementation
* Replaces Sharp with lightweight pure-JS alternatives
*/
import { BaseFormatHandler } from './base.js'
import type { FormatHandlerOptions, ProcessedData } from '../types.js'
import exifr from 'exifr'
import probeImageSize from 'probe-image-size'
import { Readable } from 'stream'
export interface ImageMetadata {
/** Image dimensions */
width: number
height: number
/** Image format (jpeg, png, webp, etc.) */
format: string
/** File size in bytes */
size: number
/** Orientation (EXIF) */
orientation?: number
/** MIME type */
mimeType?: string
}
export interface EXIFData {
/** Camera make (e.g., "Canon", "Nikon") */
make?: string
/** Camera model */
model?: string
/** Lens information */
lens?: string
/** Date/time original */
dateTimeOriginal?: Date
/** GPS latitude */
latitude?: number
/** GPS longitude */
longitude?: number
/** GPS altitude in meters */
altitude?: number
/** Exposure time (e.g., "1/250") */
exposureTime?: number
/** F-number (e.g., 2.8) */
fNumber?: number
/** ISO speed */
iso?: number
/** Focal length in mm */
focalLength?: number
/** Flash fired */
flash?: boolean
/** Copyright */
copyright?: string
/** Artist/photographer */
artist?: string
/** Image description */
imageDescription?: string
/** Software used */
software?: string
}
export interface ImageHandlerOptions extends FormatHandlerOptions {
/** Extract EXIF data (default: true) */
extractEXIF?: boolean
}
/**
* ImageImportHandler
*
* Processes image files and extracts rich metadata including EXIF data.
* Enables developers to import images into the knowledge graph with
* full metadata extraction.
*
* Pure JavaScript implementation (no native dependencies)
*/
export class ImageHandler extends BaseFormatHandler {
readonly format = 'image'
/**
* Check if this handler can process the given data
*/
canHandle(data: Buffer | string | { filename?: string; ext?: string }): boolean {
// Check by filename/extension
if (typeof data === 'object' && 'filename' in data) {
const mimeType = this.getMimeType(data)
return this.mimeTypeMatches(mimeType, ['image/*'])
}
// Check by extension
if (typeof data === 'object' && 'ext' in data && data.ext) {
return this.isImageExtension(data.ext)
}
// Check by data (Buffer magic bytes)
if (Buffer.isBuffer(data)) {
return this.detectImageFormat(data) !== null
}
return false
}
/**
* Process image file
*/
async process(
data: Buffer | string,
options: ImageHandlerOptions = {}
): Promise<ProcessedData> {
const startTime = Date.now()
try {
// Convert to Buffer
const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, 'base64')
// Extract image metadata
const metadata = await this.extractMetadata(buffer)
// Extract EXIF data (default: enabled)
let exifData: EXIFData | undefined
if (options.extractEXIF !== false) {
exifData = await this.extractEXIF(buffer)
}
// Calculate processing time
const processingTime = Date.now() - startTime
// Generate descriptive name
const imageName = options.filename
? options.filename.replace(/\.[^/.]+$/, '') // Remove extension
: `${metadata.format.toUpperCase()} Image ${metadata.width}x${metadata.height}`
// Return structured data
return {
format: 'image',
data: [
{
name: imageName,
type: 'media',
metadata: {
...metadata,
subtype: 'image',
exif: exifData
}
}
],
metadata: {
rowCount: 1,
fields: ['type', 'metadata'],
processingTime,
imageMetadata: metadata,
exifData
},
filename: options.filename
}
} catch (error) {
throw new Error(
`Image processing failed: ${error instanceof Error ? error.message : String(error)}`
)
}
}
/**
* Extract image metadata using probe-image-size (pure JS)
*/
private async extractMetadata(buffer: Buffer): Promise<ImageMetadata> {
try {
// Convert Buffer to Stream for probe-image-size
const stream = Readable.from(buffer)
const result = await probeImageSize(stream)
return {
width: result.width,
height: result.height,
format: result.type, // 'jpeg', 'png', 'webp', etc.
size: buffer.length,
mimeType: result.mime,
orientation: result.orientation
}
} catch (error) {
// Fallback: Try to detect format from magic bytes
const detectedFormat = this.detectImageFormat(buffer)
return {
width: 0,
height: 0,
format: detectedFormat || 'unknown',
size: buffer.length,
mimeType: detectedFormat ? `image/${detectedFormat}` : undefined
}
}
}
/**
* Extract EXIF data using exifr (pure JS)
*/
private async extractEXIF(buffer: Buffer): Promise<EXIFData | undefined> {
try {
const exif = await exifr.parse(buffer, {
pick: [
'Make',
'Model',
'LensModel',
'DateTimeOriginal',
'latitude',
'longitude',
'GPSAltitude',
'ExposureTime',
'FNumber',
'ISO',
'FocalLength',
'Flash',
'Copyright',
'Artist',
'ImageDescription',
'Software'
]
})
if (!exif) return undefined
return {
make: exif.Make,
model: exif.Model,
lens: exif.LensModel,
dateTimeOriginal: exif.DateTimeOriginal,
latitude: exif.latitude,
longitude: exif.longitude,
altitude: exif.GPSAltitude,
exposureTime: exif.ExposureTime,
fNumber: exif.FNumber,
iso: exif.ISO,
focalLength: exif.FocalLength,
flash: exif.Flash !== undefined ? Boolean(exif.Flash & 1) : undefined,
copyright: exif.Copyright,
artist: exif.Artist,
imageDescription: exif.ImageDescription,
software: exif.Software
}
} catch (error) {
// EXIF extraction can fail for non-JPEG images or corrupt data
return undefined
}
}
/**
* Detect image format from magic bytes
*/
private detectImageFormat(buffer: Buffer): string | null {
if (buffer.length < 4) return null
// JPEG: FF D8 FF
if (buffer[0] === 0xff && buffer[1] === 0xd8 && buffer[2] === 0xff) {
return 'jpeg'
}
// PNG: 89 50 4E 47
if (
buffer[0] === 0x89 &&
buffer[1] === 0x50 &&
buffer[2] === 0x4e &&
buffer[3] === 0x47
) {
return 'png'
}
// GIF: 47 49 46
if (buffer[0] === 0x47 && buffer[1] === 0x49 && buffer[2] === 0x46) {
return 'gif'
}
// WebP: 52 49 46 46 ... 57 45 42 50
if (
buffer[0] === 0x52 &&
buffer[1] === 0x49 &&
buffer[2] === 0x46 &&
buffer[3] === 0x46 &&
buffer.length >= 12 &&
buffer[8] === 0x57 &&
buffer[9] === 0x45 &&
buffer[10] === 0x42 &&
buffer[11] === 0x50
) {
return 'webp'
}
// TIFF: 49 49 2A 00 (little-endian) or 4D 4D 00 2A (big-endian)
if (
(buffer[0] === 0x49 && buffer[1] === 0x49 && buffer[2] === 0x2a && buffer[3] === 0x00) ||
(buffer[0] === 0x4d && buffer[1] === 0x4d && buffer[2] === 0x00 && buffer[3] === 0x2a)
) {
return 'tiff'
}
// BMP: 42 4D
if (buffer[0] === 0x42 && buffer[1] === 0x4d) {
return 'bmp'
}
return null
}
/**
* Detect if extension is an image format
*/
private isImageExtension(ext: string): boolean {
const imageExts = [
'.jpg',
'.jpeg',
'.png',
'.gif',
'.webp',
'.tiff',
'.tif',
'.bmp',
'.svg',
'.heic',
'.heif',
'.avif'
]
const normalized = ext.toLowerCase()
const withDot = normalized.startsWith('.') ? normalized : `.${normalized}`
return imageExts.includes(withDot)
}
}

33
src/augmentations/intelligentImport/index.ts
View file

@ -1,33 +0,0 @@
/**
* Intelligent Import Module
* Exports main augmentation and types
*/
export { IntelligentImportAugmentation } from './IntelligentImportAugmentation.js'
export type {
FormatHandler,
FormatHandlerOptions,
ProcessedData,
IntelligentImportConfig,
HandlerRegistry
} from './types.js'
// Format Handlers
export { CSVHandler } from './handlers/csvHandler.js'
export { ExcelHandler } from './handlers/excelHandler.js'
export { PDFHandler } from './handlers/pdfHandler.js'
export { ImageHandler } from './handlers/imageHandler.js'
// Format Handler Registry
export {
FormatHandlerRegistry,
globalHandlerRegistry
} from './FormatHandlerRegistry.js'
export type { HandlerRegistration } from './FormatHandlerRegistry.js'
// Image Handler Types
export type {
ImageMetadata,
EXIFData,
ImageHandlerOptions
} from './handlers/imageHandler.js'

875
src/augmentations/intelligentVerbScoringAugmentation.ts
View file

@ -1,875 +0,0 @@
/**
* Intelligent Verb Scoring Augmentation
*
* Enhances relationship quality through intelligent semantic scoring
* Provides context-aware relationship weights based on:
* - Semantic proximity of connected entities
* - Frequency-based amplification
* - Temporal decay modeling
* - Adaptive learning from usage patterns
*
* Critical for enterprise knowledge graphs with millions of relationships
*/
import { BaseAugmentation, AugmentationContext } from './brainyAugmentation.js'
interface VerbScoringConfig {
enabled?: boolean
// Semantic Analysis
enableSemanticScoring?: boolean // Use entity embeddings for scoring
semanticThreshold?: number // Minimum semantic similarity
semanticWeight?: number // Weight of semantic component
// Frequency Analysis
enableFrequencyAmplification?: boolean // Amplify frequently used relationships
frequencyDecay?: number // How quickly frequency importance decays
maxFrequencyBoost?: number // Maximum boost from frequency
// Temporal Analysis
enableTemporalDecay?: boolean // Apply time-based decay
temporalDecayRate?: number // Decay rate per day (0-1)
temporalWindow?: number // Time window for relevance (days)
// Learning & Adaptation
enableAdaptiveLearning?: boolean // Learn from usage patterns
learningRate?: number // How quickly to adapt (0-1)
confidenceThreshold?: number // Minimum confidence for relationships
// Weight Management
minWeight?: number // Minimum relationship weight
maxWeight?: number // Maximum relationship weight
baseWeight?: number // Default weight for new relationships
}
interface RelationshipMetrics {
count: number // How many times this relationship was created
totalWeight: number // Sum of all weights
averageWeight: number // Average weight
lastUpdated: number // Last time this relationship was scored
semanticScore: number // Semantic similarity score
frequencyScore: number // Frequency-based score
temporalScore: number // Time-based relevance score
confidenceScore: number // Overall confidence
}
interface ScoringMetrics {
relationshipsScored: number
averageSemanticScore: number
averageFrequencyScore: number
averageTemporalScore: number
averageConfidenceScore: number
adaptiveAdjustments: number
computationTimeMs: number
minScore?: number
maxScore?: number
averageScore?: number
highConfidenceCount?: number
}
export class IntelligentVerbScoringAugmentation extends BaseAugmentation {
name = 'IntelligentVerbScoring'
timing = 'around' as const
readonly metadata = {
reads: ['type', 'verb', 'source', 'target'] as string[],
writes: ['weight', 'confidence', 'intelligentScoring'] as string[]
} // Adds scoring metadata to verbs
operations = ['addVerb', 'relate'] as ('addVerb' | 'relate')[]
priority = 10 // Enhancement feature - runs after core operations
// Augmentation metadata
readonly category = 'core' as const
readonly description = 'AI-powered intelligent scoring for relationship strength analysis'
protected config: Required<VerbScoringConfig>
private relationshipStats: Map<string, RelationshipMetrics> = new Map()
private metrics: ScoringMetrics = {
relationshipsScored: 0,
averageSemanticScore: 0,
averageFrequencyScore: 0,
averageTemporalScore: 0,
averageConfidenceScore: 0,
adaptiveAdjustments: 0,
computationTimeMs: 0
}
private scoringInstance: any // Will hold IntelligentVerbScoring instance
constructor(config: VerbScoringConfig = {}) {
super()
this.config = {
enabled: config.enabled ?? true, // Smart by default!
// Semantic Analysis
enableSemanticScoring: config.enableSemanticScoring ?? true,
semanticThreshold: config.semanticThreshold ?? 0.3,
semanticWeight: config.semanticWeight ?? 0.4,
// Frequency Analysis
enableFrequencyAmplification: config.enableFrequencyAmplification ?? true,
frequencyDecay: config.frequencyDecay ?? 0.95, // 5% decay per occurrence
maxFrequencyBoost: config.maxFrequencyBoost ?? 2.0,
// Temporal Analysis
enableTemporalDecay: config.enableTemporalDecay ?? true,
temporalDecayRate: config.temporalDecayRate ?? 0.01, // 1% per day
temporalWindow: config.temporalWindow ?? 365, // 1 year
// Learning & Adaptation
enableAdaptiveLearning: config.enableAdaptiveLearning ?? true,
learningRate: config.learningRate ?? 0.1,
confidenceThreshold: config.confidenceThreshold ?? 0.3,
// Weight Management
minWeight: config.minWeight ?? 0.1,
maxWeight: config.maxWeight ?? 1.0,
baseWeight: config.baseWeight ?? 0.5
}
// Set enabled property based on config
this.enabled = this.config.enabled
}
protected async onInitialize(): Promise<void> {
if (this.config.enabled) {
this.log('Intelligent verb scoring initialized for enhanced relationship quality')
} else {
this.log('Intelligent verb scoring disabled')
}
}
/**
* Get this augmentation instance for API compatibility
* Used by Brainy to access scoring methods
*/
getScoring(): IntelligentVerbScoringAugmentation {
return this
}
shouldExecute(operation: string, params: any): boolean {
// For addVerb, params are passed as array: [sourceId, targetId, verbType, metadata, weight]
if (operation === 'addVerb' && this.config.enabled) {
return Array.isArray(params) && params.length >= 3
}
// For relate method, params might be an object
if (operation === 'relate' && this.config.enabled) {
return params.sourceId && params.targetId && params.relationType
}
return false
}
async execute<T = any>(
operation: string,
params: any,
next: () => Promise<T>
): Promise<T> {
if (!this.shouldExecute(operation, params)) {
return next()
}
const startTime = Date.now()
try {
let sourceId: string, targetId: string, relationType: string, metadata: any
let scoringResult: { weight: number; confidence: number; reasoning: string[] } | null = null
// Extract parameters based on operation type
if (operation === 'addVerb' && Array.isArray(params)) {
// addVerb params: [sourceId, targetId, verbType, metadata, weight]
[sourceId, targetId, relationType, metadata] = params
} else if (operation === 'relate') {
// relate params might be an object
sourceId = params.sourceId
targetId = params.targetId
relationType = params.relationType
metadata = params.metadata
} else {
return next()
}
// Skip if weight is already provided explicitly
if (Array.isArray(params) && params[4] !== undefined && params[4] !== null) {
return next()
}
// Get the nouns to compute scoring
const sourceNoun = await this.context?.brain.get(sourceId)
const targetNoun = await this.context?.brain.get(targetId)
// Compute intelligent scores with reasoning
scoringResult = await this.computeVerbScores(
sourceNoun,
targetNoun,
relationType
)
// For addVerb, modify the params array
if (operation === 'addVerb' && Array.isArray(params)) {
// Set the weight parameter (index 4)
params[4] = scoringResult.weight
// Enhance metadata with scoring info
params[3] = {
...params[3],
intelligentScoring: {
weight: scoringResult.weight,
confidence: scoringResult.confidence,
reasoning: scoringResult.reasoning,
scoringMethod: this.getScoringMethodsUsed(),
computedAt: Date.now()
}
}
}
// Execute with enhanced parameters
const result = await next()
// Learn from this relationship
if (this.config.enableAdaptiveLearning && scoringResult) {
await this.updateRelationshipLearning(
sourceId,
targetId,
relationType,
scoringResult.weight
)
}
// Update metrics
const computationTime = Date.now() - startTime
if (scoringResult) {
this.updateMetrics(scoringResult.weight, computationTime)
}
return result
} catch (error) {
this.log(`Intelligent verb scoring error: ${error}`, 'error')
// Fallback to original parameters
return next()
}
}
private async calculateIntelligentWeight(
sourceId: string,
targetId: string,
relationType: string,
metadata?: any
): Promise<number> {
let finalWeight = this.config.baseWeight
let scoreComponents: any = {}
// 1. Semantic Proximity Score
if (this.config.enableSemanticScoring) {
const semanticScore = await this.calculateSemanticScore(sourceId, targetId)
scoreComponents.semantic = semanticScore
finalWeight = finalWeight * (1 + semanticScore * this.config.semanticWeight)
}
// 2. Frequency Amplification Score
if (this.config.enableFrequencyAmplification) {
const frequencyScore = this.calculateFrequencyScore(sourceId, targetId, relationType)
scoreComponents.frequency = frequencyScore
finalWeight = finalWeight * (1 + frequencyScore)
}
// 3. Temporal Relevance Score
if (this.config.enableTemporalDecay) {
const temporalScore = this.calculateTemporalScore(sourceId, targetId, relationType)
scoreComponents.temporal = temporalScore
finalWeight = finalWeight * temporalScore
}
// 4. Context Awareness (from metadata)
const contextScore = this.calculateContextScore(metadata)
scoreComponents.context = contextScore
finalWeight = finalWeight * (1 + contextScore * 0.2)
// 5. Apply constraints
finalWeight = Math.max(this.config.minWeight,
Math.min(this.config.maxWeight, finalWeight))
// Store detailed scoring for analysis
this.storeDetailedScoring(sourceId, targetId, relationType, {
finalWeight,
components: scoreComponents,
timestamp: Date.now()
})
return finalWeight
}
private async calculateSemanticScore(sourceId: string, targetId: string): Promise<number> {
try {
// Get embeddings for both entities
const sourceNoun = await this.context?.brain.get(sourceId)
const targetNoun = await this.context?.brain.get(targetId)
if (!sourceNoun?.vector || !targetNoun?.vector) {
return 0
}
// Get noun types using neural detection (taxonomy-based)
const sourceType = await this.detectNounType(sourceNoun.vector)
const targetType = await this.detectNounType(targetNoun.vector)
// Calculate direct similarity
const directSimilarity = this.calculateCosineSimilarity(sourceNoun.vector, targetNoun.vector)
// Calculate taxonomy-based similarity boost
const taxonomyBoost = await this.calculateTaxonomyBoost(sourceType, targetType)
// Blend direct similarity with taxonomy guidance
// Taxonomy provides consistency while preserving flexibility
const semanticScore = directSimilarity * 0.7 + taxonomyBoost * 0.3
return Math.min(1, Math.max(0, semanticScore))
} catch (error) {
return 0
}
}
/**
* Detect noun type using neural taxonomy matching
*/
private async detectNounType(vector: number[]): Promise<string> {
// Use real neural detection from brain's type detector
if (!this.context?.brain) return 'unknown'
try {
// Access the brain's neural type detection system
const brain = this.context.brain as any
// Use the actual neural type detection if available
if (brain.neuralDetector?.detectType) {
const detectedType = await brain.neuralDetector.detectType(vector)
return detectedType || 'unknown'
}
// Fallback to pattern-based detection using actual embeddings
const patternAnalyzer = brain.patternAnalyzer || brain.neural?.patternAnalyzer
if (patternAnalyzer?.analyzeVector) {
const analysis = await patternAnalyzer.analyzeVector(vector)
return analysis.type || 'unknown'
}
// Use statistical analysis of vector characteristics
const stats = this.analyzeVectorStatistics(vector)
return this.inferTypeFromStatistics(stats)
} catch (error) {
this.log(`Type detection failed: ${(error as Error).message}`, 'warn')
return 'unknown'
}
}
/**
* Analyze vector statistics for type inference
*/
private analyzeVectorStatistics(vector: number[]): {
mean: number
variance: number
sparsity: number
entropy: number
magnitude: number
} {
const n = vector.length
if (n === 0) return { mean: 0, variance: 0, sparsity: 0, entropy: 0, magnitude: 0 }
// Calculate mean
const mean = vector.reduce((sum, val) => sum + val, 0) / n
// Calculate variance
const variance = vector.reduce((sum, val) => sum + Math.pow(val - mean, 2), 0) / n
// Calculate sparsity (percentage of non-zero elements)
const nonZeroCount = vector.filter(val => Math.abs(val) > 0.001).length
const sparsity = 1 - (nonZeroCount / n)
// Calculate entropy (information content)
const absSum = vector.reduce((sum, val) => sum + Math.abs(val), 0) || 1
const probs = vector.map(val => Math.abs(val) / absSum)
const entropy = -probs.reduce((sum, p) => {
return p > 0 ? sum + p * Math.log2(p) : sum
}, 0)
// Calculate magnitude
const magnitude = Math.sqrt(vector.reduce((sum, val) => sum + val * val, 0))
return { mean, variance, sparsity, entropy, magnitude }
}
/**
* Infer entity type from vector statistics using neural patterns
*/
private inferTypeFromStatistics(stats: {
mean: number
variance: number
sparsity: number
entropy: number
magnitude: number
}): string {
// Neural pattern recognition based on empirical analysis
// These thresholds are derived from analyzing actual embeddings
// High entropy and low sparsity often indicate abstract concepts
if (stats.entropy > 4.5 && stats.sparsity < 0.3) {
return 'concept'
}
// Moderate entropy with high magnitude indicates concrete entities
if (stats.magnitude > 8 && stats.entropy > 3 && stats.entropy < 4.5) {
return 'entity'
}
// High sparsity with focused magnitude indicates specific objects
if (stats.sparsity > 0.6 && stats.magnitude > 3) {
return 'object'
}
// Documents tend to have balanced statistics
if (stats.entropy > 3.5 && stats.entropy < 4.2 && stats.variance > 0.1) {
return 'document'
}
// Person entities have characteristic patterns
if (stats.magnitude > 5 && stats.magnitude < 10 && stats.variance > 0.15) {
return 'person'
}
// Tools and functions have lower entropy
if (stats.entropy < 3 && stats.magnitude > 4) {
return 'tool'
}
// Default to generic item for unclear patterns
return 'item'
}
/**
* Calculate taxonomy-based similarity boost
*/
private async calculateTaxonomyBoost(sourceType: string, targetType: string): Promise<number> {
// Define valid relationship patterns in taxonomy
const validPatterns: Record<string, Record<string, number>> = {
'person': { 'concept': 0.9, 'skill': 0.85, 'organization': 0.8, 'person': 0.7 },
'concept': { 'concept': 0.9, 'example': 0.85, 'application': 0.8 },
'entity': { 'entity': 0.8, 'property': 0.85, 'action': 0.75 },
'object': { 'object': 0.7, 'property': 0.8, 'location': 0.75 },
'document': { 'topic': 0.9, 'author': 0.85, 'document': 0.7 },
'tool': { 'output': 0.9, 'input': 0.85, 'user': 0.8 },
'unknown': { 'unknown': 0.5 } // Fallback
}
// Get boost from taxonomy patterns
const patterns = validPatterns[sourceType] || validPatterns['unknown']
const boost = patterns[targetType] || 0.3 // Low score for unrecognized patterns
return boost
}
private calculateCosineSimilarity(vectorA: number[], vectorB: number[]): number {
if (vectorA.length !== vectorB.length) return 0
let dotProduct = 0
let normA = 0
let normB = 0
for (let i = 0; i < vectorA.length; i++) {
dotProduct += vectorA[i] * vectorB[i]
normA += vectorA[i] * vectorA[i]
normB += vectorB[i] * vectorB[i]
}
const magnitude = Math.sqrt(normA) * Math.sqrt(normB)
return magnitude ? dotProduct / magnitude : 0
}
private calculateFrequencyScore(sourceId: string, targetId: string, relationType: string): number {
const relationshipKey = `${sourceId}:${relationType}:${targetId}`
const stats = this.relationshipStats.get(relationshipKey)
if (!stats || stats.count <= 1) return 0
// Frequency boost diminishes with each occurrence
const frequencyBoost = Math.log(stats.count) * this.config.frequencyDecay
return Math.min(this.config.maxFrequencyBoost, frequencyBoost)
}
private calculateTemporalScore(sourceId: string, targetId: string, relationType: string): number {
const relationshipKey = `${sourceId}:${relationType}:${targetId}`
const stats = this.relationshipStats.get(relationshipKey)
if (!stats) return 1.0 // New relationship - full temporal score
const daysSinceUpdate = (Date.now() - stats.lastUpdated) / (1000 * 60 * 60 * 24)
const decayFactor = Math.pow(1 - this.config.temporalDecayRate, daysSinceUpdate)
// Relationships older than temporal window get minimum score
if (daysSinceUpdate > this.config.temporalWindow) {
return this.config.minWeight / this.config.baseWeight
}
return Math.max(0.1, decayFactor)
}
private calculateContextScore(metadata?: any): number {
if (!metadata) return 0
let contextScore = 0
// Boost for explicit importance
if (metadata.importance) {
contextScore += Math.min(0.5, metadata.importance)
}
// Boost for confidence
if (metadata.confidence) {
contextScore += Math.min(0.3, metadata.confidence)
}
// Boost for source quality
if (metadata.sourceQuality) {
contextScore += Math.min(0.2, metadata.sourceQuality)
}
return contextScore
}
private async updateRelationshipLearning(
sourceId: string,
targetId: string,
relationType: string,
weight: number
): Promise<void> {
const relationshipKey = `${sourceId}:${relationType}:${targetId}`
let stats = this.relationshipStats.get(relationshipKey)
if (!stats) {
stats = {
count: 0,
totalWeight: 0,
averageWeight: this.config.baseWeight,
lastUpdated: Date.now(),
semanticScore: 0,
frequencyScore: 0,
temporalScore: 1.0,
confidenceScore: this.config.baseWeight
}
}
// Update statistics with learning rate
stats.count++
stats.totalWeight += weight
stats.averageWeight = stats.averageWeight * (1 - this.config.learningRate) +
weight * this.config.learningRate
stats.lastUpdated = Date.now()
// Update confidence based on consistency
const weightVariance = Math.abs(weight - stats.averageWeight)
const consistencyScore = 1 - Math.min(1, weightVariance)
stats.confidenceScore = stats.confidenceScore * (1 - this.config.learningRate) +
consistencyScore * this.config.learningRate
this.relationshipStats.set(relationshipKey, stats)
this.metrics.adaptiveAdjustments++
}
private getConfidenceScore(sourceId: string, targetId: string, relationType: string): number {
const relationshipKey = `${sourceId}:${relationType}:${targetId}`
const stats = this.relationshipStats.get(relationshipKey)
return stats ? stats.confidenceScore : this.config.baseWeight
}
private getScoringMethodsUsed(): string[] {
const methods = []
if (this.config.enableSemanticScoring) methods.push('semantic')
if (this.config.enableFrequencyAmplification) methods.push('frequency')
if (this.config.enableTemporalDecay) methods.push('temporal')
if (this.config.enableAdaptiveLearning) methods.push('adaptive')
return methods
}
private storeDetailedScoring(
sourceId: string,
targetId: string,
relationType: string,
scoring: any
): void {
// Store detailed scoring for analysis and debugging
// In production, this might be sent to analytics system
}
private updateMetrics(weight: number, computationTime: number): void {
this.metrics.relationshipsScored++
// Update average computation time with exponential moving average
const alpha = 0.1 // Smoothing factor
this.metrics.computationTimeMs =
alpha * computationTime + (1 - alpha) * this.metrics.computationTimeMs
// Track score distribution for analysis
this.updateScoreDistribution(weight)
}
/**
* Update score distribution statistics
*/
private updateScoreDistribution(weight: number): void {
// Track min/max scores
if (!this.metrics.minScore || weight < this.metrics.minScore) {
this.metrics.minScore = weight
}
if (!this.metrics.maxScore || weight > this.metrics.maxScore) {
this.metrics.maxScore = weight
}
// Update average score with running average
const n = this.metrics.relationshipsScored
this.metrics.averageScore = ((this.metrics.averageScore || 0) * (n - 1) + weight) / n
// Track confidence levels
if (weight > this.config.baseWeight * 1.5) {
this.metrics.highConfidenceCount = (this.metrics.highConfidenceCount || 0) + 1
}
}
/**
* Get intelligent verb scoring statistics
*/
getStats(): ScoringMetrics & {
totalRelationships: number
averageConfidence: number
highConfidenceRelationships: number
learningEfficiency: number
} {
let totalConfidence = 0
let highConfidenceCount = 0
for (const stats of this.relationshipStats.values()) {
totalConfidence += stats.confidenceScore
if (stats.confidenceScore >= this.config.confidenceThreshold * 2) {
highConfidenceCount++
}
}
const totalRelationships = this.relationshipStats.size
const averageConfidence = totalRelationships > 0 ? totalConfidence / totalRelationships : 0
const learningEfficiency = this.metrics.adaptiveAdjustments / Math.max(1, this.metrics.relationshipsScored)
return {
...this.metrics,
totalRelationships,
averageConfidence,
highConfidenceRelationships: highConfidenceCount,
learningEfficiency
}
}
/**
* Export relationship statistics for analysis
*/
exportRelationshipStats(): Array<{
relationship: string
metrics: RelationshipMetrics
}> {
return Array.from(this.relationshipStats.entries()).map(([key, metrics]) => ({
relationship: key,
metrics
}))
}
/**
* Import relationship statistics from previous sessions
*/
importRelationshipStats(stats: Array<{ relationship: string, metrics: RelationshipMetrics }>): void {
for (const { relationship, metrics } of stats) {
this.relationshipStats.set(relationship, metrics)
}
this.log(`Imported ${stats.length} relationship statistics`)
}
/**
* Get learning statistics for monitoring and debugging
* Required for Brainy.getVerbScoringStats()
*/
getLearningStats(): {
totalRelationships: number
averageConfidence: number
feedbackCount: number
topRelationships: Array<{
relationship: string
count: number
averageWeight: number
}>
} {
const relationships = Array.from(this.relationshipStats.entries())
const totalRelationships = relationships.length
const feedbackCount = relationships.reduce((sum, [, stats]) => sum + stats.count, 0)
const averageWeight = relationships.reduce((sum, [, stats]) => sum + stats.averageWeight, 0) / totalRelationships || 0
const averageConfidence = Math.min(averageWeight + 0.2, 1.0)
const topRelationships = relationships
.map(([key, stats]) => ({
relationship: key,
count: stats.count,
averageWeight: stats.averageWeight
}))
.sort((a, b) => b.count - a.count)
.slice(0, 10)
return {
totalRelationships,
averageConfidence,
feedbackCount,
topRelationships
}
}
/**
* Export learning data for backup or analysis
* Required for Brainy.exportVerbScoringLearningData()
*/
exportLearningData(): string {
const data = {
config: this.config,
stats: Array.from(this.relationshipStats.entries()).map(([key, stats]) => ({
relationship: key,
...stats
})),
exportedAt: new Date().toISOString(),
version: '1.0'
}
return JSON.stringify(data, null, 2)
}
/**
* Import learning data from backup
* Required for Brainy.importVerbScoringLearningData()
*/
importLearningData(jsonData: string): void {
try {
const data = JSON.parse(jsonData)
if (data.stats && Array.isArray(data.stats)) {
for (const stat of data.stats) {
if (stat.relationship) {
this.relationshipStats.set(stat.relationship, {
count: stat.count || 1,
totalWeight: stat.totalWeight || stat.averageWeight || 0.5,
averageWeight: stat.averageWeight || 0.5,
lastUpdated: stat.lastUpdated || Date.now(),
semanticScore: stat.semanticScore || 0.5,
frequencyScore: stat.frequencyScore || 0.5,
temporalScore: stat.temporalScore || 1.0,
confidenceScore: stat.confidenceScore || 0.5
})
}
}
}
this.log(`Imported learning data: ${this.relationshipStats.size} relationships`)
} catch (error) {
console.error('Failed to import learning data:', error)
throw new Error(`Failed to import learning data: ${error}`)
}
}
/**
* Provide feedback on a relationship's weight
* Required for Brainy.provideVerbScoringFeedback()
*/
async provideFeedback(
sourceId: string,
targetId: string,
relationType: string,
feedback: number,
feedbackType: 'correction' | 'validation' | 'enhancement' = 'correction'
): Promise<void> {
const key = `${sourceId}-${relationType}-${targetId}`
const stats = this.relationshipStats.get(key) || {
count: 0,
totalWeight: 0,
averageWeight: 0.5,
lastUpdated: Date.now(),
semanticScore: 0.5,
frequencyScore: 0.5,
temporalScore: 1.0,
confidenceScore: 0.5
}
// Update statistics based on feedback
if (feedbackType === 'correction') {
// Direct correction - heavily weight the feedback
stats.averageWeight = stats.averageWeight * 0.3 + feedback * 0.7
} else if (feedbackType === 'validation') {
// Validation - slightly adjust towards feedback
stats.averageWeight = stats.averageWeight * 0.8 + feedback * 0.2
} else {
// Enhancement - minor adjustment
stats.averageWeight = stats.averageWeight * 0.9 + feedback * 0.1
}
stats.count++
stats.totalWeight += feedback
stats.lastUpdated = Date.now()
this.relationshipStats.set(key, stats)
this.metrics.adaptiveAdjustments++
}
/**
* Compute intelligent scores for a verb relationship
* Used internally during verb creation
*/
async computeVerbScores(
sourceNoun: any,
targetNoun: any,
relationType: string
): Promise<{
weight: number
confidence: number
reasoning: string[]
}> {
const reasoning: string[] = []
let totalScore = 0
let components = 0
// Semantic scoring
if (this.config.enableSemanticScoring && sourceNoun?.vector && targetNoun?.vector) {
const similarity = this.calculateCosineSimilarity(sourceNoun.vector, targetNoun.vector)
const semanticScore = Math.max(similarity, this.config.semanticThreshold)
totalScore += semanticScore * this.config.semanticWeight
components++
reasoning.push(`Semantic similarity: ${(similarity * 100).toFixed(1)}%`)
}
// Frequency scoring
const key = `${sourceNoun?.id}-${relationType}-${targetNoun?.id}`
const stats = this.relationshipStats.get(key)
if (this.config.enableFrequencyAmplification && stats) {
const frequencyScore = Math.min(1 + (stats.count - 1) * 0.1, this.config.maxFrequencyBoost)
totalScore += frequencyScore * 0.3
components++
reasoning.push(`Frequency boost: ${frequencyScore.toFixed(2)}x`)
}
// Temporal decay scoring
if (this.config.enableTemporalDecay) {
reasoning.push(`Temporal decay applied (rate: ${this.config.temporalDecayRate})`)
}
// Calculate final weight
const weight = components > 0
? Math.min(Math.max(totalScore / components, this.config.minWeight), this.config.maxWeight)
: this.config.baseWeight
const confidence = Math.min(weight + 0.2, 1.0)
return { weight, confidence, reasoning }
}
protected async onShutdown(): Promise<void> {
const stats = this.getStats()
this.log(`Intelligent verb scoring shutdown: ${stats.relationshipsScored} relationships scored, ${Math.round(stats.averageConfidence * 100)}% avg confidence`)
}
}

207
src/augmentations/manifest.ts
View file

@ -1,207 +0,0 @@
/**
* Augmentation Manifest Types
*
* Defines the manifest structure for augmentation discovery and configuration
* Enables tools like brain-cloud to discover and configure augmentations
*/
/**
* JSON Schema type for configuration validation
*/
export interface JSONSchema {
type?: 'object' | 'array' | 'string' | 'number' | 'boolean' | 'null'
properties?: Record<string, JSONSchema>
items?: JSONSchema
required?: string[]
default?: any
description?: string
minimum?: number
maximum?: number
minLength?: number
maxLength?: number
pattern?: string
enum?: any[]
additionalProperties?: boolean | JSONSchema
}
/**
* Augmentation manifest for discovery and configuration
*/
export interface AugmentationManifest {
/**
* Unique identifier for the augmentation (e.g., 'wal', 'cache')
*/
id: string
/**
* Display name for the augmentation
*/
name: string
/**
* Semantic version (e.g., '2.0.0')
*/
version: string
/**
* Author or organization
*/
author?: string
/**
* Short description of what the augmentation does
*/
description: string
/**
* Detailed description for documentation
*/
longDescription?: string
/**
* Augmentation category for organization
*/
category: 'storage' | 'performance' | 'analytics' | 'integration' | 'internal' | 'core' | 'premium' | 'community' | 'external'
/**
* JSON Schema for configuration options
* Used to generate configuration UIs and validate configuration
*/
configSchema?: JSONSchema
/**
* Default configuration values
*/
configDefaults?: Record<string, any>
/**
* Configuration examples for documentation
*/
configExamples?: Array<{
name: string
description: string
config: Record<string, any>
}>
/**
* Minimum Brainy version required
*/
minBrainyVersion?: string
/**
* Maximum Brainy version supported
*/
maxBrainyVersion?: string
/**
* Other augmentations this one depends on
*/
dependencies?: Array<{
id: string
version?: string
optional?: boolean
}>
/**
* Keywords for search and discovery
*/
keywords?: string[]
/**
* URL to documentation
*/
documentation?: string
/**
* URL to source code repository
*/
repository?: string
/**
* License identifier (e.g., 'MIT', 'Apache-2.0')
*/
license?: string
/**
* UI hints for tools and configuration interfaces
*/
ui?: {
/**
* Icon for the augmentation (emoji or URL)
*/
icon?: string
/**
* Color theme for UI
*/
color?: string
/**
* Custom React component name for configuration
*/
configComponent?: string
/**
* URL to dashboard or control panel
*/
dashboardUrl?: string
/**
* Hide from UI listings
*/
hidden?: boolean
}
/**
* Performance characteristics
*/
performance?: {
/**
* Estimated memory usage
*/
memoryUsage?: 'low' | 'medium' | 'high'
/**
* CPU intensity
*/
cpuUsage?: 'low' | 'medium' | 'high'
/**
* Network usage
*/
networkUsage?: 'none' | 'low' | 'medium' | 'high'
}
/**
* Feature flags this augmentation provides
*/
features?: string[]
/**
* Operations this augmentation enhances
*/
enhancedOperations?: string[]
/**
* Metrics this augmentation exposes
*/
metrics?: Array<{
name: string
type: 'counter' | 'gauge' | 'histogram'
description: string
}>
/**
* Status of the augmentation
*/
status?: 'stable' | 'beta' | 'experimental' | 'deprecated'
/**
* Deprecation notice if applicable
*/
deprecation?: {
since: string
alternative?: string
removalDate?: string
}
}

204
src/augmentations/metadataEnforcer.ts
View file

@ -1,204 +0,0 @@
/**
* Runtime enforcement of metadata contracts
* Ensures augmentations only access declared fields
*/
import { BrainyAugmentation, MetadataAccess } from './brainyAugmentation.js'
export class MetadataEnforcer {
/**
* Enforce metadata access based on augmentation contract
* Returns a wrapped metadata object that enforces the contract
*/
static enforce(
augmentation: BrainyAugmentation,
metadata: any,
operation: 'read' | 'write' = 'write'
): any {
// Handle simple contracts
if (augmentation.metadata === 'none') {
// No access at all
if (operation === 'read') return null
throw new Error(`Augmentation '${augmentation.name}' has metadata='none' - cannot access metadata`)
}
if (augmentation.metadata === 'readonly') {
if (operation === 'read') {
// Return frozen deep clone for read-only access
return deepFreeze(deepClone(metadata))
}
throw new Error(`Augmentation '${augmentation.name}' has metadata='readonly' - cannot write`)
}
// Handle specific field access
const access = augmentation.metadata as MetadataAccess
if (operation === 'read') {
// For reads, filter to allowed fields
if (access.reads === '*') {
return deepClone(metadata) // Can read everything
}
if (!access.reads) {
return {} // No read access
}
// Filter to specific fields
const filtered: any = {}
for (const field of access.reads) {
if (field.includes('.')) {
// Handle nested fields like '_brainy.deleted'
const parts = field.split('.')
let source = metadata
let target = filtered
for (let i = 0; i < parts.length - 1; i++) {
const part = parts[i]
if (!source[part]) break
if (!target[part]) target[part] = {}
source = source[part]
target = target[part]
}
const lastPart = parts[parts.length - 1]
if (source && lastPart in source) {
target[lastPart] = source[lastPart]
}
} else {
// Simple field
if (field in metadata) {
filtered[field] = metadata[field]
}
}
}
return filtered
}
// For writes, create a proxy that validates
return new Proxy(metadata, {
set(target, prop, value) {
const field = String(prop)
// Check if write is allowed
if (access.writes === '*') {
// Can write anything
target[prop] = value
return true
}
if (!access.writes || !access.writes.includes(field)) {
throw new Error(
`Augmentation '${augmentation.name}' cannot write to field '${field}'. ` +
`Allowed writes: ${access.writes?.join(', ') || 'none'}`
)
}
// Check namespace if specified
if (access.namespace && !field.startsWith(access.namespace)) {
console.warn(
`Augmentation '${augmentation.name}' writing outside its namespace. ` +
`Expected: ${access.namespace}.*, got: ${field}`
)
}
target[prop] = value
return true
},
deleteProperty(target, prop) {
const field = String(prop)
// Deletion counts as a write
if (access.writes === '*' || access.writes?.includes(field)) {
delete target[prop]
return true
}
throw new Error(
`Augmentation '${augmentation.name}' cannot delete field '${field}'`
)
}
})
}
/**
* Validate that an augmentation's actual behavior matches its contract
* Used in testing to verify contracts are accurate
*/
static async validateContract(
augmentation: BrainyAugmentation,
testMetadata: any = { test: 'data', _brainy: { deleted: false } }
): Promise<{ valid: boolean; violations: string[] }> {
const violations: string[] = []
// Test read access
try {
const readable = this.enforce(augmentation, testMetadata, 'read')
if (augmentation.metadata === 'none' && readable !== null) {
violations.push(`Contract says 'none' but got readable metadata`)
}
} catch (error) {
violations.push(`Read enforcement error: ${error}`)
}
// Test write access
try {
const writable = this.enforce(augmentation, testMetadata, 'write')
if (augmentation.metadata === 'none') {
violations.push(`Contract says 'none' but got writable metadata`)
}
if (augmentation.metadata === 'readonly') {
// Try to write - should fail
try {
writable.testWrite = 'value'
violations.push(`Contract says 'readonly' but write succeeded`)
} catch {
// Expected to fail
}
}
} catch (error) {
// Expected for 'none' and 'readonly' on write
if (augmentation.metadata !== 'none' && augmentation.metadata !== 'readonly') {
violations.push(`Write enforcement error: ${error}`)
}
}
return {
valid: violations.length === 0,
violations
}
}
}
// Helper functions
function deepClone(obj: any): any {
if (obj === null || typeof obj !== 'object') return obj
if (obj instanceof Date) return new Date(obj)
if (obj instanceof Array) return obj.map(item => deepClone(item))
const cloned: any = {}
for (const key in obj) {
if (obj.hasOwnProperty(key)) {
cloned[key] = deepClone(obj[key])
}
}
return cloned
}
function deepFreeze(obj: any): any {
Object.freeze(obj)
Object.getOwnPropertyNames(obj).forEach(prop => {
if (obj[prop] !== null &&
(typeof obj[prop] === 'object' || typeof obj[prop] === 'function') &&
!Object.isFrozen(obj[prop])) {
deepFreeze(obj[prop])
}
})
return obj
}

356
src/augmentations/metricsAugmentation.ts
View file

@ -1,356 +0,0 @@
/**
* Metrics Augmentation - Optional Performance & Usage Metrics
*
* IMPORTANT: This is SEPARATE from core counting (brain.counts.*) which is always enabled.
*
* Core counting provides O(1) entity/relationship counts for scalability.
* This augmentation provides performance analytics and observability.
*
* Features:
* - Search performance tracking (latency, throughput)
* - Cache hit/miss rates
* - Operation timing analysis
* - Usage pattern insights
*
* Zero-config: Automatically enabled for observability
* Can be disabled via augmentation registry without affecting core performance
*/
import { BaseAugmentation, AugmentationContext } from './brainyAugmentation.js'
import { StatisticsCollector } from '../utils/statisticsCollector.js'
import type { BaseStorageAdapter as StorageAdapter } from '../storage/adapters/baseStorageAdapter.js'
export interface MetricsConfig {
enabled?: boolean
trackSearches?: boolean
trackContentTypes?: boolean
trackVerbTypes?: boolean
trackStorageSizes?: boolean
persistMetrics?: boolean
metricsInterval?: number
silent?: boolean // New: Silent mode support
}
/**
* MetricsAugmentation - Makes metrics collection optional and pluggable
*
* Features:
* - Performance tracking (search latency, throughput)
* - Usage patterns (content types, verb types)
* - Storage metrics (sizes, counts)
* - Zero-config with smart defaults
*/
export class MetricsAugmentation extends BaseAugmentation {
readonly metadata = 'readonly' as const // Reads metadata for metrics
readonly name = 'metrics'
readonly timing = 'after' as const
operations = ['add', 'update', 'search', 'find', 'similar', 'delete', 'relate', 'unrelate', 'clear', 'all'] as ('add' | 'update' | 'search' | 'find' | 'similar' | 'delete' | 'relate' | 'unrelate' | 'clear' | 'all')[]
readonly priority = 40 // Low priority, runs after other augmentations
private statisticsCollector: StatisticsCollector | null = null
protected config: MetricsConfig
private metricsTimer: NodeJS.Timeout | null = null
constructor(config: MetricsConfig = {}) {
super()
this.config = {
enabled: true,
trackSearches: true,
trackContentTypes: true,
trackVerbTypes: true,
trackStorageSizes: true,
persistMetrics: true,
metricsInterval: 60000, // Update metrics every minute
...config
}
}
protected async onInitialize(): Promise<void> {
if (!this.config.enabled) {
this.log('Metrics augmentation disabled by configuration')
return
}
// Initialize statistics collector
this.statisticsCollector = new StatisticsCollector()
// Load existing metrics from storage if available
if (this.config.persistMetrics && this.context?.storage) {
try {
const storage = this.context.storage as StorageAdapter
const existingStats = await storage.getStatistics?.()
if (existingStats) {
this.statisticsCollector.mergeFromStorage(existingStats)
this.log('Loaded existing metrics from storage')
}
} catch (e) {
this.log('Could not load existing metrics', 'info')
}
}
// Start metrics update timer
if (this.config.metricsInterval && this.config.metricsInterval > 0) {
this.startMetricsTimer()
}
this.log('Metrics augmentation initialized')
}
protected async onShutdown(): Promise<void> {
// Stop metrics timer
if (this.metricsTimer) {
clearInterval(this.metricsTimer)
this.metricsTimer = null
}
// Persist final metrics
if (this.config.persistMetrics && this.statisticsCollector && this.context?.storage) {
try {
await this.persistMetrics()
} catch (error) {
this.log('Error persisting metrics during shutdown', 'warn')
}
}
this.statisticsCollector = null
this.log('Metrics augmentation shut down')
}
/**
* Execute augmentation - track metrics for operations
*/
async execute<T = any>(
operation: string,
params: any,
next: () => Promise<T>
): Promise<T> {
// If metrics disabled, just pass through
if (!this.statisticsCollector || !this.config.enabled) {
return next()
}
// Track operation timing
const startTime = Date.now()
try {
const result = await next()
const duration = Date.now() - startTime
// Track metrics based on operation
switch (operation) {
case 'add':
this.handleAdd(params, duration)
break
case 'search':
this.handleSearch(params, duration)
break
case 'delete':
this.handleDelete(duration)
break
case 'clear':
this.handleClear()
break
}
return result
} catch (error) {
// Error tracking removed - StatisticsCollector doesn't have trackError method
// Could be added later if needed
throw error
}
}
/**
* Handle add operation metrics
*/
private handleAdd(params: any, duration: number): void {
if (!this.statisticsCollector) return
// Track update
this.statisticsCollector.trackUpdate()
// Track content type if available
if (this.config.trackContentTypes && params.metadata?.noun) {
this.statisticsCollector.trackContentType(params.metadata.noun)
}
// Track verb type if it's a verb operation
if (this.config.trackVerbTypes && params.metadata?.verb) {
this.statisticsCollector.trackVerbType(params.metadata.verb)
}
this.log(`Add operation completed in ${duration}ms`, 'info')
}
/**
* Handle search operation metrics
*/
private handleSearch(params: any, duration: number): void {
if (!this.statisticsCollector || !this.config.trackSearches) return
const { query } = params
this.statisticsCollector.trackSearch(query || '', duration)
this.log(`Search completed in ${duration}ms`, 'info')
}
/**
* Handle delete operation metrics
*/
private handleDelete(duration: number): void {
if (!this.statisticsCollector) return
this.statisticsCollector.trackUpdate()
this.log(`Delete operation completed in ${duration}ms`, 'info')
}
/**
* Handle clear operation - reset metrics
*/
private handleClear(): void {
if (!this.statisticsCollector) return
// Reset statistics when all data is cleared
this.statisticsCollector = new StatisticsCollector()
this.log('Metrics reset due to clear operation')
}
/**
* Start periodic metrics update timer
*/
private startMetricsTimer(): void {
if (this.metricsTimer) return
this.metricsTimer = setInterval(async () => {
await this.updateStorageMetrics()
if (this.config.persistMetrics) {
await this.persistMetrics()
}
}, this.config.metricsInterval!)
}
/**
* Update storage size metrics
*/
private async updateStorageMetrics(): Promise<void> {
if (!this.statisticsCollector || !this.config.trackStorageSizes) return
if (!this.context?.storage) return
try {
const storage = this.context.storage as StorageAdapter
const stats = await storage.getStatistics?.()
if (stats) {
// Estimate sizes based on counts
const avgNounSize = 1024 // 1KB average
const avgVerbSize = 256 // 256B average
this.statisticsCollector.updateStorageSizes({
nouns: (stats.totalNodes || 0) * avgNounSize,
verbs: (stats.totalEdges || 0) * avgVerbSize,
metadata: (stats.totalNodes || 0) * 512, // 512B per metadata
index: (stats.hnswIndexSize || 0) // Use HNSW index size from stats
})
}
} catch (e) {
this.log('Could not update storage metrics', 'info')
}
}
/**
* Persist metrics to storage
*/
private async persistMetrics(): Promise<void> {
if (!this.statisticsCollector || !this.context?.storage) return
try {
const stats = this.statisticsCollector.getStatistics()
// Storage adapters can optionally store these metrics
// This is a no-op for adapters that don't support it
this.log('Metrics persisted to storage', 'info')
} catch (e) {
this.log('Could not persist metrics', 'info')
}
}
/**
* Get current metrics (performance analytics only)
*
* NOTE: For production counting (entities, relationships), use:
* - brain.counts.entities() - O(1) total count
* - brain.counts.byType() - O(1) type-specific counts
* - brain.counts.relationships() - O(1) relationship counts
*/
getStatistics() {
if (!this.statisticsCollector) {
return {
enabled: false,
note: 'For core counting, use brain.counts.* APIs which are always available',
totalSearches: 0,
totalUpdates: 0,
contentTypes: {},
verbTypes: {},
searchPerformance: {
averageLatency: 0,
p95Latency: 0,
p99Latency: 0
}
}
}
return {
enabled: true,
note: 'Performance analytics only. Core counting available via brain.counts.*',
...this.statisticsCollector.getStatistics()
}
}
/**
* Record cache hit (called by cache augmentation)
* Note: Cache metrics are tracked internally by StatisticsCollector
*/
recordCacheHit(): void {
// StatisticsCollector doesn't have trackCacheHit method
// Cache metrics would need to be implemented if needed
this.log('Cache hit recorded', 'info')
}
/**
* Record cache miss (called by cache augmentation)
* Note: Cache metrics are tracked internally by StatisticsCollector
*/
recordCacheMiss(): void {
// StatisticsCollector doesn't have trackCacheMiss method
// Cache metrics would need to be implemented if needed
this.log('Cache miss recorded', 'info')
}
/**
* Track custom metric
* Note: Custom metrics would need to be implemented in StatisticsCollector
*/
trackCustomMetric(name: string, value: number): void {
// StatisticsCollector doesn't have trackCustomMetric method
// Could be added later if needed
this.log(`Custom metric recorded: ${name}=${value}`, 'info')
}
/**
* Reset all metrics
*/
reset(): void {
if (this.statisticsCollector) {
this.statisticsCollector = new StatisticsCollector()
this.log('Metrics manually reset')
}
}
}
/**
* Factory function for zero-config metrics augmentation
*/
export function createMetricsAugmentation(config?: MetricsConfig): MetricsAugmentation {
return new MetricsAugmentation(config)
}

270
src/augmentations/monitoringAugmentation.ts
View file

@ -1,270 +0,0 @@
/**
* Monitoring Augmentation - Optional Health & Performance Monitoring
*
* Replaces the hardcoded HealthMonitor in Brainy with an optional augmentation.
* Provides health checks, performance monitoring, and distributed system tracking.
*
* Zero-config: Automatically enabled for distributed deployments
* Can be disabled or customized via augmentation registry
*/
import { BaseAugmentation, AugmentationContext } from './brainyAugmentation.js'
import { HealthMonitor } from '../distributed/healthMonitor.js'
import { DistributedConfigManager as ConfigManager } from '../distributed/configManager.js'
export interface MonitoringConfig {
enabled?: boolean
healthCheckInterval?: number
metricsInterval?: number
trackLatency?: boolean
trackErrors?: boolean
trackCacheMetrics?: boolean
exposeHealthEndpoint?: boolean
}
/**
* MonitoringAugmentation - Makes health monitoring optional and pluggable
*
* Features:
* - Health status tracking
* - Performance monitoring
* - Error rate tracking
* - Distributed system health
* - Zero-config with smart defaults
*/
export class MonitoringAugmentation extends BaseAugmentation {
readonly metadata = 'readonly' as const // Reads metadata for monitoring
readonly name = 'monitoring'
readonly timing = 'after' as const
operations = ['search', 'find', 'similar', 'add', 'update', 'delete', 'relate', 'unrelate', 'all'] as ('search' | 'find' | 'similar' | 'add' | 'update' | 'delete' | 'relate' | 'unrelate' | 'all')[]
readonly priority = 30 // Low priority, observability layer
private healthMonitor: HealthMonitor | null = null
private configManager: ConfigManager | null = null
protected config: MonitoringConfig
private requestStartTimes = new Map<string, number>()
constructor(config: MonitoringConfig = {}) {
super()
this.config = {
enabled: true,
healthCheckInterval: 30000, // 30 seconds
metricsInterval: 60000, // 1 minute
trackLatency: true,
trackErrors: true,
trackCacheMetrics: true,
exposeHealthEndpoint: true,
...config
}
}
protected async onInitialize(): Promise<void> {
if (!this.config.enabled) {
this.log('Monitoring augmentation disabled by configuration')
return
}
// Initialize config manager and health monitor (requires storage)
if (this.context?.storage) {
this.configManager = new ConfigManager(this.context.storage as any)
this.healthMonitor = new HealthMonitor(this.configManager)
this.healthMonitor.start()
} else {
this.log('Storage not available - health monitoring disabled', 'warn')
}
this.log('Monitoring augmentation initialized')
}
protected async onShutdown(): Promise<void> {
if (this.healthMonitor) {
this.healthMonitor.stop()
this.healthMonitor = null
}
this.configManager = null
this.requestStartTimes.clear()
this.log('Monitoring augmentation shut down')
}
/**
* Execute augmentation - track health metrics
*/
async execute<T = any>(
operation: string,
params: any,
next: () => Promise<T>
): Promise<T> {
// If monitoring disabled, just pass through
if (!this.healthMonitor || !this.config.enabled) {
return next()
}
// Generate request ID for tracking
const requestId = `${operation}-${Date.now()}-${Math.random()}`
// Track request start time
if (this.config.trackLatency) {
this.requestStartTimes.set(requestId, Date.now())
}
try {
// Execute operation
const result = await next()
// Track successful operation
if (this.config.trackLatency) {
const startTime = this.requestStartTimes.get(requestId)
if (startTime) {
const latency = Date.now() - startTime
this.healthMonitor.recordRequest(latency, false)
this.requestStartTimes.delete(requestId)
}
}
// Update vector count for 'add' operations
if (operation === 'add' && this.context?.brain) {
try {
const count = await this.context.brain.getNounCount()
this.healthMonitor.updateVectorCount(count)
} catch (e) {
// Ignore count update errors
}
}
// Track cache metrics for search operations
if (operation === 'search' && this.config.trackCacheMetrics) {
// Check if result came from cache (would be set by cache augmentation)
const fromCache = (params as any)._fromCache || false
this.healthMonitor.recordCacheAccess(fromCache)
}
return result
} catch (error) {
// Track error
if (this.config.trackErrors) {
const startTime = this.requestStartTimes.get(requestId)
if (startTime) {
const latency = Date.now() - startTime
this.healthMonitor.recordRequest(latency, true)
this.requestStartTimes.delete(requestId)
} else {
this.healthMonitor.recordRequest(0, true)
}
}
throw error
}
}
/**
* Get health status
*/
getHealthStatus() {
if (!this.healthMonitor) {
return {
status: 'disabled',
enabled: false,
uptime: 0,
vectorCount: 0,
requestRate: 0,
errorRate: 0,
cacheHitRate: 0
}
}
return {
status: 'healthy',
enabled: true,
...this.healthMonitor.getHealthEndpointData()
}
}
/**
* Get health endpoint data (for API exposure)
*/
getHealthEndpointData() {
if (!this.healthMonitor) {
return {
status: 'disabled',
timestamp: new Date().toISOString()
}
}
return this.healthMonitor.getHealthEndpointData()
}
/**
* Update vector count manually
*/
updateVectorCount(count: number): void {
if (this.healthMonitor) {
this.healthMonitor.updateVectorCount(count)
}
}
/**
* Record custom health metric
*/
recordCustomMetric(name: string, value: number): void {
if (this.healthMonitor) {
// Health monitor could be extended to track custom metrics
this.log(`Custom metric recorded: ${name}=${value}`, 'info')
}
}
/**
* Check if system is healthy
*/
isHealthy(): boolean {
if (!this.healthMonitor) return true // If disabled, assume healthy
const data = this.healthMonitor.getHealthEndpointData()
// Define health criteria
const errorRateThreshold = 0.05 // 5% error rate
const minUptime = 60000 // 1 minute
return (
data.errorRate < errorRateThreshold &&
data.uptime > minUptime
)
}
/**
* Get uptime in milliseconds
*/
getUptime(): number {
if (!this.healthMonitor) return 0
const data = this.healthMonitor.getHealthEndpointData()
return data.uptime || 0
}
/**
* Force health check
*/
async checkHealth(): Promise<boolean> {
if (!this.healthMonitor) return true
// Perform active health check
try {
// Could ping storage, check memory, etc.
if (this.context?.storage) {
await this.context.storage.getStatistics?.()
}
return true
} catch (error) {
this.log('Health check failed', 'warn')
return false
}
}
}
/**
* Factory function for zero-config monitoring augmentation
*/
export function createMonitoringAugmentation(config?: MonitoringConfig): MonitoringAugmentation {
return new MonitoringAugmentation(config)
}

384
src/augmentations/rateLimitAugmentation.ts
View file

@ -1,384 +0,0 @@
/**
* Rate Limiting Augmentation
* Provides configurable rate limiting for Brainy operations
*/
import { BaseAugmentation } from './brainyAugmentation.js'
import { AugmentationManifest } from './manifest.js'
export interface RateLimitConfig {
enabled?: boolean
limits?: {
searches?: number // Per minute
writes?: number // Per minute
reads?: number // Per minute
deletes?: number // Per minute
}
windowMs?: number // Time window in milliseconds
skipSuccessfulRequests?: boolean
skipFailedRequests?: boolean
keyGenerator?: (context: any) => string
}
interface RateLimitEntry {
count: number
resetTime: number
}
/**
* Rate Limit Augmentation
*/
export class RateLimitAugmentation extends BaseAugmentation {
readonly name = 'rateLimiter'
readonly timing = 'before' as const
readonly metadata = 'none' as const
operations = ['search', 'find', 'add', 'update', 'delete', 'get'] as any
readonly priority = 10 // High priority, runs early
// Augmentation metadata
readonly category = 'core' as const // Use 'core' as security isn't a valid category
readonly description = 'Provides rate limiting for Brainy operations'
private limiters: Map<string, Map<string, RateLimitEntry>> = new Map()
private windowMs: number
constructor(config: RateLimitConfig = {}) {
super(config)
// Merge with defaults
this.config = {
enabled: config.enabled ?? true,
limits: {
searches: config.limits?.searches ?? 1000,
writes: config.limits?.writes ?? 100,
reads: config.limits?.reads ?? 5000,
deletes: config.limits?.deletes ?? 50
},
windowMs: config.windowMs ?? 60000, // 1 minute default
skipSuccessfulRequests: config.skipSuccessfulRequests ?? false,
skipFailedRequests: config.skipFailedRequests ?? true,
keyGenerator: config.keyGenerator || this.defaultKeyGenerator
}
this.windowMs = this.config.windowMs!
// Initialize operation limiters
this.initializeLimiters()
}
getManifest(): AugmentationManifest {
return {
id: 'rate-limiter',
name: 'Rate Limiter',
version: '1.0.0',
description: 'Configurable rate limiting for API operations',
longDescription: 'Provides per-operation rate limiting with configurable windows and limits. Helps prevent abuse and ensures fair resource usage.',
category: 'core',
configSchema: {
type: 'object',
properties: {
enabled: {
type: 'boolean',
default: true,
description: 'Enable or disable rate limiting'
},
limits: {
type: 'object',
properties: {
searches: {
type: 'number',
default: 1000,
description: 'Search operations per minute'
},
writes: {
type: 'number',
default: 100,
description: 'Write operations per minute'
},
reads: {
type: 'number',
default: 5000,
description: 'Read operations per minute'
},
deletes: {
type: 'number',
default: 50,
description: 'Delete operations per minute'
}
}
},
windowMs: {
type: 'number',
default: 60000,
description: 'Time window in milliseconds'
}
}
},
configDefaults: {
enabled: true,
limits: {
searches: 1000,
writes: 100,
reads: 5000,
deletes: 50
},
windowMs: 60000
},
minBrainyVersion: '3.0.0',
keywords: ['rate-limit', 'security', 'throttle'],
documentation: 'https://docs.brainy.dev/augmentations/rate-limit',
status: 'stable',
performance: {
memoryUsage: 'low',
cpuUsage: 'low',
networkUsage: 'none'
},
features: ['per-operation-limits', 'configurable-windows', 'key-based-limiting'],
enhancedOperations: ['search', 'add', 'update', 'delete', 'get'],
metrics: [
{
name: 'rate_limit_exceeded',
type: 'counter',
description: 'Number of rate limit violations'
},
{
name: 'rate_limit_requests',
type: 'counter',
description: 'Total requests checked'
}
]
}
}
/**
* Initialize rate limiters for each operation type
*/
private initializeLimiters(): void {
const operations = ['searches', 'writes', 'reads', 'deletes']
for (const op of operations) {
this.limiters.set(op, new Map())
}
}
/**
* Default key generator (could be IP, user ID, etc.)
*/
private defaultKeyGenerator(_context: any): string {
// In a real implementation, this would extract IP or user ID
return 'default'
}
/**
* Check if request should be rate limited
*/
private checkRateLimit(operation: string, key: string): boolean {
const limiter = this.limiters.get(operation)
if (!limiter) return false
const limit = (this.config.limits as any)[operation]
if (!limit) return false
const now = Date.now()
let entry = limiter.get(key)
// Initialize or reset entry
if (!entry || now >= entry.resetTime) {
entry = {
count: 0,
resetTime: now + this.windowMs
}
limiter.set(key, entry)
}
// Check if limit exceeded
if (entry.count >= limit) {
return true // Rate limited
}
// Increment counter
entry.count++
return false
}
/**
* Get remaining requests for an operation
*/
private getRemainingRequests(operation: string, key: string): number {
const limiter = this.limiters.get(operation)
if (!limiter) return -1
const limit = (this.config.limits as any)[operation]
if (!limit) return -1
const entry = limiter.get(key)
if (!entry) return limit
const now = Date.now()
if (now >= entry.resetTime) return limit
return Math.max(0, limit - entry.count)
}
/**
* Get time until reset
*/
private getResetTime(operation: string, key: string): number {
const limiter = this.limiters.get(operation)
if (!limiter) return 0
const entry = limiter.get(key)
if (!entry) return 0
const now = Date.now()
return Math.max(0, entry.resetTime - now)
}
protected async onInitialize(): Promise<void> {
if (!this.config.enabled) {
this.log('Rate limiter disabled by configuration')
return
}
this.log(`Rate limiter initialized (window: ${this.windowMs}ms)`)
// Start cleanup timer
setInterval(() => {
this.cleanup()
}, this.windowMs)
}
protected async onShutdown(): Promise<void> {
this.clear()
this.log('Rate limiter shut down')
}
/**
* Execute augmentation - apply rate limiting
*/
async execute<T = any>(
operation: string,
params: any,
next: () => Promise<T>
): Promise<T> {
// If rate limiting is disabled, just pass through
if (!this.config.enabled) {
return next()
}
// Map operations to rate limit categories
let rateLimitOperation: string
switch (operation) {
case 'search':
case 'find':
case 'similar':
rateLimitOperation = 'searches'
break
case 'add':
case 'update':
rateLimitOperation = 'writes'
break
case 'delete':
rateLimitOperation = 'deletes'
break
case 'get':
rateLimitOperation = 'reads'
break
default:
return next() // Don't rate limit unknown operations
}
const key = (this.config.keyGenerator as any)(params)
if (this.checkRateLimit(rateLimitOperation, key)) {
const error = new Error(`Rate limit exceeded for ${operation}`)
;(error as any).statusCode = 429
;(error as any).retryAfter = this.getResetTime(rateLimitOperation, key)
;(error as any).rateLimit = {
limit: (this.config.limits as any)[rateLimitOperation],
remaining: 0,
reset: Date.now() + this.getResetTime(rateLimitOperation, key)
}
throw error
}
try {
const result = await next()
// Add rate limit info to result if possible
if (result && typeof result === 'object' && !Array.isArray(result)) {
(result as any)._rateLimit = {
limit: (this.config.limits as any)[rateLimitOperation],
remaining: this.getRemainingRequests(rateLimitOperation, key),
reset: Date.now() + this.getResetTime(rateLimitOperation, key)
}
}
return result
} catch (error) {
// Optionally don't count failed requests
if (this.config.skipFailedRequests) {
const limiter = this.limiters.get(rateLimitOperation)!
const entry = limiter.get(key)
if (entry && entry.count > 0) entry.count--
}
throw error
}
}
/**
* Get rate limit statistics
*/
getStats(): {
operations: Record<string, {
activeKeys: number
totalRequests: number
}>
} {
const stats: any = { operations: {} }
for (const [operation, limiter] of this.limiters) {
let totalRequests = 0
for (const entry of limiter.values()) {
totalRequests += entry.count
}
stats.operations[operation] = {
activeKeys: limiter.size,
totalRequests
}
}
return stats
}
/**
* Clear all rate limit entries
*/
clear(): void {
for (const limiter of this.limiters.values()) {
limiter.clear()
}
}
/**
* Clear expired entries (cleanup)
*/
cleanup(): void {
const now = Date.now()
for (const limiter of this.limiters.values()) {
for (const [key, entry] of limiter) {
if (now >= entry.resetTime) {
limiter.delete(key)
}
}
}
}
}
/**
* Create rate limit augmentation
*/
export function createRateLimitAugmentation(config?: RateLimitConfig): RateLimitAugmentation {
return new RateLimitAugmentation(config)
}

216
src/augmentations/requestDeduplicatorAugmentation.ts
View file

@ -1,216 +0,0 @@
/**
* Request Deduplicator Augmentation
*
* Prevents duplicate concurrent requests to improve performance by 3x
* Automatically deduplicates identical operations
*/
import { BaseAugmentation, AugmentationContext } from './brainyAugmentation.js'
interface PendingRequest<T> {
promise: Promise<T>
timestamp: number
count: number
}
interface DeduplicatorConfig {
enabled?: boolean
ttl?: number // Time to live for cached requests (ms)
maxSize?: number // Maximum number of cached requests
}
export class RequestDeduplicatorAugmentation extends BaseAugmentation {
name = 'RequestDeduplicator'
timing = 'around' as const
metadata = 'none' as const // Doesn't access metadata
operations = ['search', 'find', 'similar', 'searchText', 'searchByNounTypes', 'findSimilar', 'get'] as ('search' | 'find' | 'similar' | 'searchText' | 'searchByNounTypes' | 'findSimilar' | 'get')[]
priority = 50 // Performance optimization
private pendingRequests: Map<string, PendingRequest<any>> = new Map()
protected config: Required<DeduplicatorConfig>
private cleanupInterval?: NodeJS.Timeout
constructor(config: DeduplicatorConfig = {}) {
super()
this.config = {
enabled: config.enabled ?? true,
ttl: config.ttl ?? 5000, // 5 second default
maxSize: config.maxSize ?? 1000
}
}
protected async onInitialize(): Promise<void> {
if (this.config.enabled) {
this.log('Request deduplicator initialized for 3x performance boost')
// Start cleanup interval
this.cleanupInterval = setInterval(() => {
this.cleanup()
}, this.config.ttl)
} else {
this.log('Request deduplicator disabled')
}
}
shouldExecute(operation: string, params: any): boolean {
// Only execute if enabled and for read operations that benefit from deduplication
return this.config.enabled && (
operation === 'search' ||
operation === 'searchText' ||
operation === 'searchByNounTypes' ||
operation === 'findSimilar' ||
operation === 'get'
)
}
async execute<T = any>(
operation: string,
params: any,
next: () => Promise<T>
): Promise<T> {
if (!this.config.enabled) {
return next()
}
// Create a unique key for this request
const key = this.createRequestKey(operation, params)
// Check if we already have this request pending
const existing = this.pendingRequests.get(key)
if (existing) {
existing.count++
this.log(`Deduplicating request: ${key} (${existing.count} total)`)
return existing.promise
}
// Execute the request and cache the promise
const promise = next()
this.pendingRequests.set(key, {
promise,
timestamp: Date.now(),
count: 1
})
// Clean up when done
promise.finally(() => {
// Use setTimeout to allow other concurrent requests to use the result
setTimeout(() => {
this.pendingRequests.delete(key)
}, 100)
})
return promise
}
/**
* Create a unique key for the request based on operation and parameters
*/
private createRequestKey(operation: string, params: any): string {
// Create a stable string representation of the operation and params
const paramsKey = this.serializeParams(params)
return `${operation}:${paramsKey}`
}
/**
* Serialize parameters to a consistent string
*/
private serializeParams(params: any): string {
if (!params) return 'null'
if (typeof params === 'string' || typeof params === 'number') {
return String(params)
}
if (Array.isArray(params)) {
// For arrays, create a hash-like representation
if (params.length > 100) {
// For large arrays (like vectors), use length + first/last elements
return `[${params.length}:${params[0]}...${params[params.length - 1]}]`
}
return `[${params.join(',')}]`
}
if (typeof params === 'object') {
// Sort keys for consistent serialization
const keys = Object.keys(params).sort()
const keyValues = keys.map(key => `${key}:${this.serializeParams(params[key])}`)
return `{${keyValues.join(',')}}`
}
return String(params)
}
/**
* Clean up expired requests
*/
private cleanup(): void {
const now = Date.now()
const expired = []
for (const [key, request] of this.pendingRequests) {
if (now - request.timestamp > this.config.ttl) {
expired.push(key)
}
}
for (const key of expired) {
this.pendingRequests.delete(key)
}
// Also enforce max size
if (this.pendingRequests.size > this.config.maxSize) {
const entries = Array.from(this.pendingRequests.entries())
.sort(([,a], [,b]) => a.timestamp - b.timestamp) // Oldest first
// Remove oldest entries
const toRemove = entries.slice(0, entries.length - this.config.maxSize)
for (const [key] of toRemove) {
this.pendingRequests.delete(key)
}
}
if (expired.length > 0) {
this.log(`Cleaned up ${expired.length} expired requests`)
}
}
/**
* Get statistics about request deduplication
*/
getStats(): {
activePendingRequests: number
totalDeduplicationHits: number
memoryUsage: string
efficiency: string
} {
const requests = Array.from(this.pendingRequests.values())
const totalRequests = requests.reduce((sum, req) => sum + req.count, 0)
const actualRequests = requests.length
const savedRequests = totalRequests - actualRequests
return {
activePendingRequests: actualRequests,
totalDeduplicationHits: savedRequests,
memoryUsage: `${Math.round(JSON.stringify(requests).length / 1024)}KB`,
efficiency: actualRequests > 0 ? `${Math.round((savedRequests / totalRequests) * 100)}% reduction` : '0%'
}
}
/**
* Force clear all pending requests (for testing)
*/
clear(): void {
this.pendingRequests.clear()
}
protected async onShutdown(): Promise<void> {
if (this.cleanupInterval) {
clearInterval(this.cleanupInterval)
}
const stats = this.getStats()
this.log(`Request deduplicator shutdown: ${stats.efficiency} efficiency achieved`)
this.pendingRequests.clear()
}
}

119
src/augmentations/storageAugmentation.ts
View file

@ -1,119 +0,0 @@
/**
* Storage Augmentation Base Classes
*
* Unifies storage adapters and augmentations into a single system.
* All storage backends are now augmentations for consistency and extensibility.
*/
import { BaseAugmentation, BrainyAugmentation, AugmentationContext } from './brainyAugmentation.js'
import { StorageAdapter } from '../coreTypes.js'
/**
* Base class for all storage augmentations
* Provides the storage adapter to the brain during initialization
*/
export abstract class StorageAugmentation extends BaseAugmentation implements BrainyAugmentation {
readonly timing = 'replace' as const
readonly metadata = 'none' as const // Storage doesn't directly access metadata
operations = ['storage'] as ('storage')[] // Make mutable for TypeScript compatibility
readonly priority = 100 // High priority for storage
protected storageAdapter: StorageAdapter | null = null
// Storage augmentations must provide their name via readonly property
constructor(config?: any) {
super(config)
}
/**
* Provide the storage adapter before full initialization
* This is called during the storage resolution phase
*/
abstract provideStorage(): Promise<StorageAdapter>
/**
* Initialize the augmentation with context
* Called after storage has been resolved
*/
async initialize(context: AugmentationContext): Promise<void> {
await super.initialize(context)
// Storage adapter should already be provided
if (!this.storageAdapter) {
this.storageAdapter = await this.provideStorage()
}
}
/**
* Execute storage operations
* For storage augmentations, this replaces the default storage
*/
async execute<T = any>(
operation: string,
params: any,
next: () => Promise<T>
): Promise<T> {
if (operation === 'storage') {
// Return our storage adapter
return this.storageAdapter as any as T
}
// Pass through all other operations
return next()
}
/**
* Shutdown and cleanup
*/
async shutdown(): Promise<void> {
// Cleanup storage adapter if needed
if (this.storageAdapter && typeof (this.storageAdapter as any).close === 'function') {
await (this.storageAdapter as any).close()
}
await super.shutdown()
}
}
/**
* Dynamic storage augmentation that wraps any storage adapter
* Used for backward compatibility and zero-config
*/
export class DynamicStorageAugmentation extends StorageAugmentation {
readonly name = 'dynamic-storage'
constructor(private adapter: StorageAdapter) {
super()
this.storageAdapter = adapter
}
async provideStorage(): Promise<StorageAdapter> {
return this.adapter
}
protected async onInitialize(): Promise<void> {
// Adapter is already provided in constructor
await this.adapter.init()
this.log(`${this.name} initialized`)
}
}
/**
* Create a storage augmentation from configuration
* Maintains backward compatibility with existing storage config
*/
export async function createStorageAugmentationFromConfig(
config: any
): Promise<StorageAugmentation | null> {
// Import storage factory dynamically to avoid circular deps
const { createStorage } = await import('../storage/storageFactory.js')
try {
// Create storage adapter from config
const adapter = await createStorage(config)
// Wrap in augmentation
return new DynamicStorageAugmentation(adapter)
} catch (error) {
console.warn('Failed to create storage augmentation from config:', error)
return null
}
}

496
src/augmentations/storageAugmentations.ts
View file

@ -1,496 +0,0 @@
/**
* Storage Augmentations - Concrete Implementations
*
* These augmentations provide different storage backends for Brainy.
* Each wraps an existing storage adapter for backward compatibility.
*/
import { StorageAugmentation } from './storageAugmentation.js'
import { StorageAdapter } from '../coreTypes.js'
import { AugmentationManifest } from './manifest.js'
import { MemoryStorage } from '../storage/adapters/memoryStorage.js'
import { OPFSStorage } from '../storage/adapters/opfsStorage.js'
import { S3CompatibleStorage } from '../storage/adapters/s3CompatibleStorage.js'
import { R2Storage } from '../storage/adapters/r2Storage.js'
import { AzureBlobStorage } from '../storage/adapters/azureBlobStorage.js'
/**
* Memory Storage Augmentation - Fast in-memory storage
*/
export class MemoryStorageAugmentation extends StorageAugmentation {
readonly name = 'memory-storage'
readonly category = 'core' as const
readonly description = 'High-performance in-memory storage for development and testing'
constructor(config?: any) {
super(config)
}
getManifest(): AugmentationManifest {
return {
id: 'memory-storage',
name: 'Memory Storage',
version: '2.0.0',
description: 'Fast in-memory storage with no persistence',
longDescription: 'Perfect for development, testing, and temporary data. All data is lost when the process ends. Provides the fastest possible performance with zero I/O overhead.',
category: 'storage',
configSchema: {
type: 'object',
properties: {
maxSize: {
type: 'number',
default: 104857600, // 100MB
minimum: 1048576, // 1MB
maximum: 1073741824, // 1GB
description: 'Maximum memory usage in bytes'
},
gcInterval: {
type: 'number',
default: 60000, // 1 minute
minimum: 1000, // 1 second
maximum: 3600000, // 1 hour
description: 'Garbage collection interval in milliseconds'
},
enableStats: {
type: 'boolean',
default: false,
description: 'Enable memory usage statistics'
}
},
additionalProperties: false
},
configDefaults: {
maxSize: 104857600,
gcInterval: 60000,
enableStats: false
},
minBrainyVersion: '2.0.0',
keywords: ['storage', 'memory', 'ram', 'volatile', 'fast'],
documentation: 'https://docs.brainy.dev/augmentations/memory-storage',
status: 'stable',
performance: {
memoryUsage: 'high',
cpuUsage: 'low',
networkUsage: 'none'
},
features: ['fast-access', 'zero-latency', 'no-persistence'],
ui: {
icon: '💾',
color: '#4CAF50'
}
}
}
async provideStorage(): Promise<StorageAdapter> {
const storage = new MemoryStorage()
this.storageAdapter = storage
return storage
}
protected async onInitialize(): Promise<void> {
await this.storageAdapter!.init()
this.log(`Memory storage initialized (max size: ${this.config.maxSize} bytes)`)
}
}
/**
* FileSystem Storage Augmentation - Node.js persistent storage
*/
export class FileSystemStorageAugmentation extends StorageAugmentation {
readonly name = 'filesystem-storage'
readonly category = 'core' as const
readonly description = 'Persistent file-based storage for Node.js environments'
constructor(config?: any) {
super(config)
}
getManifest(): AugmentationManifest {
return {
id: 'filesystem-storage',
name: 'FileSystem Storage',
version: '2.0.0',
description: 'Persistent storage using the local filesystem',
longDescription: 'Stores data as files on the local filesystem. Perfect for Node.js applications, desktop apps, and servers. Provides persistent storage with good performance and unlimited capacity.',
category: 'storage',
configSchema: {
type: 'object',
properties: {
rootDirectory: {
type: 'string',
default: './brainy-data',
description: 'Root directory for storing data files'
},
compression: {
type: 'boolean',
default: false,
description: 'Enable gzip compression for stored files'
},
maxFileSize: {
type: 'number',
default: 10485760, // 10MB
minimum: 1048576, // 1MB
maximum: 104857600, // 100MB
description: 'Maximum size per file in bytes'
},
autoBackup: {
type: 'boolean',
default: false,
description: 'Enable automatic backups'
},
backupInterval: {
type: 'number',
default: 3600000, // 1 hour
minimum: 60000, // 1 minute
maximum: 86400000, // 24 hours
description: 'Backup interval in milliseconds'
}
},
additionalProperties: false
},
configDefaults: {
rootDirectory: './brainy-data',
compression: false,
maxFileSize: 10485760,
autoBackup: false,
backupInterval: 3600000
},
minBrainyVersion: '2.0.0',
keywords: ['storage', 'filesystem', 'persistent', 'node', 'disk'],
documentation: 'https://docs.brainy.dev/augmentations/filesystem-storage',
status: 'stable',
performance: {
memoryUsage: 'low',
cpuUsage: 'low',
networkUsage: 'none'
},
features: ['persistence', 'unlimited-capacity', 'file-based', 'compression-support'],
ui: {
icon: '📁',
color: '#FF9800'
}
}
}
async provideStorage(): Promise<StorageAdapter> {
try {
// Dynamically import for Node.js environments
const { FileSystemStorage } = await import('../storage/adapters/fileSystemStorage.js')
const storage = new FileSystemStorage(this.config.rootDirectory)
this.storageAdapter = storage
return storage
} catch (error) {
this.log('FileSystemStorage not available, falling back to memory', 'warn')
// Fall back to memory storage
const storage = new MemoryStorage()
this.storageAdapter = storage
return storage
}
}
protected async onInitialize(): Promise<void> {
await this.storageAdapter!.init()
this.log(`FileSystem storage initialized at ${this.config.rootDirectory}`)
if (this.config.compression) {
this.log('Compression enabled for stored files')
}
}
}
/**
* OPFS Storage Augmentation - Browser persistent storage
*/
export class OPFSStorageAugmentation extends StorageAugmentation {
readonly name = 'opfs-storage'
readonly category = 'core' as const
readonly description = 'Persistent browser storage using Origin Private File System'
constructor(config?: any) {
super(config)
}
getManifest(): AugmentationManifest {
return {
id: 'opfs-storage',
name: 'OPFS Storage',
version: '2.0.0',
description: 'Modern browser storage with file system capabilities',
longDescription: 'Uses the Origin Private File System API for persistent browser storage. Provides file-like storage in modern browsers with better performance than IndexedDB and unlimited storage quota.',
category: 'storage',
configSchema: {
type: 'object',
properties: {
requestPersistent: {
type: 'boolean',
default: false,
description: 'Request persistent storage permission from browser'
},
directoryName: {
type: 'string',
default: 'brainy-data',
description: 'Directory name within OPFS'
},
chunkSize: {
type: 'number',
default: 1048576, // 1MB
minimum: 65536, // 64KB
maximum: 10485760, // 10MB
description: 'Chunk size for file operations in bytes'
},
enableCache: {
type: 'boolean',
default: true,
description: 'Enable in-memory caching for frequently accessed data'
}
},
additionalProperties: false
},
configDefaults: {
requestPersistent: false,
directoryName: 'brainy-data',
chunkSize: 1048576,
enableCache: true
},
minBrainyVersion: '2.0.0',
keywords: ['storage', 'browser', 'opfs', 'persistent', 'web'],
documentation: 'https://docs.brainy.dev/augmentations/opfs-storage',
status: 'stable',
performance: {
memoryUsage: 'medium',
cpuUsage: 'low',
networkUsage: 'none'
},
features: ['browser-persistent', 'file-system-api', 'unlimited-quota', 'async-operations'],
ui: {
icon: '🌐',
color: '#2196F3'
}
}
}
async provideStorage(): Promise<StorageAdapter> {
const storage = new OPFSStorage()
if (!storage.isOPFSAvailable()) {
this.log('OPFS not available, falling back to memory', 'warn')
const memStorage = new MemoryStorage()
this.storageAdapter = memStorage
return memStorage
}
this.storageAdapter = storage
return storage
}
protected async onInitialize(): Promise<void> {
await this.storageAdapter!.init()
if (this.config.requestPersistent && this.storageAdapter instanceof OPFSStorage) {
const granted = await this.storageAdapter.requestPersistentStorage()
this.log(`Persistent storage ${granted ? 'granted' : 'denied'}`)
}
this.log(`OPFS storage initialized in directory: ${this.config.directoryName}`)
}
}
/**
* S3 Storage Augmentation - Amazon S3 cloud storage
*/
export class S3StorageAugmentation extends StorageAugmentation {
readonly name = 's3-storage'
protected config: {
bucketName: string
region?: string
accessKeyId: string
secretAccessKey: string
sessionToken?: string
cacheConfig?: any
operationConfig?: any
}
constructor(config: {
bucketName: string
region?: string
accessKeyId: string
secretAccessKey: string
sessionToken?: string
cacheConfig?: any
operationConfig?: any
}) {
super()
this.config = config
}
async provideStorage(): Promise<StorageAdapter> {
const storage = new S3CompatibleStorage({
...this.config,
serviceType: 's3'
})
this.storageAdapter = storage
return storage
}
protected async onInitialize(): Promise<void> {
await this.storageAdapter!.init()
this.log(`S3 storage initialized with bucket ${this.config.bucketName}`)
}
}
/**
* R2 Storage Augmentation - Cloudflare R2 storage
*/
export class R2StorageAugmentation extends StorageAugmentation {
readonly name = 'r2-storage'
protected config: {
bucketName: string
accountId: string
accessKeyId: string
secretAccessKey: string
cacheConfig?: any
}
constructor(config: {
bucketName: string
accountId: string
accessKeyId: string
secretAccessKey: string
cacheConfig?: any
}) {
super()
this.config = config
}
async provideStorage(): Promise<StorageAdapter> {
const storage = new R2Storage({
...this.config
// serviceType not needed - R2Storage is dedicated
})
this.storageAdapter = storage
return storage
}
protected async onInitialize(): Promise<void> {
await this.storageAdapter!.init()
this.log(`R2 storage initialized with bucket ${this.config.bucketName}`)
}
}
/**
* GCS Storage Augmentation - Google Cloud Storage
*/
export class GCSStorageAugmentation extends StorageAugmentation {
readonly name = 'gcs-storage'
protected config: {
bucketName: string
region?: string
accessKeyId: string
secretAccessKey: string
endpoint?: string
cacheConfig?: any
}
constructor(config: {
bucketName: string
region?: string
accessKeyId: string
secretAccessKey: string
endpoint?: string
cacheConfig?: any
}) {
super()
this.config = config
}
async provideStorage(): Promise<StorageAdapter> {
const storage = new S3CompatibleStorage({
...this.config,
endpoint: this.config.endpoint || 'https://storage.googleapis.com',
serviceType: 'gcs'
})
this.storageAdapter = storage
return storage
}
protected async onInitialize(): Promise<void> {
await this.storageAdapter!.init()
this.log(`GCS storage initialized with bucket ${this.config.bucketName}`)
}
}
/**
* Azure Blob Storage Augmentation - Microsoft Azure
*/
export class AzureStorageAugmentation extends StorageAugmentation {
readonly name = 'azure-storage'
protected config: {
containerName: string
accountName?: string
accountKey?: string
connectionString?: string
sasToken?: string
cacheConfig?: any
}
constructor(config: {
containerName: string
accountName?: string
accountKey?: string
connectionString?: string
sasToken?: string
cacheConfig?: any
}) {
super()
this.config = config
}
async provideStorage(): Promise<StorageAdapter> {
const storage = new AzureBlobStorage({
...this.config
})
this.storageAdapter = storage
return storage
}
protected async onInitialize(): Promise<void> {
await this.storageAdapter!.init()
this.log(`Azure Blob Storage initialized with container ${this.config.containerName}`)
}
}
/**
* Auto-select the best storage augmentation for the environment
* Maintains zero-config philosophy
*/
export async function createAutoStorageAugmentation(options: {
rootDirectory?: string
requestPersistentStorage?: boolean
} = {}): Promise<StorageAugmentation> {
// Detect environment
const isNodeEnv = (globalThis as any).__ENV__?.isNode || (
typeof process !== 'undefined' &&
process.versions != null &&
process.versions.node != null
)
if (isNodeEnv) {
// Node.js environment - use FileSystem
return new FileSystemStorageAugmentation({
rootDirectory: options.rootDirectory || './brainy-data'
})
} else {
// Browser environment - try OPFS, fall back to memory
const opfsAug = new OPFSStorageAugmentation({
requestPersistent: options.requestPersistentStorage || false
})
// Test if OPFS is available
const testStorage = new OPFSStorage()
if (testStorage.isOPFSAvailable()) {
return opfsAug
} else {
// Fall back to memory
return new MemoryStorageAugmentation()
}
}
}

467
src/augmentations/synapseAugmentation.ts
View file

@ -1,467 +0,0 @@
/**
* Base Synapse Augmentation
*
* Synapses are special augmentations that provide bidirectional data sync
* with external platforms (Notion, Salesforce, Slack, etc.)
*
* Like biological synapses that transmit signals between neurons, these
* connect Brainy to external data sources, enabling seamless information flow.
*
* They are managed through the Brain Cloud augmentation registry alongside
* other augmentations, enabling unified discovery, installation, and updates.
*
* Example synapses:
* - NotionSynapse: Sync pages, databases, and blocks
* - SalesforceSynapse: Sync contacts, leads, opportunities
* - SlackSynapse: Sync messages, channels, users
* - GoogleDriveSynapse: Sync documents, sheets, presentations
*/
import {
AugmentationResponse
} from '../types/augmentations.js'
import { BaseAugmentation, BrainyAugmentation, AugmentationContext } from './brainyAugmentation.js'
import { NeuralImportAugmentation } from './neuralImport.js'
/**
* Base class for all synapse augmentations
* Provides common functionality for external data synchronization
*/
export abstract class SynapseAugmentation extends BaseAugmentation {
// BrainyAugmentation properties
readonly timing = 'after' as const
readonly operations = ['all'] as ('all')[]
readonly priority = 10
readonly metadata = {
reads: '*' as '*', // Needs to read for syncing
writes: ['_synapse', '_syncedAt'] as string[]
} // Adds synapse tracking metadata
// Synapse-specific properties
abstract readonly synapseId: string
abstract readonly supportedTypes: string[]
// State management
protected syncInProgress = false
protected lastSyncId?: string
protected syncStats = {
totalSyncs: 0,
totalItems: 0,
lastSync: undefined as string | undefined
}
// Neural Import integration
protected neuralImport?: NeuralImportAugmentation
protected useNeuralImport = true // Enable by default
protected async onInit(): Promise<void> {
// Initialize Neural Import if available
if (this.useNeuralImport && this.context?.brain) {
try {
// Check if neural import is already loaded
const existingNeuralImport = this.context.brain.augmentations?.get('neural-import')
if (existingNeuralImport) {
this.neuralImport = existingNeuralImport as NeuralImportAugmentation
} else {
// Create a new instance for this synapse
this.neuralImport = new NeuralImportAugmentation()
// NeuralImport will be initialized when the synapse is added to Brainy
// await this.neuralImport.initialize()
}
} catch (error) {
console.warn(`[${this.synapseId}] Neural Import not available, using basic import`)
this.useNeuralImport = false
}
}
await this.onInitialize()
}
/**
* Synapse-specific initialization
* Override this in implementations
*/
protected abstract onInitialize(): Promise<void>
/**
* BrainyAugmentation execute method
* Intercepts operations to sync external data when relevant
*/
async execute<T = any>(
operation: string,
params: any,
next: () => Promise<T>
): Promise<T> {
// Execute the main operation first
const result = await next()
// After certain operations, check if we should sync
if (this.shouldSync(operation, params)) {
// Start async sync in background
this.backgroundSync().catch(error => {
console.error(`[${this.synapseId}] Background sync failed:`, error)
})
}
return result
}
/**
* Determine if sync should be triggered after an operation
*/
protected shouldSync(operation: string, params: any): boolean {
// Override in implementations for specific sync triggers
return false
}
/**
* Background sync process
*/
protected async backgroundSync(): Promise<void> {
if (this.syncInProgress) return
this.syncInProgress = true
try {
await this.incrementalSync(this.lastSyncId)
} finally {
this.syncInProgress = false
}
}
protected async onShutdown(): Promise<void> {
if (this.syncInProgress) {
await this.stopSync()
}
await this.onSynapseShutdown()
}
protected async onSynapseShutdown(): Promise<void> {
// Override in implementations for cleanup
}
// getSynapseStatus implemented below with full response
/**
* ISynapseAugmentation methods
*/
abstract testConnection(): Promise<AugmentationResponse<boolean>>
abstract startSync(options?: Record<string, unknown>): Promise<AugmentationResponse<{
synced: number
failed: number
skipped: number
duration: number
errors?: Array<{ item: string; error: string }>
}>>
async stopSync(): Promise<void> {
this.syncInProgress = false
}
abstract incrementalSync(lastSyncId?: string): Promise<AugmentationResponse<{
synced: number
failed: number
skipped: number
duration: number
hasMore: boolean
nextSyncId?: string
}>>
abstract previewSync(limit?: number): Promise<AugmentationResponse<{
items: Array<{
type: string
title: string
preview: string
}>
totalCount: number
estimatedDuration: number
}>>
async getSynapseStatus(): Promise<AugmentationResponse<{
status: 'connected' | 'disconnected' | 'syncing' | 'error'
lastSync?: string
nextSync?: string
totalSyncs: number
totalItems: number
}>> {
const connectionTest = await this.testConnection()
return {
success: true,
data: {
status: this.syncInProgress ? 'syncing' :
connectionTest.success ? 'connected' : 'disconnected',
lastSync: this.syncStats.lastSync,
totalSyncs: this.syncStats.totalSyncs,
totalItems: this.syncStats.totalItems
}
}
}
/**
* Helper method to store synced data in Brainy
* Optionally uses Neural Import for intelligent processing
*/
protected async storeInBrainy(
content: string | Record<string, any>,
metadata: Record<string, any>,
options: {
useNeuralImport?: boolean
dataType?: string
rawData?: Buffer | string
} = {}
): Promise<void> {
if (!this.context?.brain) {
throw new Error('Brainy context not initialized')
}
// Add synapse source metadata
const enrichedMetadata = {
...metadata,
_synapse: this.synapseId,
_syncedAt: new Date().toISOString()
}
// Use Neural Import for intelligent processing if available
if (this.neuralImport && (options.useNeuralImport ?? this.useNeuralImport)) {
try {
// Process through Neural Import for entity/relationship detection
const rawData = options.rawData ||
(typeof content === 'string' ? content : JSON.stringify(content))
const neuralResult = await this.neuralImport.processRawData(
rawData,
options.dataType || 'json',
{
sourceSystem: this.synapseId,
metadata: enrichedMetadata
}
)
if (neuralResult.success && neuralResult.data) {
// Store detected nouns (entities)
for (const noun of neuralResult.data.nouns) {
await this.context.brain.add({
text: noun,
metadata: {
...enrichedMetadata,
_neuralConfidence: neuralResult.data.confidence,
_neuralInsights: neuralResult.data.insights
}
})
}
// Store detected verbs (relationships)
for (const verb of neuralResult.data.verbs) {
// Parse verb format: "source->relation->target"
const parts = verb.split('->')
if (parts.length === 3) {
await this.context.brain.relate(
parts[0], // source
parts[2], // target
parts[1], // verb type
enrichedMetadata
)
}
}
// Store original content with neural metadata
if (typeof content === 'string') {
await this.context.brain.add({
text: content,
metadata: {
...enrichedMetadata,
category: 'Content',
_neuralProcessed: true,
_neuralConfidence: neuralResult.data.confidence,
_detectedEntities: neuralResult.data.nouns.length,
_detectedRelationships: neuralResult.data.verbs.length
}
})
}
return // Successfully processed with Neural Import
}
} catch (error) {
console.warn(`[${this.synapseId}] Neural Import processing failed, falling back to basic import:`, error)
}
}
// Fallback to basic storage
if (typeof content === 'string') {
await this.context.brain.add({
text: content,
metadata: {
...enrichedMetadata,
category: 'Content'
}
})
} else {
// For structured data, store as JSON
await this.context.brain.add({
text: JSON.stringify(content),
metadata: {
...enrichedMetadata,
category: 'Content'
}
})
}
}
/**
* Helper method to query existing synced data
*/
protected async queryBrainy(
filter: { connector?: string; [key: string]: any }
): Promise<any[]> {
if (!this.context?.brain) {
throw new Error('Brainy context not initialized')
}
const searchFilter = {
...filter,
_synapse: this.synapseId
}
return this.context.brain.find({
where: searchFilter
})
}
}
/**
* Example implementation for reference
* Real synapses would be in Brain Cloud registry
*/
export class ExampleFileSystemSynapse extends SynapseAugmentation {
readonly name = 'example-filesystem-synapse'
readonly description = 'Example synapse for local file system with Neural Import intelligence'
readonly synapseId = 'filesystem'
readonly supportedTypes = ['text', 'markdown', 'json', 'csv']
protected async onInitialize(): Promise<void> {
// Initialize file system watcher, etc.
}
async testConnection(): Promise<AugmentationResponse<boolean>> {
// Test if we can access the configured directory
return {
success: true,
data: true
}
}
async startSync(options?: Record<string, unknown>): Promise<AugmentationResponse<{
synced: number
failed: number
skipped: number
duration: number
errors?: Array<{ item: string; error: string }>
}>> {
const startTime = Date.now()
// Example: Read files from a directory and sync to Brainy
// This would normally scan a directory, but here's a conceptual example:
const exampleFiles = [
{
path: '/data/notes.md',
content: '# Project Notes\nDiscuss roadmap with team\nReview Q1 metrics',
type: 'markdown'
},
{
path: '/data/contacts.json',
content: { name: 'John Doe', role: 'Developer', team: 'Engineering' },
type: 'json'
}
]
let synced = 0
const errors: Array<{ item: string; error: string }> = []
for (const file of exampleFiles) {
try {
// Use Neural Import for intelligent processing
await this.storeInBrainy(
file.content,
{
path: file.path,
fileType: file.type,
syncedFrom: 'filesystem'
},
{
useNeuralImport: true, // Enable AI processing
dataType: file.type
}
)
synced++
} catch (error) {
errors.push({
item: file.path,
error: error instanceof Error ? error.message : 'Unknown error'
})
}
}
this.syncStats.totalSyncs++
this.syncStats.totalItems += synced
this.syncStats.lastSync = new Date().toISOString()
return {
success: true,
data: {
synced,
failed: errors.length,
skipped: 0,
duration: Date.now() - startTime,
errors: errors.length > 0 ? errors : undefined
}
}
}
async incrementalSync(lastSyncId?: string): Promise<AugmentationResponse<{
synced: number
failed: number
skipped: number
duration: number
hasMore: boolean
nextSyncId?: string
}>> {
const startTime = Date.now()
// Example: Check for modified files since last sync
return {
success: true,
data: {
synced: 0,
failed: 0,
skipped: 0,
duration: Date.now() - startTime,
hasMore: false
}
}
}
async previewSync(limit: number = 10): Promise<AugmentationResponse<{
items: Array<{
type: string
title: string
preview: string
}>
totalCount: number
estimatedDuration: number
}>> {
// Example: List files that would be synced
return {
success: true,
data: {
items: [],
totalCount: 0,
estimatedDuration: 0
}
}
}
}

579
src/augmentations/typeMatching/brainyTypes.ts
View file

@ -1,579 +0,0 @@
/**
* BrainyTypes - Intelligent type detection using semantic embeddings
*
* This module uses our existing TransformerEmbedding and similarity functions
* to intelligently match data to our 42 noun types and 127 verb types.
*
* Features:
* - Semantic similarity matching using embeddings
* - Context-aware type detection
* - Confidence scoring
* - Caching for performance
*/
import { NounType, VerbType } from '../../types/graphTypes.js'
import { TransformerEmbedding } from '../../utils/embedding.js'
import { cosineDistance } from '../../utils/distance.js'
import { Vector } from '../../coreTypes.js'
import { getNounTypeEmbeddings, getVerbTypeEmbeddings } from '../../neural/embeddedTypeEmbeddings.js'
/**
* Type descriptions for semantic matching
* These descriptions are used to generate embeddings for each type
*/
const NOUN_TYPE_DESCRIPTIONS: Record<string, string> = {
// Core Entity Types (7)
[NounType.Person]: 'person human individual user employee customer citizen member author creator agent actor participant',
[NounType.Organization]: 'organization company business corporation institution agency department team group committee board',
[NounType.Location]: 'location place address city country region area zone coordinate position site venue building',
[NounType.Thing]: 'thing object item product device equipment tool instrument asset artifact material physical tangible',
[NounType.Concept]: 'concept idea theory principle philosophy belief value abstract intangible notion thought',
[NounType.Event]: 'event occurrence incident activity happening meeting conference celebration milestone timestamp date',
[NounType.Agent]: 'agent AI bot automated system automation software assistant service daemon daemon worker processor',
// Biological Types (1) - Stage 3
[NounType.Organism]: 'organism animal plant bacteria fungi species living biological life creature being microorganism',
// Material Types (1) - Stage 3
[NounType.Substance]: 'substance material matter chemical element compound liquid gas solid molecule atom material',
// Property & Quality Types (1) - Stage 3
[NounType.Quality]: 'quality property attribute characteristic feature aspect trait dimension parameter value',
// Temporal Types (1) - Stage 3
[NounType.TimeInterval]: 'timeInterval period duration span range epoch era season quarter interval window',
// Functional Types (1) - Stage 3
[NounType.Function]: 'function purpose role capability capacity utility objective goal aim intent',
// Informational Types (1) - Stage 3
[NounType.Proposition]: 'proposition statement claim assertion declaration premise conclusion hypothesis theory',
// Digital/Content Types
[NounType.Document]: 'document file report article paper text pdf word contract agreement record documentation',
[NounType.Media]: 'media image photo video audio music podcast multimedia graphic visualization animation',
[NounType.File]: 'file digital data binary code script program software archive package bundle',
[NounType.Message]: 'message email chat communication notification alert announcement broadcast transmission',
// Collection Types
[NounType.Collection]: 'collection group set list array category folder directory catalog inventory database',
[NounType.Dataset]: 'dataset data table spreadsheet database records statistics metrics measurements analysis',
// Business/Application Types
[NounType.Product]: 'product item merchandise offering service feature application software solution package',
[NounType.Service]: 'service offering subscription support maintenance utility function capability',
[NounType.Task]: 'task action todo item job assignment duty responsibility activity step procedure',
[NounType.Project]: 'project initiative program campaign effort endeavor plan scheme venture undertaking',
// Descriptive Types
[NounType.Process]: 'process workflow procedure method algorithm sequence pipeline operation routine protocol',
[NounType.State]: 'state status condition phase stage mode situation circumstance configuration setting',
[NounType.Role]: 'role position title function responsibility duty job capacity designation authority',
[NounType.Language]: 'language dialect locale tongue vernacular communication speech linguistics vocabulary',
[NounType.Currency]: 'currency money dollar euro pound yen bitcoin payment financial monetary unit',
[NounType.Measurement]: 'measurement metric quantity value amount size dimension weight height volume distance',
// Scientific/Research Types
[NounType.Hypothesis]: 'hypothesis theory proposition thesis assumption premise conjecture speculation prediction',
[NounType.Experiment]: 'experiment test trial study research investigation analysis observation examination',
// Legal/Regulatory Types
[NounType.Contract]: 'contract agreement deal treaty pact covenant license terms conditions policy',
[NounType.Regulation]: 'regulation law rule policy standard compliance requirement guideline ordinance statute',
// Technical Infrastructure Types
[NounType.Interface]: 'interface API endpoint protocol specification contract schema definition connection',
[NounType.Resource]: 'resource infrastructure server database storage compute memory bandwidth capacity asset',
// Custom/Extensible (1) - Stage 3
[NounType.Custom]: 'custom special unique particular specific domain-specific specialized tailored bespoke',
// Social Structures (3) - Stage 3
[NounType.SocialGroup]: 'socialGroup community tribe clan network circle collective society crowd gathering',
[NounType.Institution]: 'institution establishment foundation organization system structure framework tradition practice',
[NounType.Norm]: 'norm convention standard custom tradition protocol etiquette rule practice expectation',
// Information Theory (2) - Stage 3
[NounType.InformationContent]: 'informationContent story narrative knowledge data schema pattern model structure',
[NounType.InformationBearer]: 'informationBearer carrier medium vehicle signal token representation document',
// Meta-Level (1) - Stage 3
[NounType.Relationship]: 'relationship connection relation association link bond tie interaction dynamic'
}
const VERB_TYPE_DESCRIPTIONS: Record<string, string> = {
// Core Relationship Types
[VerbType.RelatedTo]: 'related connected associated linked correlated relevant pertinent applicable',
[VerbType.Contains]: 'contains includes holds stores encompasses comprises consists incorporates',
[VerbType.PartOf]: 'part component element member piece portion section segment constituent',
[VerbType.LocatedAt]: 'located situated positioned placed found exists resides occupies',
[VerbType.References]: 'references cites mentions points links refers quotes sources',
// Temporal/Causal Types
[VerbType.Precedes]: 'precedes before earlier prior previous antecedent preliminary foregoing',
[VerbType.Causes]: 'causes triggers induces produces generates results influences affects',
[VerbType.DependsOn]: 'depends requires needs relies necessitates contingent prerequisite',
[VerbType.Requires]: 'requires needs demands necessitates mandates obliges compels entails',
// Creation/Transformation Types
[VerbType.Creates]: 'creates makes produces generates builds constructs forms establishes',
[VerbType.Transforms]: 'transforms converts changes modifies alters transitions morphs evolves',
[VerbType.Becomes]: 'becomes turns evolves transforms changes transitions develops grows',
[VerbType.Modifies]: 'modifies changes updates alters edits revises adjusts adapts',
[VerbType.Consumes]: 'consumes uses utilizes depletes expends absorbs takes processes',
// Ownership/Attribution Types
[VerbType.Owns]: 'owns possesses holds controls manages administers governs maintains',
[VerbType.AttributedTo]: 'attributed credited assigned ascribed authored written composed',
// Social/Organizational Types
[VerbType.MemberOf]: 'member participant affiliate associate belongs joined enrolled registered',
[VerbType.WorksWith]: 'works collaborates cooperates partners teams assists helps supports',
[VerbType.FriendOf]: 'friend companion buddy pal acquaintance associate connection relationship',
[VerbType.Follows]: 'follows subscribes tracks monitors watches observes trails pursues',
[VerbType.Likes]: 'likes enjoys appreciates favors prefers admires values endorses',
[VerbType.ReportsTo]: 'reports answers subordinate accountable responsible supervised managed',
[VerbType.Mentors]: 'mentors teaches guides coaches instructs trains advises counsels',
[VerbType.Communicates]: 'communicates talks speaks messages contacts interacts corresponds exchanges',
// Descriptive/Functional Types
[VerbType.Describes]: 'describes explains details documents specifies outlines depicts characterizes',
[VerbType.Defines]: 'defines specifies establishes determines sets declares identifies designates',
[VerbType.Categorizes]: 'categorizes classifies groups sorts organizes arranges labels tags',
[VerbType.Measures]: 'measures quantifies gauges assesses evaluates calculates determines counts',
[VerbType.Evaluates]: 'evaluates assesses analyzes reviews examines appraises judges rates',
[VerbType.Uses]: 'uses utilizes employs applies operates handles manipulates exploits',
[VerbType.Implements]: 'implements executes realizes performs accomplishes carries delivers completes',
[VerbType.Extends]: 'extends expands enhances augments amplifies broadens enlarges develops',
// Enhanced Relationships
[VerbType.Inherits]: 'inherits derives extends receives obtains acquires succeeds legacy',
[VerbType.Conflicts]: 'conflicts contradicts opposes clashes disputes disagrees incompatible inconsistent',
[VerbType.Synchronizes]: 'synchronizes coordinates aligns harmonizes matches corresponds parallels coincides',
[VerbType.Competes]: 'competes rivals contends contests challenges opposes vies struggles'
}
/**
* Result of type matching with confidence scores
*/
export interface TypeMatchResult {
type: string
confidence: number
reasoning: string
alternatives: Array<{
type: string
confidence: number
}>
}
/**
* BrainyTypes - Intelligent type detection for nouns and verbs
* PRODUCTION OPTIMIZATION: Uses pre-computed type embeddings
* Type embeddings are loaded instantly; only input objects are embedded at runtime
*/
export class BrainyTypes {
private embedder: TransformerEmbedding // Only for embedding input objects
private nounEmbeddings: Map<string, Vector> = new Map()
private verbEmbeddings: Map<string, Vector> = new Map()
private initialized = false
private cache: Map<string, TypeMatchResult> = new Map()
constructor() {
// Embedder only used for input objects, NOT for type embeddings
this.embedder = new TransformerEmbedding({ verbose: false })
}
/**
* Initialize the type matcher by loading pre-computed embeddings
* INSTANT - type embeddings are loaded from pre-computed data
* Only the model for input embedding needs initialization
*/
async init(): Promise<void> {
if (this.initialized) return
// Initialize embedder for input objects only
await this.embedder.init()
// Load pre-computed type embeddings (instant, no computation)
const nounEmbeddings = getNounTypeEmbeddings()
const verbEmbeddings = getVerbTypeEmbeddings()
// Convert NounType/VerbType keys to strings for lookup
for (const [type, embedding] of nounEmbeddings.entries()) {
this.nounEmbeddings.set(type, embedding)
}
for (const [type, embedding] of verbEmbeddings.entries()) {
this.verbEmbeddings.set(type, embedding)
}
this.initialized = true
}
/**
* Match an object to the most appropriate noun type
*/
async matchNounType(obj: any): Promise<TypeMatchResult> {
await this.init()
// Create a text representation of the object for embedding
const textRepresentation = this.createTextRepresentation(obj)
// Check cache
const cacheKey = `noun:${textRepresentation}`
if (this.cache.has(cacheKey)) {
return this.cache.get(cacheKey)!
}
// Generate embedding for the input
const inputEmbedding = await this.embedder.embed(textRepresentation)
// Calculate similarities to all noun types
const similarities: Array<{ type: string; similarity: number }> = []
for (const [type, typeEmbedding] of this.nounEmbeddings.entries()) {
// Convert cosine distance to similarity (1 - distance)
const similarity = 1 - cosineDistance(inputEmbedding, typeEmbedding)
similarities.push({ type, similarity })
}
// Sort by similarity (highest first)
similarities.sort((a, b) => b.similarity - a.similarity)
// Apply heuristic rules for common patterns
const heuristicType = this.applyNounHeuristics(obj)
if (heuristicType) {
// Boost the heuristic type's confidence
const heuristicIndex = similarities.findIndex(s => s.type === heuristicType)
if (heuristicIndex > 0) {
similarities[heuristicIndex].similarity *= 1.2 // 20% boost
similarities.sort((a, b) => b.similarity - a.similarity)
}
}
// Create result
const result: TypeMatchResult = {
type: similarities[0].type,
confidence: similarities[0].similarity,
reasoning: this.generateReasoning(obj, similarities[0].type, 'noun'),
alternatives: similarities.slice(1, 4).map(s => ({
type: s.type,
confidence: s.similarity
}))
}
// Cache result
this.cache.set(cacheKey, result)
return result
}
/**
* Match a relationship to the most appropriate verb type
*/
async matchVerbType(
sourceObj: any,
targetObj: any,
relationshipHint?: string
): Promise<TypeMatchResult> {
await this.init()
// Create text representation of the relationship
const textRepresentation = this.createRelationshipText(sourceObj, targetObj, relationshipHint)
// Check cache
const cacheKey = `verb:${textRepresentation}`
if (this.cache.has(cacheKey)) {
return this.cache.get(cacheKey)!
}
// Generate embedding
const inputEmbedding = await this.embedder.embed(textRepresentation)
// Calculate similarities to all verb types
const similarities: Array<{ type: string; similarity: number }> = []
for (const [type, typeEmbedding] of this.verbEmbeddings.entries()) {
const similarity = 1 - cosineDistance(inputEmbedding, typeEmbedding)
similarities.push({ type, similarity })
}
// Sort by similarity
similarities.sort((a, b) => b.similarity - a.similarity)
// Apply heuristic rules
const heuristicType = this.applyVerbHeuristics(sourceObj, targetObj, relationshipHint)
if (heuristicType) {
const heuristicIndex = similarities.findIndex(s => s.type === heuristicType)
if (heuristicIndex > 0) {
similarities[heuristicIndex].similarity *= 1.2
similarities.sort((a, b) => b.similarity - a.similarity)
}
}
// Create result
const result: TypeMatchResult = {
type: similarities[0].type,
confidence: similarities[0].similarity,
reasoning: this.generateReasoning(
{ source: sourceObj, target: targetObj, hint: relationshipHint },
similarities[0].type,
'verb'
),
alternatives: similarities.slice(1, 4).map(s => ({
type: s.type,
confidence: s.similarity
}))
}
// Cache result
this.cache.set(cacheKey, result)
return result
}
/**
* Create text representation of an object for embedding
*/
private createTextRepresentation(obj: any): string {
const parts: string[] = []
// Add type if available
if (typeof obj === 'object' && obj !== null) {
// Add field names and values
for (const [key, value] of Object.entries(obj)) {
parts.push(key)
if (typeof value === 'string') {
parts.push(value.slice(0, 100)) // Limit string length
} else if (typeof value === 'number' || typeof value === 'boolean') {
parts.push(String(value))
}
}
// Add special fields with higher weight
const importantFields = ['type', 'kind', 'category', 'class', 'name', 'title', 'description']
for (const field of importantFields) {
if (obj[field]) {
parts.push(String(obj[field]))
parts.push(String(obj[field])) // Double weight for important fields
}
}
} else if (typeof obj === 'string') {
parts.push(obj)
} else {
parts.push(String(obj))
}
return parts.join(' ')
}
/**
* Create text representation of a relationship
*/
private createRelationshipText(
sourceObj: any,
targetObj: any,
relationshipHint?: string
): string {
const parts: string[] = []
if (relationshipHint) {
parts.push(relationshipHint)
parts.push(relationshipHint) // Double weight for explicit hint
}
// Add source context
if (sourceObj) {
parts.push('source:')
parts.push(this.getObjectSummary(sourceObj))
}
// Add target context
if (targetObj) {
parts.push('target:')
parts.push(this.getObjectSummary(targetObj))
}
return parts.join(' ')
}
/**
* Get a brief summary of an object
*/
private getObjectSummary(obj: any): string {
if (typeof obj === 'string') return obj.slice(0, 50)
if (typeof obj !== 'object' || obj === null) return String(obj)
const summary: string[] = []
const fields = ['type', 'name', 'title', 'id', 'category', 'kind']
for (const field of fields) {
if (obj[field]) {
summary.push(String(obj[field]))
}
}
return summary.join(' ').slice(0, 100)
}
/**
* Apply heuristic rules for noun type detection
*/
private applyNounHeuristics(obj: any): string | null {
if (typeof obj !== 'object' || obj === null) return null
// Person heuristics
if (obj.email || obj.firstName || obj.lastName || obj.username || obj.age || obj.gender) {
return NounType.Person
}
// Organization heuristics
if (obj.companyName || obj.organizationId || obj.employees || obj.industry) {
return NounType.Organization
}
// Location heuristics
if (obj.latitude || obj.longitude || obj.address || obj.city || obj.country || obj.coordinates) {
return NounType.Location
}
// Document heuristics
if (obj.content && (obj.title || obj.author) || obj.documentType || obj.pages) {
return NounType.Document
}
// Event heuristics
if (obj.startTime || obj.endTime || obj.date || obj.eventType || obj.attendees) {
return NounType.Event
}
// Product heuristics
if (obj.price || obj.sku || obj.inventory || obj.productId) {
return NounType.Product
}
// Task heuristics
if (obj.status && (obj.assignee || obj.dueDate) || obj.priority || obj.completed !== undefined) {
return NounType.Task
}
// Media heuristics
if (obj.url && (obj.url.match(/\.(jpg|jpeg|png|gif|mp4|mp3|wav)/i))) {
return NounType.Media
}
// Dataset heuristics
if (Array.isArray(obj.data) || obj.rows || obj.columns || obj.schema) {
return NounType.Dataset
}
return null
}
/**
* Apply heuristic rules for verb type detection
*/
private applyVerbHeuristics(
sourceObj: any,
targetObj: any,
relationshipHint?: string
): string | null {
if (!relationshipHint) return null
const hint = relationshipHint.toLowerCase()
// Ownership patterns
if (hint.includes('own') || hint.includes('possess') || hint.includes('has')) {
return VerbType.Owns
}
// Creation patterns
if (hint.includes('create') || hint.includes('made') || hint.includes('authored')) {
return VerbType.Creates
}
// Containment patterns
if (hint.includes('contain') || hint.includes('include') || hint.includes('has')) {
return VerbType.Contains
}
// Membership patterns
if (hint.includes('member') || hint.includes('belong') || hint.includes('part')) {
return VerbType.MemberOf
}
// Reference patterns
if (hint.includes('refer') || hint.includes('cite') || hint.includes('link')) {
return VerbType.References
}
// Dependency patterns
if (hint.includes('depend') || hint.includes('require') || hint.includes('need')) {
return VerbType.DependsOn
}
return null
}
/**
* Generate human-readable reasoning for the type selection
*/
private generateReasoning(
obj: any,
selectedType: string,
typeKind: 'noun' | 'verb'
): string {
const descriptions = typeKind === 'noun' ? NOUN_TYPE_DESCRIPTIONS : VERB_TYPE_DESCRIPTIONS
const typeDesc = descriptions[selectedType]
// Handle missing descriptions gracefully
if (!typeDesc) {
if (typeKind === 'noun') {
const fields = Object.keys(obj).slice(0, 3).join(', ')
return `Matched to ${selectedType} based on semantic similarity and object fields: ${fields}`
} else {
return `Matched to ${selectedType} based on semantic similarity and relationship context`
}
}
if (typeKind === 'noun') {
const fields = Object.keys(obj).slice(0, 3).join(', ')
return `Matched to ${selectedType} based on semantic similarity to "${typeDesc.split(' ').slice(0, 5).join(' ')}..." and object fields: ${fields}`
} else {
return `Matched to ${selectedType} based on semantic similarity to "${typeDesc.split(' ').slice(0, 5).join(' ')}..." and relationship context`
}
}
/**
* Clear the cache
*/
clearCache(): void {
this.cache.clear()
}
/**
* Dispose of resources
*/
async dispose(): Promise<void> {
await this.embedder.dispose()
this.cache.clear()
this.nounEmbeddings.clear()
this.verbEmbeddings.clear()
}
}
/**
* Singleton instance for efficient reuse
*/
let globalInstance: BrainyTypes | null = null
/**
* Get or create the global BrainyTypes instance
*/
export async function getBrainyTypes(): Promise<BrainyTypes> {
if (!globalInstance) {
globalInstance = new BrainyTypes()
await globalInstance.init()
}
return globalInstance
}

180
src/augmentations/typeMatching/intelligentTypeMatcher.ts
View file

@ -1,180 +0,0 @@
/**
* IntelligentTypeMatcher - Wrapper around BrainyTypes for testing
*
* Provides intelligent type detection using semantic embeddings
* for matching data to our 42 noun types and 127 verb types.
*/
import { NounType, VerbType } from '../../types/graphTypes.js'
import { BrainyTypes, TypeMatchResult, getBrainyTypes } from './brainyTypes.js'
export interface TypeMatchOptions {
threshold?: number
topK?: number
useCache?: boolean
}
/**
* Intelligent type matcher using semantic embeddings
*/
export class IntelligentTypeMatcher {
private brainyTypes: BrainyTypes | null = null
private cache = new Map<string, TypeMatchResult>()
constructor(private options: TypeMatchOptions = {}) {
this.options = {
threshold: 0.3,
topK: 3,
useCache: true,
...options
}
}
/**
* Initialize the type matcher
*/
async init(): Promise<void> {
this.brainyTypes = await getBrainyTypes()
await this.brainyTypes.init()
}
/**
* Dispose of resources
*/
async dispose(): Promise<void> {
if (this.brainyTypes) {
await this.brainyTypes.dispose()
this.brainyTypes = null
}
this.cache.clear()
}
/**
* Match data to a noun type
*/
async matchNounType(data: any): Promise<{
type: NounType
confidence: number
alternatives: Array<{ type: NounType; confidence: number }>
}> {
if (!this.brainyTypes) {
throw new Error('IntelligentTypeMatcher not initialized. Call init() first.')
}
// Check cache if enabled
const cacheKey = JSON.stringify(data)
if (this.options.useCache && this.cache.has(cacheKey)) {
const cached = this.cache.get(cacheKey)!
return {
type: cached.type as NounType,
confidence: cached.confidence,
alternatives: cached.alternatives?.map(alt => ({
type: alt.type as NounType,
confidence: alt.confidence
})) || []
}
}
// Detect type using BrainyTypes
const result = await this.brainyTypes.matchNounType(data)
// Convert to expected format
const response = {
type: result.type as NounType,
confidence: result.confidence,
alternatives: result.alternatives?.map(alt => ({
type: alt.type as NounType,
confidence: alt.confidence
})) || []
}
// Cache the result if enabled
if (this.options.useCache) {
this.cache.set(cacheKey, result)
}
return response
}
/**
* Match a relationship to a verb type
*/
async matchVerbType(
source: any,
target: any,
relationship?: string
): Promise<{
type: VerbType
confidence: number
alternatives: Array<{ type: VerbType; confidence: number }>
}> {
if (!this.brainyTypes) {
throw new Error('IntelligentTypeMatcher not initialized. Call init() first.')
}
// Create context for verb detection
const context = {
source,
target,
relationship: relationship || 'related',
description: relationship || ''
}
// Check cache if enabled
const cacheKey = JSON.stringify(context)
if (this.options.useCache && this.cache.has(cacheKey)) {
const cached = this.cache.get(cacheKey)!
return {
type: cached.type as VerbType || VerbType.RelatedTo,
confidence: cached.confidence,
alternatives: cached.alternatives?.map(alt => ({
type: alt.type as VerbType || VerbType.RelatedTo,
confidence: alt.confidence
})) || []
}
}
// Detect verb type using BrainyTypes
const result = await this.brainyTypes.matchVerbType(
context.source,
context.target,
context.relationship
)
// Convert to expected format
const response = {
type: result.type as VerbType || VerbType.RelatedTo,
confidence: result.confidence,
alternatives: result.alternatives?.map(alt => ({
type: alt.type as VerbType || VerbType.RelatedTo,
confidence: alt.confidence
})) || []
}
// Cache the result if enabled
if (this.options.useCache) {
this.cache.set(cacheKey, result)
}
return response
}
/**
* Clear the cache
*/
clearCache(): void {
this.cache.clear()
}
/**
* Get cache statistics
*/
getCacheStats(): { size: number; maxSize: number } {
return {
size: this.cache.size,
maxSize: 1000 // Default max cache size
}
}
}
export default IntelligentTypeMatcher

449
src/augmentations/universalDisplayAugmentation.ts
View file

@ -1,449 +0,0 @@
/**
* Universal Display Augmentation
*
* 🎨 Provides intelligent display fields for any noun or verb using AI-powered analysis
*
* Features:
* - ✅ Leverages existing BrainyTypes for semantic type detection
* - ✅ Complete icon coverage for all 42 NounTypes + 127 VerbTypes
* - ✅ Zero performance impact with lazy computation and intelligent caching
* - ✅ Perfect isolation - can be disabled, replaced, or configured
* - ✅ Clean developer experience with zero conflicts
* - ✅ TypeScript support with full autocomplete
*
* Usage:
* ```typescript
* // User data access (unchanged)
* result.firstName // "John"
* result.metadata.title // "CEO"
*
* // Enhanced display (new capabilities)
* result.getDisplay('title') // "John Doe" (AI-computed)
* result.getDisplay('description') // "CEO at Acme Corp" (enhanced)
* result.getDisplay('type') // "Person" (from AI detection)
* result.getDisplay() // All display fields
* ```
*/
import { BaseAugmentation, AugmentationContext, MetadataAccess } from './brainyAugmentation.js'
import type { VectorDocument, GraphVerb } from '../coreTypes.js'
import type {
DisplayConfig,
ComputedDisplayFields,
EnhancedVectorDocument,
EnhancedGraphVerb,
DisplayAugmentationStats
} from './display/types.js'
import { IntelligentComputationEngine } from './display/intelligentComputation.js'
import { DisplayCache, RequestDeduplicator, getGlobalDisplayCache } from './display/cache.js'
import { getNounIcon, getVerbIcon, getIconCoverage } from './display/iconMappings.js'
/**
* Universal Display Augmentation
*
* Self-contained augmentation that provides intelligent display fields
* for any data type using existing Brainy AI infrastructure
*/
export class UniversalDisplayAugmentation extends BaseAugmentation {
readonly name = 'display'
readonly version = '1.0.0'
readonly timing = 'after' as const // Enhance results after main operations
readonly priority = 50 // Medium priority - after core operations
readonly metadata: MetadataAccess = {
reads: '*', // Read all user data for intelligent analysis
writes: ['_display'] // Cache computed fields in isolated namespace
}
operations = ['get', 'search', 'find', 'similar', 'findSimilar', 'getVerb' as any, 'add', 'addNoun', 'addVerb', 'relate'] as any
// Augmentation metadata
readonly category = 'core' as const
readonly description = 'AI-powered intelligent display fields for enhanced data visualization'
// Computed fields declaration for TypeScript support and discovery
computedFields = {
display: {
title: { type: 'string' as const, description: 'Primary display name (AI-computed)' },
description: { type: 'string' as const, description: 'Enhanced description with context' },
type: { type: 'string' as const, description: 'Human-readable type (from AI detection)' },
tags: { type: 'array' as const, description: 'Generated display tags' },
relationship: { type: 'string' as const, description: 'Human-readable relationship (verbs only)' },
confidence: { type: 'number' as const, description: 'AI confidence score (0-1)' }
}
}
// Core components (all self-contained)
private computationEngine: IntelligentComputationEngine
private displayCache: DisplayCache
private requestDeduplicator: RequestDeduplicator
protected config: DisplayConfig
protected context: AugmentationContext | undefined
constructor(config: Partial<DisplayConfig> = {}) {
super()
// Merge with defaults
this.config = {
enabled: true,
cacheSize: 1000,
lazyComputation: true,
batchSize: 50,
confidenceThreshold: 0.7,
customFieldMappings: {},
priorityFields: {},
debugMode: false,
...config
}
// Initialize components
this.computationEngine = new IntelligentComputationEngine(this.config)
this.displayCache = getGlobalDisplayCache(this.config.cacheSize)
this.requestDeduplicator = new RequestDeduplicator(this.config.batchSize)
}
/**
* Initialize the augmentation with AI components
* @param context Brainy context
*/
async initialize(context: AugmentationContext): Promise<void> {
if (!this.config.enabled) {
this.log('🎨 Universal Display augmentation disabled')
return
}
this.context = context
try {
// Initialize AI-powered computation engine
await this.computationEngine.initialize()
this.log('🎨 Universal Display augmentation initialized successfully')
this.log(` Cache size: ${this.config.cacheSize}`)
this.log(` Lazy computation: ${this.config.lazyComputation}`)
this.log(` Coverage: ${this.getCoverageInfo()}`)
} catch (error) {
this.log('⚠️ Display augmentation initialization warning:', 'warn')
this.log(` ${error}`, 'warn')
this.log(' Falling back to basic mode', 'warn')
}
}
/**
* Execute augmentation - attach display capabilities to results
* @param operation The operation being performed
* @param params Operation parameters
* @param next Function to execute main operation
* @returns Enhanced result with display capabilities
*/
async execute<T = any>(
operation: string,
params: any,
next: () => Promise<T>
): Promise<T> {
// Always execute main operation first
const result = await next()
// Only enhance if enabled and operation is relevant
if (!this.config.enabled || !this.shouldEnhanceOperation(operation)) {
return result
}
try {
// Enhance result with display capabilities
return this.enhanceWithDisplayCapabilities(result, operation) as T
} catch (error) {
this.log(`Display enhancement failed for ${operation}: ${error}`, 'warn')
return result // Return unenhanced result on error
}
}
/**
* Check if operation should be enhanced
* @param operation Operation name
* @returns True if should enhance
*/
private shouldEnhanceOperation(operation: string): boolean {
const enhanceableOps = ['get', 'search', 'findSimilar', 'getVerb']
return enhanceableOps.includes(operation)
}
/**
* Enhance result with display capabilities
* @param result The operation result
* @param operation The operation type
* @returns Enhanced result
*/
private enhanceWithDisplayCapabilities(result: any, operation: string): any {
if (!result) return result
// Handle different result types
if (Array.isArray(result)) {
// Array of results (search, findSimilar)
return result.map(item => this.enhanceEntity(item))
} else if (result.id || result.metadata) {
// Single entity (get, getVerb)
return this.enhanceEntity(result)
}
return result
}
/**
* Enhance a single entity with display capabilities
* @param entity The entity to enhance
* @returns Enhanced entity
*/
private enhanceEntity(entity: any): EnhancedVectorDocument | EnhancedGraphVerb {
if (!entity) return entity
// Determine if it's a noun or verb
const isVerb = this.isVerbEntity(entity)
// Add display methods
const enhanced = {
...entity,
getDisplay: this.createGetDisplayMethod(entity, isVerb),
getAvailableFields: this.createGetAvailableFieldsMethod(),
getAvailableAugmentations: this.createGetAvailableAugmentationsMethod(),
explore: this.createExploreMethod(entity)
}
return enhanced
}
/**
* Create getDisplay method for an entity
* @param entity The entity
* @param isVerb Whether it's a verb entity
* @returns getDisplay function
*/
private createGetDisplayMethod(entity: any, isVerb: boolean) {
return async (field?: keyof ComputedDisplayFields): Promise<any> => {
// Generate cache key
const cacheKey = this.displayCache.generateKey(
entity.id,
entity.metadata || entity,
isVerb ? 'verb' : 'noun'
)
// Use request deduplicator to prevent duplicate computations
const computedFields = await this.requestDeduplicator.deduplicate(
cacheKey,
async () => {
// Check cache first
let cached = this.displayCache.get(cacheKey)
if (cached) return cached
// Compute display fields
const startTime = Date.now()
let computed: ComputedDisplayFields
if (isVerb) {
computed = await this.computationEngine.computeVerbDisplay(entity as GraphVerb)
} else {
computed = await this.computationEngine.computeNounDisplay(
entity.metadata || entity,
entity.id
)
}
// Cache the result
const computationTime = Date.now() - startTime
this.displayCache.set(cacheKey, computed, computationTime)
return computed
}
)
// Return specific field or all fields
return field ? computedFields[field] : computedFields
}
}
/**
* Create getAvailableFields method
* @returns getAvailableFields function
*/
private createGetAvailableFieldsMethod() {
return (namespace: string): string[] => {
if (namespace === 'display') {
return ['title', 'description', 'type', 'tags', 'relationship', 'confidence']
}
return []
}
}
/**
* Create getAvailableAugmentations method
* @returns getAvailableAugmentations function
*/
private createGetAvailableAugmentationsMethod() {
return (): string[] => {
return ['display'] // This augmentation provides 'display' namespace
}
}
/**
* Create explore method for debugging
* @param entity The entity
* @returns explore function
*/
private createExploreMethod(entity: any) {
return async (): Promise<void> => {
// Respect silent mode
if (this.config.silent) {
return
}
console.log(`\n📋 Entity Exploration: ${entity.id || 'unknown'}`)
console.log('━'.repeat(50))
// Show user data
console.log('\n👤 User Data:')
const userData = entity.metadata || entity
for (const [key, value] of Object.entries(userData)) {
if (!key.startsWith('_')) {
console.log(` • ${key}: ${JSON.stringify(value)}`)
}
}
// Show computed display fields
try {
console.log('\n🎨 Display Fields:')
const displayMethod = this.createGetDisplayMethod(entity, this.isVerbEntity(entity))
const displayFields = await displayMethod()
for (const [key, value] of Object.entries(displayFields)) {
console.log(` • ${key}: ${JSON.stringify(value)}`)
}
} catch (error) {
console.log(` Error computing display fields: ${error}`)
}
console.log('')
}
}
/**
* Check if an entity is a verb
* @param entity The entity to check
* @returns True if it's a verb
*/
private isVerbEntity(entity: any): boolean {
return !!(entity.sourceId && entity.targetId) ||
!!(entity.source && entity.target) ||
!!entity.verb
}
/**
* Get coverage information
* @returns Coverage info string
*/
private getCoverageInfo(): string {
return 'Clean display - focuses on AI-powered content'
}
/**
* Get augmentation statistics
* @returns Performance and usage statistics
*/
getStats(): DisplayAugmentationStats {
return this.displayCache.getStats()
}
/**
* Configure the augmentation at runtime
* @param newConfig Partial configuration to merge
*/
configure(newConfig: Partial<DisplayConfig>): void {
this.config = { ...this.config, ...newConfig }
if (!this.config.enabled) {
this.displayCache.clear()
}
}
/**
* Clear all cached display data
*/
clearCache(): void {
this.displayCache.clear()
}
/**
* Precompute display fields for a batch of entities
* @param entities Array of entities to precompute
*/
async precomputeBatch(entities: Array<{ id: string; data: any }>): Promise<void> {
const computeRequests = entities.map(({ id, data }) => ({
key: this.displayCache.generateKey(id, data, 'noun'),
computeFn: () => this.computationEngine.computeNounDisplay(data, id)
}))
await this.displayCache.batchPrecompute(computeRequests)
}
/**
* Optional check if this augmentation should run
* @param operation Operation name
* @param params Operation parameters
* @returns True if should execute
*/
shouldExecute(operation: string, params: any): boolean {
return this.config.enabled && this.shouldEnhanceOperation(operation)
}
/**
* Cleanup when augmentation is shut down
*/
async shutdown(): Promise<void> {
try {
// Cleanup computation engine
await this.computationEngine.shutdown()
// Cleanup request deduplicator
this.requestDeduplicator.shutdown()
// Clear cache if configured to do so
if (this.config.debugMode) {
const stats = this.getStats()
this.log(`🎨 Display augmentation shutdown statistics:`)
this.log(` Total computations: ${stats.totalComputations}`)
this.log(` Cache hit ratio: ${(stats.cacheHitRatio * 100).toFixed(1)}%`)
this.log(` Average computation time: ${stats.averageComputationTime.toFixed(1)}ms`)
}
this.log('🎨 Universal Display augmentation shut down')
} catch (error) {
this.log(`Display augmentation shutdown error: ${error}`, 'error')
}
}
}
/**
* Factory function to create display augmentation with default config
* @param config Optional configuration overrides
* @returns Configured display augmentation instance
*/
export function createDisplayAugmentation(config: Partial<DisplayConfig> = {}): UniversalDisplayAugmentation {
return new UniversalDisplayAugmentation(config)
}
/**
* Default configuration for the display augmentation
*/
export const DEFAULT_DISPLAY_CONFIG: DisplayConfig = {
enabled: true,
cacheSize: 1000,
lazyComputation: true,
batchSize: 50,
confidenceThreshold: 0.7,
customFieldMappings: {},
priorityFields: {},
debugMode: false
}
/**
* Export for easy import and registration
*/
export default UniversalDisplayAugmentation

125
src/brainy.ts
View file

@ -18,8 +18,6 @@ import {
} from './utils/index.js'
import { embeddingManager } from './embeddings/EmbeddingManager.js'
import { matchesMetadataFilter } from './utils/metadataFilter.js'
import { AugmentationRegistry, AugmentationContext } from './augmentations/brainyAugmentation.js'
import { createDefaultAugmentations } from './augmentations/defaultAugmentations.js'
import { ImprovedNeuralAPI } from './neural/improvedNeuralAPI.js'
import { NaturalLanguageProcessor } from './neural/naturalLanguageProcessor.js'
import { NeuralEntityExtractor, ExtractedEntity } from './neural/entityExtractor.js'
@ -130,7 +128,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
private transactionManager: TransactionManager
private embedder: EmbeddingFunction
private distance: DistanceFunction
private augmentationRegistry: AugmentationRegistry
private config: Required<BrainyConfig>
// Distributed components (optional)
@ -192,7 +189,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
// Setup core components
this.distance = cosineDistance
this.embedder = this.setupEmbedder()
this.augmentationRegistry = this.setupAugmentations()
this.transactionManager = new TransactionManager()
// Setup distributed components if enabled
@ -230,7 +226,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
...configOverrides,
storage: { ...this.config.storage, ...configOverrides.storage },
index: { ...this.config.index, ...configOverrides.index },
augmentations: { ...this.config.augmentations, ...configOverrides.augmentations },
verbose: configOverrides.verbose ?? this.config.verbose,
silent: configOverrides.silent ?? this.config.silent
}
@ -332,23 +327,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
// Rebuild indexes if needed for existing data
await this.rebuildIndexesIfNeeded()
// Initialize augmentations
await this.augmentationRegistry.initializeAll({
brain: this,
storage: this.storage,
config: this.config,
log: (message: string, level = 'info') => {
// Simple logging for now
if (level === 'error') {
console.error(message)
} else if (level === 'warn') {
console.warn(message)
} else {
console.log(message)
}
}
})
// Connect distributed components to storage
await this.connectDistributedStorage()
@ -698,8 +676,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
)
}
// Execute through augmentation pipeline
return this.augmentationRegistry.execute('add', params, async () => {
// Prepare metadata for storage (backward compat format - unchanged)
const storageMetadata = {
...(typeof params.data === 'object' && params.data !== null && !Array.isArray(params.data) ? params.data : {}),
@ -776,7 +752,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
})
return id
})
}
/**
@ -914,7 +889,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
async get(id: string, options?: GetOptions): Promise<Entity<T> | null> {
await this.ensureInitialized()
return this.augmentationRegistry.execute('get', { id, options }, async () => {
// Route to metadata-only or full entity based on options
const includeVectors = options?.includeVectors ?? false // Default: metadata-only (fast)
@ -935,7 +909,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
}
return this.convertMetadataToEntity(id, metadata)
}
})
}
/**
@ -1104,7 +1077,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
// Zero-config validation (static import for performance)
validateUpdateParams(params)
return this.augmentationRegistry.execute('update', params, async () => {
// Get existing entity with vectors (fix for regression)
// We need includeVectors: true because:
// 1. SaveNounOperation requires the vector
@ -1244,7 +1216,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
new AddToMetadataIndexOperation(this.metadataIndex, params.id, entityForIndexing)
)
})
})
}
/**
@ -1258,7 +1229,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
await this.ensureInitialized()
return this.augmentationRegistry.execute('delete', { id }, async () => {
// Get entity metadata and related verbs before deletion
const metadata = await this.storage.getNounMetadata(id)
const noun = await this.storage.getNoun(id)
@ -1310,7 +1280,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
)
}
})
})
}
// ============= RELATIONSHIP OPERATIONS =============
@ -1465,7 +1434,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
(v, i) => (v + toEntity.vector[i]) / 2
)
return this.augmentationRegistry.execute('relate', params, async () => {
// Prepare verb metadata
// CRITICAL: Include verb type in metadata for count tracking
const verbMetadata = {
@ -1551,7 +1519,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
})
return id
})
}
/**
@ -1560,7 +1527,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
async unrelate(id: string): Promise<void> {
await this.ensureInitialized()
return this.augmentationRegistry.execute('unrelate', { id }, async () => {
// Get verb data before deletion for rollback
const verb = await this.storage.getVerb(id)
@ -1578,7 +1544,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
new DeleteVerbMetadataOperation(this.storage, id)
)
})
})
}
/**
@ -1865,7 +1830,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
validateFindParams(params)
const startTime = Date.now()
const result = await this.augmentationRegistry.execute('find', params, async () => {
const result = await (async () => {
let results: Result<T>[] = []
// Distinguish between search criteria (need vector search) and filter criteria (metadata only)
@ -2201,7 +2166,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
// Efficient pagination - only slice what we need (limit already defined above)
return results.slice(finalOffset, finalOffset + limit)
})
})()
// Record performance for auto-tuning
const duration = Date.now() - startTime
@ -2771,7 +2736,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
async clear(): Promise<void> {
await this.ensureInitialized()
return this.augmentationRegistry.execute('clear', {}, async () => {
// Clear storage
await this.storage.clear()
@ -2832,7 +2796,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
// VFS was never used, reset flag for clean state
this._vfsInitialized = false
}
})
}
// ============= COW (COPY-ON-WRITE) API =============
@ -2887,7 +2850,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
}): Promise<Brainy<T>> {
await this.ensureInitialized()
return this.augmentationRegistry.execute('fork', { branch, options }, async () => {
const branchName = branch || `fork-${Date.now()}`
// Lazy COW initialization - enable automatically on first fork()
@ -2978,25 +2940,11 @@ export class Brainy<T = any> implements BrainyInterface<T> {
clone.graphIndex = await (clone.storage as any).getGraphIndex()
// getGraphIndex() will rebuild automatically if data exists (via _initializeGraphIndex)
// Setup augmentations
clone.augmentationRegistry = this.setupAugmentations()
await clone.augmentationRegistry.initializeAll({
brain: clone,
storage: clone.storage,
config: clone.config,
log: (message: string, level?: 'info' | 'warn' | 'error') => {
if (!clone.config.silent) {
console[level || 'info'](message)
}
}
})
// Mark as initialized
clone.initialized = true
clone.dimensions = this.dimensions
return clone
})
}
/**
@ -3012,7 +2960,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
async listBranches(): Promise<string[]> {
await this.ensureInitialized()
return this.augmentationRegistry.execute('listBranches', {}, async () => {
if (!('refManager' in this.storage)) {
throw new Error('Branch management requires COW-enabled storage')
}
@ -3024,7 +2971,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
return refs
.filter((ref: any) => ref.name.startsWith('refs/heads/'))
.map((ref: any) => ref.name.replace('refs/heads/', ''))
})
}
/**
@ -3040,13 +2986,11 @@ export class Brainy<T = any> implements BrainyInterface<T> {
async getCurrentBranch(): Promise<string> {
await this.ensureInitialized()
return this.augmentationRegistry.execute('getCurrentBranch', {}, async () => {
if (!('currentBranch' in this.storage)) {
return 'main' // Default branch
}
return (this.storage as any).currentBranch || 'main'
})
}
/**
@ -3062,7 +3006,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
async checkout(branch: string): Promise<void> {
await this.ensureInitialized()
return this.augmentationRegistry.execute('checkout', { branch }, async () => {
if (!('refManager' in this.storage)) {
throw new Error('Branch management requires COW-enabled storage')
}
@ -3106,7 +3049,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
this._vfs = new VirtualFileSystem(this)
await this._vfs.init()
}
})
}
/**
@ -3131,7 +3073,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
}): Promise<string> {
await this.ensureInitialized()
return this.augmentationRegistry.execute('commit', { options }, async () => {
if (!('refManager' in this.storage) || !('commitLog' in this.storage) || !('blobStorage' in this.storage)) {
throw new Error('Commit requires COW-enabled storage')
}
@ -3187,7 +3128,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
})
return commitHash
})
}
/**
@ -3380,7 +3320,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
async deleteBranch(branch: string): Promise<void> {
await this.ensureInitialized()
return this.augmentationRegistry.execute('deleteBranch', { branch }, async () => {
if (!('refManager' in this.storage)) {
throw new Error('Branch management requires COW-enabled storage')
}
@ -3392,7 +3331,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
const refManager = (this.storage as any).refManager
await refManager.deleteRef(branch)
})
}
/**
@ -3421,7 +3359,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
}>> {
await this.ensureInitialized()
return this.augmentationRegistry.execute('getHistory', { options }, async () => {
if (!('commitLog' in this.storage) || !('refManager' in this.storage)) {
throw new Error('History requires COW-enabled storage')
}
@ -3444,7 +3381,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
timestamp: commit.timestamp,
metadata: commit.metadata
}))
})
}
/**
@ -3950,21 +3886,12 @@ export class Brainy<T = any> implements BrainyInterface<T> {
}) => void
}
) {
// Execute through augmentation pipeline (Enables IntelligentImportAugmentation)
// If source is an ImportSource object (not a Buffer), spread it so augmentations can access properties
const params = typeof source === 'object' && !Buffer.isBuffer(source)
? { ...source as object, ...options } // Spread ImportSource: { type, data, filename, ...options }
: { source, ...options } // Wrap Buffer/string: { source, ...options }
return this.augmentationRegistry.execute('import', params, async () => {
// Lazy load ImportCoordinator
const { ImportCoordinator } = await import('./import/ImportCoordinator.js')
const coordinator = new ImportCoordinator(this)
await coordinator.init()
// Pass augmentation-modified params (contains _intelligentImport, _extractedData, etc)
return await coordinator.import(source, { ...options, ...params })
})
return await coordinator.import(source, options)
}
/**
@ -4690,17 +4617,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
}
}
/**
* Augmentations API - Clean and simple
*/
get augmentations() {
return {
list: () => this.augmentationRegistry.getAll().map(a => a.name),
get: (name: string) => this.augmentationRegistry.getAll().find(a => a.name === name),
has: (name: string) => this.augmentationRegistry.getAll().some(a => a.name === name)
}
}
/**
* Get complete statistics - convenience method
* For more granular counting, use brain.counts API
@ -6282,32 +6198,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
return persistMode
}
/**
* Setup augmentations
*/
private setupAugmentations(): AugmentationRegistry {
const registry = new AugmentationRegistry()
// Register default augmentations with silent mode support
const augmentationConfig = {
...this.config.augmentations,
// Pass silent mode to all augmentations
...(this.config.silent && {
cache: this.config.augmentations?.cache !== false ? { ...this.config.augmentations?.cache, silent: true } : false,
metrics: this.config.augmentations?.metrics !== false ? { ...this.config.augmentations?.metrics, silent: true } : false,
display: this.config.augmentations?.display !== false ? { ...this.config.augmentations?.display, silent: true } : false,
monitoring: this.config.augmentations?.monitoring !== false ? { ...this.config.augmentations?.monitoring, silent: true } : false
})
}
const defaults = createDefaultAugmentations(augmentationConfig)
for (const aug of defaults) {
registry.register(aug)
}
return registry
}
/**
* Normalize and validate configuration
*/
@ -6356,7 +6246,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
storage: config?.storage || { type: 'auto' },
index: config?.index || {},
cache: config?.cache ?? true,
augmentations: config?.augmentations || {},
distributed: distributedConfig as any, // Type will be fixed when used
warmup: config?.warmup ?? false,
realtime: config?.realtime ?? false,
@ -6670,14 +6559,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
// Deactivate plugins
await this.pluginRegistry.deactivateAll()
// Shutdown augmentations
const augs = this.augmentationRegistry.getAll()
for (const aug of augs) {
if ('shutdown' in aug && typeof aug.shutdown === 'function') {
await aug.shutdown()
}
}
// Restore console methods if silent mode was enabled
if (this.config.silent && this.originalConsole) {
console.log = this.originalConsole.log as typeof console.log

18
src/cli/commands/core.ts
View file

@ -148,21 +148,9 @@ export const coreCommands = {
}
nounType = options.type as NounType
} else {
// Use AI to suggest type
spinner.text = 'Detecting type with AI...'
const suggestion = await BrainyTypes.suggestNoun(
typeof text === 'string' ? { content: text, ...metadata } : text
)
if (suggestion.confidence < 0.6) {
spinner.fail('Could not determine type with confidence')
console.log(chalk.yellow(`Suggestion: ${suggestion.type} (${(suggestion.confidence * 100).toFixed(1)}%)`))
console.log(chalk.dim('Use --type flag to specify explicitly'))
process.exit(1)
}
nounType = suggestion.type as NounType
spinner.text = `Using detected type: ${nounType}`
// Default to Thing when no type specified
nounType = NounType.Thing
spinner.text = `No type specified, using default: ${nounType}`
}
// Add with explicit type

100
src/cli/commands/types.ts
View file

@ -1,12 +1,9 @@
/**
* CLI Commands for Type Management
* Consistent with BrainyTypes public API
*/
import chalk from 'chalk'
import ora from 'ora'
import inquirer from 'inquirer'
import Table from 'cli-table3'
import { BrainyTypes, NounType, VerbType } from '../../index.js'
/**
@ -30,7 +27,7 @@ export async function types(options: { json?: boolean, noun?: boolean, verb?: bo
// Display nouns
if (showNouns) {
console.log(chalk.bold.cyan('\n📚 Noun Types (42):\n'))
console.log(chalk.bold.cyan(`\nNoun Types (${BrainyTypes.nouns.length}):\n`))
const nounChunks = []
for (let i = 0; i < BrainyTypes.nouns.length; i += 3) {
nounChunks.push(BrainyTypes.nouns.slice(i, i + 3))
@ -43,7 +40,7 @@ export async function types(options: { json?: boolean, noun?: boolean, verb?: bo
// Display verbs
if (showVerbs) {
console.log(chalk.bold.cyan('\n🔗 Verb Types (127):\n'))
console.log(chalk.bold.cyan(`\nVerb Types (${BrainyTypes.verbs.length}):\n`))
const verbChunks = []
for (let i = 0; i < BrainyTypes.verbs.length; i += 3) {
verbChunks.push(BrainyTypes.verbs.slice(i, i + 3))
@ -54,99 +51,6 @@ export async function types(options: { json?: boolean, noun?: boolean, verb?: bo
}
}
console.log(chalk.dim('\n💡 Use "brainy suggest <data>" to get AI-powered type suggestions'))
} catch (error: any) {
console.error(chalk.red('Error:', error.message))
process.exit(1)
}
}
/**
* Suggest type - matches BrainyTypes.suggestNoun() and suggestVerb()
* Usage: brainy suggest <data>
* Interactive if data not provided
*/
export async function suggest(
data?: string,
options: {
verb?: boolean,
json?: boolean
} = {}
) {
try {
// Interactive mode if no data provided
if (!data) {
const answers = await inquirer.prompt([
{
type: 'list',
name: 'kind',
message: 'What type do you want to suggest?',
choices: ['Noun', 'Verb'],
default: 'Noun'
},
{
type: 'input',
name: 'data',
message: 'Enter data (JSON or text):',
validate: (input) => input.length > 0 || 'Data is required'
},
{
type: 'input',
name: 'hint',
message: 'Relationship hint (optional):',
when: (answers) => answers.kind === 'Verb'
}
])
data = answers.data
options.verb = answers.kind === 'Verb'
// For verbs, parse source/target if provided as JSON
if (options.verb && answers.hint) {
data = JSON.stringify({ hint: answers.hint })
}
}
const spinner = ora('Analyzing with AI...').start()
let parsedData: any
try {
parsedData = JSON.parse(data)
} catch {
parsedData = { content: data }
}
let suggestion
if (options.verb) {
// For verb suggestions, need source and target
const source = parsedData.source || { type: 'unknown' }
const target = parsedData.target || { type: 'unknown' }
const hint = parsedData.hint || parsedData.relationship || parsedData.verb
suggestion = await BrainyTypes.suggestVerb(source, target, hint)
spinner.succeed('Verb type analyzed')
} else {
suggestion = await BrainyTypes.suggestNoun(parsedData)
spinner.succeed('Noun type analyzed')
}
if (options.json) {
console.log(JSON.stringify(suggestion, null, 2))
return
}
// Display results
console.log(chalk.bold.green(`\n✨ Suggested: ${suggestion.type}`))
console.log(chalk.cyan(`Confidence: ${(suggestion.confidence * 100).toFixed(1)}%`))
if (suggestion.alternatives && suggestion.alternatives.length > 0) {
console.log(chalk.yellow('\nAlternatives:'))
for (const alt of suggestion.alternatives.slice(0, 3)) {
console.log(` ${alt.type} (${(alt.confidence * 100).toFixed(1)}%)`)
}
}
} catch (error: any) {
console.error(chalk.red('Error:', error.message))
process.exit(1)

97
src/augmentations/neuralImport.ts → src/cortex/neuralImportAugmentation.ts
View file

@ -1,17 +1,12 @@
/**
* Neural Import Augmentation - AI-Powered Data Understanding
* Neural Import - AI-Powered Data Understanding
*
* 🧠 Built-in AI augmentation for intelligent data processing
* ⚛️ Always free, always included, always enabled
*
* Now using the unified BrainyAugmentation interface!
* Standalone implementation for intelligent data processing.
*/
import { BaseAugmentation, AugmentationContext } from './brainyAugmentation.js'
import { NounType, VerbType } from '../types/graphTypes.js'
import * as fs from '../universal/fs.js'
import * as path from '../universal/path.js'
import { BrainyTypes, getBrainyTypes } from './typeMatching/brainyTypes.js'
import { prodLog } from '../utils/logger.js'
// Neural Import Analysis Types
@ -62,22 +57,15 @@ export interface NeuralImportConfig {
* Neural Import Augmentation - Unified Implementation
* Processes data with AI before storage operations
*/
export class NeuralImportAugmentation extends BaseAugmentation {
export class NeuralImportAugmentation {
readonly name = 'neural-import'
readonly timing = 'before' as const // Process data before storage
readonly metadata = {
reads: '*' as '*', // Needs to read data for analysis
writes: ['_neuralProcessed', '_neuralConfidence', '_detectedEntities', '_detectedRelationships', '_neuralInsights', 'nounType', 'verbType'] as string[]
} // Enriches metadata with neural analysis
operations = ['add', 'addNoun', 'addVerb', 'all'] as ('add' | 'addNoun' | 'addVerb' | 'all')[] // Use 'all' to catch batch operations
readonly priority = 80 // High priority for data processing
private operations = ['add', 'addNoun', 'addVerb', 'all']
protected config: NeuralImportConfig
private analysisCache = new Map<string, NeuralAnalysisResult>()
private typeMatcher: BrainyTypes | null = null
private context?: { brain: any }
constructor(config: Partial<NeuralImportConfig> = {}) {
super()
this.config = {
confidenceThreshold: 0.7,
enableWeights: true,
@ -87,18 +75,12 @@ export class NeuralImportAugmentation extends BaseAugmentation {
}
}
protected async onInitialize(): Promise<void> {
try {
this.typeMatcher = await getBrainyTypes()
this.log('🧠 Neural Import augmentation initialized with intelligent type matching')
} catch (error) {
this.log('⚠️ Failed to initialize type matcher, falling back to heuristics', 'warn')
}
async init(): Promise<void> {
// No external dependencies to initialize
}
protected async onShutdown(): Promise<void> {
this.analysisCache.clear()
this.log('🧠 Neural Import augmentation shut down')
private log(message: string, _level?: string): void {
// Silent by default
}
/**
@ -455,22 +437,29 @@ export class NeuralImportAugmentation extends BaseAugmentation {
}
/**
* Infer noun type from object structure using intelligent type matching
* Infer noun type from object structure using field heuristics
*/
private async inferNounType(obj: any): Promise<string> {
if (!this.typeMatcher) {
// Initialize type matcher if not available
this.typeMatcher = await getBrainyTypes()
if (typeof obj !== 'object' || obj === null) return NounType.Thing
// Check for explicit type field
if (obj.type && typeof obj.type === 'string') {
const normalized = obj.type.charAt(0).toUpperCase() + obj.type.slice(1)
if (Object.values(NounType).includes(normalized as NounType)) {
return normalized as NounType
}
}
const result = await this.typeMatcher.matchNounType(obj)
if (obj.email || obj.firstName || obj.lastName || obj.username) return NounType.Person
if (obj.companyName || obj.organizationId || obj.employees) return NounType.Organization
if (obj.latitude || obj.longitude || obj.address || obj.city) return NounType.Location
if ((obj.content && (obj.title || obj.author)) || obj.pages) return NounType.Document
if (obj.startTime || obj.endTime || obj.date || obj.attendees) return NounType.Event
if (obj.price || obj.sku || obj.productId) return NounType.Product
if ((obj.status && obj.assignee) || obj.priority) return NounType.Task
if (Array.isArray(obj.data) || obj.rows || obj.columns) return NounType.Dataset
// Log if confidence is low for debugging
if (result.confidence < 0.5) {
this.log(`Low confidence (${result.confidence.toFixed(2)}) for noun type: ${result.type}`, 'warn')
}
return result.type
return NounType.Thing
}
/**
@ -511,22 +500,28 @@ export class NeuralImportAugmentation extends BaseAugmentation {
}
/**
* Infer verb type from field name using intelligent type matching
* Infer verb type from field name using common patterns
*/
private async inferVerbType(fieldName: string, sourceObj?: any, targetObj?: any): Promise<string> {
if (!this.typeMatcher) {
// Initialize type matcher if not available
this.typeMatcher = await getBrainyTypes()
private async inferVerbType(fieldName: string, _sourceObj?: any, _targetObj?: any): Promise<string> {
const field = fieldName.toLowerCase()
if (field.includes('parent') || field.includes('child') || field.includes('contain')) {
return VerbType.Contains
}
if (field.includes('owner') || field.includes('created') || field.includes('author')) {
return VerbType.Creates
}
if (field.includes('member') || field.includes('belong')) {
return VerbType.MemberOf
}
if (field.includes('depend') || field.includes('require')) {
return VerbType.DependsOn
}
if (field.includes('ref') || field.includes('link') || field.includes('source')) {
return VerbType.References
}
const result = await this.typeMatcher.matchVerbType(sourceObj, targetObj, fieldName)
// Log if confidence is low for debugging
if (result.confidence < 0.5) {
this.log(`Low confidence (${result.confidence.toFixed(2)}) for verb type: ${result.type}`, 'warn')
}
return result.type
return VerbType.RelatedTo
}
/**

4
src/importers/SmartCSVImporter.ts
View file

@ -16,8 +16,8 @@ import { NeuralEntityExtractor, ExtractedEntity } from '../neural/entityExtracto
import { NaturalLanguageProcessor } from '../neural/naturalLanguageProcessor.js'
import { SmartRelationshipExtractor } from '../neural/SmartRelationshipExtractor.js'
import { NounType, VerbType } from '../types/graphTypes.js'
import { CSVHandler } from '../augmentations/intelligentImport/handlers/csvHandler.js'
import type { FormatHandlerOptions } from '../augmentations/intelligentImport/types.js'
import { CSVHandler } from './handlers/csvHandler.js'
import type { FormatHandlerOptions } from './handlers/types.js'
export interface SmartCSVOptions extends FormatHandlerOptions {
/** Enable neural entity extraction */

4
src/importers/SmartExcelImporter.ts
View file

@ -14,8 +14,8 @@ import { NeuralEntityExtractor, ExtractedEntity } from '../neural/entityExtracto
import { NaturalLanguageProcessor } from '../neural/naturalLanguageProcessor.js'
import { SmartRelationshipExtractor } from '../neural/SmartRelationshipExtractor.js'
import { NounType, VerbType } from '../types/graphTypes.js'
import { ExcelHandler } from '../augmentations/intelligentImport/handlers/excelHandler.js'
import type { FormatHandlerOptions } from '../augmentations/intelligentImport/types.js'
import { ExcelHandler } from './handlers/excelHandler.js'
import type { FormatHandlerOptions } from './handlers/types.js'
export interface SmartExcelOptions extends FormatHandlerOptions {
/** Enable neural entity extraction */

4
src/importers/SmartPDFImporter.ts
View file

@ -15,8 +15,8 @@ import { NeuralEntityExtractor, ExtractedEntity } from '../neural/entityExtracto
import { NaturalLanguageProcessor } from '../neural/naturalLanguageProcessor.js'
import { SmartRelationshipExtractor } from '../neural/SmartRelationshipExtractor.js'
import { NounType, VerbType } from '../types/graphTypes.js'
import { PDFHandler } from '../augmentations/intelligentImport/handlers/pdfHandler.js'
import type { FormatHandlerOptions } from '../augmentations/intelligentImport/types.js'
import { PDFHandler } from './handlers/pdfHandler.js'
import type { FormatHandlerOptions } from './handlers/types.js'
export interface SmartPDFOptions extends FormatHandlerOptions {
/** Enable neural entity extraction */

4
src/augmentations/intelligentImport/handlers/base.ts → src/importers/handlers/base.ts
View file

@ -5,8 +5,8 @@
* Uses MimeTypeDetector for comprehensive file type detection (2000+ types)
*/
import { FormatHandler, FormatHandlerOptions, ProcessedData } from '../types.js'
import { mimeDetector } from '../../../vfs/MimeTypeDetector.js'
import { FormatHandler, FormatHandlerOptions, ProcessedData } from './types.js'
import { mimeDetector } from '../../vfs/MimeTypeDetector.js'
export abstract class BaseFormatHandler implements FormatHandler {
abstract readonly format: string

2
src/augmentations/intelligentImport/handlers/csvHandler.ts → src/importers/handlers/csvHandler.ts
View file

@ -10,7 +10,7 @@
import { parse } from 'csv-parse/sync'
import { detect as detectEncoding } from 'chardet'
import { BaseFormatHandler } from './base.js'
import { FormatHandlerOptions, ProcessedData } from '../types.js'
import { FormatHandlerOptions, ProcessedData } from './types.js'
export class CSVHandler extends BaseFormatHandler {
readonly format = 'csv'

2
src/augmentations/intelligentImport/handlers/excelHandler.ts → src/importers/handlers/excelHandler.ts
View file

@ -9,7 +9,7 @@
import * as XLSX from 'xlsx'
import { BaseFormatHandler } from './base.js'
import { FormatHandlerOptions, ProcessedData } from '../types.js'
import { FormatHandlerOptions, ProcessedData } from './types.js'
export class ExcelHandler extends BaseFormatHandler {
readonly format = 'excel'

2
src/augmentations/intelligentImport/handlers/pdfHandler.ts → src/importers/handlers/pdfHandler.ts
View file

@ -9,7 +9,7 @@
import * as pdfjsLib from 'pdfjs-dist/legacy/build/pdf.mjs'
import { BaseFormatHandler } from './base.js'
import { FormatHandlerOptions, ProcessedData } from '../types.js'
import { FormatHandlerOptions, ProcessedData } from './types.js'
// Use built-in worker for Node.js environments
// In production, this can be customized via options

0
src/augmentations/intelligentImport/types.ts → src/importers/handlers/types.ts
View file

121
src/index.ts
View file

@ -5,7 +5,7 @@
* Core Components:
* - Brainy: The unified database with Triple Intelligence
* - Triple Intelligence: Seamless fusion of vector + graph + field search
* - Augmentations: Extensible plugin system
* - Plugins: Extensible plugin system (cortex, storage adapters)
* - Neural API: AI-powered clustering and analysis
*/
@ -54,21 +54,6 @@ export {
getPresetDescription
} from './config/index.js'
// Export augmentation types
export type {
BrainyAugmentations,
IAugmentation,
ISenseAugmentation,
IConduitAugmentation,
ICognitionAugmentation,
IMemoryAugmentation,
IPerceptionAugmentation,
IDialogAugmentation,
IActivationAugmentation,
IWebSocketSupport,
AugmentationType
} from './types/augmentations.js'
// Export Neural Import (AI data understanding)
export { NeuralImport } from './cortex/neuralImport.js'
export type {
@ -96,10 +81,6 @@ export type {
SmartRelationshipExtractorOptions
} from './neural/SmartRelationshipExtractor.js'
// Import Manager removed - use brain.import() instead (available on all Brainy instances)
// Augmentation types are already exported later in the file
// Export distance functions for convenience
import {
euclideanDistance,
@ -234,7 +215,6 @@ export {
export { FileSystemStorage } from './storage/adapters/fileSystemStorage.js'
// Export COW (Copy-on-Write) infrastructure
// Enables premium augmentations to implement temporal features
import { CommitLog } from './storage/cow/CommitLog.js'
import { CommitObject, CommitBuilder } from './storage/cow/CommitObject.js'
import { BlobStorage } from './storage/cow/BlobStorage.js'
@ -255,7 +235,6 @@ export {
import {
Pipeline,
pipeline,
augmentationPipeline,
ExecutionMode,
PipelineOptions,
PipelineResult,
@ -266,85 +245,21 @@ import {
StreamlinedPipelineResult
} from './pipeline.js'
// Sequential pipeline removed - use unified pipeline instead
// REMOVED: Old augmentation factory for 2.0 clean architecture
export {
// Unified pipeline exports
Pipeline,
pipeline,
augmentationPipeline,
ExecutionMode,
// Factory functions
createPipeline,
createStreamingPipeline,
StreamlinedExecutionMode,
// Augmentation factory exports (REMOVED in 2.0 - Use BrainyAugmentation interface)
// createSenseAugmentation, // → Use BaseAugmentation class
// addWebSocketSupport, // → Use APIServerAugmentation
// executeAugmentation, // → Use brain.augmentations.execute()
// loadAugmentationModule // → Use dynamic imports
StreamlinedExecutionMode
}
export type {
PipelineOptions,
PipelineResult,
StreamlinedPipelineOptions,
StreamlinedPipelineResult
// AugmentationOptions - REMOVED in 2.0 (use BaseAugmentation config)
}
// Augmentation registry removed - use Brainy's built-in augmentation system
// Export augmentation implementations
import {
StorageAugmentation,
DynamicStorageAugmentation,
createStorageAugmentationFromConfig
} from './augmentations/storageAugmentation.js'
import {
MemoryStorageAugmentation,
FileSystemStorageAugmentation,
OPFSStorageAugmentation,
S3StorageAugmentation,
R2StorageAugmentation,
GCSStorageAugmentation,
createAutoStorageAugmentation
} from './augmentations/storageAugmentations.js'
import {
WebSocketConduitAugmentation
} from './augmentations/conduitAugmentations.js'
// Storage augmentation exports
export {
// Base classes
StorageAugmentation,
DynamicStorageAugmentation,
// Concrete implementations
MemoryStorageAugmentation,
FileSystemStorageAugmentation,
OPFSStorageAugmentation,
S3StorageAugmentation,
R2StorageAugmentation,
GCSStorageAugmentation,
// Factory functions
createAutoStorageAugmentation,
createStorageAugmentationFromConfig
}
// Other augmentation exports
export {
WebSocketConduitAugmentation
}
// LLM augmentations are optional and not imported by default
// They can be imported directly from their module if needed:
// import { LLMCognitionAugmentation, LLMActivationAugmentation, createLLMAugmentations } from './augmentations/llmAugmentations.js'
// Export types
import type {
Vector,
@ -377,25 +292,6 @@ export type {
StorageAdapter
}
// Export augmentation types
import type {
AugmentationResponse,
BrainyAugmentation,
BaseAugmentation,
AugmentationContext
} from './types/augmentations.js'
// Export augmentation manager for type-safe augmentation management
export { AugmentationManager, type AugmentationInfo } from './augmentationManager.js'
// Export only the clean augmentation types for 2.0
export type {
AugmentationResponse,
BrainyAugmentation,
BaseAugmentation,
AugmentationContext
}
// Export graph types
import type {
GraphNoun,
@ -496,8 +392,8 @@ export type {
// Export type utility functions
import { getNounTypes, getVerbTypes, getNounTypeMap, getVerbTypeMap } from './utils/typeUtils.js'
// Export BrainyTypes for complete type management
import { BrainyTypes, TypeSuggestion, suggestType } from './utils/brainyTypes.js'
// Export BrainyTypes for type validation and lookup
import { BrainyTypes } from './utils/brainyTypes.js'
export {
NounType,
@ -506,13 +402,8 @@ export {
getVerbTypes,
getNounTypeMap,
getVerbTypeMap,
// BrainyTypes - complete type management
BrainyTypes,
suggestType
}
export type {
TypeSuggestion
// BrainyTypes - type validation and lookup
BrainyTypes
}
// Export MCP (Model Control Protocol) components

44
src/integrations/core/IntegrationBase.ts
View file

@ -5,11 +5,6 @@
* for event subscriptions, tabular export, and lifecycle management.
*/
import {
BaseAugmentation,
AugmentationContext
} from '../../augmentations/brainyAugmentation.js'
import { AugmentationManifest } from '../../augmentations/manifest.js'
import { EventBus } from './EventBus.js'
import { TabularExporter } from './TabularExporter.js'
import {
@ -48,17 +43,12 @@ import { Entity, Relation, FindParams } from '../../types/brainy.types.js'
* }
* ```
*/
export abstract class IntegrationBase extends BaseAugmentation {
// Augmentation interface implementation
readonly timing = 'after' as const
readonly metadata = 'readonly' as const
readonly operations: ('all')[] = ['all']
readonly priority = 5 // Low priority - runs after main operations
category: 'internal' | 'core' | 'premium' | 'community' | 'external' = 'core'
export abstract class IntegrationBase {
// Shared infrastructure
protected eventBus: EventBus
protected exporter: TabularExporter
protected config: IntegrationConfig
protected context?: { brain: any; storage: any; config: any; log: (message: string, level?: string) => void }
// Integration state
protected isRunning = false
@ -80,11 +70,35 @@ export abstract class IntegrationBase extends BaseAugmentation {
config?: IntegrationConfig,
exporterConfig?: TabularExporterConfig
) {
super(config)
this.config = config || {}
this.eventBus = new EventBus()
this.exporter = new TabularExporter(exporterConfig)
}
/**
* Log a message
*/
protected log(message: string, level: string = 'info'): void {
if (this.context?.log) {
this.context.log(message, level)
}
}
/**
* Initialize the integration with context
*/
async initialize(ctx: { brain: any; storage: any; config: any; log: (message: string, level?: string) => void }): Promise<void> {
this.context = ctx
await this.onInitialize()
}
/**
* Shutdown the integration
*/
async shutdown(): Promise<void> {
await this.onShutdown()
}
/**
* Integration name (must be unique)
*/
@ -311,7 +325,7 @@ export abstract class IntegrationBase extends BaseAugmentation {
* Get manifest for this integration
* Subclasses should override this
*/
getManifest(): AugmentationManifest {
getManifest(): Record<string, any> {
return {
id: this.name,
name: this.name,

9
src/integrations/odata/ODataIntegration.ts
View file

@ -15,7 +15,6 @@ import {
HTTPIntegration
} from '../core/IntegrationBase.js'
import { IntegrationConfig, ODataQueryOptions } from '../core/types.js'
import { AugmentationManifest } from '../../augmentations/manifest.js'
import { Entity, Relation, FindParams } from '../../types/brainy.types.js'
import { NounType } from '../../types/graphTypes.js'
import {
@ -94,11 +93,11 @@ interface ODataResponse {
*
* @example
* ```typescript
* // Register with Brainy
* brain.augmentations.register(new ODataIntegration({
* const odata = new ODataIntegration({
* basePath: '/odata',
* maxPageSize: 1000
* }))
* })
* await odata.initialize()
*
* // Connect from Excel Power Query:
* // Data → Get Data → From OData Feed → http://localhost:3000/odata
@ -249,7 +248,7 @@ export class ODataIntegration extends IntegrationBase implements HTTPIntegration
/**
* Get augmentation manifest
*/
getManifest(): AugmentationManifest {
getManifest(): Record<string, any> {
return {
id: 'odata',
name: 'OData Integration',

8
src/integrations/sheets/GoogleSheetsIntegration.ts
View file

@ -15,7 +15,6 @@
import { IntegrationBase, HTTPIntegration } from '../core/IntegrationBase.js'
import { IntegrationConfig } from '../core/types.js'
import { AugmentationManifest } from '../../augmentations/manifest.js'
import { Entity, FindParams } from '../../types/brainy.types.js'
import { NounType, VerbType } from '../../types/graphTypes.js'
@ -92,10 +91,11 @@ interface SheetsResponse {
*
* @example
* ```typescript
* brain.augmentations.register(new GoogleSheetsIntegration({
* const sheets = new GoogleSheetsIntegration({
* basePath: '/sheets',
* maxResults: 1000
* }))
* })
* await sheets.initialize()
* ```
*/
export class GoogleSheetsIntegration
@ -283,7 +283,7 @@ export class GoogleSheetsIntegration
/**
* Get manifest
*/
getManifest(): AugmentationManifest {
getManifest(): Record<string, any> {
return {
id: 'sheets',
name: 'Google Sheets Integration',

8
src/integrations/sse/SSEIntegration.ts
View file

@ -15,7 +15,6 @@ import {
EventFilter,
BrainyEvent
} from '../core/types.js'
import { AugmentationManifest } from '../../augmentations/manifest.js'
/**
* SSE integration configuration
@ -63,10 +62,11 @@ interface SSEClient {
*
* @example
* ```typescript
* brain.augmentations.register(new SSEIntegration({
* const sse = new SSEIntegration({
* basePath: '/events',
* heartbeatInterval: 30000
* }))
* })
* await sse.initialize()
*
* // Client-side:
* const source = new EventSource('/events?types=noun&operations=create,update')
@ -292,7 +292,7 @@ export class SSEIntegration
/**
* Get manifest
*/
getManifest(): AugmentationManifest {
getManifest(): Record<string, any> {
return {
id: 'sse',
name: 'SSE Streaming',

5
src/integrations/webhooks/WebhookIntegration.ts
View file

@ -15,7 +15,6 @@ import {
WebhookDeliveryResult,
BrainyEvent
} from '../core/types.js'
import { AugmentationManifest } from '../../augmentations/manifest.js'
/**
* Webhook integration configuration
@ -69,7 +68,7 @@ interface PendingDelivery {
* @example
* ```typescript
* const webhooks = new WebhookIntegration()
* brain.augmentations.register(webhooks)
* await webhooks.initialize()
*
* // Register a webhook
* await webhooks.register({
@ -263,7 +262,7 @@ export class WebhookIntegration extends IntegrationBase {
/**
* Get manifest
*/
getManifest(): AugmentationManifest {
getManifest(): Record<string, any> {
return {
id: 'webhooks',
name: 'Webhooks',

11
src/mcp/mcpAugmentationToolset.ts
View file

@ -12,10 +12,6 @@ import {
MCPTool,
MCP_VERSION
} from '../types/mcpTypes.js'
import { AugmentationType } from '../types/augmentations.js'
// Import the augmentation pipeline
import { augmentationPipeline } from '../pipeline.js'
export class MCPAugmentationToolset {
/**
@ -146,9 +142,8 @@ export class MCPAugmentationToolset {
const { args = [], options = {} } = parameters
// Note: This MCP toolset needs to be updated to use the new brain.augmentations API
// For now, return a placeholder response to fix compilation
throw new Error(`MCP toolset requires update to use brain.augmentations API. Method '${method}' not available.`)
// Augmentation pipeline has been removed — operations are called directly on brain
throw new Error(`MCP augmentation toolset is deprecated. Method '${method}' not available. Use brain API directly.`)
}
/**
@ -157,7 +152,7 @@ export class MCPAugmentationToolset {
* @returns Whether the augmentation type is valid
*/
private isValidAugmentationType(type: string): boolean {
return Object.values(AugmentationType).includes(type as AugmentationType)
return false
}
/**

4
src/pipeline.ts
View file

@ -1,8 +1,7 @@
/**
* Pipeline - Augmentation execution pipeline
* Pipeline - Execution pipeline
*
* Provides Pipeline class, execution modes, and factory functions.
* All augmentation management should use brain.augmentations API.
*/
/**
@ -28,7 +27,6 @@ export interface PipelineOptions {
/**
* Minimal Pipeline class for backward compatibility.
* All augmentation management should use brain.augmentations API.
*/
export class Pipeline {
private static instance?: Pipeline

130
src/shared/default-augmentations.ts
View file

@ -1,130 +0,0 @@
/**
* Default Augmentation Registry
*
* 🧠⚛️ Pre-installed augmentations that come with every Brainy installation
* These are the core "sensory organs" of the atomic age brain-in-jar system
*/
import { BrainyInterface } from '../types/brainyInterface.js'
/**
* Default augmentations that ship with Brainy
* These are automatically registered on startup
*/
export class DefaultAugmentationRegistry {
private brainy: BrainyInterface
constructor(brainy: BrainyInterface) {
this.brainy = brainy
}
/**
* Initialize all default augmentations
* Called during Brainy startup to register core functionality
*/
async initializeDefaults(): Promise<void> {
console.log('🧠⚛️ Initializing default augmentations...')
// Register Neural Import as default SENSE augmentation
await this.registerNeuralImport()
console.log('🧠⚛️ Default augmentations initialized')
}
/**
* Neural Import - Default SENSE Augmentation
* AI-powered data understanding and entity extraction (always free)
*/
private async registerNeuralImport(): Promise<void> {
try {
// Import the Neural Import augmentation
const { NeuralImportAugmentation } = await import('../augmentations/neuralImport.js')
// Note: The actual registration is commented out since Brainy doesn't have addAugmentation method yet
// This would create instance with default configuration
/*
const neuralImport = new NeuralImportAugmentation(this.brainy as any, {
confidenceThreshold: 0.7,
enableWeights: true,
skipDuplicates: true
})
// Add as SENSE augmentation to Brainy (when method is available)
if (this.brainy.addAugmentation) {
await this.brainy.addAugmentation('SENSE', cortex, {
position: 1, // First in the SENSE pipeline
name: 'cortex',
autoStart: true
})
}
*/
console.log('🧠⚛️ Cortex module loaded (awaiting Brainy augmentation support)')
} catch (error) {
console.error('❌ Failed to register Cortex:', error instanceof Error ? error.message : String(error))
// Don't throw - Brainy should still work without Neural Import
}
}
/**
* Check if Cortex is available and working
*/
async checkCortexHealth(): Promise<{
available: boolean
status: string
version?: string
}> {
try {
// Check if Cortex is registered as an augmentation
// Note: hasAugmentation method doesn't exist yet in Brainy
const hasCortex = false // this.brainy.hasAugmentation && this.brainy.hasAugmentation('SENSE', 'cortex')
return {
available: hasCortex || false,
status: hasCortex ? 'active' : 'not registered (awaiting Brainy support)',
version: '1.0.0'
}
} catch (error) {
return {
available: false,
status: `Error: ${error instanceof Error ? error.message : String(error)}`
}
}
}
/**
* Reinstall Cortex if it's missing or corrupted
*/
async reinstallCortex(): Promise<void> {
try {
// Remove existing if present
// Note: removeAugmentation method doesn't exist yet in Brainy
/*
if (this.brainy.removeAugmentation) {
try {
await this.brainy.removeAugmentation('SENSE', 'cortex')
} catch (error) {
// Ignore errors if augmentation doesn't exist
}
}
*/
// Re-register (method exists on base class)
// await this.registerCortex()
console.log('🧠⚛️ Cortex reinstalled successfully')
} catch (error) {
throw new Error(`Failed to reinstall Cortex: ${error instanceof Error ? error.message : String(error)}`)
}
}
}
/**
* Helper function to initialize default augmentations for any Brainy instance
*/
export async function initializeDefaultAugmentations(brainy: BrainyInterface): Promise<DefaultAugmentationRegistry> {
const registry = new DefaultAugmentationRegistry(brainy)
await registry.initializeDefaults()
return registry
}

220
src/types/augmentations.ts
View file

@ -1,220 +0,0 @@
/**
* Brainy 2.0 Augmentation Types
*
* This file contains only the minimal types needed for augmentations.
* The main augmentation interfaces are now in augmentations/brainyAugmentation.ts
*/
/**
* WebSocket connection type for conduit augmentations
*/
export type WebSocketConnection = {
connectionId: string
url: string
status: 'connected' | 'disconnected' | 'error'
send?: (data: string | ArrayBufferLike | Blob | ArrayBufferView) => Promise<void>
close?: () => Promise<void>
_streamMessageHandler?: (event: { data: unknown }) => void
_messageHandlerWrapper?: (data: unknown) => void
}
/**
* Generic augmentation response type
*/
export type AugmentationResponse<T> = {
success: boolean
data: T
error?: string
}
/**
* Data callback type for subscriptions
*/
export type DataCallback<T> = (data: T) => void
/**
* Import types for re-export (avoiding circular dependencies)
*/
import type {
BrainyAugmentation as BA,
BaseAugmentation as BaseA,
AugmentationContext as AC
} from '../augmentations/brainyAugmentation.js'
export type BrainyAugmentation = BA
export type BaseAugmentation = BaseA
export type AugmentationContext = AC
// REMOVED: Old augmentation type system for 2.0 clean architecture
// All augmentations now use the unified BrainyAugmentation interface from brainyAugmentation.ts
/**
* @deprecated Use BrainyAugmentation from '../augmentations/brainyAugmentation.js' instead
*
* @example
* // ❌ Old way (v2.x):
* import { IAugmentation } from '@soulcraft/brainy/types/augmentations'
*
* // ✅ New way (v3.x):
* import { BrainyAugmentation } from '@soulcraft/brainy'
*/
export type IAugmentation = BrainyAugmentation
/**
* @deprecated Augmentation types are now unified under BrainyAugmentation interface
*
* @example
* // ❌ Old way (v2.x):
* import { AugmentationType } from '@soulcraft/brainy/types/augmentations'
* if (augmentation.type === AugmentationType.SENSE) { ... }
*
* // ✅ New way (v3.x):
* import { BrainyAugmentation } from '@soulcraft/brainy'
* // Use the unified BrainyAugmentation interface directly
*/
export enum AugmentationType { SENSE = 'sense', CONDUIT = 'conduit', COGNITION = 'cognition', MEMORY = 'memory', PERCEPTION = 'perception', DIALOG = 'dialog', ACTIVATION = 'activation', WEBSOCKET = 'webSocket', SYNAPSE = 'synapse' }
/**
* @deprecated Use BrainyAugmentation interface directly instead of namespace types
*
* @example
* // ❌ Old way (v2.x):
* import { BrainyAugmentations } from '@soulcraft/brainy/types/augmentations'
* class MyAugmentation implements BrainyAugmentations.ISenseAugmentation { ... }
*
* // ✅ New way (v3.x):
* import { BrainyAugmentation } from '@soulcraft/brainy'
* class MyAugmentation implements BrainyAugmentation { ... }
*/
export namespace BrainyAugmentations {
/** @deprecated Use BrainyAugmentation instead */
export type ISenseAugmentation = BrainyAugmentation
/** @deprecated Use BrainyAugmentation instead */
export type IConduitAugmentation = BrainyAugmentation
/** @deprecated Use BrainyAugmentation instead */
export type ICognitionAugmentation = BrainyAugmentation
/** @deprecated Use BrainyAugmentation instead */
export type IMemoryAugmentation = BrainyAugmentation
/** @deprecated Use BrainyAugmentation instead */
export type IPerceptionAugmentation = BrainyAugmentation
/** @deprecated Use BrainyAugmentation instead */
export type IDialogAugmentation = BrainyAugmentation
/** @deprecated Use BrainyAugmentation instead */
export type IActivationAugmentation = BrainyAugmentation
/** @deprecated Use BrainyAugmentation instead */
export type ISynapseAugmentation = BrainyAugmentation
}
/**
* @deprecated Use BrainyAugmentation from '../augmentations/brainyAugmentation.js' instead
*
* @example
* // ❌ Old way (v2.x):
* import { ISenseAugmentation } from '@soulcraft/brainy/types/augmentations'
* class MySense implements ISenseAugmentation { ... }
*
* // ✅ New way (v3.x):
* import { BrainyAugmentation } from '@soulcraft/brainy'
* class MySense implements BrainyAugmentation { ... }
*/
export type ISenseAugmentation = BrainyAugmentation
/**
* @deprecated Use BrainyAugmentation from '../augmentations/brainyAugmentation.js' instead
*
* @example
* // ❌ Old way (v2.x):
* import { IConduitAugmentation } from '@soulcraft/brainy/types/augmentations'
*
* // ✅ New way (v3.x):
* import { BrainyAugmentation } from '@soulcraft/brainy'
*/
export type IConduitAugmentation = BrainyAugmentation
/**
* @deprecated Use BrainyAugmentation from '../augmentations/brainyAugmentation.js' instead
*
* @example
* // ❌ Old way (v2.x):
* import { ICognitionAugmentation } from '@soulcraft/brainy/types/augmentations'
*
* // ✅ New way (v3.x):
* import { BrainyAugmentation } from '@soulcraft/brainy'
*/
export type ICognitionAugmentation = BrainyAugmentation
/**
* @deprecated Use BrainyAugmentation from '../augmentations/brainyAugmentation.js' instead
*
* @example
* // ❌ Old way (v2.x):
* import { IMemoryAugmentation } from '@soulcraft/brainy/types/augmentations'
*
* // ✅ New way (v3.x):
* import { BrainyAugmentation } from '@soulcraft/brainy'
*/
export type IMemoryAugmentation = BrainyAugmentation
/**
* @deprecated Use BrainyAugmentation from '../augmentations/brainyAugmentation.js' instead
*
* @example
* // ❌ Old way (v2.x):
* import { IPerceptionAugmentation } from '@soulcraft/brainy/types/augmentations'
*
* // ✅ New way (v3.x):
* import { BrainyAugmentation } from '@soulcraft/brainy'
*/
export type IPerceptionAugmentation = BrainyAugmentation
/**
* @deprecated Use BrainyAugmentation from '../augmentations/brainyAugmentation.js' instead
*
* @example
* // ❌ Old way (v2.x):
* import { IDialogAugmentation } from '@soulcraft/brainy/types/augmentations'
*
* // ✅ New way (v3.x):
* import { BrainyAugmentation } from '@soulcraft/brainy'
*/
export type IDialogAugmentation = BrainyAugmentation
/**
* @deprecated Use BrainyAugmentation from '../augmentations/brainyAugmentation.js' instead
*
* @example
* // ❌ Old way (v2.x):
* import { IActivationAugmentation } from '@soulcraft/brainy/types/augmentations'
*
* // ✅ New way (v3.x):
* import { BrainyAugmentation } from '@soulcraft/brainy'
*/
export type IActivationAugmentation = BrainyAugmentation
/**
* @deprecated Use BrainyAugmentation from '../augmentations/brainyAugmentation.js' instead
*
* @example
* // ❌ Old way (v2.x):
* import { ISynapseAugmentation } from '@soulcraft/brainy/types/augmentations'
*
* // ✅ New way (v3.x):
* import { BrainyAugmentation } from '@soulcraft/brainy'
*/
export type ISynapseAugmentation = BrainyAugmentation
/**
* @deprecated WebSocket support is now built into BrainyAugmentation interface
*
* @example
* // ❌ Old way (v2.x):
* import { IWebSocketSupport } from '@soulcraft/brainy/types/augmentations'
* class MyAugmentation implements IWebSocketSupport { ... }
*
* // ✅ New way (v3.x):
* import { BrainyAugmentation } from '@soulcraft/brainy'
* class MyAugmentation implements BrainyAugmentation {
* // WebSocket functionality is now part of the unified interface
* }
*/
export interface IWebSocketSupport {}

3
src/types/brainy.types.ts
View file

@ -693,9 +693,6 @@ export interface BrainyConfig {
ttl?: number
}
// Augmentations
augmentations?: Record<string, any>
// Distributed configuration
distributed?: {
enabled: boolean

33
src/types/pipelineTypes.ts
View file

@ -1,33 +0,0 @@
/**
* Pipeline Types
*
* This module provides shared types for the pipeline system to avoid circular dependencies.
*/
import {
BrainyAugmentations,
IWebSocketSupport,
IAugmentation
} from './augmentations.js'
/**
* Type definitions for the augmentation registry
*/
export type AugmentationRegistry = {
sense: BrainyAugmentations.ISenseAugmentation[];
conduit: BrainyAugmentations.IConduitAugmentation[];
cognition: BrainyAugmentations.ICognitionAugmentation[];
memory: BrainyAugmentations.IMemoryAugmentation[];
perception: BrainyAugmentations.IPerceptionAugmentation[];
dialog: BrainyAugmentations.IDialogAugmentation[];
activation: BrainyAugmentations.IActivationAugmentation[];
webSocket: IWebSocketSupport[];
}
/**
* Interface for the Pipeline class
* This is used to break circular dependencies between pipeline.ts and augmentationRegistry.ts
*/
export interface IPipeline {
register<T extends IAugmentation>(augmentation: T): IPipeline;
}

190
src/utils/brainyTypes.ts
View file

@ -1,8 +1,8 @@
/**
* BrainyTypes - Complete type management for Brainy
* BrainyTypes - Type validation and lookup for Brainy
*
* Provides type lists, validation, and intelligent suggestions
* for nouns and verbs using semantic embeddings.
* Provides type lists and validation for the 42 noun types and 127 verb types.
* Source of truth: graphTypes.ts
*
* @example
* ```typescript
@ -15,50 +15,20 @@
* // Validate types
* BrainyTypes.isValidNoun('Person') // true
* BrainyTypes.isValidVerb('Unknown') // false
*
* // Get intelligent suggestions
* const personData = {
* name: 'John Doe',
* email: 'john@example.com'
* }
* const suggestion = await BrainyTypes.suggestNoun(personData)
* console.log(suggestion.type) // 'Person'
* console.log(suggestion.confidence) // 0.92
* ```
*/
import { NounType, VerbType } from '../types/graphTypes.js'
import { BrainyTypes as InternalBrainyTypes, TypeMatchResult } from '../augmentations/typeMatching/brainyTypes.js'
/**
* Type suggestion result
*/
export interface TypeSuggestion {
/** The suggested type */
type: NounType | VerbType
/** Confidence score between 0 and 1 */
confidence: number
/** Human-readable explanation */
reason?: string
/** Alternative suggestions */
alternatives?: Array<{
type: NounType | VerbType
confidence: number
}>
}
/**
* BrainyTypes - Complete type management for Brainy
* BrainyTypes - Type validation and lookup for Brainy
*
* Static class providing type lists, validation, and intelligent suggestions.
* Static class providing type lists and validation.
* No instantiation needed - all methods are static.
*/
export class BrainyTypes {
private static instance: InternalBrainyTypes | null = null
private static initialized = false
/**
* All available noun types
* All available noun types (42)
* @example
* ```typescript
* BrainyTypes.nouns.forEach(type => console.log(type))
@ -68,7 +38,7 @@ export class BrainyTypes {
static readonly nouns: readonly NounType[] = Object.freeze(Object.values(NounType))
/**
* All available verb types
* All available verb types (127)
* @example
* ```typescript
* BrainyTypes.verbs.forEach(type => console.log(type))
@ -77,18 +47,6 @@ export class BrainyTypes {
*/
static readonly verbs: readonly VerbType[] = Object.freeze(Object.values(VerbType))
/**
* Get or create the internal matcher instance
*/
private static async getInternalMatcher(): Promise<InternalBrainyTypes> {
if (!this.instance) {
this.instance = new InternalBrainyTypes()
await this.instance.init()
this.initialized = true
}
return this.instance
}
/**
* Check if a string is a valid noun type
*
@ -123,85 +81,6 @@ export class BrainyTypes {
return (this.verbs as readonly string[]).includes(type)
}
/**
* Suggest the most appropriate noun type for an object
*
* @param data The object or data to analyze
* @returns Promise resolving to type suggestion with confidence score
*
* @example
* ```typescript
* const data = {
* title: 'Quarterly Report',
* author: 'Jane Smith',
* pages: 42
* }
* const suggestion = await BrainyTypes.suggestNoun(data)
* console.log(suggestion.type) // 'Document'
* console.log(suggestion.confidence) // 0.88
*
* // Check alternatives if confidence is low
* if (suggestion.confidence < 0.8) {
* console.log('Also consider:', suggestion.alternatives)
* }
* ```
*/
static async suggestNoun(data: any): Promise<TypeSuggestion> {
const matcher = await this.getInternalMatcher()
const result = await matcher.matchNounType(data)
return {
type: result.type as NounType,
confidence: result.confidence,
reason: result.reasoning,
alternatives: result.alternatives?.map(alt => ({
type: alt.type as NounType,
confidence: alt.confidence
}))
}
}
/**
* Suggest the most appropriate verb type for a relationship
*
* @param source The source entity
* @param target The target entity
* @param hint Optional hint about the relationship
* @returns Promise resolving to type suggestion with confidence score
*
* @example
* ```typescript
* const source = { type: 'Person', name: 'Alice' }
* const target = { type: 'Document', title: 'Research Paper' }
*
* const suggestion = await BrainyTypes.suggestVerb(source, target, 'authored')
* console.log(suggestion.type) // 'CreatedBy'
* console.log(suggestion.confidence) // 0.91
*
* // Without hint
* const suggestion2 = await BrainyTypes.suggestVerb(source, target)
* console.log(suggestion2.type) // 'RelatedTo' (more generic)
* ```
*/
static async suggestVerb(
source: any,
target: any,
hint?: string
): Promise<TypeSuggestion> {
const matcher = await this.getInternalMatcher()
const result = await matcher.matchVerbType(source, target, hint)
return {
type: result.type as VerbType,
confidence: result.confidence,
reason: result.reasoning,
alternatives: result.alternatives?.map(alt => ({
type: alt.type as VerbType,
confidence: alt.confidence
}))
}
}
/**
* Get a noun type by name (with validation)
*
@ -242,26 +121,6 @@ export class BrainyTypes {
return name as VerbType
}
/**
* Clear the internal cache
* Useful when processing many different types of data
*/
static clearCache(): void {
this.instance?.clearCache()
}
/**
* Dispose of resources
* Call when completely done using BrainyTypes
*/
static async dispose(): Promise<void> {
if (this.instance) {
await this.instance.dispose()
this.instance = null
this.initialized = false
}
}
/**
* Get noun types as a plain object (for iteration)
* @returns Object with noun type names as keys
@ -289,38 +148,3 @@ export class BrainyTypes {
// Re-export the enums for convenience
export { NounType, VerbType }
/**
* Helper function to validate and suggest types in one call
*
* @example
* ```typescript
* import { suggestType } from '@soulcraft/brainy'
*
* // For nouns
* const nounSuggestion = await suggestType('noun', data)
*
* // For verbs
* const verbSuggestion = await suggestType('verb', source, target)
* ```
*/
export async function suggestType(
kind: 'noun',
data: any
): Promise<TypeSuggestion>
export async function suggestType(
kind: 'verb',
source: any,
target: any,
hint?: string
): Promise<TypeSuggestion>
export async function suggestType(
kind: 'noun' | 'verb',
...args: any[]
): Promise<TypeSuggestion> {
if (kind === 'noun') {
return BrainyTypes.suggestNoun(args[0])
} else {
return BrainyTypes.suggestVerb(args[0], args[1], args[2])
}
}

281
tests/augmentations/intelligentImport/csvHandler.unit.test.ts
View file

@ -1,281 +0,0 @@
/**
* CSV Handler Tests
* Comprehensive tests for CSV parsing, encoding detection, and type inference
*/
import { describe, it, expect, beforeAll } from 'vitest'
import { CSVHandler } from '../../../src/augmentations/intelligentImport/handlers/csvHandler.js'
import { promises as fs } from 'fs'
import * as path from 'path'
describe('CSVHandler', () => {
let handler: CSVHandler
const fixturesPath = path.join(process.cwd(), 'tests/fixtures/import')
beforeAll(() => {
handler = new CSVHandler()
})
describe('canHandle', () => {
it('should handle .csv extension', () => {
expect(handler.canHandle({ filename: 'data.csv' })).toBe(true)
})
it('should handle .tsv extension', () => {
expect(handler.canHandle({ filename: 'data.tsv' })).toBe(true)
})
it('should handle .txt extension', () => {
expect(handler.canHandle({ filename: 'data.txt' })).toBe(true)
})
it('should handle CSV content (comma)', () => {
const content = 'name,age,email\nJohn,30,john@example.com'
expect(handler.canHandle(content)).toBe(true)
})
it('should handle CSV content (semicolon)', () => {
const content = 'name;age;email\nJohn;30;john@example.com'
expect(handler.canHandle(content)).toBe(true)
})
it('should not handle non-CSV content', () => {
const content = 'This is just plain text without any structure'
expect(handler.canHandle(content)).toBe(false)
})
it('should not handle .xlsx extension', () => {
expect(handler.canHandle({ filename: 'data.xlsx' })).toBe(false)
})
})
describe('process - simple CSV', () => {
it('should parse simple comma-delimited CSV', async () => {
const filePath = path.join(fixturesPath, 'simple.csv')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'simple.csv' })
expect(result.format).toBe('csv')
expect(result.data).toHaveLength(4)
expect(result.data[0]).toEqual({
name: 'John Doe',
age: 30,
email: 'john@example.com',
active: true
})
expect(result.metadata.rowCount).toBe(4)
expect(result.metadata.fields).toEqual(['name', 'age', 'email', 'active'])
expect(result.metadata.delimiter).toBe(',')
expect(result.metadata.hasHeaders).toBe(true)
})
it('should infer correct types', async () => {
const filePath = path.join(fixturesPath, 'simple.csv')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'simple.csv' })
expect(typeof result.data[0].name).toBe('string')
expect(typeof result.data[0].age).toBe('number')
expect(typeof result.data[0].active).toBe('boolean')
})
})
describe('process - delimiter detection', () => {
it('should detect semicolon delimiter', async () => {
const filePath = path.join(fixturesPath, 'semicolon.csv')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'semicolon.csv' })
expect(result.metadata.delimiter).toBe(';')
expect(result.data).toHaveLength(4)
expect(result.data[0]).toEqual({
product: 'Laptop',
price: 999.99,
quantity: 10,
inStock: true
})
})
it('should detect tab delimiter', async () => {
const filePath = path.join(fixturesPath, 'tab-delimited.csv')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'tab-delimited.csv' })
expect(result.metadata.delimiter).toBe('\t')
expect(result.data).toHaveLength(4)
expect(result.data[0].city).toBe('Tokyo')
expect(result.data[0].population).toBe(13960000)
})
it('should use specified delimiter when provided', async () => {
const content = 'name|age|email\nJohn|30|john@example.com'
const result = await handler.process(content, {
csvDelimiter: '|',
filename: 'pipe.csv'
})
expect(result.data).toHaveLength(1)
expect(result.data[0].name).toBe('John')
expect(result.data[0].age).toBe(30)
})
})
describe('process - type inference', () => {
it('should infer integer, float, date, and boolean types', async () => {
const filePath = path.join(fixturesPath, 'types.csv')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'types.csv' })
expect(result.metadata.types).toBeDefined()
expect(result.metadata.types.id).toBe('integer')
expect(result.metadata.types.name).toBe('string')
// Score is 95 (integer), not 95.0, so it's detected as integer
expect(['integer', 'float']).toContain(result.metadata.types.score)
expect(result.metadata.types.active).toBe('boolean')
// Check actual values
expect(result.data[0].id).toBe(1)
expect([95, 95.0]).toContain(result.data[0].score) // Can be integer or float
expect(result.data[0].active).toBe(true)
expect(result.data[1].score).toBe(87.5) // This one is definitely a float
})
it('should handle null values', async () => {
const content = 'name,age,email\nJohn,30,john@example.com\nJane,,jane@example.com\nBob,45,'
const result = await handler.process(content, { filename: 'nulls.csv' })
expect(result.data[1].age).toBeNull()
expect(result.data[2].email).toBeNull()
})
})
describe('process - encoding detection', () => {
it('should detect UTF-8 encoding', async () => {
const content = 'name,description\nProduct 1,Description with émojis 🎉'
const result = await handler.process(content, { filename: 'utf8.csv' })
// Encoding can be 'UTF-8', 'utf-8', or 'utf8' depending on detection
expect(result.metadata.encoding.toLowerCase()).toContain('utf')
expect(result.data[0].description).toContain('émojis')
expect(result.data[0].description).toContain('🎉')
})
it('should use specified encoding when provided', async () => {
const content = 'name,age\nJohn,30'
const result = await handler.process(content, {
encoding: 'ascii',
filename: 'ascii.csv'
})
expect(result.metadata.encoding).toBe('ascii')
})
})
describe('process - options', () => {
it('should respect csvHeaders=false option', async () => {
const content = 'John,30,john@example.com\nJane,25,jane@example.com'
const result = await handler.process(content, {
csvHeaders: false,
filename: 'no-headers.csv'
})
expect(result.metadata.hasHeaders).toBe(false)
// csv-parse will generate column names like '0', '1', '2'
expect(Object.keys(result.data[0])).toHaveLength(3)
})
it('should respect maxRows option', async () => {
const filePath = path.join(fixturesPath, 'simple.csv')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, {
maxRows: 2,
filename: 'simple.csv'
})
expect(result.data).toHaveLength(2)
expect(result.metadata.rowCount).toBe(2)
})
})
describe('process - edge cases', () => {
it('should handle empty CSV', async () => {
const content = ''
const result = await handler.process(content, { filename: 'empty.csv' })
expect(result.data).toHaveLength(0)
expect(result.metadata.rowCount).toBe(0)
expect(result.metadata.fields).toEqual([])
})
it('should handle CSV with only headers', async () => {
const content = 'name,age,email'
const result = await handler.process(content, { filename: 'headers-only.csv' })
expect(result.data).toHaveLength(0)
expect(result.metadata.rowCount).toBe(0)
})
it('should handle CSV with single row', async () => {
const content = 'name,age,email\nJohn,30,john@example.com'
const result = await handler.process(content, { filename: 'single-row.csv' })
expect(result.data).toHaveLength(1)
expect(result.data[0].name).toBe('John')
})
it('should handle CSV with varying column counts', async () => {
const content = 'name,age,email\nJohn,30,john@example.com\nJane,25\nBob,45,bob@example.com,extra'
const result = await handler.process(content, { filename: 'varying.csv' })
// csv-parse with relax_column_count should handle this
expect(result.data).toHaveLength(3)
})
it('should trim whitespace', async () => {
const content = 'name, age , email \n John , 30 , john@example.com '
const result = await handler.process(content, { filename: 'whitespace.csv' })
expect(result.data[0].name).toBe('John')
expect(result.data[0].age).toBe(30)
expect(result.data[0].email).toBe('john@example.com')
})
})
describe('process - performance', () => {
it('should measure processing time', async () => {
const filePath = path.join(fixturesPath, 'simple.csv')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'simple.csv' })
expect(result.metadata.processingTime).toBeGreaterThanOrEqual(0)
expect(typeof result.metadata.processingTime).toBe('number')
})
})
describe('process - error handling', () => {
it('should throw error for malformed CSV', async () => {
const content = 'name,age,email\nJohn,30,"unclosed quote'
await expect(
handler.process(content, { filename: 'malformed.csv' })
).rejects.toThrow('CSV parsing failed')
})
})
})

269
tests/augmentations/intelligentImport/excelHandler.unit.test.ts
View file

@ -1,269 +0,0 @@
/**
* Excel Handler Tests
* Comprehensive tests for Excel parsing, multi-sheet extraction, and type inference
*/
import { describe, it, expect, beforeAll } from 'vitest'
import { ExcelHandler } from '../../../src/augmentations/intelligentImport/handlers/excelHandler.js'
import { promises as fs } from 'fs'
import * as path from 'path'
describe('ExcelHandler', () => {
let handler: ExcelHandler
const fixturesPath = path.join(process.cwd(), 'tests/fixtures/import')
beforeAll(() => {
handler = new ExcelHandler()
})
describe('canHandle', () => {
it('should handle .xlsx extension', () => {
expect(handler.canHandle({ filename: 'data.xlsx' })).toBe(true)
})
it('should handle .xls extension', () => {
expect(handler.canHandle({ filename: 'data.xls' })).toBe(true)
})
it('should handle .xlsb extension', () => {
expect(handler.canHandle({ filename: 'data.xlsb' })).toBe(true)
})
it('should handle .xlsm extension', () => {
expect(handler.canHandle({ filename: 'data.xlsm' })).toBe(true)
})
it('should not handle .csv extension', () => {
expect(handler.canHandle({ filename: 'data.csv' })).toBe(false)
})
it('should not handle .pdf extension', () => {
expect(handler.canHandle({ filename: 'data.pdf' })).toBe(false)
})
})
describe('process - simple Excel', () => {
it('should parse simple single-sheet Excel file', async () => {
const filePath = path.join(fixturesPath, 'simple.xlsx')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'simple.xlsx' })
expect(result.format).toBe('excel')
expect(result.data).toHaveLength(4)
expect(result.data[0]).toMatchObject({
Name: 'Alice Johnson',
Age: 28,
Department: 'Engineering',
Salary: 95000,
Active: true
})
expect(result.metadata.rowCount).toBe(4)
expect(result.metadata.sheetCount).toBe(1)
expect(result.metadata.sheets).toEqual(['Employees'])
})
it('should infer correct types from Excel data', async () => {
const filePath = path.join(fixturesPath, 'simple.xlsx')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'simple.xlsx' })
expect(result.metadata.types).toBeDefined()
expect(result.metadata.types.Name).toBe('string')
expect(result.metadata.types.Age).toBe('integer')
expect(['integer', 'float']).toContain(result.metadata.types.Salary)
expect(result.metadata.types.Active).toBe('boolean')
})
})
describe('process - multi-sheet Excel', () => {
it('should extract data from all sheets by default', async () => {
const filePath = path.join(fixturesPath, 'multi-sheet.xlsx')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'multi-sheet.xlsx' })
expect(result.metadata.sheetCount).toBe(3)
expect(result.metadata.sheets).toEqual(['Products', 'Orders', 'Customers'])
// Total rows from all sheets
const totalRows = 4 + 3 + 3 // Products + Orders + Customers
expect(result.data).toHaveLength(totalRows)
// Check _sheet field is added
expect(result.data[0]._sheet).toBe('Products')
})
it('should filter data by sheet name', async () => {
const filePath = path.join(fixturesPath, 'multi-sheet.xlsx')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, {
filename: 'multi-sheet.xlsx',
excelSheets: ['Products']
})
expect(result.metadata.sheetCount).toBe(1)
expect(result.metadata.sheets).toEqual(['Products'])
expect(result.data).toHaveLength(4)
expect(result.data.every(row => row._sheet === 'Products')).toBe(true)
})
it('should extract data from multiple specified sheets', async () => {
const filePath = path.join(fixturesPath, 'multi-sheet.xlsx')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, {
filename: 'multi-sheet.xlsx',
excelSheets: ['Products', 'Customers']
})
expect(result.metadata.sheetCount).toBe(2)
expect(result.metadata.sheets).toEqual(['Products', 'Customers'])
expect(result.data).toHaveLength(7) // 4 + 3
const sheets = new Set(result.data.map(row => row._sheet))
expect(sheets.has('Products')).toBe(true)
expect(sheets.has('Customers')).toBe(true)
expect(sheets.has('Orders')).toBe(false)
})
it('should include sheet metadata for each sheet', async () => {
const filePath = path.join(fixturesPath, 'multi-sheet.xlsx')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'multi-sheet.xlsx' })
expect(result.metadata.sheetMetadata).toBeDefined()
expect(result.metadata.sheetMetadata.Products).toMatchObject({
rowCount: 4,
columnCount: 5
})
expect(result.metadata.sheetMetadata.Products.headers).toEqual([
'ID', 'Product', 'Price', 'Stock', 'Category'
])
})
})
describe('process - type inference', () => {
it('should infer integer, float, date, and boolean types', async () => {
const filePath = path.join(fixturesPath, 'types.xlsx')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'types.xlsx' })
expect(result.metadata.types).toBeDefined()
expect(result.metadata.types.ID).toBe('integer')
expect(result.metadata.types.Name).toBe('string')
expect(['integer', 'float']).toContain(result.metadata.types.Score)
expect(['integer', 'float']).toContain(result.metadata.types.Percentage)
expect(result.metadata.types.Active).toBe('boolean')
// Check actual values
expect(result.data[0].ID).toBe(1)
expect(result.data[0].Active).toBe(true)
expect(typeof result.data[0].Score).toBe('number')
})
it('should handle null/empty values', async () => {
const filePath = path.join(fixturesPath, 'empty.xlsx')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'empty.xlsx' })
expect(result.data).toHaveLength(0)
expect(result.metadata.rowCount).toBe(0)
})
})
describe('process - workbook metadata', () => {
it('should extract workbook information', async () => {
const filePath = path.join(fixturesPath, 'multi-sheet.xlsx')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'multi-sheet.xlsx' })
expect(result.metadata.workbookInfo).toBeDefined()
expect(result.metadata.workbookInfo.sheetNames).toEqual([
'Products', 'Orders', 'Customers'
])
})
})
describe('process - edge cases', () => {
it('should handle Excel file with only headers', async () => {
const filePath = path.join(fixturesPath, 'empty.xlsx')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'empty.xlsx' })
expect(result.data).toHaveLength(0)
expect(result.metadata.sheetCount).toBe(1)
})
it('should skip non-existent sheets when filtering', async () => {
const filePath = path.join(fixturesPath, 'simple.xlsx')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, {
filename: 'simple.xlsx',
excelSheets: ['Employees', 'NonExistent']
})
expect(result.metadata.sheets).toEqual(['Employees'])
expect(result.data).toHaveLength(4)
})
})
describe('process - performance', () => {
it('should measure processing time', async () => {
const filePath = path.join(fixturesPath, 'simple.xlsx')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'simple.xlsx' })
expect(result.metadata.processingTime).toBeGreaterThanOrEqual(0)
expect(typeof result.metadata.processingTime).toBe('number')
})
})
describe('process - field sanitization', () => {
it('should sanitize column names', async () => {
const filePath = path.join(fixturesPath, 'simple.xlsx')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'simple.xlsx' })
// All field names should be valid identifiers
const fields = Object.keys(result.data[0]).filter(k => k !== '_sheet')
for (const field of fields) {
expect(field).toMatch(/^[a-zA-Z_][a-zA-Z0-9_]*$/)
}
})
})
describe('process - error handling', () => {
it('should handle corrupted Excel file gracefully', async () => {
// XLSX library is very forgiving, so we test that it returns empty data
// rather than crashing
const invalidData = Buffer.from('This is not an Excel file')
const result = await handler.process(invalidData, { filename: 'invalid.xlsx' })
// Should return empty data instead of crashing
expect(result.data).toHaveLength(0)
expect(result.metadata.rowCount).toBe(0)
})
it('should handle binary garbage', async () => {
const garbage = Buffer.from(new Uint8Array(256).fill(255))
const result = await handler.process(garbage, { filename: 'garbage.xlsx' })
// Should not crash
expect(result).toBeDefined()
expect(result.format).toBe('excel')
})
})
})

292
tests/augmentations/intelligentImport/pdfHandler.unit.test.ts
View file

@ -1,292 +0,0 @@
/**
* PDF Handler Tests
* Comprehensive tests for PDF text extraction, table detection, and metadata
*/
import { describe, it, expect, beforeAll } from 'vitest'
import { PDFHandler } from '../../../src/augmentations/intelligentImport/handlers/pdfHandler.js'
import { promises as fs } from 'fs'
import * as path from 'path'
describe('PDFHandler', () => {
let handler: PDFHandler
const fixturesPath = path.join(process.cwd(), 'tests/fixtures/import')
beforeAll(() => {
handler = new PDFHandler()
})
describe('canHandle', () => {
it('should handle .pdf extension', () => {
expect(handler.canHandle({ filename: 'document.pdf' })).toBe(true)
})
it('should handle PDF magic bytes', () => {
const pdfBuffer = Buffer.from('%PDF-1.4\n')
expect(handler.canHandle(pdfBuffer)).toBe(true)
})
it('should not handle .xlsx extension', () => {
expect(handler.canHandle({ filename: 'data.xlsx' })).toBe(false)
})
it('should not handle .csv extension', () => {
expect(handler.canHandle({ filename: 'data.csv' })).toBe(false)
})
it('should not handle non-PDF buffer', () => {
const buffer = Buffer.from('This is not a PDF')
expect(handler.canHandle(buffer)).toBe(false)
})
})
describe('process - simple text PDF', () => {
it('should extract text from simple PDF', async () => {
const filePath = path.join(fixturesPath, 'simple.pdf')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'simple.pdf' })
expect(result.format).toBe('pdf')
expect(result.data.length).toBeGreaterThan(0)
// Should have extracted paragraphs
const paragraphs = result.data.filter(item => item._type === 'paragraph')
expect(paragraphs.length).toBeGreaterThan(0)
// Check that text was extracted
const hasText = paragraphs.some(p => p.text && p.text.length > 0)
expect(hasText).toBe(true)
})
it('should include page numbers', async () => {
const filePath = path.join(fixturesPath, 'simple.pdf')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'simple.pdf' })
// All items should have _page field
expect(result.data.every(item => typeof item._page === 'number')).toBe(true)
expect(result.data.every(item => item._page >= 1)).toBe(true)
})
it('should count pages correctly', async () => {
const filePath = path.join(fixturesPath, 'simple.pdf')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'simple.pdf' })
expect(result.metadata.pageCount).toBe(1)
})
})
describe('process - multi-page PDF', () => {
it('should extract text from all pages', async () => {
const filePath = path.join(fixturesPath, 'multi-page.pdf')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'multi-page.pdf' })
expect(result.metadata.pageCount).toBe(3)
// Should have content from multiple pages
const pages = new Set(result.data.map(item => item._page))
expect(pages.size).toBeGreaterThan(1)
expect(pages.has(1)).toBe(true)
expect(pages.has(2)).toBe(true)
expect(pages.has(3)).toBe(true)
})
it('should preserve page order', async () => {
const filePath = path.join(fixturesPath, 'multi-page.pdf')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'multi-page.pdf' })
// Page numbers should be in order
let lastPage = 0
let pageChanged = false
for (const item of result.data) {
if (item._page !== lastPage && lastPage > 0) {
pageChanged = true
}
expect(item._page).toBeGreaterThanOrEqual(lastPage)
lastPage = item._page
}
expect(pageChanged).toBe(true) // Should have moved through pages
})
})
describe('process - table detection', () => {
it('should detect tables in PDF', async () => {
const filePath = path.join(fixturesPath, 'table.pdf')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'table.pdf' })
// Should have detected at least one table
expect(result.metadata.tableCount).toBeGreaterThan(0)
})
it('should extract table rows when tables detected', async () => {
const filePath = path.join(fixturesPath, 'table.pdf')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, {
filename: 'table.pdf',
pdfExtractTables: true
})
// Should have table_row items
const tableRows = result.data.filter(item => item._type === 'table_row')
expect(tableRows.length).toBeGreaterThan(0)
// Table rows should have structured data
if (tableRows.length > 0) {
const firstRow = tableRows[0]
const fields = Object.keys(firstRow).filter(k => !k.startsWith('_'))
expect(fields.length).toBeGreaterThan(0)
}
})
it('should skip table extraction when disabled', async () => {
const filePath = path.join(fixturesPath, 'table.pdf')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, {
filename: 'table.pdf',
pdfExtractTables: false
})
// Should not have detected tables
expect(result.metadata.tableCount).toBe(0)
// Should not have table_row items
const tableRows = result.data.filter(item => item._type === 'table_row')
expect(tableRows.length).toBe(0)
})
})
describe('process - metadata extraction', () => {
it('should extract PDF metadata', async () => {
const filePath = path.join(fixturesPath, 'metadata.pdf')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'metadata.pdf' })
expect(result.metadata.pdfMetadata).toBeDefined()
expect(result.metadata.pdfMetadata.title).toBe('Test Document')
expect(result.metadata.pdfMetadata.author).toBe('Test Author')
expect(result.metadata.pdfMetadata.creator).toBe('Brainy Test Suite')
})
it('should handle PDFs without metadata gracefully', async () => {
const filePath = path.join(fixturesPath, 'simple.pdf')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'simple.pdf' })
expect(result.metadata.pdfMetadata).toBeDefined()
// Some fields may be null
expect(result.metadata.pdfMetadata).toHaveProperty('title')
expect(result.metadata.pdfMetadata).toHaveProperty('author')
})
})
describe('process - text statistics', () => {
it('should track total text length', async () => {
const filePath = path.join(fixturesPath, 'simple.pdf')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'simple.pdf' })
expect(result.metadata.textLength).toBeGreaterThan(0)
expect(typeof result.metadata.textLength).toBe('number')
})
it('should count extracted items', async () => {
const filePath = path.join(fixturesPath, 'simple.pdf')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'simple.pdf' })
expect(result.metadata.rowCount).toBe(result.data.length)
expect(result.metadata.rowCount).toBeGreaterThan(0)
})
})
describe('process - edge cases', () => {
it('should handle empty PDF', async () => {
const filePath = path.join(fixturesPath, 'empty.pdf')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'empty.pdf' })
expect(result.format).toBe('pdf')
expect(result.metadata.pageCount).toBe(1)
// Empty PDF may have 0 or minimal content
expect(result.data).toBeDefined()
})
it('should measure processing time', async () => {
const filePath = path.join(fixturesPath, 'simple.pdf')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'simple.pdf' })
expect(result.metadata.processingTime).toBeGreaterThan(0)
expect(typeof result.metadata.processingTime).toBe('number')
})
})
describe('process - data structure', () => {
it('should include type indicators', async () => {
const filePath = path.join(fixturesPath, 'simple.pdf')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'simple.pdf' })
// All items should have _type field
expect(result.data.every(item => '_type' in item)).toBe(true)
// Types should be valid
const validTypes = ['paragraph', 'table_row']
expect(result.data.every(item => validTypes.includes(item._type))).toBe(true)
})
it('should include index for paragraphs', async () => {
const filePath = path.join(fixturesPath, 'simple.pdf')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'simple.pdf' })
const paragraphs = result.data.filter(item => item._type === 'paragraph')
// Paragraphs should have _index field
expect(paragraphs.every(p => typeof p._index === 'number')).toBe(true)
})
})
describe('process - error handling', () => {
it('should throw error for invalid PDF', async () => {
const invalidData = Buffer.from('This is not a PDF file')
await expect(
handler.process(invalidData, { filename: 'invalid.pdf' })
).rejects.toThrow('PDF parsing failed')
})
it('should throw error for corrupted PDF', async () => {
const corruptedPDF = Buffer.concat([
Buffer.from('%PDF-1.4\n'),
Buffer.from('corrupted data that is not valid PDF structure')
])
await expect(
handler.process(corruptedPDF, { filename: 'corrupted.pdf' })
).rejects.toThrow('PDF parsing failed')
})
})
})

1
tests/benchmarks/benchmark-real-world.js
View file

@ -60,7 +60,6 @@ async function runBrainyBenchmark() {
const brain = new Brainy({
storage: { type: 'memory' },
augmentations: {},
model: {
type: 'fast', // Using real transformer model
precision: 'Q8' // Quantized for speed

6
tests/benchmarks/benchmark-vs-industry.js
View file

@ -158,12 +158,6 @@ async function runBrainyBenchmark() {
const brain = new Brainy({
storage: { type: 'memory' },
augmentations: {
cache: false,
metrics: false,
display: false,
index: false
},
embedder: mockEmbedder,
warmup: false
})

7
tests/benchmarks/perf-final.js
View file

@ -14,15 +14,8 @@ async function runBenchmark() {
console.log('🧠 Brainy v3 Performance Benchmark')
console.log('═'.repeat(60))
// Disable all augmentations for raw performance
const brain = new Brainy({
storage: { type: 'memory' },
augmentations: {
cache: false,
metrics: false,
display: false,
index: false
},
embedder: mockEmbedder,
warmup: false
})

1
tests/benchmarks/perf-simple.js
View file

@ -16,7 +16,6 @@ async function benchmark() {
const brain = new Brainy({
storage: { type: 'memory' },
augmentations: {},
embedder: mockEmbedder
})

4
tests/benchmarks/performance-comprehensive.js
View file

@ -22,7 +22,7 @@ async function runV2Benchmark() {
const brain = new Brainy({
storage: { type: 'memory' },
embeddingFunction: mockEmbedder,
augmentations: false // Disable for raw performance
// Raw performance test
})
await brain.init()
@ -111,7 +111,6 @@ async function runV3Benchmark() {
const brain = new Brainy({
storage: { type: 'memory' },
augmentations: {}, // Minimal augmentations
embedder: mockEmbedder
})
@ -216,7 +215,6 @@ async function runScaleTest() {
const brain = new Brainy({
storage: { type: 'memory' },
augmentations: {},
embedder: mockEmbedder
})

31
tests/benchmarks/performance-profile.js
View file

@ -77,30 +77,21 @@ async function profilePerformance() {
// Initialize Brainy with different configurations
console.log('Initializing Brainy configurations...')
// 1. Minimal config (no augmentations)
// 1. Minimal config
const minimalBrain = new Brainy({
storage: new MemoryStorage(),
augmentations: false // Disable all augmentations
storage: new MemoryStorage()
})
await minimalBrain.init()
// 2. Default config (with augmentations)
// 2. Default config
const defaultBrain = new Brainy({
storage: new MemoryStorage()
})
await defaultBrain.init()
// 3. Full augmentations config
// 3. Full config
const fullBrain = new Brainy({
storage: new MemoryStorage(),
augmentations: {
batchProcessing: { enabled: true, maxBatchSize: 1000 },
connectionPool: { enabled: true },
cache: { enabled: true },
index: { enabled: true },
entityRegistry: { enabled: true },
monitoring: { enabled: true }
}
storage: new MemoryStorage()
})
await fullBrain.init()
@ -188,11 +179,7 @@ async function profilePerformance() {
verbCounter++
}, 100)
console.log('\n🔧 Testing Augmentation Overhead\n')
// Test individual augmentations
const augmentations = defaultBrain.augmentations.getAugmentationTypes()
console.log(`Active augmentations: ${augmentations.join(', ')}\n`)
console.log('\n🔧 Testing Operation Overhead\n')
// Measure raw storage performance
const storage = new MemoryStorage()
@ -261,8 +248,8 @@ async function profilePerformance() {
console.log('Overhead breakdown:')
console.log(` Base operation: ${minimalPerf.perOpMs.toFixed(2)}ms`)
console.log(` Default augmentations: +${(defaultPerf.perOpMs - minimalPerf.perOpMs).toFixed(2)}ms`)
console.log(` Full augmentations: +${(fullPerf.perOpMs - defaultPerf.perOpMs).toFixed(2)}ms`)
console.log(` Default config: +${(defaultPerf.perOpMs - minimalPerf.perOpMs).toFixed(2)}ms`)
console.log(` Full config: +${(fullPerf.perOpMs - defaultPerf.perOpMs).toFixed(2)}ms`)
console.log(` Embedding generation: ${embedPerf.perOpMs.toFixed(2)}ms`)
console.log(` Without embeddings: ${vectorPerf.perOpMs.toFixed(2)}ms`)
@ -290,7 +277,7 @@ async function profilePerformance() {
console.log(' 1. Fake/stub operations that returned immediately')
console.log(' 2. No actual embedding generation')
console.log(' 3. No real storage operations')
console.log(' 4. No augmentation processing')
console.log(' 4. Direct operation calls')
console.log('\n With real implementations:')
console.log(` - Raw storage: ${profiler.results['Raw storage.saveNoun']?.opsPerSec || 'N/A'} ops/sec`)
console.log(` - With embeddings: ${defaultPerf.opsPerSec} ops/sec`)

2
tests/benchmarks/performance-v3.js
View file

@ -17,7 +17,6 @@ async function benchmarkV2() {
const brain = new Brainy({
storage: { type: 'memory' },
augmentations: false,
embeddingFunction: async () => new Array(384).fill(0).map(() => Math.random())
})
await brain.init()
@ -92,7 +91,6 @@ async function benchmarkV3() {
const brain = new Brainy({
storage: { type: 'memory' },
augmentations: {},
warmup: false,
embedder: async () => new Array(384).fill(0).map(() => Math.random())
})

1
tests/benchmarks/quick-benchmark-v3.js
View file

@ -13,7 +13,6 @@ async function testV3() {
const brain = new Brainy({
storage: { type: 'memory' },
augmentations: {},
warmup: false,
embedder: async () => new Array(384).fill(0).map(() => Math.random())
})

19
tests/benchmarks/quick-perf.js
View file

@ -37,7 +37,7 @@ async function quickPerf() {
const brain = new Brainy({
storage: new MemoryStorage(),
embeddingFunction: mockEmbed,
augmentations: false // Disable augmentations
// Minimal config
})
await brain.init()
@ -55,12 +55,11 @@ async function quickPerf() {
const brainyOps = Math.round(1000 / ((end2 - start2) / 1000))
console.log(` ✅ Brainy: ${brainyOps.toLocaleString()} ops/sec\n`)
// Test 3: With augmentations
console.log('3️⃣ Brainy with Augmentations')
// Test 3: Default config
console.log('3️⃣ Brainy with Default Config')
const brain2 = new Brainy({
storage: new MemoryStorage(),
embeddingFunction: mockEmbed
// Default augmentations enabled
})
await brain2.init()
@ -74,13 +73,12 @@ async function quickPerf() {
}
const end3 = performance.now()
const augOps = Math.round(1000 / ((end3 - start3) / 1000))
console.log(` ✅ With Augmentations: ${augOps.toLocaleString()} ops/sec\n`)
console.log(` ✅ Default Config: ${augOps.toLocaleString()} ops/sec\n`)
// Test 4: Real embeddings (the killer)
console.log('4️⃣ With Real Embeddings (10 samples)')
const brain3 = new Brainy({
storage: new MemoryStorage(),
augmentations: false
// Uses real embedding function
})
await brain3.init()
@ -119,15 +117,10 @@ async function quickPerf() {
console.log(' ❌ Embeddings are the primary bottleneck')
console.log(' Each embedding takes ~' + Math.round((end4 - start4) / 10) + 'ms')
}
if (augOverhead > 50) {
console.log(' ⚠️ Augmentations add significant overhead')
}
console.log('\n🎯 The 500,000 ops/sec claim was achievable with:')
console.log(' 1. Pre-computed vectors (no embedding)')
console.log(' 2. Minimal augmentations')
console.log(' 3. In-memory storage')
console.log(' 4. Batch operations')
console.log(' 2. In-memory storage')
console.log(' 3. Batch operations')
await brain.close()
await brain2.close()

4
tests/benchmarks/quick-verify.js
View file

@ -69,10 +69,6 @@ async function quickVerify() {
console.log(`❌ Verb storage: Expected 1 verb, got ${stats.totalVerbs}`)
}
// Check augmentations
const augTypes = brain.augmentations.getAugmentationTypes()
console.log(`✅ Active augmentations: ${augTypes.join(', ')}`)
// Close
await brain.close()
console.log('\n✅ All tests passed - implementations are REAL!')

55
tests/benchmarks/scale-test.js
View file

@ -33,34 +33,14 @@ console.log(`🔧 Mode: ${CURRENT_SCALE}\n`)
async function runScaleTest() {
const startTime = Date.now()
// Initialize Brainy with real augmentations
// Initialize Brainy
const brain = new Brainy({
storage: new MemoryStorage(),
augmentations: {
// These should all be REAL implementations now
batchProcessing: {
enabled: true,
maxBatchSize: BATCH_SIZE,
adaptiveBatching: true
},
connectionPool: {
enabled: true,
maxConnections: 50,
minConnections: 5
},
cache: {
enabled: true,
maxSize: 10000
},
index: {
enabled: true
}
}
storage: new MemoryStorage()
})
await brain.init()
console.log('✅ Brainy initialized with real augmentations\n')
console.log('✅ Brainy initialized\n')
// Test 1: Batch Insert Performance
console.log('📝 Test 1: Batch Insert Performance')
@ -160,35 +140,6 @@ async function runScaleTest() {
console.log(`✅ Created ${verbCount.toLocaleString()} relationships in ${verbTime}ms`)
console.log(`📈 Rate: ${verbRate.toLocaleString()} relationships/second\n`)
// Test 4: Verify Augmentations Are Real
console.log('🔧 Test 4: Verify Real Implementations')
// Check BatchProcessing stats
const batchAug = brain.augmentations.getAugmentation('BatchProcessing')
if (batchAug && typeof batchAug.getStats === 'function') {
const stats = batchAug.getStats()
console.log(` BatchProcessing: ${stats.batchesProcessed} batches processed`)
console.log(` Average batch size: ${Math.round(stats.averageBatchSize)}`)
console.log(` Throughput: ${stats.throughputPerSecond} ops/sec`)
}
// Check ConnectionPool stats
const poolAug = brain.augmentations.getAugmentation('ConnectionPool')
if (poolAug && typeof poolAug.getStats === 'function') {
const stats = poolAug.getStats()
console.log(` ConnectionPool: ${stats.totalConnections} connections`)
console.log(` Pool utilization: ${stats.poolUtilization}`)
console.log(` Total requests: ${stats.totalRequests}`)
}
// Check Cache stats
const cacheAug = brain.augmentations.getAugmentation('cache')
if (cacheAug && typeof cacheAug.getStats === 'function') {
const stats = cacheAug.getStats()
console.log(` Cache: ${stats.hits} hits, ${stats.misses} misses`)
console.log(` Hit rate: ${Math.round((stats.hits / (stats.hits + stats.misses)) * 100)}%`)
}
// Final Statistics
const totalTime = Date.now() - startTime
const stats = brain.getStats()

14
tests/brainy-3.test.ts
View file

@ -11,13 +11,7 @@ describe('Brainy 3.0 API', () => {
beforeEach(async () => {
brain = new Brainy({
storage: { type: 'memory' },
augmentations: {
cache: false,
metrics: false,
display: false,
monitoring: false
}
storage: { type: 'memory' }
})
await brain.init()
})
@ -426,7 +420,6 @@ describe('Brainy 3.0 Neural API', () => {
beforeEach(async () => {
brain = new Brainy({
storage: { type: 'memory' },
augmentations: { metrics: false, display: false }
})
await brain.init()
})
@ -440,9 +433,4 @@ describe('Brainy 3.0 Neural API', () => {
expect(typeof brain.neural).toBe('object')
})
it('should provide augmentations API access', () => {
expect(brain.augmentations).toBeDefined()
expect(brain.augmentations.list).toBeDefined()
expect(typeof brain.augmentations.list).toBe('function')
})
})

1
tests/comprehensive/public-api-complete.test.ts
View file

@ -701,7 +701,6 @@ describe('Brainy Public API - Complete Coverage', () => {
expect(health.status).toBe('healthy')
expect(health.storage).toBeDefined()
expect(health.storage.status).toBe('connected')
expect(health.augmentations).toBeDefined()
expect(health.memory).toBeDefined()
})

361
tests/integration/image-import.test.ts
View file

@ -1,361 +0,0 @@
/**
* Image Import Integration Test (v5.2.0)
*
* Tests that ImageHandler works as a built-in handler with IntelligentImportAugmentation
*/
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
import { Brainy } from '../../src/brainy.js'
import sharp from 'sharp'
describe('Image Import Integration (v5.2.0)', () => {
let brain: Brainy
// Create test image
const createTestImage = async (width: number, height: number): Promise<Buffer> => {
const channels = 3
const pixelData = Buffer.alloc(width * height * channels)
// Fill with gradient
for (let y = 0; y < height; y++) {
for (let x = 0; x < width; x++) {
const i = (y * width + x) * channels
pixelData[i] = Math.floor((x / width) * 255)
pixelData[i + 1] = Math.floor((y / height) * 255)
pixelData[i + 2] = 128
}
}
return sharp(pixelData, {
raw: { width, height, channels }
})
.jpeg({ quality: 90 })
.toBuffer()
}
beforeEach(async () => {
// IntelligentImportAugmentation is enabled by default with all handlers
// Configure it to only enable image handler for focused testing
brain = new Brainy({
silent: true,
augmentations: {
intelligentImport: {
enableCSV: false,
enableExcel: false,
enablePDF: false,
enableImage: true // Only enable image handler for this test
}
}
})
await brain.init()
})
afterEach(async () => {
await brain.close()
})
describe('Built-in Image Handler', () => {
it('should automatically handle image import via IntelligentImportAugmentation', async () => {
const imageBuffer = await createTestImage(800, 600)
// Import image - should be automatically processed by ImageHandler
const result = await brain.import({
type: 'buffer',
data: imageBuffer,
filename: 'test-photo.jpg'
})
expect(result).toBeDefined()
expect(result.entities).toHaveLength(1)
const imageEntity = result.entities[0]
expect(imageEntity.type).toBe('media')
expect(imageEntity.metadata?.subtype).toBe('image')
expect(imageEntity.metadata).toBeDefined()
})
it('should extract image metadata via built-in handler', async () => {
const imageBuffer = await createTestImage(1920, 1080)
const result = await brain.import({
type: 'buffer',
data: imageBuffer,
filename: 'hd-photo.jpg'
})
const imageEntity = result.entities[0]
expect(imageEntity).toBeDefined()
expect(imageEntity.metadata).toBeDefined()
expect(typeof imageEntity.metadata).toBe('object')
expect(imageEntity.metadata.width).toBe(1920)
expect(imageEntity.metadata.height).toBe(1080)
expect(imageEntity.metadata.format).toBe('jpeg')
})
it('should extract EXIF data when available', async () => {
const imageBuffer = await createTestImage(400, 300)
const result = await brain.import({
type: 'buffer',
data: imageBuffer,
filename: 'camera-photo.jpg',
options: {
extractEXIF: true
}
})
const imageEntity = result.entities[0]
// Test image won't have EXIF, but should not crash
expect(imageEntity).toBeDefined()
expect(imageEntity.type).toBe('media')
expect(imageEntity.metadata?.subtype).toBe('image')
})
it('should allow disabling EXIF extraction via config', async () => {
const imageBuffer = await createTestImage(400, 300)
const result = await brain.import({
type: 'buffer',
data: imageBuffer,
filename: 'no-exif.jpg',
options: {
extractEXIF: false
}
})
expect(result.entities).toHaveLength(1)
expect(result.entities[0].type).toBe('media')
expect(result.entities[0].metadata?.subtype).toBe('image')
})
it('should handle PNG images', async () => {
const pngBuffer = await sharp(Buffer.alloc(100 * 100 * 4), {
raw: { width: 100, height: 100, channels: 4 }
})
.png()
.toBuffer()
const result = await brain.import({
type: 'buffer',
data: pngBuffer,
filename: 'logo.png'
})
const imageEntity = result.entities[0]
expect(imageEntity.metadata.format).toBe('png')
expect(imageEntity.metadata.hasAlpha).toBe(true)
})
it('should handle WebP images', async () => {
const webpBuffer = await sharp(Buffer.alloc(200 * 200 * 3), {
raw: { width: 200, height: 200, channels: 3 }
})
.webp({ quality: 90 })
.toBuffer()
const result = await brain.import({
type: 'buffer',
data: webpBuffer,
filename: 'modern.webp'
})
const imageEntity = result.entities[0]
expect(imageEntity.metadata.format).toBe('webp')
})
})
describe('Configuration', () => {
it('should allow disabling image handler', async () => {
// Create new brain with image handler disabled
const brainNoImage = new Brainy({
silent: true,
augmentations: {
intelligentImport: {
enableImage: false
}
}
})
await brainNoImage.init()
const imageBuffer = await createTestImage(400, 300)
// Import should still work, but won't be processed by ImageHandler
const result = await brainNoImage.import({
type: 'buffer',
data: imageBuffer,
filename: 'unprocessed.jpg'
})
// Should create entity but not extract image metadata
expect(result).toBeDefined()
await brainNoImage.close()
})
it('should apply imageDefaults config', async () => {
const brainWithDefaults = new Brainy({
silent: true,
augmentations: {
intelligentImport: {
enableImage: true,
imageDefaults: {
extractEXIF: false // Default to no EXIF extraction
}
}
}
})
await brainWithDefaults.init()
const imageBuffer = await createTestImage(400, 300)
const result = await brainWithDefaults.import({
type: 'buffer',
data: imageBuffer,
filename: 'default-config.jpg'
})
expect(result.entities[0].type).toBe('media')
expect(result.entities[0].metadata?.subtype).toBe('image')
await brainWithDefaults.close()
})
})
describe('Image Format Detection', () => {
it('should detect JPEG by filename', async () => {
const imageBuffer = await createTestImage(400, 300)
const result = await brain.import({
type: 'buffer',
data: imageBuffer,
filename: 'photo.jpg'
})
expect(result.entities[0].metadata.format).toBe('jpeg')
})
it('should detect JPEG by .jpeg extension', async () => {
const imageBuffer = await createTestImage(400, 300)
const result = await brain.import({
type: 'buffer',
data: imageBuffer,
filename: 'photo.jpeg'
})
expect(result.entities[0].metadata.format).toBe('jpeg')
})
it('should handle various image sizes', async () => {
const sizes = [
[100, 100],
[800, 600],
[1920, 1080],
[4000, 3000]
]
for (const [width, height] of sizes) {
const imageBuffer = await createTestImage(width, height)
const result = await brain.import({
type: 'buffer',
data: imageBuffer,
filename: `image-${width}x${height}.jpg`
})
const imageEntity = result.entities[0]
expect(imageEntity.metadata.width).toBe(width)
expect(imageEntity.metadata.height).toBe(height)
}
})
})
describe('Error Handling', () => {
it('should handle invalid image data gracefully', async () => {
const invalidBuffer = Buffer.from('not an image')
// Brainy is resilient - invalid images still import with fallback minimal metadata
const result = await brain.import({
type: 'buffer',
data: invalidBuffer,
filename: 'invalid.jpg'
})
expect(result).toBeDefined()
expect(result.entities).toHaveLength(1)
expect(result.entities[0].type).toBe('media')
expect(result.entities[0].name).toBe('invalid.jpg')
})
it('should handle empty image buffer', async () => {
const emptyBuffer = Buffer.alloc(0)
// Brainy is resilient - empty buffers still import with fallback minimal metadata
const result = await brain.import({
type: 'buffer',
data: emptyBuffer,
filename: 'empty.jpg'
})
expect(result).toBeDefined()
expect(result.entities).toHaveLength(1)
expect(result.entities[0].type).toBe('media')
expect(result.entities[0].name).toBe('empty.jpg')
})
})
describe('Integration with Knowledge Graph', () => {
it('should store image entities in knowledge graph', async () => {
const imageBuffer = await createTestImage(800, 600)
const result = await brain.import({
type: 'buffer',
data: imageBuffer,
filename: 'stored-image.jpg'
})
// Verify entity was created with metadata in import result
expect(result.entities).toHaveLength(1)
expect(result.entities[0].metadata.width).toBe(800)
expect(result.entities[0].metadata.height).toBe(600)
// Query for image entities - verifies it's in the knowledge graph
const images = await brain.find({ type: 'media' })
expect(images.length).toBeGreaterThan(0)
// Verify media entities have the expected type
const mediaEntities = images.filter(img => img.type === 'media')
expect(mediaEntities.length).toBeGreaterThan(0)
})
it('should support querying by image metadata', async () => {
// Import multiple images and capture metadata from results
const result1 = await brain.import({
type: 'buffer',
data: await createTestImage(1920, 1080),
filename: 'hd.jpg'
})
const result2 = await brain.import({
type: 'buffer',
data: await createTestImage(800, 600),
filename: 'sd.jpg'
})
// Verify metadata in import results
expect(result1.entities[0].metadata.width).toBe(1920)
expect(result1.entities[0].metadata.height).toBe(1080)
expect(result2.entities[0].metadata.width).toBe(800)
expect(result2.entities[0].metadata.height).toBe(600)
// Query all media entities - verifies they're in the knowledge graph
const allImages = await brain.find({ type: 'media' })
expect(allImages.length).toBeGreaterThan(0)
// Verify multiple media entities were stored
const mediaEntities = allImages.filter(img => img.type === 'media')
expect(mediaEntities.length).toBeGreaterThanOrEqual(2)
})
})
})

1
tests/streaming-pipeline.test.ts
View file

@ -14,7 +14,6 @@ describe('Streaming Pipeline', () => {
beforeEach(async () => {
brain = new Brainy({
storage: { type: 'memory' },
augmentations: {},
warmup: false
})
await brain.init()

535
tests/unit/augmentations/augmentations-simplified.test.ts
View file

@ -1,535 +0,0 @@
import { describe, it, expect, beforeEach } from 'vitest'
import { Brainy } from '../../../src/brainy'
import { createAddParams } from '../../helpers/test-factory'
import { NounType, VerbType } from '../../../src/types/graphTypes'
/**
* Comprehensive test suite for Brainy's built-in augmentation system
* Tests the actual augmentation functionality that's available in production
*/
describe('Brainy Built-in Augmentations', () => {
let brain: Brainy<any>
beforeEach(async () => {
brain = new Brainy({
storage: { type: 'memory' },
augmentations: {
cache: { enabled: true, maxSize: 1000 },
display: { enabled: true },
metrics: { enabled: true }
}
})
await brain.init()
})
describe('1. Augmentation Registry', () => {
it('should list all registered augmentations', async () => {
const augmentations = brain.augmentations.list()
expect(Array.isArray(augmentations)).toBe(true)
expect(augmentations.length).toBeGreaterThan(0)
})
it('should have default augmentations', async () => {
const augmentations = brain.augmentations.list()
expect(augmentations).toContain('cache')
expect(augmentations).toContain('display')
expect(augmentations).toContain('metrics')
})
it('should get augmentation by name', async () => {
const cacheAug = brain.augmentations.get('cache')
expect(cacheAug).toBeDefined()
expect(cacheAug?.name).toBe('cache')
const displayAug = brain.augmentations.get('display')
expect(displayAug).toBeDefined()
expect(displayAug?.name).toBe('display')
const metricsAug = brain.augmentations.get('metrics')
expect(metricsAug).toBeDefined()
expect(metricsAug?.name).toBe('metrics')
})
it('should check if augmentation exists', async () => {
expect(brain.augmentations.has('cache')).toBe(true)
expect(brain.augmentations.has('display')).toBe(true)
expect(brain.augmentations.has('metrics')).toBe(true)
expect(brain.augmentations.has('non-existent')).toBe(false)
})
it('should return undefined for non-existent augmentation', async () => {
const nonExistent = brain.augmentations.get('does-not-exist')
expect(nonExistent).toBeUndefined()
})
})
describe('2. Cache Augmentation', () => {
it('should cache get operations', async () => {
const id = await brain.add(createAddParams({
data: 'cached entity',
metadata: { cached: true }
}))
// First get - should load from storage
const start1 = Date.now()
const entity1 = await brain.get(id)
const duration1 = Date.now() - start1
// Second get - should be cached (faster)
const start2 = Date.now()
const entity2 = await brain.get(id)
const duration2 = Date.now() - start2
// Compare key properties instead of full object equality
expect(entity1?.id).toBe(entity2?.id)
expect(entity1?.data).toBe(entity2?.data)
expect(entity1?.data).toBe('cached entity')
expect(entity1?.metadata?.cached).toBe(true)
// Cache should make second call faster (though this might not be reliable)
// expect(duration2).toBeLessThanOrEqual(duration1)
})
it('should handle cache misses gracefully', async () => {
// Try to get non-existent entity (v5.1.0: use valid UUID format)
const nonExistent = await brain.get('00000000-0000-0000-0000-000000000099')
expect(nonExistent).toBeNull()
})
it('should work with find operations', async () => {
await brain.add(createAddParams({ data: 'findable content 1' }))
await brain.add(createAddParams({ data: 'findable content 2' }))
// First search
const results1 = await brain.find({ query: 'findable' })
// Second search (may be cached)
const results2 = await brain.find({ query: 'findable' })
expect(results1.length).toBe(results2.length)
expect(results1.length).toBeGreaterThanOrEqual(2)
})
})
describe('3. Display Augmentation', () => {
it('should provide display functionality for entities', async () => {
const id = await brain.add(createAddParams({
data: 'Entity with display capabilities',
type: NounType.Document,
metadata: {
title: 'Test Document',
author: 'Test Author',
category: 'test'
}
}))
const entity = await brain.get(id)
expect(entity).toBeDefined()
// Check if getDisplay method is available
if (entity && typeof entity.getDisplay === 'function') {
const display = entity.getDisplay()
expect(display).toBeDefined()
expect(typeof display).toBe('object')
}
})
it('should work with different entity types', async () => {
const personId = await brain.add(createAddParams({
data: 'John Doe',
type: NounType.Person,
metadata: { role: 'developer', age: 30 }
}))
const locationId = await brain.add(createAddParams({
data: 'San Francisco',
type: NounType.Location,
metadata: { country: 'USA', population: 900000 }
}))
const person = await brain.get(personId)
const location = await brain.get(locationId)
expect(person).toBeDefined()
expect(location).toBeDefined()
// Display should work for all types
if (person && typeof person.getDisplay === 'function') {
const personDisplay = person.getDisplay()
expect(personDisplay).toBeDefined()
}
if (location && typeof location.getDisplay === 'function') {
const locationDisplay = location.getDisplay()
expect(locationDisplay).toBeDefined()
}
})
it('should handle entities without metadata', async () => {
const id = await brain.add(createAddParams({
data: 'Simple entity without metadata'
}))
const entity = await brain.get(id)
expect(entity).toBeDefined()
if (entity && typeof entity.getDisplay === 'function') {
const display = entity.getDisplay()
expect(display).toBeDefined()
}
})
it('should provide schema information', async () => {
const id = await brain.add(createAddParams({
data: 'Entity with schema',
metadata: {
stringField: 'text',
numberField: 42,
booleanField: true,
arrayField: [1, 2, 3]
}
}))
const entity = await brain.get(id)
expect(entity).toBeDefined()
if (entity && typeof entity.getSchema === 'function') {
const schema = entity.getSchema()
expect(schema).toBeDefined()
expect(typeof schema).toBe('object')
}
})
})
describe('4. Metrics Augmentation', () => {
it('should track operations without interfering', async () => {
// Perform various operations
const id1 = await brain.add(createAddParams({ data: 'metrics test 1' }))
const id2 = await brain.add(createAddParams({ data: 'metrics test 2' }))
const id3 = await brain.add(createAddParams({ data: 'metrics test 3' }))
// Read operations
await brain.get(id1)
await brain.get(id2)
await brain.find({ query: 'metrics' })
// Write operations
await brain.update({ id: id1, data: 'updated metrics test 1' })
await brain.delete(id3)
// Relationship operations
await brain.relate({
from: id1,
to: id2,
type: VerbType.RelatedTo
})
// All operations should complete successfully
const entity1 = await brain.get(id1)
const entity2 = await brain.get(id2)
const entity3 = await brain.get(id3)
expect(entity1).toBeDefined()
expect(entity1?.data).toBe('updated metrics test 1')
expect(entity2).toBeDefined()
expect(entity3).toBeNull() // Deleted
// Check relationships (may be empty if relationship storage isn't implemented)
const relations = await brain.getRelations(id1)
expect(Array.isArray(relations)).toBe(true)
})
it('should handle high-frequency operations', async () => {
// Create many entities rapidly
const promises = Array.from({ length: 50 }, (_, i) =>
brain.add(createAddParams({
data: `High frequency test ${i}`,
metadata: { index: i }
}))
)
const ids = await Promise.all(promises)
expect(ids.length).toBe(50)
// Perform many reads
const readPromises = ids.map(id => brain.get(id))
const entities = await Promise.all(readPromises)
expect(entities.length).toBe(50)
expect(entities.every(e => e !== null)).toBe(true)
})
it('should handle error scenarios gracefully', async () => {
// v5.1.0: Invalid UUID formats throw errors
await expect(brain.get('invalid-id')).rejects.toThrow('Invalid UUID format')
await expect(brain.delete('invalid-id')).rejects.toThrow('Invalid UUID format')
// Try invalid find parameters
await expect(brain.find({
query: 'test',
limit: -1
} as any)).rejects.toThrow()
await expect(brain.find({
query: 'test',
offset: -1
} as any)).rejects.toThrow()
})
})
describe('5. Augmentation Integration', () => {
it('should apply all augmentations to add operations', async () => {
const id = await brain.add(createAddParams({
data: 'Full integration test',
type: NounType.Document,
metadata: {
title: 'Integration Test',
priority: 'high',
tags: ['test', 'integration']
}
}))
expect(id).toBeDefined()
expect(typeof id).toBe('string')
// Verify entity was created correctly
const entity = await brain.get(id)
expect(entity).toBeDefined()
expect(entity?.data).toBe('Full integration test')
expect(entity?.type).toBe(NounType.Document)
expect(entity?.metadata?.title).toBe('Integration Test')
})
it('should apply all augmentations to find operations', async () => {
// Add test data
const id1 = await brain.add(createAddParams({
data: 'Integration search test 1',
metadata: { category: 'integration' }
}))
const id2 = await brain.add(createAddParams({
data: 'Integration search test 2',
metadata: { category: 'integration' }
}))
await brain.add(createAddParams({
data: 'Different content',
metadata: { category: 'other' }
}))
// Test text search
const textResults = await brain.find({ query: 'Integration search' })
expect(textResults.length).toBeGreaterThanOrEqual(2)
// Test metadata filtering
const categoryResults = await brain.find({
query: '',
where: { category: 'integration' }
})
expect(categoryResults.length).toBe(2)
// Verify entities have display capabilities
const firstResult = textResults[0]
if (firstResult?.entity && typeof firstResult.entity.getDisplay === 'function') {
const display = firstResult.entity.getDisplay()
expect(display).toBeDefined()
}
})
it('should apply augmentations to update operations', async () => {
const id = await brain.add(createAddParams({
data: 'Original data',
metadata: { version: 1 }
}))
// Update should work with all augmentations
await brain.update({
id,
data: 'Updated data',
metadata: { version: 2, updated: true }
})
const updatedEntity = await brain.get(id)
expect(updatedEntity?.data).toBe('Updated data')
expect(updatedEntity?.metadata?.version).toBe(2)
expect(updatedEntity?.metadata?.updated).toBe(true)
})
it('should apply augmentations to relationship operations', async () => {
const id1 = await brain.add(createAddParams({ data: 'Entity 1' }))
const id2 = await brain.add(createAddParams({ data: 'Entity 2' }))
// Create relationship
await brain.relate({
from: id1,
to: id2,
type: VerbType.RelatedTo,
metadata: { strength: 0.8 }
})
// Verify relationship exists (may be empty depending on storage implementation)
const relations = await brain.getRelations(id1)
expect(Array.isArray(relations)).toBe(true)
// If relationships are stored, verify the properties
if (relations.length > 0) {
const relation = relations.find(r => r.to === id2)
if (relation) {
expect(relation.type).toBe(VerbType.RelatedTo)
expect(relation.metadata?.strength).toBe(0.8)
}
}
})
})
describe('6. Performance with Augmentations', () => {
it('should perform well with many entities', async () => {
const start = Date.now()
// Create 100 entities
const createPromises = Array.from({ length: 100 }, (_, i) =>
brain.add(createAddParams({
data: `Performance test entity ${i}`,
metadata: { index: i, batch: 'performance' }
}))
)
const ids = await Promise.all(createPromises)
const createDuration = Date.now() - start
expect(ids.length).toBe(100)
expect(createDuration).toBeLessThan(5000) // Should complete in under 5 seconds
// Search should be fast
const searchStart = Date.now()
const searchResults = await brain.find({
query: 'Performance test',
limit: 50
})
const searchDuration = Date.now() - searchStart
expect(searchResults.length).toBeGreaterThan(0)
expect(searchResults.length).toBeLessThanOrEqual(50) // May return fewer due to relevance
expect(searchDuration).toBeLessThan(1000) // Should complete in under 1 second
})
it('should handle concurrent operations efficiently', async () => {
const start = Date.now()
// Perform 50 concurrent operations
const operations = Array.from({ length: 50 }, (_, i) => {
if (i % 4 === 0) {
return brain.add(createAddParams({ data: `Concurrent add ${i}` }))
} else if (i % 4 === 1) {
return brain.find({ query: 'concurrent', limit: 5 })
} else if (i % 4 === 2) {
return brain.add(createAddParams({ data: `Another add ${i}` }))
} else {
return brain.find({ query: 'test', limit: 3 })
}
})
const results = await Promise.all(operations)
const duration = Date.now() - start
expect(results.length).toBe(50)
expect(duration).toBeLessThan(3000) // Should handle concurrency well
// Verify some results
const addResults = results.filter(r => typeof r === 'string')
const findResults = results.filter(r => Array.isArray(r))
expect(addResults.length).toBeGreaterThan(0)
expect(findResults.length).toBeGreaterThan(0)
})
})
describe('7. Error Handling with Augmentations', () => {
it('should handle errors gracefully without breaking augmentations', async () => {
// v5.1.0: Invalid UUID formats throw errors (these should not crash the system)
await expect(brain.get('')).rejects.toThrow('UUID is required')
await expect(brain.update({
id: 'non-existent',
data: 'new data'
})).rejects.toThrow('Invalid UUID format')
// Normal operations should still work
const id = await brain.add(createAddParams({ data: 'After error test' }))
const entity = await brain.get(id)
expect(entity?.data).toBe('After error test')
})
it('should handle edge case data gracefully', async () => {
// Add entity with minimal but valid data
const id = await brain.add(createAddParams({
data: 'minimal', // Valid minimal data
metadata: { test: true }
}))
expect(id).toBeDefined()
const entity = await brain.get(id)
expect(entity).toBeDefined()
expect(entity?.data).toBe('minimal')
})
})
describe('8. Configuration and Customization', () => {
it('should respect augmentation configuration', async () => {
// Test that augmentations can be configured
const configuredBrain = new Brainy({
storage: { type: 'memory' },
augmentations: {
cache: {
enabled: true,
maxSize: 50,
ttl: 60000
},
display: {
enabled: true,
lazyComputation: true
},
metrics: {
enabled: true
}
}
})
await configuredBrain.init()
// Should work with custom configuration
const id = await configuredBrain.add(createAddParams({
data: 'Configured test'
}))
const entity = await configuredBrain.get(id)
expect(entity?.data).toBe('Configured test')
await configuredBrain.close()
})
it('should work with disabled augmentations', async () => {
const minimalBrain = new Brainy({
storage: { type: 'memory' },
augmentations: {
cache: { enabled: false },
display: { enabled: false },
metrics: { enabled: false }
}
})
await minimalBrain.init()
// Should still work with augmentations disabled
const id = await minimalBrain.add(createAddParams({
data: 'Minimal test'
}))
const entity = await minimalBrain.get(id)
expect(entity?.data).toBe('Minimal test')
await minimalBrain.close()
})
})
})

437
tests/unit/augmentations/format-handler-registry.test.ts
View file

@ -1,437 +0,0 @@
/**
* FormatHandlerRegistry Tests (v5.2.0)
*
* Tests for MIME-based format handler registration and routing
*/
import { describe, it, expect, beforeEach } from 'vitest'
import { FormatHandlerRegistry } from '../../../src/augmentations/intelligentImport/FormatHandlerRegistry.js'
import type { FormatHandler, ProcessedData } from '../../../src/augmentations/intelligentImport/types.js'
describe('FormatHandlerRegistry (v5.2.0)', () => {
let registry: FormatHandlerRegistry
// Mock handlers for testing
const createMockHandler = (format: string): FormatHandler => ({
format,
async process(): Promise<ProcessedData> {
return {
format,
data: [],
metadata: {
rowCount: 0,
fields: [],
processingTime: 0
}
}
},
canHandle(): boolean {
return true
}
})
beforeEach(() => {
registry = new FormatHandlerRegistry()
})
describe('Handler Registration', () => {
it('should register handler with MIME types and extensions', () => {
const csvHandler = createMockHandler('csv')
registry.registerHandler({
name: 'csv',
mimeTypes: ['text/csv', 'application/csv'],
extensions: ['.csv', '.tsv'],
loader: async () => csvHandler
})
expect(registry.hasHandler('csv')).toBe(true)
expect(registry.getRegisteredHandlers()).toContain('csv')
})
it('should register multiple handlers', () => {
registry.registerHandler({
name: 'csv',
mimeTypes: ['text/csv'],
extensions: ['.csv'],
loader: async () => createMockHandler('csv')
})
registry.registerHandler({
name: 'excel',
mimeTypes: ['application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'],
extensions: ['.xlsx'],
loader: async () => createMockHandler('excel')
})
const handlers = registry.getRegisteredHandlers()
expect(handlers).toContain('csv')
expect(handlers).toContain('excel')
expect(handlers).toHaveLength(2)
})
it('should normalize extensions (remove leading dots)', () => {
registry.registerHandler({
name: 'test',
mimeTypes: [],
extensions: ['.txt', 'md'], // Mixed with/without dots
loader: async () => createMockHandler('test')
})
expect(registry.hasHandler('test')).toBe(true)
})
})
describe('MIME Type Routing', () => {
beforeEach(() => {
registry.registerHandler({
name: 'csv',
mimeTypes: ['text/csv', 'application/csv'],
extensions: ['.csv', '.tsv'],
loader: async () => createMockHandler('csv')
})
registry.registerHandler({
name: 'excel',
mimeTypes: [
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
'application/vnd.ms-excel'
],
extensions: ['.xlsx', '.xls'],
loader: async () => createMockHandler('excel')
})
registry.registerHandler({
name: 'pdf',
mimeTypes: ['application/pdf'],
extensions: ['.pdf'],
loader: async () => createMockHandler('pdf')
})
})
it('should get handler by MIME type', async () => {
const handler = await registry.getHandlerByMimeType('text/csv')
expect(handler).toBeDefined()
expect(handler?.format).toBe('csv')
})
it('should get handler by Excel MIME type', async () => {
const handler = await registry.getHandlerByMimeType(
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
)
expect(handler).toBeDefined()
expect(handler?.format).toBe('excel')
})
it('should get handler by legacy Excel MIME type', async () => {
const handler = await registry.getHandlerByMimeType('application/vnd.ms-excel')
expect(handler).toBeDefined()
expect(handler?.format).toBe('excel')
})
it('should return null for unknown MIME type', async () => {
const handler = await registry.getHandlerByMimeType('application/unknown')
expect(handler).toBeNull()
})
it('should list handlers for a MIME type', () => {
const handlers = registry.getHandlersForMimeType('text/csv')
expect(handlers).toContain('csv')
})
})
describe('Extension Routing', () => {
beforeEach(() => {
registry.registerHandler({
name: 'csv',
mimeTypes: ['text/csv'],
extensions: ['.csv', '.tsv'],
loader: async () => createMockHandler('csv')
})
registry.registerHandler({
name: 'excel',
mimeTypes: ['application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'],
extensions: ['.xlsx', '.xls'],
loader: async () => createMockHandler('excel')
})
})
it('should get handler by extension (with dot)', async () => {
const handler = await registry.getHandlerByExtension('.csv')
expect(handler).toBeDefined()
expect(handler?.format).toBe('csv')
})
it('should get handler by extension (without dot)', async () => {
const handler = await registry.getHandlerByExtension('csv')
expect(handler).toBeDefined()
expect(handler?.format).toBe('csv')
})
it('should handle case-insensitive extensions', async () => {
const handler = await registry.getHandlerByExtension('.XLSX')
expect(handler).toBeDefined()
expect(handler?.format).toBe('excel')
})
it('should return null for unknown extension', async () => {
const handler = await registry.getHandlerByExtension('.xyz')
expect(handler).toBeNull()
})
})
describe('Filename-Based Handler Selection', () => {
beforeEach(() => {
registry.registerHandler({
name: 'csv',
mimeTypes: ['text/csv'],
extensions: ['.csv'],
loader: async () => createMockHandler('csv')
})
registry.registerHandler({
name: 'excel',
mimeTypes: ['application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'],
extensions: ['.xlsx'],
loader: async () => createMockHandler('excel')
})
})
it('should get handler by filename (MIME detection)', async () => {
const handler = await registry.getHandler('data.xlsx')
expect(handler).toBeDefined()
expect(handler?.format).toBe('excel')
})
it('should get handler by filename (extension fallback)', async () => {
const handler = await registry.getHandler('report.csv')
expect(handler).toBeDefined()
expect(handler?.format).toBe('csv')
})
it('should handle full paths', async () => {
const handler = await registry.getHandler('/path/to/file.xlsx')
expect(handler).toBeDefined()
expect(handler?.format).toBe('excel')
})
it('should return null for unsupported filename', async () => {
const handler = await registry.getHandler('document.txt')
expect(handler).toBeNull()
})
})
describe('Lazy Loading', () => {
it('should lazy-load handlers on first access', async () => {
let loadCount = 0
registry.registerHandler({
name: 'csv',
mimeTypes: ['text/csv'],
extensions: ['.csv'],
loader: async () => {
loadCount++
return createMockHandler('csv')
}
})
expect(loadCount).toBe(0)
const handler1 = await registry.getHandlerByName('csv')
expect(loadCount).toBe(1)
expect(handler1?.format).toBe('csv')
// Second access should use cached instance
const handler2 = await registry.getHandlerByName('csv')
expect(loadCount).toBe(1) // Still 1, not 2
expect(handler2).toBe(handler1) // Same instance
})
it('should cache handler instances', async () => {
registry.registerHandler({
name: 'csv',
mimeTypes: ['text/csv'],
extensions: ['.csv'],
loader: async () => createMockHandler('csv')
})
const handler1 = await registry.getHandlerByExtension('.csv')
const handler2 = await registry.getHandlerByMimeType('text/csv')
expect(handler2).toBe(handler1) // Same cached instance
})
it('should handle loader errors gracefully', async () => {
registry.registerHandler({
name: 'failing',
mimeTypes: ['application/x-failing'],
extensions: ['.fail'],
loader: async () => {
throw new Error('Handler failed to load')
}
})
const handler = await registry.getHandlerByExtension('.fail')
expect(handler).toBeNull()
})
})
describe('Interface Compatibility', () => {
it('should implement HandlerRegistry interface', () => {
// Check required properties exist
expect(registry.handlers).toBeDefined()
expect(registry.loaded).toBeDefined()
expect(typeof registry.register).toBe('function')
expect(typeof registry.getHandler).toBe('function')
})
it('should support register() method for backward compatibility', async () => {
const csvHandler = createMockHandler('csv')
registry.register(['.csv', '.tsv'], async () => csvHandler)
const handler = await registry.getHandler('.csv')
expect(handler).toBe(csvHandler)
})
it('should populate handlers Map for backward compatibility', () => {
registry.registerHandler({
name: 'csv',
mimeTypes: ['text/csv'],
extensions: ['.csv'],
loader: async () => createMockHandler('csv')
})
expect(registry.handlers.has('csv')).toBe(true)
expect(typeof registry.handlers.get('csv')).toBe('function')
})
it('should populate loaded Map after loading', async () => {
registry.registerHandler({
name: 'csv',
mimeTypes: ['text/csv'],
extensions: ['.csv'],
loader: async () => createMockHandler('csv')
})
expect(registry.loaded.has('csv')).toBe(false)
await registry.getHandlerByName('csv')
expect(registry.loaded.has('csv')).toBe(true)
})
})
describe('Multiple Handlers Per MIME Type', () => {
it('should support multiple handlers for same MIME type', () => {
registry.registerHandler({
name: 'csv-basic',
mimeTypes: ['text/csv'],
extensions: ['.csv'],
loader: async () => createMockHandler('csv-basic')
})
registry.registerHandler({
name: 'csv-advanced',
mimeTypes: ['text/csv'],
extensions: ['.csv'],
loader: async () => createMockHandler('csv-advanced')
})
const handlers = registry.getHandlersForMimeType('text/csv')
expect(handlers).toHaveLength(2)
expect(handlers).toContain('csv-basic')
expect(handlers).toContain('csv-advanced')
})
it('should return first matching handler', async () => {
registry.registerHandler({
name: 'csv-basic',
mimeTypes: ['text/csv'],
extensions: ['.csv'],
loader: async () => createMockHandler('csv-basic')
})
registry.registerHandler({
name: 'csv-advanced',
mimeTypes: ['text/csv'],
extensions: ['.csv'],
loader: async () => createMockHandler('csv-advanced')
})
// Should return first registered handler
const handler = await registry.getHandlerByMimeType('text/csv')
expect(handler?.format).toBe('csv-basic')
})
})
describe('Registry Management', () => {
it('should clear all handlers', () => {
registry.registerHandler({
name: 'csv',
mimeTypes: ['text/csv'],
extensions: ['.csv'],
loader: async () => createMockHandler('csv')
})
expect(registry.hasHandler('csv')).toBe(true)
registry.clear()
expect(registry.hasHandler('csv')).toBe(false)
expect(registry.getRegisteredHandlers()).toHaveLength(0)
})
it('should check if handler is registered', () => {
expect(registry.hasHandler('csv')).toBe(false)
registry.registerHandler({
name: 'csv',
mimeTypes: ['text/csv'],
extensions: ['.csv'],
loader: async () => createMockHandler('csv')
})
expect(registry.hasHandler('csv')).toBe(true)
expect(registry.hasHandler('excel')).toBe(false)
})
})
describe('Edge Cases', () => {
it('should handle files without extensions', async () => {
registry.registerHandler({
name: 'csv',
mimeTypes: ['text/csv'],
extensions: ['.csv'],
loader: async () => createMockHandler('csv')
})
const handler = await registry.getHandler('data')
expect(handler).toBeNull()
})
it('should handle empty extension list', () => {
registry.registerHandler({
name: 'special',
mimeTypes: ['application/x-special'],
extensions: [],
loader: async () => createMockHandler('special')
})
expect(registry.hasHandler('special')).toBe(true)
})
it('should handle empty MIME type list', async () => {
registry.registerHandler({
name: 'extension-only',
mimeTypes: [],
extensions: ['.xyz'],
loader: async () => createMockHandler('extension-only')
})
const handler = await registry.getHandlerByExtension('.xyz')
expect(handler?.format).toBe('extension-only')
})
})
})

311
tests/unit/augmentations/image-handler.test.ts
View file

@ -1,311 +0,0 @@
/**
* ImageHandler Tests (v5.8.0 - Pure JavaScript)
*
* Tests for image processing with EXIF extraction and metadata extraction
* Using probe-image-size + exifr (no native dependencies)
*/
import { describe, it, expect, beforeEach } from 'vitest'
import { ImageHandler } from '../../../src/augmentations/intelligentImport/handlers/imageHandler.js'
import { readFileSync } from 'fs'
import { join } from 'path'
describe('ImageHandler (v5.8.0 - Pure JS)', () => {
let handler: ImageHandler
// Simple test image generators (minimal PNG/JPEG headers)
const createMinimalJPEG = (width: number = 100, height: number = 100): Buffer => {
// Minimal JPEG: SOI + SOF0 + EOI
// This is a valid but minimal JPEG structure
const soi = Buffer.from([0xFF, 0xD8]) // Start of Image
const sof0 = Buffer.from([
0xFF, 0xC0, // SOF0 marker
0x00, 0x11, // Length: 17 bytes
0x08, // Precision: 8 bits
(height >> 8) & 0xFF, height & 0xFF, // Height
(width >> 8) & 0xFF, width & 0xFF, // Width
0x03, // Components: 3 (YCbCr)
0x01, 0x22, 0x00, // Y component
0x02, 0x11, 0x01, // Cb component
0x03, 0x11, 0x01 // Cr component
])
const eoi = Buffer.from([0xFF, 0xD9]) // End of Image
return Buffer.concat([soi, sof0, eoi])
}
const createMinimalPNG = (width: number = 100, height: number = 100): Buffer => {
// PNG signature
const signature = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10])
// IHDR chunk
const ihdr = Buffer.alloc(25)
ihdr.writeUInt32BE(13, 0) // Length
ihdr.write('IHDR', 4)
ihdr.writeUInt32BE(width, 8)
ihdr.writeUInt32BE(height, 12)
ihdr.writeUInt8(8, 16) // Bit depth
ihdr.writeUInt8(2, 17) // Color type (RGB)
ihdr.writeUInt8(0, 18) // Compression
ihdr.writeUInt8(0, 19) // Filter
ihdr.writeUInt8(0, 20) // Interlace
ihdr.writeUInt32BE(0, 21) // CRC (simplified)
// IEND chunk
const iend = Buffer.from([0, 0, 0, 0, 73, 69, 78, 68, 174, 66, 96, 130])
return Buffer.concat([signature, ihdr, iend])
}
beforeEach(() => {
handler = new ImageHandler()
})
describe('Handler Detection', () => {
it('should identify as image format handler', () => {
expect(handler.format).toBe('image')
})
it('should handle JPEG files by filename', () => {
expect(handler.canHandle({ filename: 'photo.jpg' })).toBe(true)
expect(handler.canHandle({ filename: 'photo.jpeg' })).toBe(true)
})
it('should handle PNG files by filename', () => {
expect(handler.canHandle({ filename: 'logo.png' })).toBe(true)
})
it('should handle WebP files by filename', () => {
expect(handler.canHandle({ filename: 'image.webp' })).toBe(true)
})
it('should handle other image formats', () => {
expect(handler.canHandle({ filename: 'photo.gif' })).toBe(true)
expect(handler.canHandle({ filename: 'photo.tiff' })).toBe(true)
expect(handler.canHandle({ filename: 'photo.bmp' })).toBe(true)
expect(handler.canHandle({ filename: 'icon.svg' })).toBe(true)
})
it('should handle images by extension', () => {
expect(handler.canHandle({ ext: '.jpg' })).toBe(true)
expect(handler.canHandle({ ext: 'png' })).toBe(true)
})
it('should reject non-image files', () => {
expect(handler.canHandle({ filename: 'document.pdf' })).toBe(false)
expect(handler.canHandle({ filename: 'data.csv' })).toBe(false)
expect(handler.canHandle({ filename: 'text.txt' })).toBe(false)
})
it('should detect JPEG by magic bytes', () => {
const jpegBuffer = createMinimalJPEG(100, 100)
expect(handler.canHandle(jpegBuffer)).toBe(true)
})
it('should detect PNG by magic bytes', () => {
const pngBuffer = createMinimalPNG(100, 100)
expect(handler.canHandle(pngBuffer)).toBe(true)
})
})
describe('Image Metadata Extraction', () => {
it('should extract JPEG metadata', async () => {
const imageBuffer = createMinimalJPEG(800, 600)
const result = await handler.process(imageBuffer, {
extractEXIF: false
})
expect(result.format).toBe('image')
expect(result.data).toHaveLength(1)
const imageData = result.data[0]
expect(imageData.type).toBe('media')
expect(imageData.metadata).toBeDefined()
expect(imageData.metadata.subtype).toBe('image')
expect(imageData.metadata.width).toBe(800)
expect(imageData.metadata.height).toBe(600)
expect(imageData.metadata.format).toBe('jpg') // probe-image-size uses 'jpg' not 'jpeg'
})
it('should extract PNG metadata', async () => {
const imageBuffer = createMinimalPNG(400, 300)
const result = await handler.process(imageBuffer, {
extractEXIF: false
})
const imageData = result.data[0]
expect(imageData.metadata.width).toBe(400)
expect(imageData.metadata.height).toBe(300)
expect(imageData.metadata.format).toBe('png')
})
it('should include image size in bytes', async () => {
const imageBuffer = createMinimalJPEG(200, 200)
const result = await handler.process(imageBuffer, {
extractEXIF: false
})
const imageData = result.data[0]
expect(imageData.metadata.size).toBe(imageBuffer.length)
expect(imageData.metadata.size).toBeGreaterThan(0)
})
it('should include MIME type when available', async () => {
const imageBuffer = createMinimalJPEG(100, 100)
const result = await handler.process(imageBuffer, {
extractEXIF: false
})
const imageData = result.data[0]
expect(imageData.metadata.mimeType).toBeDefined()
expect(imageData.metadata.mimeType).toContain('image/')
})
it('should include processing time in metadata', async () => {
const imageBuffer = createMinimalJPEG(800, 600)
const result = await handler.process(imageBuffer)
expect(result.metadata.processingTime).toBeGreaterThanOrEqual(0)
expect(result.metadata.processingTime).toBeLessThan(5000)
})
it('should include image metadata in result metadata', async () => {
const imageBuffer = createMinimalJPEG(800, 600)
const result = await handler.process(imageBuffer)
expect(result.metadata.imageMetadata).toBeDefined()
expect(result.metadata.imageMetadata.width).toBe(800)
expect(result.metadata.imageMetadata.height).toBe(600)
})
})
describe('EXIF Data Extraction', () => {
it('should handle images without EXIF data', async () => {
const imageBuffer = createMinimalJPEG(800, 600)
const result = await handler.process(imageBuffer, {
extractEXIF: true
})
const imageData = result.data[0]
// Should not crash, EXIF will be undefined
expect(imageData.metadata.exif).toBeUndefined()
})
it('should extract EXIF by default', async () => {
const imageBuffer = createMinimalJPEG(800, 600)
// Default: EXIF extraction enabled
const result = await handler.process(imageBuffer)
// Will be undefined for test images, but should not crash
expect(result.metadata.exifData).toBeUndefined()
})
it('should allow disabling EXIF extraction', async () => {
const imageBuffer = createMinimalJPEG(800, 600)
const result = await handler.process(imageBuffer, {
extractEXIF: false
})
const imageData = result.data[0]
expect(imageData.metadata.exif).toBeUndefined()
})
})
describe('Error Handling', () => {
it('should handle invalid image data gracefully', async () => {
const invalidBuffer = Buffer.from('not an image', 'utf-8')
// Should still process but with fallback values
const result = await handler.process(invalidBuffer, { extractEXIF: false })
// Fallback detection should provide basic info
expect(result.data[0].metadata.format).toBeDefined()
expect(result.data[0].metadata.size).toBe(invalidBuffer.length)
})
it('should handle empty buffer', async () => {
const emptyBuffer = Buffer.alloc(0)
// Should not crash, but will have unknown format
const result = await handler.process(emptyBuffer, { extractEXIF: false })
expect(result.data[0].metadata.format).toBe('unknown')
})
})
describe('Format Support', () => {
it('should process JPEG images', async () => {
const imageBuffer = createMinimalJPEG(200, 200)
const result = await handler.process(imageBuffer, {
extractEXIF: false
})
expect(result.data[0].metadata.format).toBe('jpg') // probe-image-size uses 'jpg' not 'jpeg'
})
it('should process PNG images', async () => {
const imageBuffer = createMinimalPNG(200, 200)
const result = await handler.process(imageBuffer, {
extractEXIF: false
})
expect(result.data[0].metadata.format).toBe('png')
})
it('should handle various image dimensions', async () => {
const sizes = [
[100, 100],
[1920, 1080],
[400, 300],
[1000, 500]
]
for (const [width, height] of sizes) {
const imageBuffer = createMinimalJPEG(width, height)
const result = await handler.process(imageBuffer, { extractEXIF: false })
expect(result.data[0].metadata.width).toBe(width)
expect(result.data[0].metadata.height).toBe(height)
}
})
})
describe('Integration with BaseFormatHandler', () => {
it('should inherit MIME type detection from BaseFormatHandler', () => {
// getMimeType is protected, but we can verify behavior
expect(handler.canHandle({ filename: 'test.jpg' })).toBe(true)
expect(handler.canHandle({ filename: 'test.png' })).toBe(true)
})
it('should provide structured ProcessedData', async () => {
const imageBuffer = createMinimalJPEG(200, 200)
const result = await handler.process(imageBuffer, {
filename: 'test-image.jpg',
extractEXIF: false
})
// Verify ProcessedData structure
expect(result.format).toBe('image')
expect(result.data).toBeInstanceOf(Array)
expect(result.metadata).toBeDefined()
expect(result.filename).toBe('test-image.jpg')
// Verify data structure
const entity = result.data[0]
expect(entity.name).toBe('test-image')
expect(entity.type).toBe('media')
expect(entity.metadata).toBeDefined()
})
})
})

10
tests/unit/brainy-core.unit.test.ts
View file

@ -15,13 +15,7 @@ describe('Brainy 3.0 Core (Unit Tests)', () => {
beforeEach(async () => {
// Create instance with real embeddings for production-ready tests
brain = new Brainy({
storage: { type: 'memory' },
augmentations: {
cache: false,
metrics: false,
display: false,
monitoring: false
}
storage: { type: 'memory' }
})
await brain.init()
@ -196,7 +190,7 @@ describe('Brainy 3.0 Core (Unit Tests)', () => {
})
describe('Statistics and Metadata', () => {
it('should track statistics through augmentations', async () => {
it('should track statistics', async () => {
await brain.add({
data: { name: 'Test1' },
type: NounType.Concept

18
tests/unit/brainy/add.test.ts
View file

@ -482,24 +482,6 @@ describe('Brainy.add()', () => {
})
})
describe('augmentations', () => {
it('should apply augmentations during add', async () => {
// This would test augmentation pipeline if configured
// For now, just verify the entity is added correctly
const params = createAddParams({
data: 'Augmentation test',
type: 'thing',
metadata: { augment: true }
})
const id = await brain.add(params)
const entity = await brain.get(id)
expect(entity).not.toBeNull()
expect(entity!.metadata.augment).toBe(true)
})
})
describe('caching behavior', () => {
it('should retrieve consistent entities', async () => {
// Arrange (v5.1.0: use valid UUID format)

8
tests/unit/performance/add-performance.test.ts
View file

@ -18,13 +18,7 @@ describe('add() Performance Regression Tests', () => {
beforeEach(async () => {
brain = new Brainy({
storage: { type: 'memory' },
silent: true,
augmentations: {
cache: false,
metrics: false,
display: false,
monitoring: false
}
silent: true
})
await brain.init()
})

4
tests/unit/plugin.test.ts
View file

@ -214,7 +214,6 @@ describe('Brainy plugin integration', () => {
const brain = new Brainy({
storage: { type: 'memory' },
silent: true,
augmentations: { cache: false, metrics: false, display: false, monitoring: false }
})
brain.use(mockPlugin)
await brain.init()
@ -244,7 +243,6 @@ describe('Brainy plugin integration', () => {
const brain = new Brainy({
storage: { type: 'memory' },
silent: true,
augmentations: { cache: false, metrics: false, display: false, monitoring: false }
})
brain.use(mockPlugin)
await brain.init()
@ -263,7 +261,6 @@ describe('Brainy plugin integration', () => {
const brain = new Brainy({
storage: { type: 'memory' },
silent: true,
augmentations: { cache: false, metrics: false, display: false, monitoring: false }
})
brain.use(mockPlugin)
await brain.init()
@ -293,7 +290,6 @@ describe('Brainy plugin integration', () => {
const brain = new Brainy({
storage: { type: 'memory' },
silent: true,
augmentations: { cache: false, metrics: false, display: false, monitoring: false }
})
brain.use(mockPlugin)

219
tests/unit/type-matching.unit.test.ts
View file

@ -1,219 +0,0 @@
/**
* Tests for Intelligent Type Matching with embeddings
*/
import { describe, it, expect, beforeAll, afterAll } from 'vitest'
import { IntelligentTypeMatcher } from '../../src/augmentations/typeMatching/intelligentTypeMatcher.js'
import { NounType, VerbType } from '../../src/types/graphTypes.js'
describe('Intelligent Type Matching', () => {
let matcher: IntelligentTypeMatcher
beforeAll(async () => {
matcher = new IntelligentTypeMatcher()
await matcher.init()
})
afterAll(async () => {
await matcher.dispose()
})
describe('Noun Type Detection', () => {
it('should detect Person type from user data', async () => {
const result = await matcher.matchNounType({
name: 'John Doe',
email: 'john@example.com',
age: 30
})
expect(result.type).toBeDefined()
expect(result.confidence).toBeGreaterThan(0)
expect(result.alternatives).toBeDefined()
expect(result.alternatives.length).toBeGreaterThanOrEqual(0)
})
it('should detect Organization type from company data', async () => {
const result = await matcher.matchNounType({
companyName: 'Acme Corp',
employees: 500,
industry: 'Technology'
})
// With real type embeddings (v3.33.0+), type detection uses actual semantic similarity
// Results depend on the embedding quality and field patterns
expect(result.type).toBeDefined()
expect(result.confidence).toBeGreaterThan(0.1)
})
it('should detect Location type from geographic data', async () => {
const result = await matcher.matchNounType({
latitude: 37.7749,
longitude: -122.4194,
city: 'San Francisco'
})
// With real type embeddings (v3.33.0+), geographic data detection is semantic
expect(result.type).toBeDefined()
expect(result.confidence).toBeGreaterThan(0.1)
})
it('should detect Document type from text content', async () => {
const result = await matcher.matchNounType({
title: 'Research Paper',
content: 'Abstract: This paper discusses...',
author: 'Dr. Smith',
pages: 20
})
// With real type embeddings (v3.33.0+), could match various types based on semantic similarity
expect(result.type).toBeDefined()
expect(result.confidence).toBeGreaterThan(0.0)
})
it('should detect Product type from commercial data', async () => {
const result = await matcher.matchNounType({
price: 99.99,
sku: 'PROD-123',
inventory: 50,
productId: 'abc-123'
})
// With real type embeddings (v3.33.0+), commercial data uses semantic matching
expect(result.type).toBeDefined()
expect(result.confidence).toBeGreaterThan(0.1)
})
it('should detect Event type from temporal data', async () => {
const result = await matcher.matchNounType({
startTime: '2024-01-01',
endTime: '2024-01-02',
attendees: 100,
eventType: 'conference'
})
// With real type embeddings (v3.33.0+), temporal data uses semantic matching
expect(result.type).toBeDefined()
expect(result.confidence).toBeGreaterThan(0.1)
})
it('should handle ambiguous data with alternatives', async () => {
const result = await matcher.matchNounType({
value: 100,
type: 'unknown',
data: 'mixed'
})
expect(result.type).toBeDefined()
expect(result.alternatives).toBeDefined()
expect(result.alternatives.length).toBeGreaterThan(0)
})
})
describe('Verb Type Detection', () => {
it('should detect MemberOf relationship', async () => {
const result = await matcher.matchVerbType(
{ id: 'user1', type: 'person' },
{ id: 'org1', type: 'organization' },
'memberOf'
)
// With mocked embeddings, type detection is non-deterministic
expect(result.type).toBeDefined()
expect(Object.values(VerbType)).toContain(result.type)
expect(result.confidence).toBeGreaterThan(0)
})
it('should detect CreatedBy relationship', async () => {
const result = await matcher.matchVerbType(
{ id: 'doc1', type: 'document' },
{ id: 'author1', type: 'person' },
'created by'
)
// With mocked embeddings, type detection is non-deterministic
expect(result.type).toBeDefined()
expect(Object.values(VerbType)).toContain(result.type)
expect(result.confidence).toBeGreaterThan(0)
})
it('should detect Contains relationship', async () => {
const result = await matcher.matchVerbType(
{ id: 'folder1', type: 'collection' },
{ id: 'file1', type: 'file' },
'contains'
)
// With mocked embeddings, type detection is non-deterministic
expect(result.type).toBeDefined()
expect(Object.values(VerbType)).toContain(result.type)
expect(result.confidence).toBeGreaterThan(0)
})
it('should handle unknown relationships with defaults', async () => {
const result = await matcher.matchVerbType(
{ id: 'a' },
{ id: 'b' },
'somehow connected'
)
expect(result.type).toBeDefined()
expect(Object.values(VerbType)).toContain(result.type)
})
})
describe('Type Coverage', () => {
it('should have embeddings for all 42 noun types', async () => {
const nounTypes = Object.values(NounType)
expect(nounTypes.length).toBe(42)
// Test that each type can be matched
for (const nounType of nounTypes) {
const result = await matcher.matchNounType({
type: nounType,
test: 'coverage check'
})
expect(result.type).toBeDefined()
}
})
it('should have embeddings for all 127 verb types', async () => {
const verbTypes = Object.values(VerbType)
expect(verbTypes.length).toBe(127)
// Test that each type can be matched
for (const verbType of verbTypes) {
const result = await matcher.matchVerbType(
{ id: 'source' },
{ id: 'target' },
verbType
)
expect(result.type).toBeDefined()
}
})
})
describe('Cache Performance', () => {
it('should cache repeated type matches', async () => {
const testData = { name: 'Cache Test', value: 123 }
const result1 = await matcher.matchNounType(testData)
const result2 = await matcher.matchNounType(testData)
expect(result1.type).toBe(result2.type)
expect(result1.confidence).toBe(result2.confidence)
// Cache should return consistent results
expect(result2.type).toBeDefined()
})
it('should clear cache when requested', async () => {
const testData = { name: 'Clear Cache Test' }
await matcher.matchNounType(testData) // Populate cache
matcher.clearCache()
// Should still work after cache clear
const result = await matcher.matchNounType(testData)
expect(result.type).toBeDefined()
})
})
})