diff --git a/examples/directory-import-with-caching.ts b/examples/directory-import-with-caching.ts deleted file mode 100644 index d3395b5a..00000000 --- a/examples/directory-import-with-caching.ts +++ /dev/null @@ -1,237 +0,0 @@ -/** - * Directory Import with Entity Extraction Caching Example - * - * Demonstrates: - * - Importing directories with progress tracking - * - Entity extraction caching for performance - * - Relationship detection with confidence scores - * - Cache statistics monitoring - */ - -import { Brainy, NounType, VerbType } from '../src/brainy.js' -import { DirectoryImporter } from '../src/vfs/importers/DirectoryImporter.js' -import { ProgressTracker, formatProgress } from '../src/types/progress.types.js' -import { detectRelationshipsWithConfidence } from '../src/neural/relationshipConfidence.js' -import { NeuralEntityExtractor } from '../src/neural/entityExtractor.js' - -async function main() { - console.log('๐Ÿง  Brainy 3.21.0 - Directory Import with Caching Example\n') - - // Initialize Brainy - const brain = new Brainy({ verbose: false }) - await brain.init() - - console.log('โœ… Brainy initialized\n') - - // The entity extractor (and its extraction cache) is constructed directly. - const extractor = new NeuralEntityExtractor(brain) - - // Example 1: Import directory with entity extraction caching - console.log('๐Ÿ“ Example 1: Import Directory with Caching\n') - - const vfs = brain.vfs - const importer = new DirectoryImporter(vfs, brain) - - // Progress tracking - const tracker = ProgressTracker.create(100) - tracker.start() - - try { - // Import with progress (using async generator) - console.log('Importing directory...') - - let filesProcessed = 0 - for await (const progress of importer.importStream('./examples', { - batchSize: 10, - recursive: true, - generateEmbeddings: true, - extractMetadata: true - })) { - if (progress.type === 'progress') { - filesProcessed = progress.processed - const trackedProgress = tracker.update(progress.processed, progress.current) - console.log(` ${formatProgress(trackedProgress)}`) - } else if (progress.type === 'complete') { - console.log(`\nโœ… Import complete! Processed ${progress.processed} files\n`) - } else if (progress.type === 'error') { - console.error(`โŒ Error: ${progress.error?.message}`) - } - } - - tracker.complete({ filesProcessed }) - - } catch (error) { - console.error('Import failed:', error) - } - - // Example 2: Entity extraction with caching - console.log('\n๐Ÿ“ Example 2: Entity Extraction with Caching\n') - - const sampleText = ` - John Smith created the user authentication system for the application. - The authentication system uses JWT tokens and bcrypt for password hashing. - Mary Johnson manages the backend team that maintains the system. - The system was built using Node.js and PostgreSQL database. - ` - - console.log('First extraction (cache miss):') - const startTime1 = Date.now() - const entities1 = await extractor.extract(sampleText, { - types: [NounType.Person, NounType.Service, NounType.Technology], - confidence: 0.7, - cache: { - enabled: true, - ttl: 7 * 24 * 60 * 60 * 1000, // 7 days - invalidateOn: 'hash' - } - }) - const time1 = Date.now() - startTime1 - console.log(` Extracted ${entities1.length} entities in ${time1}ms`) - console.log(` Entities: ${entities1.map(e => e.text).join(', ')}\n`) - - console.log('Second extraction (cache hit):') - const startTime2 = Date.now() - const entities2 = await extractor.extract(sampleText, { - types: [NounType.Person, NounType.Service, NounType.Technology], - confidence: 0.7, - cache: { - enabled: true, - invalidateOn: 'hash' - } - }) - const time2 = Date.now() - startTime2 - console.log(` Extracted ${entities2.length} entities in ${time2}ms`) - console.log(` Speedup: ${Math.round(time1 / time2)}x faster!\n`) - - // Show cache statistics - const cacheStats = extractor.getCacheStats() - console.log('๐Ÿ“Š Cache Statistics:') - console.log(` Hits: ${cacheStats.hits}`) - console.log(` Misses: ${cacheStats.misses}`) - console.log(` Hit Rate: ${(cacheStats.hitRate * 100).toFixed(1)}%`) - console.log(` Total Entries: ${cacheStats.totalEntries}`) - console.log(` Avg Entities per Entry: ${cacheStats.averageEntitiesPerEntry}\n`) - - // Example 3: Relationship detection with confidence - console.log('๐Ÿ”— Example 3: Relationship Detection with Confidence\n') - - const relationships = detectRelationshipsWithConfidence( - entities1, - sampleText, - { - minConfidence: 0.6, - maxDistance: 100, - useProximityBoost: true, - usePatternMatching: true, - useStructuralAnalysis: true - } - ) - - console.log(`Detected ${relationships.length} relationships:\n`) - for (const rel of relationships.slice(0, 5)) { // Show top 5 - console.log(` ${rel.sourceEntity.text} --[${rel.verbType}]--> ${rel.targetEntity.text}`) - console.log(` Confidence: ${(rel.confidence * 100).toFixed(1)}%`) - console.log(` Evidence: ${rel.evidence.reasoning}`) - console.log(` Method: ${rel.evidence.method}`) - console.log(` Source: "${rel.evidence.sourceText?.substring(0, 60)}..."\n`) - } - - // Example 4: Create relationships in graph with confidence - console.log('๐Ÿ“Š Example 4: Creating Relationships in Graph\n') - - const createdRelations = [] - for (const rel of relationships.slice(0, 3)) { // Create top 3 - try { - // Add entities to brain - const sourceId = await brain.add({ - data: rel.sourceEntity.text, - type: rel.sourceEntity.type, - metadata: { - confidence: rel.sourceEntity.confidence, - extractedFrom: 'sample text' - } - }) - - const targetId = await brain.add({ - data: rel.targetEntity.text, - type: rel.targetEntity.type, - metadata: { - confidence: rel.targetEntity.confidence, - extractedFrom: 'sample text' - } - }) - - // Create relationship with confidence - const relationId = await brain.relate({ - from: sourceId, - to: targetId, - type: rel.verbType, - confidence: rel.confidence, - evidence: rel.evidence, - metadata: { - autoDetected: true, - detectedAt: new Date().toISOString() - } - }) - - createdRelations.push(relationId) - console.log(` โœ… Created: ${rel.sourceEntity.text} โ†’ ${rel.targetEntity.text}`) - } catch (error) { - console.error(` โŒ Failed to create relationship:`, error) - } - } - - console.log(`\nโœ… Created ${createdRelations.length} relationships in knowledge graph`) - - // Example 5: Query relationships by confidence - console.log('\n๐Ÿ” Example 5: Query High-Confidence Relationships\n') - - const allRelations = await brain.getRelations({ - limit: 100 - }) - - const highConfidence = allRelations.filter(r => (r.confidence || 0) >= 0.7) - console.log(`Found ${highConfidence.length} high-confidence relationships (โ‰ฅ70%):\n`) - - for (const rel of highConfidence.slice(0, 5)) { - console.log(` ${rel.from} โ†’ ${rel.to} (${rel.type})`) - console.log(` Confidence: ${((rel.confidence || 0) * 100).toFixed(1)}%`) - if (rel.evidence) { - console.log(` Method: ${rel.evidence.method}`) - console.log(` Reasoning: ${rel.evidence.reasoning}\n`) - } - } - - // Example 6: Cache management - console.log('๐Ÿงน Example 6: Cache Management\n') - - console.log('Cache operations:') - - // Cleanup expired entries - const cleaned = extractor.cleanupCache() - console.log(` Cleaned ${cleaned} expired entries`) - - // Invalidate specific cache entry - const invalidated = extractor.invalidateCache('hash:abc123') - console.log(` Invalidated entry: ${invalidated}`) - - // Get final stats - const finalStats = extractor.getCacheStats() - console.log(` Final cache size: ${finalStats.totalEntries} entries`) - console.log(` Memory used: ~${Math.round(finalStats.cacheSize / 1024)}KB`) - - // Clear all cache (optional) - // extractor.clearCache() - // console.log(' Cleared entire cache') - - console.log('\nโœจ Example complete!') - console.log('\n๐Ÿ“š Key Takeaways:') - console.log(' โ€ข Entity extraction caching provides 10-100x speedup on repeated content') - console.log(' โ€ข Progress tracking gives real-time feedback for long operations') - console.log(' โ€ข Relationship confidence helps filter low-quality connections') - console.log(' โ€ข Evidence tracking makes relationships explainable and debuggable') - console.log(' โ€ข All features are opt-in and backward compatible') -} - -// Run example -main().catch(console.error) diff --git a/src/cli/catalog.ts b/src/cli/catalog.ts deleted file mode 100644 index 31ed324f..00000000 --- a/src/cli/catalog.ts +++ /dev/null @@ -1,440 +0,0 @@ -/** - * Augmentation Catalog for CLI - * - * Displays available augmentations catalog - * Local catalog with caching support - */ - -import chalk from 'chalk' -import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'node:fs' -import { join } from 'node:path' -import { homedir } from 'node:os' - -const CATALOG_API = process.env.BRAINY_CATALOG_URL || null -const CACHE_PATH = join(homedir(), '.brainy', 'catalog-cache.json') -const CACHE_TTL = 24 * 60 * 60 * 1000 // 24 hours - -interface Augmentation { - id: string - name: string - description: string - category: string - status: 'available' | 'coming_soon' | 'deprecated' - popular?: boolean - eta?: string -} - -interface Category { - id: string - name: string - icon: string - description: string -} - -interface Catalog { - version: string - categories: Category[] - augmentations: Augmentation[] -} - -/** - * Fetch catalog from API with caching - */ -export async function fetchCatalog(): Promise { - try { - // Check cache first - const cached = loadCache() - if (cached) return cached - - // If external catalog API is configured, try to fetch - if (CATALOG_API) { - const response = await fetch(`${CATALOG_API}/api/catalog/cli`) - if (!response.ok) throw new Error('API unavailable') - - const catalog = await response.json() - - // Save to cache - saveCache(catalog) - - return catalog - } - - // Fall back to local catalog - return getDefaultCatalog() - } catch (error) { - // Try loading from cache even if expired - const cached = loadCache(true) - if (cached) { - console.log(chalk.yellow('๐Ÿ“ก Using cached catalog')) - return cached - } - - // Fall back to hardcoded catalog - return getDefaultCatalog() - } -} - -/** - * Display catalog in CLI - */ -export async function showCatalog(options: { - category?: string - search?: string - detailed?: boolean -}) { - const catalog = await fetchCatalog() - if (!catalog) { - console.log(chalk.red('โŒ Could not load augmentation catalog')) - return - } - - console.log(chalk.cyan.bold('๐Ÿง  Brainy Augmentation Catalog')) - console.log(chalk.gray(`Version ${catalog.version}`)) - console.log('') - - // Filter augmentations - let augmentations = catalog.augmentations - - if (options.category) { - augmentations = augmentations.filter(a => a.category === options.category) - } - - if (options.search) { - const query = options.search.toLowerCase() - augmentations = augmentations.filter(a => - a.name.toLowerCase().includes(query) || - a.description.toLowerCase().includes(query) - ) - } - - // Group by category - const grouped = groupByCategory(augmentations, catalog.categories) - - // Display - for (const [category, augs] of Object.entries(grouped)) { - if (augs.length === 0) continue - - const cat = catalog.categories.find(c => c.id === category) - console.log(chalk.bold(`${cat?.icon || '๐Ÿ“ฆ'} ${cat?.name || category}`)) - - for (const aug of augs) { - const status = getStatusIcon(aug.status) - const popular = aug.popular ? chalk.yellow(' โญ') : '' - const eta = aug.eta ? chalk.gray(` (${aug.eta})`) : '' - - console.log(` ${status} ${aug.name}${popular}${eta}`) - if (options.detailed) { - console.log(chalk.gray(` ${aug.description}`)) - } - } - console.log('') - } - - // Show summary - const available = augmentations.filter(a => a.status === 'available').length - const coming = augmentations.filter(a => a.status === 'coming_soon').length - - console.log(chalk.gray('โ”€'.repeat(50))) - console.log(chalk.green(`โœ… ${available} available`) + chalk.gray(` โ€ข `) + - chalk.yellow(`๐Ÿ”œ ${coming} coming soon`)) - console.log('') - console.log(chalk.dim('Configure augmentations with "brainy augment"')) - console.log(chalk.dim('Run "brainy augment info " for details')) -} - -/** - * Show detailed info about an augmentation - */ -export async function showAugmentationInfo(id: string) { - const catalog = await fetchCatalog() - if (!catalog) { - console.log(chalk.red('โŒ Could not load augmentation catalog')) - return - } - - const aug = catalog.augmentations.find(a => a.id === id) - if (!aug) { - console.log(chalk.red(`โŒ Augmentation not found: ${id}`)) - console.log('') - console.log('Available augmentations:') - catalog.augmentations.forEach(a => { - console.log(` โ€ข ${a.id}`) - }) - return - } - - // Fetch full details from API if available - try { - if (!CATALOG_API) throw new Error('No external catalog configured') - - const response = await fetch(`${CATALOG_API}/api/catalog/augmentation/${id}`) - const details = await response.json() - - console.log(chalk.cyan.bold(`๐Ÿ“ฆ ${details.name}`)) - if (details.popular) console.log(chalk.yellow('โญ Popular')) - console.log('') - - console.log(chalk.bold('Category:'), getCategoryName(details.category, catalog.categories)) - console.log(chalk.bold('Status:'), getStatusText(details.status)) - if (details.eta) console.log(chalk.bold('Expected:'), details.eta) - console.log('') - - console.log(chalk.bold('Description:')) - console.log(details.longDescription || details.description) - console.log('') - - if (details.features) { - console.log(chalk.bold('Features:')) - details.features.forEach((f: string) => console.log(` โœ“ ${f}`)) - console.log('') - } - - if (details.example) { - console.log(chalk.bold('Example:')) - console.log(chalk.gray('โ”€'.repeat(50))) - console.log(details.example.code) - console.log(chalk.gray('โ”€'.repeat(50))) - console.log('') - } - - if (details.requirements?.config) { - console.log(chalk.bold('Required Configuration:')) - details.requirements.config.forEach((c: string) => console.log(` โ€ข ${c}`)) - console.log('') - } - - if (details.pricing) { - console.log(chalk.bold('Available in:')) - details.pricing.tiers.forEach((t: string) => console.log(` โ€ข ${t}`)) - console.log('') - } - - console.log(chalk.dim('To activate: brainy augment activate')) - } catch (error) { - // Show basic info if API fails - console.log(chalk.cyan.bold(`๐Ÿ“ฆ ${aug.name}`)) - console.log(aug.description) - console.log('') - console.log(chalk.dim('Full details unavailable (no external catalog configured)')) - } -} - -/** - * Show user's available augmentations - */ -export async function showAvailable(licenseKey?: string) { - // Show local catalog as default - const catalog = await fetchCatalog() - if (!catalog) { - console.log(chalk.red('โŒ Could not load augmentation catalog')) - return - } - - console.log(chalk.cyan.bold('๐Ÿง  Available Augmentations')) - console.log('') - - const available = catalog.augmentations.filter(a => a.status === 'available') - const grouped = groupByCategory(available, catalog.categories) - - for (const [category, augs] of Object.entries(grouped)) { - if (augs.length === 0) continue - - const cat = catalog.categories.find(c => c.id === category) - console.log(chalk.bold(`${cat?.icon || '๐Ÿ“ฆ'} ${cat?.name || category}`)) - augs.forEach(aug => { - console.log(` โœ… ${aug.name}`) - console.log(chalk.gray(` ${aug.description}`)) - }) - console.log('') - } - - console.log(chalk.green(`โœ… ${available.length} augmentations available`)) - - // If external API is configured and license key provided, try to fetch personalized data - if (CATALOG_API && licenseKey) { - try { - const response = await fetch(`${CATALOG_API}/api/catalog/available`, { - headers: { 'x-license-key': licenseKey } - }) - - if (response.ok) { - const data = await response.json() - console.log(chalk.gray(`Plan: ${data.plan || 'Standard'}`)) - - if (data.operations) { - const used = data.operations.used || 0 - const limit = data.operations.limit - const percent = limit === 'unlimited' ? 0 : Math.round((used / limit) * 100) - - console.log(chalk.bold('Usage:')) - if (limit === 'unlimited') { - console.log(` Unlimited operations`) - } else { - console.log(` ${used.toLocaleString()} / ${limit.toLocaleString()} operations (${percent}%)`) - } - } - } - } catch (error) { - // Ignore external API errors - local catalog is sufficient - } - } -} - -// Helper functions - -function loadCache(ignoreExpiry = false): Catalog | null { - try { - if (!existsSync(CACHE_PATH)) return null - - const data = JSON.parse(readFileSync(CACHE_PATH, 'utf8')) - - if (!ignoreExpiry && Date.now() - data.timestamp > CACHE_TTL) { - return null - } - - return data.catalog - } catch { - return null - } -} - -function saveCache(catalog: Catalog): void { - try { - const dir = join(homedir(), '.brainy') - if (!existsSync(dir)) { - mkdirSync(dir, { recursive: true }) - } - - writeFileSync(CACHE_PATH, JSON.stringify({ - catalog, - timestamp: Date.now() - })) - } catch { - // Ignore cache save errors - } -} - -function groupByCategory(augmentations: Augmentation[], categories: Category[]) { - const grouped: Record = {} - - for (const aug of augmentations) { - if (!grouped[aug.category]) { - grouped[aug.category] = [] - } - grouped[aug.category].push(aug) - } - - // Sort by category order - const ordered: Record = {} - const categoryOrder = ['memory', 'coordination', 'enterprise', 'perception', 'dialog', 'activation', 'cognition', 'websocket'] - - for (const cat of categoryOrder) { - if (grouped[cat]) { - ordered[cat] = grouped[cat] - } - } - - return ordered -} - -function getStatusIcon(status: string): string { - switch (status) { - case 'available': return chalk.green('โœ…') - case 'coming_soon': return chalk.yellow('๐Ÿ”œ') - case 'deprecated': return chalk.red('โš ๏ธ') - default: return 'โ“' - } -} - -function getStatusText(status: string): string { - switch (status) { - case 'available': return chalk.green('Available') - case 'coming_soon': return chalk.yellow('Coming Soon') - case 'deprecated': return chalk.red('Deprecated') - default: return 'Unknown' - } -} - -function getCategoryName(categoryId: string, categories: Category[]): string { - const cat = categories.find(c => c.id === categoryId) - return cat ? `${cat.icon} ${cat.name}` : categoryId -} - -function readLicenseFile(): string | null { - try { - const licensePath = join(homedir(), '.brainy', 'license') - if (existsSync(licensePath)) { - return readFileSync(licensePath, 'utf8').trim() - } - } catch (error) { - // License file read failed, return null - console.debug('Failed to read license file:', error) - } - return null -} - -function getDefaultCatalog(): Catalog { - // Local catalog with current features - return { - version: '1.5.0', - categories: [ - { id: 'core', name: 'Core Features', icon: '๐Ÿง ', description: 'Essential brainy functionality' }, - { id: 'neural', name: 'Neural API', icon: '๐Ÿ”—', description: 'Semantic similarity and clustering' }, - { id: 'enterprise', name: 'Enterprise', icon: '๐Ÿข', description: 'Business integrations' }, - { id: 'storage', name: 'Storage', icon: '๐Ÿ’พ', description: 'Data persistence and caching' } - ], - augmentations: [ - { - id: 'vector-search', - name: 'Vector Search', - category: 'core', - description: 'High-performance semantic search with HNSW indexing', - status: 'available', - popular: true - }, - { - id: 'neural-similarity', - name: 'Neural Similarity API', - category: 'neural', - description: 'Advanced semantic similarity, clustering, and hierarchy detection', - status: 'available', - popular: true - }, - { - id: 'intelligent-verb-scoring', - name: 'Intelligent Verb Scoring', - category: 'neural', - description: 'Smart relationship scoring with taxonomy understanding', - status: 'available' - }, - { - id: 'connection-pooling', - name: 'Connection Pooling', - category: 'enterprise', - description: 'Efficient database connection management', - status: 'available' - }, - { - id: 'batch-processing', - name: 'Batch Processing', - category: 'enterprise', - description: 'High-throughput batch operations with deduplication', - status: 'available' - }, - { - id: 's3-storage', - name: 'S3 Compatible Storage', - category: 'storage', - description: 'Cloud storage with optimized batch operations', - status: 'available' - }, - { - id: 'opfs-storage', - name: 'OPFS Storage', - category: 'storage', - description: 'Browser-based persistent storage', - status: 'available' - } - ] - } -} \ No newline at end of file diff --git a/src/embeddings/CachedEmbeddings.ts b/src/embeddings/CachedEmbeddings.ts deleted file mode 100644 index 6108717a..00000000 --- a/src/embeddings/CachedEmbeddings.ts +++ /dev/null @@ -1,163 +0,0 @@ -/** - * Cached Embeddings - Performance Optimization Layer - * - * Provides pre-computed embeddings for common terms to avoid - * unnecessary model calls. Falls back to EmbeddingManager for - * unknown terms. - * - * This is purely a performance optimization - it doesn't affect - * the consistency or accuracy of embeddings. - */ - -import { Vector } from '../coreTypes.js' -import { embeddingManager } from './EmbeddingManager.js' - -// Pre-computed embeddings for top common terms -// In production, this could be loaded from a file or expanded significantly -const PRECOMPUTED_EMBEDDINGS: Record = { - // Programming languages - 'javascript': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.1)), - 'python': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.1)), - 'typescript': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.15)), - 'java': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.15)), - 'rust': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.2)), - 'go': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.2)), - 'c++': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.22)), - 'c#': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.22)), - - // Web frameworks - 'react': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.25)), - 'vue': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.25)), - 'angular': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.3)), - 'svelte': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.3)), - 'nextjs': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.32)), - 'nuxt': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.32)), - - // Databases - 'postgresql': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.35)), - 'mysql': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.35)), - 'mongodb': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.4)), - 'redis': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.4)), - 'elasticsearch': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.42)), - - // Common tech terms - 'database': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.45)), - 'api': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.45)), - 'server': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.5)), - 'client': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.5)), - 'frontend': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.55)), - 'backend': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.55)), - 'fullstack': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.57)), - 'devops': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.57)), - 'cloud': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.6)), - 'docker': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.6)), - 'kubernetes': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.62)), - 'microservices': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.62)), -} - -/** - * Simple character n-gram based embedding for short text - * This is much faster than using the model for simple terms - */ -function computeSimpleEmbedding(text: string): Vector { - const normalized = text.toLowerCase().trim() - const vector = new Array(384).fill(0) - - // Character trigrams for simple semantic similarity - for (let i = 0; i < normalized.length - 2; i++) { - const trigram = normalized.slice(i, i + 3) - const hash = trigram.charCodeAt(0) * 31 + - trigram.charCodeAt(1) * 7 + - trigram.charCodeAt(2) - const index = Math.abs(hash) % 384 - vector[index] += 1 / (normalized.length - 2) - } - - // Normalize vector - const magnitude = Math.sqrt(vector.reduce((sum, val) => sum + val * val, 0)) - if (magnitude > 0) { - for (let i = 0; i < vector.length; i++) { - vector[i] /= magnitude - } - } - - return vector -} - -/** - * Cached Embeddings with fallback to EmbeddingManager - */ -export class CachedEmbeddings { - private stats = { - cacheHits: 0, - simpleComputes: 0, - modelCalls: 0 - } - - /** - * Generate embedding with caching - */ - async embed(text: string | string[]): Promise { - if (Array.isArray(text)) { - return Promise.all(text.map(t => this.embedSingle(t))) - } - return this.embedSingle(text) - } - - /** - * Embed single text with cache lookup - */ - private async embedSingle(text: string): Promise { - const normalized = text.toLowerCase().trim() - - // 1. Check pre-computed cache (instant, zero cost) - if (PRECOMPUTED_EMBEDDINGS[normalized]) { - this.stats.cacheHits++ - return PRECOMPUTED_EMBEDDINGS[normalized] - } - - // 2. Check for partial matches in cache - for (const [term, embedding] of Object.entries(PRECOMPUTED_EMBEDDINGS)) { - if (normalized.includes(term) || term.includes(normalized)) { - this.stats.cacheHits++ - // Return slightly modified version to maintain uniqueness - return embedding.map(v => v * 0.95) - } - } - - // 3. For short text, use simple embedding (fast, low cost) - if (normalized.length < 50 && normalized.split(' ').length < 5) { - this.stats.simpleComputes++ - return computeSimpleEmbedding(normalized) - } - - // 4. Fall back to EmbeddingManager for complex text - this.stats.modelCalls++ - return await embeddingManager.embed(text) - } - - /** - * Get cache statistics - */ - getStats() { - return { - ...this.stats, - totalEmbeddings: this.stats.cacheHits + this.stats.simpleComputes + this.stats.modelCalls, - cacheHitRate: this.stats.cacheHits / - (this.stats.cacheHits + this.stats.simpleComputes + this.stats.modelCalls) || 0 - } - } - - /** - * Add custom pre-computed embeddings - */ - addPrecomputed(term: string, embedding: Vector) { - if (embedding.length !== 384) { - throw new Error('Embedding must have 384 dimensions') - } - PRECOMPUTED_EMBEDDINGS[term.toLowerCase()] = embedding - } -} - -// Export singleton instance -export const cachedEmbeddings = new CachedEmbeddings() \ No newline at end of file diff --git a/src/embeddings/index.ts b/src/embeddings/index.ts deleted file mode 100644 index e46062b5..00000000 --- a/src/embeddings/index.ts +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Embeddings Module - Clean, Unified Architecture - * - * This module provides all embedding functionality for Brainy. - * - * Main Components: - * - EmbeddingManager: Core embedding generation with Q8/FP32 support - * - CachedEmbeddings: Performance optimization layer with pre-computed embeddings - */ - -// Core embedding functionality -export { - EmbeddingManager, - embeddingManager, - embed, - getEmbeddingFunction, - getEmbeddingStats, - type ModelPrecision -} from './EmbeddingManager.js' - -// Cached embeddings for performance -export { - CachedEmbeddings, - cachedEmbeddings -} from './CachedEmbeddings.js' - -// Default export is the singleton manager -export { embeddingManager as default } from './EmbeddingManager.js' \ No newline at end of file diff --git a/src/graph/pathfinding.ts b/src/graph/pathfinding.ts deleted file mode 100644 index 46f26e9c..00000000 --- a/src/graph/pathfinding.ts +++ /dev/null @@ -1,523 +0,0 @@ -/** - * Advanced Graph Pathfinding Algorithms - * Provides shortest path, multi-hop traversal, and path ranking - */ - -// Graph pathfinding doesn't need to import from coreTypes - -export interface GraphNode { - id: string - [key: string]: any -} - -export interface GraphEdge { - source: string - target: string - type: string - weight: number - metadata?: any -} - -export interface Path { - nodes: string[] - edges: GraphEdge[] - totalWeight: number - length: number -} - -export interface PathfindingOptions { - maxDepth?: number - maxPaths?: number - bidirectional?: boolean - weightField?: string - relationshipTypes?: string[] - nodeFilter?: (node: GraphNode) => boolean - edgeFilter?: (edge: GraphEdge) => boolean -} - -export class GraphPathfinding { - private adjacencyList: Map> = new Map() - private nodes: Map = new Map() - - /** - * Add a node to the graph - */ - public addNode(node: GraphNode): void { - this.nodes.set(node.id, node) - if (!this.adjacencyList.has(node.id)) { - this.adjacencyList.set(node.id, new Map()) - } - } - - /** - * Add an edge to the graph - */ - public addEdge(edge: GraphEdge): void { - // Ensure nodes exist - if (!this.adjacencyList.has(edge.source)) { - this.adjacencyList.set(edge.source, new Map()) - } - if (!this.adjacencyList.has(edge.target)) { - this.adjacencyList.set(edge.target, new Map()) - } - - // Add edge to adjacency list - const sourceEdges = this.adjacencyList.get(edge.source)! - if (!sourceEdges.has(edge.target)) { - sourceEdges.set(edge.target, []) - } - sourceEdges.get(edge.target)!.push(edge) - } - - /** - * Find shortest path using Dijkstra's algorithm - * O((V + E) log V) with binary heap - */ - public shortestPath( - start: string, - end: string, - options: PathfindingOptions = {} - ): Path | null { - const { - maxDepth = Infinity, - relationshipTypes, - edgeFilter - } = options - - // Priority queue: [nodeId, distance, path] - const pq: Array<[string, number, string[], GraphEdge[]]> = [[start, 0, [start], []]] - const visited = new Set() - const distances = new Map([[start, 0]]) - - while (pq.length > 0) { - // Sort by distance (simple array, could optimize with heap) - pq.sort((a, b) => a[1] - b[1]) - const [current, distance, path, edges] = pq.shift()! - - if (visited.has(current)) continue - visited.add(current) - - // Found target - if (current === end) { - return { - nodes: path, - edges, - totalWeight: distance, - length: path.length - 1 - } - } - - // Max depth reached - if (path.length > maxDepth) continue - - // Explore neighbors - const neighbors = this.adjacencyList.get(current) - if (!neighbors) continue - - for (const [neighbor, edgeList] of neighbors) { - if (visited.has(neighbor)) continue - - // Find best edge to neighbor - let bestEdge: GraphEdge | null = null - let bestWeight = Infinity - - for (const edge of edgeList) { - // Apply filters - if (relationshipTypes && !relationshipTypes.includes(edge.type)) continue - if (edgeFilter && !edgeFilter(edge)) continue - - if (edge.weight < bestWeight) { - bestWeight = edge.weight - bestEdge = edge - } - } - - if (!bestEdge) continue - - const newDistance = distance + bestWeight - const currentBest = distances.get(neighbor) ?? Infinity - - if (newDistance < currentBest) { - distances.set(neighbor, newDistance) - pq.push([ - neighbor, - newDistance, - [...path, neighbor], - [...edges, bestEdge] - ]) - } - } - } - - return null // No path found - } - - /** - * Find all paths between two nodes - * Uses DFS with cycle detection - */ - public allPaths( - start: string, - end: string, - options: PathfindingOptions = {} - ): Path[] { - const { - maxDepth = 10, - maxPaths = 100, - relationshipTypes, - edgeFilter - } = options - - const paths: Path[] = [] - const visited = new Set() - - const dfs = ( - current: string, - path: string[], - edges: GraphEdge[], - weight: number - ): void => { - if (paths.length >= maxPaths) return - if (path.length > maxDepth) return - - if (current === end && path.length > 1) { - paths.push({ - nodes: [...path], - edges: [...edges], - totalWeight: weight, - length: path.length - 1 - }) - return - } - - visited.add(current) - - const neighbors = this.adjacencyList.get(current) - if (neighbors) { - for (const [neighbor, edgeList] of neighbors) { - if (visited.has(neighbor)) continue - - for (const edge of edgeList) { - // Apply filters - if (relationshipTypes && !relationshipTypes.includes(edge.type)) continue - if (edgeFilter && !edgeFilter(edge)) continue - - dfs( - neighbor, - [...path, neighbor], - [...edges, edge], - weight + edge.weight - ) - } - } - } - - visited.delete(current) - } - - dfs(start, [start], [], 0) - - // Sort paths by weight - paths.sort((a, b) => a.totalWeight - b.totalWeight) - - return paths - } - - /** - * Bidirectional search for faster pathfinding - * Searches from both start and end simultaneously - */ - public bidirectionalSearch( - start: string, - end: string, - options: PathfindingOptions = {} - ): Path | null { - const { maxDepth = 10 } = options - - // Two search frontiers - const forwardVisited = new Map() - const backwardVisited = new Map() - - forwardVisited.set(start, { path: [start], edges: [], weight: 0 }) - backwardVisited.set(end, { path: [end], edges: [], weight: 0 }) - - const forwardQueue = [start] - const backwardQueue = [end] - - let depth = 0 - - while ( - (forwardQueue.length > 0 || backwardQueue.length > 0) && - depth < maxDepth - ) { - // Expand forward frontier - const forwardNext: string[] = [] - for (const current of forwardQueue) { - const currentData = forwardVisited.get(current)! - const neighbors = this.adjacencyList.get(current) - - if (neighbors) { - for (const [neighbor, edges] of neighbors) { - if (forwardVisited.has(neighbor)) continue - - // Select edge with lowest weight for optimal path - const bestEdge = edges.reduce((best, edge) => - edge.weight < best.weight ? edge : best, edges[0]) - forwardVisited.set(neighbor, { - path: [...currentData.path, neighbor], - edges: [...currentData.edges, bestEdge], - weight: currentData.weight + bestEdge.weight - }) - - // Check if we met the backward search - if (backwardVisited.has(neighbor)) { - const forward = forwardVisited.get(neighbor)! - const backward = backwardVisited.get(neighbor)! - - // Combine paths - const fullPath = [ - ...forward.path, - ...backward.path.slice(1).reverse() - ] - - // Reverse backward edges and combine - const backwardEdgesReversed = backward.edges - .map(e => ({ - ...e, - source: e.target, - target: e.source - })) - .reverse() - - return { - nodes: fullPath, - edges: [...forward.edges, ...backwardEdgesReversed], - totalWeight: forward.weight + backward.weight, - length: fullPath.length - 1 - } - } - - forwardNext.push(neighbor) - } - } - } - - // Expand backward frontier - const backwardNext: string[] = [] - for (const current of backwardQueue) { - const currentData = backwardVisited.get(current)! - - // For backward search, we need to look at incoming edges - for (const [nodeId, neighbors] of this.adjacencyList) { - const edges = neighbors.get(current) - if (!edges) continue - - if (backwardVisited.has(nodeId)) continue - - // Select edge with lowest weight for optimal path - const bestEdge = edges.reduce((best, edge) => - edge.weight < best.weight ? edge : best, edges[0]) - backwardVisited.set(nodeId, { - path: [...currentData.path, nodeId], - edges: [...currentData.edges, bestEdge], - weight: currentData.weight + bestEdge.weight - }) - - // Check if we met the forward search - if (forwardVisited.has(nodeId)) { - const forward = forwardVisited.get(nodeId)! - const backward = backwardVisited.get(nodeId)! - - // Combine paths - const fullPath = [ - ...forward.path, - ...backward.path.slice(1).reverse() - ] - - // Reverse backward edges and combine - const backwardEdgesReversed = backward.edges - .map(e => ({ - ...e, - source: e.target, - target: e.source - })) - .reverse() - - return { - nodes: fullPath, - edges: [...forward.edges, ...backwardEdgesReversed], - totalWeight: forward.weight + backward.weight, - length: fullPath.length - 1 - } - } - - backwardNext.push(nodeId) - } - } - - forwardQueue.splice(0, forwardQueue.length, ...forwardNext) - backwardQueue.splice(0, backwardQueue.length, ...backwardNext) - depth++ - } - - return null - } - - /** - * Multi-hop traversal (e.g., friends of friends) - * Returns all nodes within N hops - */ - public multiHopTraversal( - start: string, - hops: number, - options: PathfindingOptions = {} - ): Map { - const { relationshipTypes, nodeFilter, edgeFilter } = options - - const results = new Map() - const visited = new Set() - const queue: Array<{ node: string, distance: number, path: string[], edges: GraphEdge[] }> = [ - { node: start, distance: 0, path: [start], edges: [] } - ] - - while (queue.length > 0) { - const { node, distance, path, edges } = queue.shift()! - - if (distance > hops) continue - - // Record this node - if (!results.has(node)) { - results.set(node, { distance, paths: [] }) - } - results.get(node)!.paths.push({ - nodes: path, - edges, - totalWeight: edges.reduce((sum, e) => sum + e.weight, 0), - length: path.length - 1 - }) - - if (distance === hops) continue - - // Explore neighbors - const neighbors = this.adjacencyList.get(node) - if (neighbors) { - for (const [neighbor, edgeList] of neighbors) { - // Apply node filter - if (nodeFilter) { - const neighborNode = this.nodes.get(neighbor) - if (neighborNode && !nodeFilter(neighborNode)) continue - } - - for (const edge of edgeList) { - // Apply filters - if (relationshipTypes && !relationshipTypes.includes(edge.type)) continue - if (edgeFilter && !edgeFilter(edge)) continue - - queue.push({ - node: neighbor, - distance: distance + 1, - path: [...path, neighbor], - edges: [...edges, edge] - }) - } - } - } - } - - return results - } - - /** - * Find connected components using DFS - */ - public connectedComponents(): Array> { - const visited = new Set() - const components: Array> = [] - - const dfs = (node: string, component: Set): void => { - visited.add(node) - component.add(node) - - const neighbors = this.adjacencyList.get(node) - if (neighbors) { - for (const neighbor of neighbors.keys()) { - if (!visited.has(neighbor)) { - dfs(neighbor, component) - } - } - } - } - - for (const node of this.adjacencyList.keys()) { - if (!visited.has(node)) { - const component = new Set() - dfs(node, component) - components.push(component) - } - } - - return components - } - - /** - * Calculate PageRank for all nodes - * Useful for ranking importance in the graph - */ - public pageRank(iterations: number = 100, damping: number = 0.85): Map { - const nodes = Array.from(this.adjacencyList.keys()) - const n = nodes.length - - if (n === 0) return new Map() - - // Initialize ranks - const ranks = new Map() - for (const node of nodes) { - ranks.set(node, 1 / n) - } - - // Calculate outgoing edge counts - const outDegree = new Map() - for (const [node, neighbors] of this.adjacencyList) { - let count = 0 - for (const edges of neighbors.values()) { - count += edges.length - } - outDegree.set(node, count) - } - - // Iterate PageRank algorithm - for (let i = 0; i < iterations; i++) { - const newRanks = new Map() - - for (const node of nodes) { - let rank = (1 - damping) / n - - // Sum contributions from incoming edges - for (const [source, neighbors] of this.adjacencyList) { - if (neighbors.has(node)) { - const sourceRank = ranks.get(source) ?? 0 - const sourceOutDegree = outDegree.get(source) ?? 1 - rank += damping * (sourceRank / sourceOutDegree) - } - } - - newRanks.set(node, rank) - } - - // Update ranks - for (const [node, rank] of newRanks) { - ranks.set(node, rank) - } - } - - return ranks - } - - /** - * Clear the graph - */ - public clear(): void { - this.adjacencyList.clear() - this.nodes.clear() - } -} \ No newline at end of file diff --git a/src/import/EntityDeduplicator.ts b/src/import/EntityDeduplicator.ts deleted file mode 100644 index 306e349a..00000000 --- a/src/import/EntityDeduplicator.ts +++ /dev/null @@ -1,354 +0,0 @@ -/** - * Entity Deduplicator - * - * Finds and merges duplicate entities across imports using: - * - Embedding-based similarity matching - * - Type-aware comparison - * - Confidence-weighted merging - * - Provenance tracking - * - * NO MOCKS - Production-ready implementation - */ - -import { Brainy } from '../brainy.js' -import { NounType } from '../types/graphTypes.js' - -export interface EntityCandidate { - id?: string - name: string - type: NounType - description: string - confidence: number - metadata: Record -} - -export interface DuplicateMatch { - existingId: string - existingName: string - similarity: number - shouldMerge: boolean - reason: string -} - -export interface EntityDeduplicationOptions { - /** Similarity threshold for considering entities as duplicates (0-1) */ - similarityThreshold?: number - - /** Only match entities of the same type */ - strictTypeMatching?: boolean - - /** Enable fuzzy name matching */ - enableFuzzyMatching?: boolean - - /** Minimum confidence to consider for merging */ - minConfidence?: number -} - -export interface MergeResult { - mergedEntityId: string - wasMerged: boolean - mergedWith?: string - confidence: number - provenance: string[] -} - -/** - * EntityDeduplicator - Prevents duplicate entities across imports - */ -export class EntityDeduplicator { - private brain: Brainy - - constructor(brain: Brainy) { - this.brain = brain - } - - /** - * Find duplicate entities in the knowledge graph - */ - async findDuplicates( - candidate: EntityCandidate, - options: EntityDeduplicationOptions = {} - ): Promise { - const opts = { - similarityThreshold: options.similarityThreshold || 0.85, - strictTypeMatching: options.strictTypeMatching !== false, - enableFuzzyMatching: options.enableFuzzyMatching !== false, - minConfidence: options.minConfidence || 0.6 - } - - // Skip low-confidence candidates - if (candidate.confidence < opts.minConfidence) { - return null - } - - // Search for similar entities by name and description - const searchText = `${candidate.name} ${candidate.description}`.trim() - - try { - const results = await this.brain.find({ - query: searchText, - limit: 5, - where: opts.strictTypeMatching ? { type: candidate.type } : undefined - }) - - // Check each result for potential duplicates - for (const result of results) { - const similarity = result.score || 0 - const existingName = result.entity.metadata?.name || result.id - const existingType = result.entity.metadata?.type || result.entity.metadata?.nounType || result.entity.type - - // Skip if below similarity threshold - if (similarity < opts.similarityThreshold) { - continue - } - - // Type matching check - if (opts.strictTypeMatching && existingType !== candidate.type) { - continue - } - - // Exact name match (case-insensitive) - if (this.normalizeString(candidate.name) === this.normalizeString(existingName)) { - return { - existingId: result.id, - existingName, - similarity: 1.0, - shouldMerge: true, - reason: 'Exact name match' - } - } - - // High similarity match - if (similarity >= opts.similarityThreshold) { - // Additional validation for fuzzy matching - if (opts.enableFuzzyMatching && this.areSimilarNames(candidate.name, existingName)) { - return { - existingId: result.id, - existingName, - similarity, - shouldMerge: true, - reason: `High similarity (${(similarity * 100).toFixed(1)}%)` - } - } - } - } - } catch (error) { - // If search fails, assume no duplicates - return null - } - - return null - } - - /** - * Merge entity data with existing entity - */ - async mergeEntity( - existingId: string, - candidate: EntityCandidate, - importSource: string - ): Promise { - try { - // Get existing entity - const existing = await this.brain.get(existingId) - if (!existing) { - throw new Error(`Entity ${existingId} not found`) - } - - // Update confidence (weighted average) โ€” `confidence` is a reserved - // top-level field, so it travels via the dedicated update() param, not - // the metadata bag. - const mergedConfidence = this.mergeConfidence( - existing.confidence ?? 0.5, - candidate.confidence - ) - - // Merge metadata (custom fields only โ€” reserved fields are top-level) - const mergedMetadata = { - ...existing.metadata, - // Track provenance - imports: [ - ...(existing.metadata?.imports || []), - importSource - ], - // Merge VFS paths - vfsPaths: [ - ...(existing.metadata?.vfsPaths || [existing.metadata?.vfsPath]).filter(Boolean), - candidate.metadata?.vfsPath - ].filter(Boolean), - // Merge other metadata - ...this.mergeMetadataFields(existing.metadata, candidate.metadata), - // Track last update - lastUpdated: Date.now(), - mergeCount: (existing.metadata?.mergeCount || 0) + 1 - } - - // Update entity - await this.brain.update({ - id: existingId, - confidence: mergedConfidence, - metadata: mergedMetadata, - merge: true - }) - - return { - mergedEntityId: existingId, - wasMerged: true, - mergedWith: existing.metadata?.name || existingId, - confidence: mergedConfidence, - provenance: mergedMetadata.imports - } - } catch (error) { - throw new Error(`Failed to merge entity: ${error instanceof Error ? error.message : String(error)}`) - } - } - - /** - * Create or merge entity with deduplication - */ - async createOrMerge( - candidate: EntityCandidate, - importSource: string, - options: EntityDeduplicationOptions = {} - ): Promise { - // Check for duplicates - const duplicate = await this.findDuplicates(candidate, options) - - if (duplicate && duplicate.shouldMerge) { - // Merge with existing entity - return await this.mergeEntity(duplicate.existingId, candidate, importSource) - } - - // No duplicate found, create new entity. Preserve any subtype the candidate - // carried (set by the extractor or upstream importer), else fall back to - // `'imported'` so enforcement doesn't fire (added 7.30.1). - // `confidence` is a reserved top-level field (dedicated add() param); - // creation time is system-managed โ€” neither belongs in the metadata bag. - const entityId = await this.brain.add({ - data: candidate.description || candidate.name, - type: candidate.type, - subtype: - (candidate as EntityCandidate & { subtype?: string }).subtype ?? - 'imported', - confidence: candidate.confidence, - metadata: { - ...candidate.metadata, - name: candidate.name, - imports: [importSource], - vfsPaths: [candidate.metadata?.vfsPath].filter(Boolean), - mergeCount: 0 - } - }) - - // Update candidate with new ID - candidate.id = entityId - - return { - mergedEntityId: entityId, - wasMerged: false, - confidence: candidate.confidence, - provenance: [importSource] - } - } - - /** - * Normalize string for comparison - */ - private normalizeString(str: string): string { - return str - .toLowerCase() - .trim() - .replace(/[^a-z0-9]/g, '') - } - - /** - * Check if two names are similar (fuzzy matching) - */ - private areSimilarNames(name1: string, name2: string): boolean { - const n1 = this.normalizeString(name1) - const n2 = this.normalizeString(name2) - - // Exact match - if (n1 === n2) return true - - // Length difference check - const lengthDiff = Math.abs(n1.length - n2.length) - if (lengthDiff > 3) return false - - // Levenshtein distance - const distance = this.levenshteinDistance(n1, n2) - const maxLength = Math.max(n1.length, n2.length) - const similarity = 1 - (distance / maxLength) - - return similarity >= 0.85 - } - - /** - * Calculate Levenshtein distance between two strings - */ - private levenshteinDistance(str1: string, str2: string): number { - const m = str1.length - const n = str2.length - const dp: number[][] = Array(m + 1).fill(null).map(() => Array(n + 1).fill(0)) - - for (let i = 0; i <= m; i++) dp[i][0] = i - for (let j = 0; j <= n; j++) dp[0][j] = j - - for (let i = 1; i <= m; i++) { - for (let j = 1; j <= n; j++) { - if (str1[i - 1] === str2[j - 1]) { - dp[i][j] = dp[i - 1][j - 1] - } else { - dp[i][j] = Math.min( - dp[i - 1][j] + 1, // deletion - dp[i][j - 1] + 1, // insertion - dp[i - 1][j - 1] + 1 // substitution - ) - } - } - } - - return dp[m][n] - } - - /** - * Merge confidence scores (weighted average favoring higher confidence) - */ - private mergeConfidence(existing: number, incoming: number): number { - // Weight higher confidence more heavily - const weights = existing > incoming ? [0.6, 0.4] : [0.4, 0.6] - return existing * weights[0] + incoming * weights[1] - } - - /** - * Merge metadata fields intelligently - */ - private mergeMetadataFields( - existing: Record, - incoming: Record - ): Record { - const merged: Record = {} - - // Merge arrays - const arrayFields = ['concepts', 'tags', 'categories'] - for (const field of arrayFields) { - if (existing[field] || incoming[field]) { - const combined = [ - ...(existing[field] || []), - ...(incoming[field] || []) - ] - // Deduplicate - merged[field] = [...new Set(combined)] - } - } - - // Prefer longer descriptions - if (existing.description || incoming.description) { - merged.description = (existing.description || '').length > (incoming.description || '').length - ? existing.description - : incoming.description - } - - return merged - } -} diff --git a/src/import/InstancePool.ts b/src/import/InstancePool.ts deleted file mode 100644 index 2ac26120..00000000 --- a/src/import/InstancePool.ts +++ /dev/null @@ -1,269 +0,0 @@ -/** - * InstancePool - Shared instance management for memory efficiency - * - * Production-grade instance pooling to prevent memory leaks during imports. - * Critical for scaling to billions of entities. - * - * Problem: Creating new NLP/Extractor instances in loops โ†’ memory leak - * Solution: Reuse shared instances across entire import session - * - * Memory savings: - * - Without pooling: 100K rows ร— 50MB per instance = 5TB RAM (OOM!) - * - With pooling: 50MB total (shared across all rows) - */ - -import { Brainy } from '../brainy.js' -import { NaturalLanguageProcessor } from '../neural/naturalLanguageProcessor.js' -import { NeuralEntityExtractor } from '../neural/entityExtractor.js' - -/** - * InstancePool - Manages shared instances for memory efficiency - * - * Lifecycle: - * 1. Create pool at import start - * 2. Reuse instances across all rows - * 3. Pool is garbage collected when import completes - * - * Thread safety: Not thread-safe (single import session per pool) - */ -export class InstancePool { - private brain: Brainy - - // Shared instances (created lazily) - private nlpInstance: NaturalLanguageProcessor | null = null - private extractorInstance: NeuralEntityExtractor | null = null - - // Initialization state - private nlpInitialized = false - private initializationPromise: Promise | null = null - - // Statistics - private stats = { - nlpReuses: 0, - extractorReuses: 0, - creationTime: 0 - } - - constructor(brain: Brainy) { - this.brain = brain - } - - /** - * Get shared NaturalLanguageProcessor instance - * - * Lazy initialization - created on first access - * All subsequent calls return same instance - * - * @returns Shared NLP instance - */ - async getNLP(): Promise { - if (!this.nlpInstance) { - const startTime = Date.now() - this.nlpInstance = new NaturalLanguageProcessor(this.brain) - this.stats.creationTime += Date.now() - startTime - } - - // Ensure initialized before returning - if (!this.nlpInitialized) { - await this.ensureNLPInitialized() - } - - this.stats.nlpReuses++ - return this.nlpInstance - } - - /** - * Get shared NeuralEntityExtractor instance - * - * Lazy initialization - created on first access - * All subsequent calls return same instance - * - * @returns Shared extractor instance - */ - getExtractor(): NeuralEntityExtractor { - if (!this.extractorInstance) { - const startTime = Date.now() - this.extractorInstance = new NeuralEntityExtractor(this.brain) - this.stats.creationTime += Date.now() - startTime - } - - this.stats.extractorReuses++ - return this.extractorInstance - } - - /** - * Get shared NLP instance (synchronous, may return uninitialized) - * - * Use when you need NLP synchronously and will handle initialization yourself. - * Prefer getNLP() for async code. - * - * @returns Shared NLP instance (possibly uninitialized) - */ - getNLPSync(): NaturalLanguageProcessor { - if (!this.nlpInstance) { - this.nlpInstance = new NaturalLanguageProcessor(this.brain) - } - - this.stats.nlpReuses++ - return this.nlpInstance - } - - /** - * Initialize all instances upfront - * - * Call at start of import to avoid lazy initialization overhead - * during processing. Improves predictability and first-row performance. - * - * @returns Promise that resolves when all instances are ready - */ - async init(): Promise { - // Prevent duplicate initialization - if (this.initializationPromise) { - return this.initializationPromise - } - - this.initializationPromise = this.initializeInternal() - return this.initializationPromise - } - - /** - * Internal initialization implementation - */ - private async initializeInternal(): Promise { - const startTime = Date.now() - - // Create instances - if (!this.nlpInstance) { - this.nlpInstance = new NaturalLanguageProcessor(this.brain) - } - if (!this.extractorInstance) { - this.extractorInstance = new NeuralEntityExtractor(this.brain) - } - - // Initialize NLP (loads pattern library) - await this.ensureNLPInitialized() - - this.stats.creationTime = Date.now() - startTime - } - - /** - * Ensure NLP is initialized (loads 220 patterns) - * - * Handles concurrent initialization requests safely - */ - private async ensureNLPInitialized(): Promise { - if (this.nlpInitialized) { - return - } - - if (!this.nlpInstance) { - throw new Error('NLP instance not created yet') - } - - await this.nlpInstance.init() - this.nlpInitialized = true - } - - /** - * Check if instances are initialized - * - * @returns True if NLP is initialized and ready to use - */ - isInitialized(): boolean { - return this.nlpInitialized && this.nlpInstance !== null - } - - /** - * Get pool statistics - * - * Useful for performance monitoring and memory leak detection - * - * @returns Statistics about instance reuse - */ - getStats() { - return { - ...this.stats, - nlpCreated: this.nlpInstance !== null, - extractorCreated: this.extractorInstance !== null, - initialized: this.isInitialized(), - // Memory savings estimate - memorySaved: this.calculateMemorySaved() - } - } - - /** - * Calculate estimated memory saved by pooling - * - * Assumes ~50MB per NLP instance, ~10MB per extractor instance - * - * @returns Estimated memory saved in bytes - */ - private calculateMemorySaved(): number { - const nlpSize = 50 * 1024 * 1024 // 50MB per instance - const extractorSize = 10 * 1024 * 1024 // 10MB per instance - - // Without pooling: size ร— reuses - // With pooling: size ร— 1 - // Saved: size ร— (reuses - 1) - - const nlpSaved = nlpSize * Math.max(0, this.stats.nlpReuses - 1) - const extractorSaved = extractorSize * Math.max(0, this.stats.extractorReuses - 1) - - return nlpSaved + extractorSaved - } - - /** - * Reset statistics (useful for testing) - */ - resetStats(): void { - this.stats = { - nlpReuses: 0, - extractorReuses: 0, - creationTime: 0 - } - } - - /** - * Get string representation (for debugging) - */ - toString(): string { - const stats = this.getStats() - return `InstancePool(nlp=${stats.nlpCreated}, extractor=${stats.extractorCreated}, initialized=${stats.initialized}, nlpReuses=${stats.nlpReuses}, extractorReuses=${stats.extractorReuses})` - } - - /** - * Cleanup method (for explicit resource management) - * - * Note: Usually not needed - pool is garbage collected when import completes. - * Use only if you need explicit cleanup for some reason. - */ - cleanup(): void { - // Clear references to allow garbage collection - this.nlpInstance = null - this.extractorInstance = null - this.nlpInitialized = false - this.initializationPromise = null - } -} - -/** - * Create a new instance pool - * - * Convenience factory function - * - * @param brain Brainy instance - * @param autoInit Whether to initialize instances immediately - * @returns Instance pool - */ -export async function createInstancePool( - brain: Brainy, - autoInit = true -): Promise { - const pool = new InstancePool(brain) - - if (autoInit) { - await pool.init() - } - - return pool -} diff --git a/src/import/index.ts b/src/import/index.ts deleted file mode 100644 index 8aed2f41..00000000 --- a/src/import/index.ts +++ /dev/null @@ -1,38 +0,0 @@ -/** - * Unified Import System - * - * Single entry point for importing any file format into Brainy with: - * - Auto-detection of formats - * - Dual storage (VFS + Knowledge Graph) - * - Shared entities across imports (deduplication) - * - Simple, powerful API - */ - -export { ImportCoordinator } from './ImportCoordinator.js' -export { FormatDetector, SupportedFormat, DetectionResult } from './FormatDetector.js' -export { EntityDeduplicator } from './EntityDeduplicator.js' -export { BackgroundDeduplicator } from './BackgroundDeduplicator.js' -export { ImportHistory } from './ImportHistory.js' - -export type { - ImportSource, - ImportOptions, - ImportProgress, - ImportResult -} from './ImportCoordinator.js' - -export type { - EntityCandidate, - DuplicateMatch, - EntityDeduplicationOptions, - MergeResult -} from './EntityDeduplicator.js' - -export type { - DeduplicationStats -} from './BackgroundDeduplicator.js' - -export type { - ImportHistoryEntry, - RollbackResult -} from './ImportHistory.js' diff --git a/src/importers/SmartImportOrchestrator.ts b/src/importers/SmartImportOrchestrator.ts deleted file mode 100644 index 2084250d..00000000 --- a/src/importers/SmartImportOrchestrator.ts +++ /dev/null @@ -1,789 +0,0 @@ -/** - * Smart Import Orchestrator - * - * Coordinates the entire smart import pipeline: - * 1. Extract entities/relationships using SmartExcelImporter - * 2. Create entities and relationships in Brainy - * 3. Organize into VFS structure using VFSStructureGenerator - * - * NO MOCKS - Production-ready implementation - */ - -import { Brainy } from '../brainy.js' -import { VirtualFileSystem } from '../vfs/VirtualFileSystem.js' -import { NounType, VerbType } from '../types/graphTypes.js' -import { splitNounMetadataRecord } from '../types/reservedFields.js' -import { SmartExcelImporter, SmartExcelOptions, SmartExcelResult } from './SmartExcelImporter.js' -import { SmartPDFImporter, SmartPDFOptions, SmartPDFResult } from './SmartPDFImporter.js' -import { SmartCSVImporter, SmartCSVOptions, SmartCSVResult } from './SmartCSVImporter.js' -import { SmartJSONImporter, SmartJSONOptions, SmartJSONResult } from './SmartJSONImporter.js' -import { SmartMarkdownImporter, SmartMarkdownOptions, SmartMarkdownResult } from './SmartMarkdownImporter.js' -import { VFSStructureGenerator, VFSStructureOptions } from './VFSStructureGenerator.js' - -export interface SmartImportOptions extends SmartExcelOptions { - /** Create VFS structure */ - createVFSStructure?: boolean - - /** VFS root path */ - vfsRootPath?: string - - /** VFS grouping strategy */ - vfsGroupBy?: 'type' | 'sheet' | 'flat' | 'custom' - - /** Create entities in Brainy */ - createEntities?: boolean - - /** Create relationships in Brainy */ - createRelationships?: boolean - - /** Source filename */ - filename?: string - - /** - * Default subtype tag for entities + relationships this importer creates when - * the extractor doesn't set one. See `ValidImportOptions.defaultSubtype` โ€” - * same semantics, same precedence (extractor > caller default > `'imported'`). - * Added 7.30.1. - */ - defaultSubtype?: string -} - -export interface SmartImportProgress { - phase: 'parsing' | 'extracting' | 'creating' | 'relationships' | 'organizing' | 'complete' - message: string - processed: number - total: number - entities: number - relationships: number -} - -export interface SmartImportResult { - success: boolean - - /** Extraction results */ - extraction: SmartExcelResult - - /** Created entity IDs */ - entityIds: string[] - - /** Created relationship IDs */ - relationshipIds: string[] - - /** VFS structure created */ - vfsStructure?: { - rootPath: string - directories: string[] - files: number - } - - /** Overall statistics */ - stats: { - rowsProcessed: number - entitiesCreated: number - relationshipsCreated: number - filesCreated: number - totalTime: number - } - - /** Any errors encountered */ - errors: string[] -} - -/** - * SmartImportOrchestrator - Main entry point for smart imports - */ -export class SmartImportOrchestrator { - private brain: Brainy - private excelImporter: SmartExcelImporter - private pdfImporter: SmartPDFImporter - private csvImporter: SmartCSVImporter - private jsonImporter: SmartJSONImporter - private markdownImporter: SmartMarkdownImporter - private vfsGenerator: VFSStructureGenerator - - constructor(brain: Brainy) { - this.brain = brain - this.excelImporter = new SmartExcelImporter(brain) - this.pdfImporter = new SmartPDFImporter(brain) - this.csvImporter = new SmartCSVImporter(brain) - this.jsonImporter = new SmartJSONImporter(brain) - this.markdownImporter = new SmartMarkdownImporter(brain) - this.vfsGenerator = new VFSStructureGenerator(brain) - } - - /** - * Initialize the orchestrator - */ - async init(): Promise { - await this.excelImporter.init() - await this.pdfImporter.init() - await this.csvImporter.init() - await this.jsonImporter.init() - await this.markdownImporter.init() - await this.vfsGenerator.init() - } - - /** - * Import Excel file with full pipeline - */ - async importExcel( - buffer: Buffer, - options: SmartImportOptions = {}, - onProgress?: (progress: SmartImportProgress) => void - ): Promise { - const startTime = Date.now() - const result: SmartImportResult = { - success: false, - // Typed boundary: populated in the extraction phase below. If extraction - // throws, the error path returns with this still null (pre-existing - // contract โ€” consumers check `success`/`errors` before reading it). - extraction: null as unknown as SmartExcelResult, - entityIds: [], - relationshipIds: [], - stats: { - rowsProcessed: 0, - entitiesCreated: 0, - relationshipsCreated: 0, - filesCreated: 0, - totalTime: 0 - }, - errors: [] - } - - try { - // Phase 1: Extract entities and relationships - onProgress?.({ - phase: 'extracting', - message: 'Extracting entities and relationships...', - processed: 0, - total: 0, - entities: 0, - relationships: 0 - }) - - result.extraction = await this.excelImporter.extract(buffer, { - ...options, - onProgress: (stats) => { - onProgress?.({ - phase: 'extracting', - message: `Processing row ${stats.processed}/${stats.total}...`, - processed: stats.processed, - total: stats.total, - entities: stats.entities, - relationships: stats.relationships - }) - } - }) - - result.stats.rowsProcessed = result.extraction.rowsProcessed - - // Phase 2: Create entities in Brainy - if (options.createEntities !== false) { - onProgress?.({ - phase: 'creating', - message: 'Creating entities in knowledge graph...', - processed: 0, - total: result.extraction.rows.length, - entities: 0, - relationships: 0 - }) - - for (let i = 0; i < result.extraction.rows.length; i++) { - const extracted = result.extraction.rows[i] - - try { - // Create main entity. Subtype precedence: extractor-set โ†’ caller default - // โ†’ Brainy default `'imported'` (added 7.30.1). - const entityId = await this.brain.add({ - data: extracted.entity.description, - type: extracted.entity.type, - subtype: - (extracted.entity as typeof extracted.entity & { subtype?: string }) - .subtype ?? options.defaultSubtype ?? 'imported', - confidence: extracted.entity.confidence, // reserved field โ€” dedicated param, not metadata - metadata: { - // Strip reserved keys an extractor may have smuggled into the bag - // (8.0 reservedFieldPolicy defaults to 'throw'). - ...splitNounMetadataRecord(extracted.entity.metadata).custom, - name: extracted.entity.name, - importedFrom: 'smart-import' - } - }) - - result.entityIds.push(entityId) - result.stats.entitiesCreated++ - - // Update entity ID in extraction result - extracted.entity.id = entityId - - onProgress?.({ - phase: 'creating', - message: `Created entity: ${extracted.entity.name}`, - processed: i + 1, - total: result.extraction.rows.length, - entities: result.entityIds.length, - relationships: result.relationshipIds.length - }) - } catch (error: any) { - result.errors.push(`Failed to create entity ${extracted.entity.name}: ${error.message}`) - } - } - } - - // Phase 3: Create relationships - if (options.createRelationships !== false && options.createEntities !== false) { - onProgress?.({ - phase: 'creating', - message: 'Preparing relationships...', - processed: 0, - total: result.extraction.rows.length, - entities: result.entityIds.length, - relationships: 0 - }) - - // Build entity name -> ID map - const entityMap = new Map() - for (const extracted of result.extraction.rows) { - entityMap.set(extracted.entity.name.toLowerCase(), extracted.entity.id) - } - - // Collect all relationship parameters - const relationshipParams: Array<{from: string; to: string; type: VerbType; subtype?: string; confidence?: number; metadata?: any}> = [] - - for (const extracted of result.extraction.rows) { - for (const rel of extracted.relationships) { - try { - // Find target entity ID - let toEntityId: string | undefined - - // Try to find by name in our extracted entities - for (const otherExtracted of result.extraction.rows) { - if (rel.to.toLowerCase().includes(otherExtracted.entity.name.toLowerCase()) || - otherExtracted.entity.name.toLowerCase().includes(rel.to.toLowerCase())) { - toEntityId = otherExtracted.entity.id - break - } - } - - // If not found, create a placeholder entity. `import-placeholder` marks - // these as synthetic targets so consumers can distinguish them from real - // imports and downstream dedup can consolidate (added 7.30.1). - if (!toEntityId) { - toEntityId = await this.brain.add({ - data: rel.to, - type: NounType.Thing, - subtype: 'import-placeholder', - metadata: { - name: rel.to, - placeholder: true, - extractedFrom: extracted.entity.name - } - }) - result.entityIds.push(toEntityId) - } - - // Collect relationship parameter. Subtype precedence: extractor-set rel - // subtype โ†’ caller default โ†’ Brainy default `'imported'` (added 7.30.1). - relationshipParams.push({ - from: extracted.entity.id, - to: toEntityId, - type: rel.type, - subtype: - (rel as typeof rel & { subtype?: string }).subtype ?? - options.defaultSubtype ?? 'imported', - confidence: rel.confidence, // reserved field โ€” dedicated param, not metadata - metadata: { - evidence: rel.evidence - } - }) - } catch (error: any) { - result.errors.push(`Failed to prepare relationship: ${error.message}`) - } - } - } - - // Batch create all relationships with progress - if (relationshipParams.length > 0) { - onProgress?.({ - phase: 'relationships', - message: 'Building relationships...', - processed: 0, - total: relationshipParams.length, - entities: result.entityIds.length, - relationships: 0 - }) - - try { - const relationshipIds = await this.brain.relateMany({ - items: relationshipParams, - parallel: true, - chunkSize: 100, - continueOnError: true, - onProgress: (done, total) => { - onProgress?.({ - phase: 'relationships', - message: `Building relationships: ${done}/${total}`, - processed: done, - total: total, - entities: result.entityIds.length, - relationships: done - }) - } - }) - - result.relationshipIds = relationshipIds - result.stats.relationshipsCreated = relationshipIds.length - } catch (error: any) { - result.errors.push(`Failed to create relationships: ${error.message}`) - } - } - } - - // Phase 4: Create VFS structure - if (options.createVFSStructure !== false) { - onProgress?.({ - phase: 'organizing', - message: 'Organizing into file structure...', - processed: 0, - total: result.extraction.rows.length, - entities: result.entityIds.length, - relationships: result.relationshipIds.length - }) - - const vfsOptions: VFSStructureOptions = { - rootPath: options.vfsRootPath || '/imports/' + (options.filename || 'import'), - groupBy: options.vfsGroupBy || 'type', - preserveSource: true, - sourceBuffer: buffer, - sourceFilename: options.filename || 'import.xlsx', - createRelationshipFile: true, - createMetadataFile: true - } - - const vfsResult = await this.vfsGenerator.generate(result.extraction, vfsOptions) - - result.vfsStructure = { - rootPath: vfsResult.rootPath, - directories: vfsResult.directories, - files: vfsResult.files.length - } - - result.stats.filesCreated = vfsResult.files.length - } - - // Complete - result.success = result.errors.length === 0 - result.stats.totalTime = Date.now() - startTime - - onProgress?.({ - phase: 'complete', - message: `Import complete: ${result.stats.entitiesCreated} entities, ${result.stats.relationshipsCreated} relationships`, - processed: result.extraction.rows.length, - total: result.extraction.rows.length, - entities: result.stats.entitiesCreated, - relationships: result.stats.relationshipsCreated - }) - - } catch (error: any) { - result.errors.push(`Import failed: ${error.message}`) - result.success = false - } - - return result - } - - /** - * Import PDF file with full pipeline - */ - async importPDF( - buffer: Buffer, - options: SmartImportOptions & SmartPDFOptions = {}, - onProgress?: (progress: SmartImportProgress) => void - ): Promise { - const startTime = Date.now() - const result: SmartImportResult = { - success: false, - // Typed boundary: populated after extraction (see importExcel). - extraction: null as unknown as SmartExcelResult, - entityIds: [], - relationshipIds: [], - stats: { - rowsProcessed: 0, - entitiesCreated: 0, - relationshipsCreated: 0, - filesCreated: 0, - totalTime: 0 - }, - errors: [] - } - - try { - // Phase 1: Extract from PDF - onProgress?.({ phase: 'extracting', message: 'Extracting from PDF...', processed: 0, total: 0, entities: 0, relationships: 0 }) - - const pdfResult = await this.pdfImporter.extract(buffer, options) - - // Convert PDF result to Excel-like format for processing - result.extraction = this.convertPDFToExcelFormat(pdfResult) - result.stats.rowsProcessed = pdfResult.sectionsProcessed - - // Phase 2 & 3: Create entities and relationships - await this.createEntitiesAndRelationships(result, options, onProgress) - - // Phase 4: Create VFS structure - if (options.createVFSStructure !== false) { - const vfsOptions: VFSStructureOptions = { - rootPath: options.vfsRootPath || '/imports/' + (options.filename || 'import'), - groupBy: options.vfsGroupBy || 'type', - preserveSource: true, - sourceBuffer: buffer, - sourceFilename: options.filename || 'import.pdf', - createRelationshipFile: true, - createMetadataFile: true - } - const vfsResult = await this.vfsGenerator.generate(result.extraction, vfsOptions) - result.vfsStructure = { rootPath: vfsResult.rootPath, directories: vfsResult.directories, files: vfsResult.files.length } - result.stats.filesCreated = vfsResult.files.length - } - - result.success = result.errors.length === 0 - result.stats.totalTime = Date.now() - startTime - onProgress?.({ phase: 'complete', message: `Import complete: ${result.stats.entitiesCreated} entities, ${result.stats.relationshipsCreated} relationships`, processed: result.stats.rowsProcessed, total: result.stats.rowsProcessed, entities: result.stats.entitiesCreated, relationships: result.stats.relationshipsCreated }) - - } catch (error: any) { - result.errors.push(`PDF import failed: ${error.message}`) - result.success = false - } - - return result - } - - /** - * Import CSV file with full pipeline - */ - async importCSV( - buffer: Buffer, - options: SmartImportOptions & SmartCSVOptions = {}, - onProgress?: (progress: SmartImportProgress) => void - ): Promise { - // CSV is very similar to Excel, can reuse importExcel logic - return this.importExcel(buffer, options, onProgress) - } - - /** - * Import JSON data with full pipeline - */ - async importJSON( - data: any, - options: SmartImportOptions & SmartJSONOptions = {}, - onProgress?: (progress: SmartImportProgress) => void - ): Promise { - const startTime = Date.now() - const result: SmartImportResult = { - success: false, - // Typed boundary: populated after extraction (see importExcel). - extraction: null as unknown as SmartExcelResult, - entityIds: [], - relationshipIds: [], - stats: { - rowsProcessed: 0, - entitiesCreated: 0, - relationshipsCreated: 0, - filesCreated: 0, - totalTime: 0 - }, - errors: [] - } - - try { - onProgress?.({ phase: 'extracting', message: 'Extracting from JSON...', processed: 0, total: 0, entities: 0, relationships: 0 }) - - const jsonResult = await this.jsonImporter.extract(data, options) - - result.extraction = this.convertJSONToExcelFormat(jsonResult) - result.stats.rowsProcessed = jsonResult.nodesProcessed - - await this.createEntitiesAndRelationships(result, options, onProgress) - - if (options.createVFSStructure !== false) { - const sourceBuffer = Buffer.from(typeof data === 'string' ? data : JSON.stringify(data, null, 2)) - const vfsOptions: VFSStructureOptions = { - rootPath: options.vfsRootPath || '/imports/' + (options.filename || 'import'), - groupBy: options.vfsGroupBy || 'type', - preserveSource: true, - sourceBuffer, - sourceFilename: options.filename || 'import.json', - createRelationshipFile: true, - createMetadataFile: true - } - const vfsResult = await this.vfsGenerator.generate(result.extraction, vfsOptions) - result.vfsStructure = { rootPath: vfsResult.rootPath, directories: vfsResult.directories, files: vfsResult.files.length } - result.stats.filesCreated = vfsResult.files.length - } - - result.success = result.errors.length === 0 - result.stats.totalTime = Date.now() - startTime - onProgress?.({ phase: 'complete', message: `Import complete: ${result.stats.entitiesCreated} entities, ${result.stats.relationshipsCreated} relationships`, processed: result.stats.rowsProcessed, total: result.stats.rowsProcessed, entities: result.stats.entitiesCreated, relationships: result.stats.relationshipsCreated }) - - } catch (error: any) { - result.errors.push(`JSON import failed: ${error.message}`) - result.success = false - } - - return result - } - - /** - * Import Markdown content with full pipeline - */ - async importMarkdown( - markdown: string, - options: SmartImportOptions & SmartMarkdownOptions = {}, - onProgress?: (progress: SmartImportProgress) => void - ): Promise { - const startTime = Date.now() - const result: SmartImportResult = { - success: false, - // Typed boundary: populated after extraction (see importExcel). - extraction: null as unknown as SmartExcelResult, - entityIds: [], - relationshipIds: [], - stats: { - rowsProcessed: 0, - entitiesCreated: 0, - relationshipsCreated: 0, - filesCreated: 0, - totalTime: 0 - }, - errors: [] - } - - try { - onProgress?.({ phase: 'extracting', message: 'Extracting from Markdown...', processed: 0, total: 0, entities: 0, relationships: 0 }) - - const mdResult = await this.markdownImporter.extract(markdown, options) - - result.extraction = this.convertMarkdownToExcelFormat(mdResult) - result.stats.rowsProcessed = mdResult.sectionsProcessed - - await this.createEntitiesAndRelationships(result, options, onProgress) - - if (options.createVFSStructure !== false) { - const sourceBuffer = Buffer.from(markdown, 'utf-8') - const vfsOptions: VFSStructureOptions = { - rootPath: options.vfsRootPath || '/imports/' + (options.filename || 'import'), - groupBy: options.vfsGroupBy || 'type', - preserveSource: true, - sourceBuffer, - sourceFilename: options.filename || 'import.md', - createRelationshipFile: true, - createMetadataFile: true - } - const vfsResult = await this.vfsGenerator.generate(result.extraction, vfsOptions) - result.vfsStructure = { rootPath: vfsResult.rootPath, directories: vfsResult.directories, files: vfsResult.files.length } - result.stats.filesCreated = vfsResult.files.length - } - - result.success = result.errors.length === 0 - result.stats.totalTime = Date.now() - startTime - onProgress?.({ phase: 'complete', message: `Import complete: ${result.stats.entitiesCreated} entities, ${result.stats.relationshipsCreated} relationships`, processed: result.stats.rowsProcessed, total: result.stats.rowsProcessed, entities: result.stats.entitiesCreated, relationships: result.stats.relationshipsCreated }) - - } catch (error: any) { - result.errors.push(`Markdown import failed: ${error.message}`) - result.success = false - } - - return result - } - - /** - * Helper: Create entities and relationships from extraction result - */ - private async createEntitiesAndRelationships( - result: SmartImportResult, - options: SmartImportOptions, - onProgress?: (progress: SmartImportProgress) => void - ): Promise { - if (options.createEntities !== false) { - onProgress?.({ phase: 'creating', message: 'Creating entities in knowledge graph...', processed: 0, total: result.extraction.rows.length, entities: 0, relationships: 0 }) - - for (let i = 0; i < result.extraction.rows.length; i++) { - const extracted = result.extraction.rows[i] - try { - // Subtype precedence: extractor โ†’ caller default โ†’ `'imported'` (7.30.1). - const entityId = await this.brain.add({ - data: extracted.entity.description, - type: extracted.entity.type, - subtype: - (extracted.entity as typeof extracted.entity & { subtype?: string }) - .subtype ?? options.defaultSubtype ?? 'imported', - confidence: extracted.entity.confidence, // reserved field โ€” dedicated param, not metadata - metadata: { ...splitNounMetadataRecord(extracted.entity.metadata).custom, name: extracted.entity.name, importedFrom: 'smart-import' } - }) - result.entityIds.push(entityId) - result.stats.entitiesCreated++ - extracted.entity.id = entityId - } catch (error: any) { - result.errors.push(`Failed to create entity ${extracted.entity.name}: ${error.message}`) - } - } - } - - if (options.createRelationships !== false && options.createEntities !== false) { - onProgress?.({ phase: 'creating', message: 'Preparing relationships...', processed: 0, total: result.extraction.rows.length, entities: result.entityIds.length, relationships: 0 }) - - // Collect all relationship parameters - const relationshipParams: Array<{from: string; to: string; type: VerbType; subtype?: string; confidence?: number; metadata?: any}> = [] - - for (const extracted of result.extraction.rows) { - for (const rel of extracted.relationships) { - try { - let toEntityId: string | undefined - for (const otherExtracted of result.extraction.rows) { - if (rel.to.toLowerCase().includes(otherExtracted.entity.name.toLowerCase()) || otherExtracted.entity.name.toLowerCase().includes(rel.to.toLowerCase())) { - toEntityId = otherExtracted.entity.id - break - } - } - if (!toEntityId) { - // Subtype `import-placeholder` marks synthetic targets (7.30.1). - toEntityId = await this.brain.add({ data: rel.to, type: NounType.Thing, subtype: 'import-placeholder', metadata: { name: rel.to, placeholder: true, extractedFrom: extracted.entity.name } }) - result.entityIds.push(toEntityId) - } - // Relationship subtype precedence: extractor โ†’ caller default โ†’ `'imported'` (7.30.1). - // `confidence` is a reserved top-level field โ€” dedicated relate() param, not metadata - relationshipParams.push({ from: extracted.entity.id, to: toEntityId, type: rel.type, subtype: (rel as typeof rel & { subtype?: string }).subtype ?? options.defaultSubtype ?? 'imported', confidence: rel.confidence, metadata: { evidence: rel.evidence } }) - } catch (error: any) { - result.errors.push(`Failed to prepare relationship: ${error.message}`) - } - } - } - - // Batch create all relationships with progress - if (relationshipParams.length > 0) { - onProgress?.({ phase: 'relationships', message: 'Building relationships...', processed: 0, total: relationshipParams.length, entities: result.entityIds.length, relationships: 0 }) - - try { - const relationshipIds = await this.brain.relateMany({ - items: relationshipParams, - parallel: true, - chunkSize: 100, - continueOnError: true, - onProgress: (done, total) => { - onProgress?.({ phase: 'relationships', message: `Building relationships: ${done}/${total}`, processed: done, total: total, entities: result.entityIds.length, relationships: done }) - } - }) - - result.relationshipIds = relationshipIds - result.stats.relationshipsCreated = relationshipIds.length - } catch (error: any) { - result.errors.push(`Failed to create relationships: ${error.message}`) - } - } - } - } - - /** - * Helper: Convert PDF result to Excel-like format - */ - private convertPDFToExcelFormat(pdfResult: SmartPDFResult): Omit & { rows: any[] } { - const rows = pdfResult.sections.flatMap(section => - section.entities.map(entity => ({ - entity, - relatedEntities: [], - relationships: section.relationships.filter(r => r.from === entity.id), - concepts: section.concepts - })) - ) - - return { - rowsProcessed: pdfResult.sectionsProcessed, - entitiesExtracted: pdfResult.entitiesExtracted, - relationshipsInferred: pdfResult.relationshipsInferred, - rows, - entityMap: pdfResult.entityMap, - processingTime: pdfResult.processingTime, - stats: pdfResult.stats - } - } - - /** - * Helper: Convert JSON result to Excel-like format - */ - private convertJSONToExcelFormat(jsonResult: SmartJSONResult): Omit & { rows: any[] } { - const rows = jsonResult.entities.map(entity => ({ - entity, - relatedEntities: [], - relationships: jsonResult.relationships.filter(r => r.from === entity.id), - concepts: entity.metadata.concepts || [] - })) - - return { - rowsProcessed: jsonResult.nodesProcessed, - entitiesExtracted: jsonResult.entitiesExtracted, - relationshipsInferred: jsonResult.relationshipsInferred, - rows, - entityMap: jsonResult.entityMap, - processingTime: jsonResult.processingTime, - stats: jsonResult.stats - } - } - - /** - * Helper: Convert Markdown result to Excel-like format - */ - private convertMarkdownToExcelFormat(mdResult: SmartMarkdownResult): Omit & { rows: any[] } { - const rows = mdResult.sections.flatMap(section => - section.entities.map(entity => ({ - entity, - relatedEntities: [], - relationships: section.relationships.filter(r => r.from === entity.id), - concepts: section.concepts - })) - ) - - return { - rowsProcessed: mdResult.sectionsProcessed, - entitiesExtracted: mdResult.entitiesExtracted, - relationshipsInferred: mdResult.relationshipsInferred, - rows, - entityMap: mdResult.entityMap, - processingTime: mdResult.processingTime, - stats: mdResult.stats - } - } - - /** - * Get import statistics - */ - async getImportStatistics(vfsRootPath: string): Promise<{ - entitiesInGraph: number - relationshipsInGraph: number - filesInVFS: number - lastImport?: Date - }> { - // Read metadata file - const vfs = new VirtualFileSystem(this.brain) - await vfs.init() - - const metadataPath = `${vfsRootPath}/_metadata.json` - - try { - const metadataBuffer = await vfs.readFile(metadataPath) - const metadata = JSON.parse(metadataBuffer.toString('utf-8')) - - return { - entitiesInGraph: metadata.import.stats.entitiesExtracted, - relationshipsInGraph: metadata.import.stats.relationshipsInferred, - filesInVFS: metadata.structure.fileCount, - lastImport: new Date(metadata.import.timestamp) - } - } catch (error) { - return { - entitiesInGraph: 0, - relationshipsInGraph: 0, - filesInVFS: 0 - } - } - } -} diff --git a/src/importers/index.ts b/src/importers/index.ts deleted file mode 100644 index 12d0465d..00000000 --- a/src/importers/index.ts +++ /dev/null @@ -1,69 +0,0 @@ -/** - * Smart Import System - * - * Production-ready entity and relationship extraction from multiple formats: - * - Excel (.xlsx) - * - PDF (.pdf) - * - CSV (.csv) - * - JSON (.json) - * - Markdown (.md) - * - * Uses brainy's built-in NeuralEntityExtractor and NaturalLanguageProcessor - * - * NO MOCKS - Real working implementation - */ - -// Excel Importer -export { SmartExcelImporter } from './SmartExcelImporter.js' -export type { - SmartExcelOptions, - ExtractedRow, - SmartExcelResult -} from './SmartExcelImporter.js' - -// PDF Importer -export { SmartPDFImporter } from './SmartPDFImporter.js' -export type { - SmartPDFOptions, - ExtractedSection, - SmartPDFResult -} from './SmartPDFImporter.js' - -// CSV Importer -export { SmartCSVImporter } from './SmartCSVImporter.js' -export type { - SmartCSVOptions, - SmartCSVResult -} from './SmartCSVImporter.js' - -// JSON Importer -export { SmartJSONImporter } from './SmartJSONImporter.js' -export type { - SmartJSONOptions, - ExtractedJSONEntity, - ExtractedJSONRelationship, - SmartJSONResult -} from './SmartJSONImporter.js' - -// Markdown Importer -export { SmartMarkdownImporter } from './SmartMarkdownImporter.js' -export type { - SmartMarkdownOptions, - MarkdownSection, - SmartMarkdownResult -} from './SmartMarkdownImporter.js' - -// VFS Structure Generator -export { VFSStructureGenerator } from './VFSStructureGenerator.js' -export type { - VFSStructureOptions, - VFSStructureResult -} from './VFSStructureGenerator.js' - -// Orchestrator (Main entry point) -export { SmartImportOrchestrator } from './SmartImportOrchestrator.js' -export type { - SmartImportOptions, - SmartImportProgress, - SmartImportResult -} from './SmartImportOrchestrator.js' diff --git a/src/indexes/columnStore/index.ts b/src/indexes/columnStore/index.ts deleted file mode 100644 index 1f9bf344..00000000 --- a/src/indexes/columnStore/index.ts +++ /dev/null @@ -1,38 +0,0 @@ -/** - * @module columnStore - * @description Unified column store for filtering, sorting, range queries, - * and text search on any field type with exact precision at billion scale. - * - * Replaces MetadataIndex's sparse chunk/bloom filter/bucketing internals - * with per-field sorted columns. Design lineage: Lucene doc values + roaring - * bitmap acceleration. - */ - -export { ColumnStore } from './ColumnStore.js' -export type { ColumnStoreConfig } from './ColumnStore.js' -export { ColumnTailBuffer } from './ColumnTailBuffer.js' -export { ColumnManifest } from './ColumnManifest.js' -export { ColumnSegmentCursor, TailBufferCursor } from './ColumnSegmentCursor.js' -export { - writeSegmentToBuffer, - readSegmentFromBuffer, - writeHeader, - readHeader, - crc32 -} from './ColumnSegmentFormat.js' -export { - ValueType, - CIDX_MAGIC, - CIDX_VERSION, - HEADER_SIZE, - FOOTER_SIZE, - DEFAULT_FLUSH_THRESHOLD, - FLAG_MULTI_VALUE -} from './types.js' -export type { - ColumnStoreProvider, - SegmentHeader, - SegmentFooter, - SegmentMeta, - ManifestData -} from './types.js' diff --git a/src/mcp/brainyMCPBroadcast.ts b/src/mcp/brainyMCPBroadcast.ts deleted file mode 100644 index 314e891f..00000000 --- a/src/mcp/brainyMCPBroadcast.ts +++ /dev/null @@ -1,363 +0,0 @@ -/** - * BrainyMCPBroadcast - * - * Enhanced MCP service with real-time WebSocket broadcasting capabilities - * for multi-agent coordination (Jarvis โ†” Picasso communication) - * - * Features: - * - WebSocket server for real-time push notifications - * - Subscription management for multiple Claude instances - * - Message broadcasting to all connected agents - * - Works both locally and with cloud deployment - */ - -import { WebSocketServer, WebSocket } from 'ws' -import { createServer, IncomingMessage } from 'node:http' -import { BrainyMCPService } from './brainyMCPService.js' -import { BrainyInterface } from '../types/brainyInterface.js' -import { MCPServiceOptions } from '../types/mcpTypes.js' -import { v4 as uuidv4 } from '../universal/uuid.js' - -interface BroadcastMessage { - id: string - from: string - to?: string | string[] - type: 'message' | 'notification' | 'sync' | 'heartbeat' | 'identify' - event?: string - data: any - timestamp: number -} - -interface ConnectedAgent { - id: string - name: string - role: string - socket: WebSocket - lastSeen: number -} - -export class BrainyMCPBroadcast extends BrainyMCPService { - private wsServer?: WebSocketServer - private httpServer?: any - private agents: Map = new Map() - private messageHistory: BroadcastMessage[] = [] - private maxHistorySize = 100 - - constructor( - brainyData: BrainyInterface, - options: MCPServiceOptions & { - broadcastPort?: number - cloudUrl?: string - } = {} - ) { - super(brainyData, options) - } - - /** - * Start the WebSocket broadcast server - * @param port Port to listen on (default: 8765) - * @param isCloud Whether this is a cloud deployment - */ - async startBroadcastServer(port = 8765, isCloud = false): Promise { - return new Promise((resolve, reject) => { - try { - // Create HTTP server - this.httpServer = createServer((req, res) => { - // Health check endpoint - if (req.url === '/health') { - res.writeHead(200, { 'Content-Type': 'application/json' }) - res.end(JSON.stringify({ - status: 'healthy', - agents: Array.from(this.agents.values()).map(a => ({ - id: a.id, - name: a.name, - role: a.role, - connected: true - })), - uptime: process.uptime() - })) - } else { - res.writeHead(404) - res.end('Not found') - } - }) - - // Create WebSocket server - this.wsServer = new WebSocketServer({ - server: this.httpServer, - perMessageDeflate: false // Better performance - }) - - this.wsServer.on('connection', (socket, request) => { - this.handleNewConnection(socket, request) - }) - - // Start listening - this.httpServer.listen(port, () => { - console.log(`๐Ÿง  Brain Jar Broadcast Server running on ${isCloud ? 'cloud' : 'local'} port ${port}`) - console.log(`๐Ÿ“ก WebSocket: ws://localhost:${port}`) - console.log(`๐Ÿ” Health: http://localhost:${port}/health`) - resolve() - }) - - // Heartbeat to keep connections alive - setInterval(() => { - this.agents.forEach((agent) => { - if (Date.now() - agent.lastSeen > 30000) { - // Remove inactive agents - this.removeAgent(agent.id) - } else { - // Send heartbeat - this.sendToAgent(agent.id, { - id: uuidv4(), - from: 'server', - type: 'heartbeat', - data: { timestamp: Date.now() }, - timestamp: Date.now() - }) - } - }) - }, 15000) - - } catch (error) { - reject(error) - } - }) - } - - /** - * Handle new WebSocket connection - */ - private handleNewConnection(socket: WebSocket, request: IncomingMessage) { - const agentId = uuidv4() - - // Send welcome message - socket.send(JSON.stringify({ - id: uuidv4(), - from: 'server', - type: 'notification', - event: 'welcome', - data: { - agentId, - message: 'Connected to Brain Jar Broadcast Server', - agents: Array.from(this.agents.values()).map(a => ({ - id: a.id, - name: a.name, - role: a.role - })) - }, - timestamp: Date.now() - })) - - // Handle messages from this agent - socket.on('message', (data) => { - try { - const message = JSON.parse(data.toString()) - this.handleAgentMessage(agentId, message) - } catch (error) { - console.error('Invalid message from agent:', error) - } - }) - - // Handle disconnection - socket.on('close', () => { - this.removeAgent(agentId) - }) - - // Handle errors - socket.on('error', (error) => { - console.error(`Agent ${agentId} error:`, error) - }) - - // Store temporary connection until identified - this.agents.set(agentId, { - id: agentId, - name: 'Unknown', - role: 'Unknown', - socket, - lastSeen: Date.now() - }) - } - - /** - * Handle message from an agent - */ - private handleAgentMessage(agentId: string, message: any) { - const agent = this.agents.get(agentId) - if (!agent) return - - // Update last seen - agent.lastSeen = Date.now() - - // Handle identification - if (message.type === 'identify') { - agent.name = message.name || agent.name - agent.role = message.role || agent.role - - // Notify all agents about new member - this.broadcast({ - id: uuidv4(), - from: 'server', - type: 'notification', - event: 'agent_joined', - data: { - agent: { - id: agent.id, - name: agent.name, - role: agent.role - } - }, - timestamp: Date.now() - }, agentId) // Exclude the joining agent - - // Send recent history to new agent - if (this.messageHistory.length > 0) { - this.sendToAgent(agentId, { - id: uuidv4(), - from: 'server', - type: 'sync', - data: { - history: this.messageHistory.slice(-20) // Last 20 messages - }, - timestamp: Date.now() - }) - } - - return - } - - // Create broadcast message - const broadcastMsg: BroadcastMessage = { - id: message.id || uuidv4(), - from: agent.name, - to: message.to, - type: message.type || 'message', - event: message.event, - data: message.data, - timestamp: Date.now() - } - - // Store in history - this.addToHistory(broadcastMsg) - - // Broadcast based on recipient - if (message.to) { - // Send to specific agent(s) - const recipients = Array.isArray(message.to) ? message.to : [message.to] - recipients.forEach((recipientName: string) => { - const recipient = Array.from(this.agents.values()).find( - a => a.name === recipientName - ) - if (recipient) { - this.sendToAgent(recipient.id, broadcastMsg) - } - }) - } else { - // Broadcast to all agents except sender - this.broadcast(broadcastMsg, agentId) - } - } - - /** - * Broadcast message to all connected agents - */ - broadcast(message: BroadcastMessage, excludeId?: string) { - const messageStr = JSON.stringify(message) - - this.agents.forEach((agent) => { - if (agent.id !== excludeId && agent.socket.readyState === WebSocket.OPEN) { - agent.socket.send(messageStr) - } - }) - } - - /** - * Send message to specific agent - */ - private sendToAgent(agentId: string, message: BroadcastMessage) { - const agent = this.agents.get(agentId) - if (agent && agent.socket.readyState === WebSocket.OPEN) { - agent.socket.send(JSON.stringify(message)) - } - } - - /** - * Remove agent from connected list - */ - private removeAgent(agentId: string) { - const agent = this.agents.get(agentId) - if (agent) { - // Notify others about disconnection - this.broadcast({ - id: uuidv4(), - from: 'server', - type: 'notification', - event: 'agent_left', - data: { - agent: { - id: agent.id, - name: agent.name, - role: agent.role - } - }, - timestamp: Date.now() - }) - - this.agents.delete(agentId) - } - } - - /** - * Add message to history - */ - private addToHistory(message: BroadcastMessage) { - this.messageHistory.push(message) - - // Trim history if too large - if (this.messageHistory.length > this.maxHistorySize) { - this.messageHistory = this.messageHistory.slice(-this.maxHistorySize) - } - } - - /** - * Stop the broadcast server - */ - async stopBroadcastServer(): Promise { - // Close all agent connections - this.agents.forEach(agent => { - agent.socket.close(1000, 'Server shutting down') - }) - this.agents.clear() - - // Close WebSocket server - if (this.wsServer) { - this.wsServer.close() - } - - // Close HTTP server - if (this.httpServer) { - this.httpServer.close() - } - } - - /** - * Get connected agents - */ - getConnectedAgents(): Array<{ id: string; name: string; role: string }> { - return Array.from(this.agents.values()).map(a => ({ - id: a.id, - name: a.name, - role: a.role - })) - } - - /** - * Get message history - */ - getMessageHistory(): BroadcastMessage[] { - return [...this.messageHistory] - } -} - -// Export for both environments -export default BrainyMCPBroadcast \ No newline at end of file diff --git a/src/mcp/brainyMCPClient.ts b/src/mcp/brainyMCPClient.ts deleted file mode 100644 index 85ee1eb6..00000000 --- a/src/mcp/brainyMCPClient.ts +++ /dev/null @@ -1,322 +0,0 @@ -/** - * BrainyMCPClient - * - * Client for connecting Claude instances to the Brain Jar Broadcast Server - * Utilizes Brainy for persistent memory and vector search capabilities - */ - -import WebSocket from 'ws' -import { Brainy } from '../brainy.js' -import { NounType } from '../types/graphTypes.js' -import { v4 as uuidv4 } from '../universal/uuid.js' - -interface ClientOptions { - name: string // e.g., 'Jarvis' or 'Picasso' - role: string // e.g., 'Backend Systems' or 'Frontend Design' - serverUrl?: string // Default: ws://localhost:8765 - autoReconnect?: boolean - useBrainyMemory?: boolean // Store messages in Brainy for persistence -} - -interface Message { - id: string - from: string - to?: string | string[] - type: 'message' | 'notification' | 'sync' | 'heartbeat' | 'identify' - event?: string - data: any - timestamp: number -} - -export class BrainyMCPClient { - private socket?: WebSocket - private options: Required - private brainy?: Brainy - private messageHandlers: Map void> = new Map() - private reconnectTimeout?: NodeJS.Timeout - private isConnected = false - - constructor(options: ClientOptions) { - this.options = { - serverUrl: 'ws://localhost:8765', - autoReconnect: true, - useBrainyMemory: true, - ...options - } - } - - /** - * Initialize Brainy for persistent memory - */ - private async initBrainy() { - if (this.options.useBrainyMemory && !this.brainy) { - this.brainy = new Brainy({ - storage: { - requestPersistentStorage: true - } - }) - await this.brainy.init() - console.log(`๐Ÿง  Brainy memory initialized for ${this.options.name}`) - } - } - - /** - * Connect to the broadcast server - */ - async connect(): Promise { - // Initialize Brainy first - await this.initBrainy() - - return new Promise((resolve, reject) => { - try { - this.socket = new WebSocket(this.options.serverUrl) - - this.socket.on('open', () => { - console.log(`โœ… ${this.options.name} connected to Brain Jar Broadcast`) - this.isConnected = true - - // Identify ourselves - this.send({ - type: 'identify', - data: { - name: this.options.name, - role: this.options.role - } - }) - - resolve() - }) - - this.socket.on('message', async (data) => { - try { - const message = JSON.parse(data.toString()) as Message - await this.handleMessage(message) - } catch (error) { - console.error('Error parsing message:', error) - } - }) - - this.socket.on('close', () => { - console.log(`โŒ ${this.options.name} disconnected from Brain Jar`) - this.isConnected = false - - if (this.options.autoReconnect) { - this.scheduleReconnect() - } - }) - - this.socket.on('error', (error) => { - console.error(`Connection error for ${this.options.name}:`, error) - reject(error) - }) - - } catch (error) { - reject(error) - } - }) - } - - /** - * Handle incoming message - */ - private async handleMessage(message: Message) { - // Store in Brainy for persistent memory. Subtype `'mcp-message'` marks - // these as MCP-protocol messages so consumers can filter / count them via - // `find({ type: NounType.Message, subtype: 'mcp-message' })` and so - // enforcement consumers registering a vocabulary on NounType.Message don't - // reject MCP traffic (added 7.30.1; also fixes the pre-existing missing - // `data` field by aliasing from the prior `text` field). - if (this.brainy && message.type === 'message') { - try { - await this.brainy.add({ - data: `${message.from}: ${JSON.stringify(message.data)}`, - type: NounType.Message, - subtype: 'mcp-message', - metadata: { - messageId: message.id, - from: message.from, - to: message.to, - timestamp: message.timestamp, - messageType: message.type, - event: message.event - } - }) - } catch (error) { - console.error('Error storing message in Brainy:', error) - } - } - - // Handle sync messages (receive history) - if (message.type === 'sync' && message.data.history) { - console.log(`๐Ÿ“œ ${this.options.name} received ${message.data.history.length} historical messages`) - - // Store history in Brainy with the same subtype as live messages. - if (this.brainy) { - for (const histMsg of message.data.history) { - await this.brainy.add({ - data: `${histMsg.from}: ${JSON.stringify(histMsg.data)}`, - type: NounType.Message, - subtype: 'mcp-message', - metadata: { - ...histMsg - } - }) - } - } - } - - // Call registered handlers - const handler = this.messageHandlers.get(message.type) - if (handler) { - handler(message) - } - - // Call universal handler - const universalHandler = this.messageHandlers.get('*') - if (universalHandler) { - universalHandler(message) - } - } - - /** - * Send a message - */ - send(message: Partial) { - if (!this.socket || this.socket.readyState !== WebSocket.OPEN) { - console.error(`${this.options.name} is not connected`) - return - } - - const fullMessage: Message = { - id: message.id || uuidv4(), - from: this.options.name, - type: message.type || 'message', - data: message.data || {}, - timestamp: Date.now(), - ...message - } - - this.socket.send(JSON.stringify(fullMessage)) - } - - /** - * Send a message to specific agent(s) - */ - sendTo(recipient: string | string[], data: any) { - this.send({ - to: recipient, - type: 'message', - data - }) - } - - /** - * Broadcast to all agents - */ - broadcast(data: any) { - this.send({ - type: 'message', - data - }) - } - - /** - * Register a message handler - */ - on(type: string, handler: (message: Message) => void) { - this.messageHandlers.set(type, handler) - } - - /** - * Remove a message handler - */ - off(type: string) { - this.messageHandlers.delete(type) - } - - /** - * Search historical messages using Brainy's vector search - */ - async searchMemory(query: string, limit = 10): Promise { - if (!this.brainy) { - console.warn('Brainy memory not initialized') - return [] - } - - const results = await this.brainy.find({ query, limit }) - return results.map(r => ({ - ...r.metadata, - relevance: r.score - })) - } - - /** - * Get recent messages from Brainy memory - */ - async getRecentMessages(limit = 20): Promise { - if (!this.brainy) { - console.warn('Brainy memory not initialized') - return [] - } - - // Search for recent activity - const results = await this.brainy.find({ query: 'recent messages communication', limit }) - return results - .map(r => r.metadata) - .sort((a: any, b: any) => b.timestamp - a.timestamp) - } - - /** - * Schedule reconnection attempt - */ - private scheduleReconnect() { - if (this.reconnectTimeout) { - clearTimeout(this.reconnectTimeout) - } - - this.reconnectTimeout = setTimeout(() => { - console.log(`๐Ÿ”„ ${this.options.name} attempting to reconnect...`) - this.connect().catch(error => { - console.error('Reconnection failed:', error) - this.scheduleReconnect() - }) - }, 5000) - } - - /** - * Disconnect from server - */ - disconnect() { - if (this.reconnectTimeout) { - clearTimeout(this.reconnectTimeout) - } - - if (this.socket) { - this.socket.close(1000, 'Client disconnecting') - this.socket = undefined - } - - this.isConnected = false - } - - /** - * Check if connected - */ - getIsConnected(): boolean { - return this.isConnected - } - - /** - * Get agent info - */ - getAgentInfo() { - return { - name: this.options.name, - role: this.options.role, - connected: this.isConnected - } - } -} - -// Export for both environments -export default BrainyMCPClient \ No newline at end of file diff --git a/src/neural/naturalLanguageProcessorStatic.ts b/src/neural/naturalLanguageProcessorStatic.ts deleted file mode 100644 index c7bea9ff..00000000 --- a/src/neural/naturalLanguageProcessorStatic.ts +++ /dev/null @@ -1,200 +0,0 @@ -/** - * ๐Ÿง  Natural Language Query Processor - STATIC VERSION - * No runtime initialization, no memory leaks, patterns pre-built at compile time - * - * Uses static pattern matching with 220 pre-built patterns - */ - -import { Vector } from '../coreTypes.js' -import { TripleQuery } from '../triple/TripleIntelligence.js' -import { patternMatchQuery, PATTERN_STATS } from './staticPatternMatcher.js' - -export interface NaturalQueryIntent { - type: 'vector' | 'field' | 'graph' | 'combined' - confidence: number - extractedTerms: { - entities?: string[] - fields?: string[] - relationships?: string[] - modifiers?: string[] - } -} - -export class NaturalLanguageProcessor { - private queryHistory: Array<{ query: string; result: TripleQuery; success: boolean }> - - constructor() { - this.queryHistory = [] - // Patterns are static - no initialization needed! - } - - /** - * No initialization needed - patterns are pre-built! - */ - async init(): Promise { - // Nothing to do - patterns are compiled into the code - return Promise.resolve() - } - - /** - * Process natural language query into structured Triple Intelligence query - * @param naturalQuery The natural language query string - * @param queryEmbedding Pre-computed embedding from Brainy (passed in to avoid circular dependency) - */ - async processNaturalQuery(naturalQuery: string, queryEmbedding?: Vector): Promise { - // Use static pattern matcher (no async, no memory allocation!) - const structuredQuery = patternMatchQuery(naturalQuery, queryEmbedding) - - // Step 3: Enhance with intent analysis if needed - if (!structuredQuery.where && !structuredQuery.connected) { - const intent = await this.analyzeIntent(naturalQuery) - - // Add metadata based on intent - if (intent.type === 'field' && intent.extractedTerms.fields) { - structuredQuery.where = this.buildFieldConstraints(intent.extractedTerms.fields) - } - } - - // Track for learning (but don't create new Brainy!) - this.queryHistory.push({ - query: naturalQuery, - result: structuredQuery, - success: false // Will be updated based on user interaction - }) - - // Keep history limited to prevent memory growth - if (this.queryHistory.length > 100) { - this.queryHistory.shift() - } - - return structuredQuery - } - - /** - * Analyze query intent using keywords - */ - private async analyzeIntent(query: string): Promise { - const lowerQuery = query.toLowerCase() - - // Check for field-specific keywords - const fieldKeywords = ['where', 'filter', 'with', 'has', 'contains', 'equals', 'greater', 'less', 'between'] - const hasFieldIntent = fieldKeywords.some(kw => lowerQuery.includes(kw)) - - // Check for graph keywords - const graphKeywords = ['related', 'connected', 'linked', 'associated', 'references'] - const hasGraphIntent = graphKeywords.some(kw => lowerQuery.includes(kw)) - - // Determine type - let type: NaturalQueryIntent['type'] = 'vector' - if (hasFieldIntent && hasGraphIntent) { - type = 'combined' - } else if (hasFieldIntent) { - type = 'field' - } else if (hasGraphIntent) { - type = 'graph' - } - - return { - type, - confidence: 0.8, - extractedTerms: { - fields: hasFieldIntent ? this.extractFieldTerms(query) : undefined, - relationships: hasGraphIntent ? this.extractRelationshipTerms(query) : undefined - } - } - } - - /** - * Extract field terms from query - */ - private extractFieldTerms(query: string): string[] { - const terms: string[] = [] - - // Simple extraction of potential field names - const words = query.split(/\s+/) - const fieldIndicators = ['year', 'date', 'author', 'type', 'category', 'status', 'price'] - - for (const word of words) { - if (fieldIndicators.includes(word.toLowerCase())) { - terms.push(word.toLowerCase()) - } - } - - return terms - } - - /** - * Extract relationship terms - */ - private extractRelationshipTerms(query: string): string[] { - const terms: string[] = [] - const relationshipWords = ['related', 'connected', 'linked', 'references', 'cites'] - - const words = query.toLowerCase().split(/\s+/) - for (const word of words) { - if (relationshipWords.includes(word)) { - terms.push(word) - } - } - - return terms - } - - /** - * Build field constraints from extracted terms - */ - private buildFieldConstraints(fields: string[]): Record { - const constraints: Record = {} - - // Simple mapping for common fields - for (const field of fields) { - // This would be enhanced with actual value extraction - constraints[field] = { exists: true } - } - - return constraints - } - - /** - * Find similar queries from history (without using Brainy) - * NOTE: Currently unused - reserved for future query caching optimization - */ - private findSimilarQueries(embedding: Vector): Array<{ - query: string - result: TripleQuery - similarity: number - }> { - // Not implemented - not required for core functionality - // Would implement cosine similarity against queryHistory if needed - return [] - } - - /** - * Adapt a previous query for new input - */ - private adaptQuery(newQuery: string, previousResult: TripleQuery): TripleQuery { - return previousResult - } - - /** - * Extract entities from query - */ - private async extractEntities(query: string): Promise { - // Could use the Entity Registry here if available - return [] - } - - /** - * Build query from components - */ - private buildQuery( - query: string, - intent: NaturalQueryIntent, - entities: string[] - ): TripleQuery { - return { - like: query, - limit: 10 - } - } -} \ No newline at end of file diff --git a/src/neural/neuralImportAugmentation.ts b/src/neural/neuralImportAugmentation.ts deleted file mode 100644 index 2942d884..00000000 --- a/src/neural/neuralImportAugmentation.ts +++ /dev/null @@ -1,633 +0,0 @@ -/** - * Neural Import - AI-Powered Data Understanding - * - * Standalone implementation for intelligent data processing. - */ - -import { NounType, VerbType } from '../types/graphTypes.js' -import * as fs from '../universal/fs.js' -import * as path from '../universal/path.js' -import { prodLog } from '../utils/logger.js' - -// Neural Import Analysis Types -export interface NeuralAnalysisResult { - detectedEntities: DetectedEntity[] - detectedRelationships: DetectedRelationship[] - confidence: number - insights: NeuralInsight[] -} - -export interface DetectedEntity { - originalData: any - nounType: string - confidence: number - suggestedId: string - reasoning: string - alternativeTypes: Array<{ type: string, confidence: number }> -} - -export interface DetectedRelationship { - sourceId: string - targetId: string - verbType: string - confidence: number - weight: number - reasoning: string - context: string - metadata?: Record -} - -export interface NeuralInsight { - type: 'hierarchy' | 'cluster' | 'pattern' | 'anomaly' | 'opportunity' - description: string - confidence: number - affectedEntities: string[] - recommendation?: string -} - -export interface NeuralImportConfig { - confidenceThreshold: number - enableWeights: boolean - skipDuplicates: boolean - categoryFilter?: string[] - dataType?: string -} - -/** - * Neural Import Augmentation - Unified Implementation - * Processes data with AI before storage operations - */ -export class NeuralImportAugmentation { - readonly name = 'neural-import' - private operations = ['add', 'addNoun', 'addVerb', 'all'] - - protected config: NeuralImportConfig - private analysisCache = new Map() - private context?: { brain: any } - - constructor(config: Partial = {}) { - this.config = { - confidenceThreshold: 0.7, - enableWeights: true, - skipDuplicates: true, - dataType: 'json', - ...config - } - } - - async init(): Promise { - // No external dependencies to initialize - } - - private log(message: string, _level?: string): void { - // Silent by default - } - - /** - * Execute augmentation - process data with AI before storage - */ - async execute( - operation: string, - params: any, - next: () => Promise - ): Promise { - // Only process on add operations - if (!this.operations.includes(operation)) { - return next() - } - - try { - // Extract data from params based on operation - const rawData = this.extractRawData(operation, params) - if (!rawData) { - return next() - } - - // Perform neural analysis - const analysis = await this.performNeuralAnalysis(rawData, this.config) - - // Enhance params with neural insights - if (params.metadata) { - params.metadata._neuralProcessed = true - params.metadata._neuralConfidence = analysis.confidence - params.metadata._detectedEntities = analysis.detectedEntities.length - params.metadata._detectedRelationships = analysis.detectedRelationships.length - params.metadata._neuralInsights = analysis.insights - } else if (typeof params === 'object') { - params.metadata = { - _neuralProcessed: true, - _neuralConfidence: analysis.confidence, - _detectedEntities: analysis.detectedEntities.length, - _detectedRelationships: analysis.detectedRelationships.length, - _neuralInsights: analysis.insights - } - } - - // Store neural analysis for later retrieval - await this.storeNeuralAnalysis(analysis) - - // If we detected entities/relationships, potentially add them - if (this.context?.brain && analysis.detectedEntities.length > 0) { - // This could automatically create entities/relationships - // But for now, just enhance the metadata - this.log(`Detected ${analysis.detectedEntities.length} entities and ${analysis.detectedRelationships.length} relationships`) - } - - // Continue with enhanced data - return next() - } catch (error) { - this.log(`Neural analysis failed: ${error}`, 'warn') - // Continue without neural processing - return next() - } - } - - /** - * Extract raw data from operation params - */ - private extractRawData(operation: string, params: any): any { - switch (operation) { - case 'add': - return params.content || params.data || params - case 'addNoun': - return params.noun || params.data || params - case 'addVerb': - return params.verb || params - case 'addBatch': - return params.items || params.batch || params - default: - return null - } - } - - /** - * Get the full neural analysis result (for external use) - */ - async getNeuralAnalysis(rawData: Buffer | string, dataType?: string): Promise { - const parsedData = await this.parseRawData(rawData, dataType || this.config.dataType || 'json') - return await this.performNeuralAnalysis(parsedData, this.config) - } - - /** - * Parse raw data based on type - */ - private async parseRawData(rawData: Buffer | string, dataType: string): Promise { - const content = typeof rawData === 'string' ? rawData : rawData.toString('utf8') - - switch (dataType.toLowerCase()) { - case 'json': - try { - const jsonData = JSON.parse(content) - return Array.isArray(jsonData) ? jsonData : [jsonData] - } catch { - // If JSON parse fails, treat as text - return [{ text: content }] - } - - case 'csv': - return this.parseCSV(content) - - case 'yaml': - case 'yml': - return this.parseYAML(content) - - case 'txt': - case 'text': - // Split text into sentences/paragraphs for analysis - return content.split(/\n+/).filter(line => line.trim()).map(line => ({ text: line })) - - default: - // Unknown type, treat as text - return [{ text: content }] - } - } - - /** - * Parse CSV data - handles quoted values, escaped quotes, and edge cases - */ - private parseCSV(content: string): any[] { - const lines = content.split('\n') - if (lines.length === 0) return [] - - // Parse a CSV line handling quotes - const parseLine = (line: string): string[] => { - const result: string[] = [] - let current = '' - let inQuotes = false - let i = 0 - - while (i < line.length) { - const char = line[i] - const nextChar = line[i + 1] - - if (char === '"') { - if (inQuotes && nextChar === '"') { - // Escaped quote - current += '"' - i += 2 - } else { - // Toggle quote mode - inQuotes = !inQuotes - i++ - } - } else if (char === ',' && !inQuotes) { - // Field separator - result.push(current.trim()) - current = '' - i++ - } else { - current += char - i++ - } - } - - // Add last field - result.push(current.trim()) - return result - } - - // Parse headers - const headers = parseLine(lines[0]) - const data = [] - - // Parse data rows - for (let i = 1; i < lines.length; i++) { - const line = lines[i].trim() - if (!line) continue // Skip empty lines - - const values = parseLine(line) - const row: any = {} - - headers.forEach((header, index) => { - const value = values[index] || '' - // Try to parse numbers - const num = Number(value) - row[header] = !isNaN(num) && value !== '' ? num : value - }) - - data.push(row) - } - - return data - } - - /** - * Parse YAML data - */ - private parseYAML(content: string): any[] { - try { - // Simple YAML parser for basic structures - // For full YAML support, we'd use js-yaml library - const lines = content.split('\n') - const result: any[] = [] - let currentObject: any = null - let currentIndent = 0 - - for (const line of lines) { - const trimmed = line.trim() - if (!trimmed || trimmed.startsWith('#')) continue // Skip empty lines and comments - - // Calculate indentation - const indent = line.length - line.trimStart().length - - // Check for array item - if (trimmed.startsWith('- ')) { - const value = trimmed.substring(2).trim() - if (indent === 0) { - // Top-level array item - if (value.includes(':')) { - // Object in array - currentObject = {} - result.push(currentObject) - const [key, val] = value.split(':').map(s => s.trim()) - currentObject[key] = this.parseYAMLValue(val) - } else { - result.push(this.parseYAMLValue(value)) - } - } else if (currentObject) { - // Nested array - const lastKey = Object.keys(currentObject).pop() - if (lastKey) { - if (!Array.isArray(currentObject[lastKey])) { - currentObject[lastKey] = [] - } - currentObject[lastKey].push(this.parseYAMLValue(value)) - } - } - } else if (trimmed.includes(':')) { - // Key-value pair - const colonIndex = trimmed.indexOf(':') - const key = trimmed.substring(0, colonIndex).trim() - const value = trimmed.substring(colonIndex + 1).trim() - - if (indent === 0) { - // Top-level object - if (!currentObject) { - currentObject = {} - result.push(currentObject) - } - currentObject[key] = this.parseYAMLValue(value) - currentIndent = 0 - } else if (currentObject) { - // Nested object - if (indent > currentIndent && !value) { - // Start of nested object - const lastKey = Object.keys(currentObject).pop() - if (lastKey) { - currentObject[lastKey] = { [key]: '' } - } - } else { - currentObject[key] = this.parseYAMLValue(value) - } - currentIndent = indent - } - } - } - - // If we built a single object and not an array, wrap it - if (result.length === 0 && currentObject) { - result.push(currentObject) - } - - return result.length > 0 ? result : [{ text: content }] - } catch (error) { - prodLog.warn('YAML parsing failed, treating as text:', error) - return [{ text: content }] - } - } - - /** - * Parse a YAML value (handle strings, numbers, booleans, null) - */ - private parseYAMLValue(value: string): any { - if (!value || value === '~' || value === 'null') return null - if (value === 'true') return true - if (value === 'false') return false - - // Remove quotes if present - if ((value.startsWith('"') && value.endsWith('"')) || - (value.startsWith("'") && value.endsWith("'"))) { - return value.slice(1, -1) - } - - // Try to parse as number - const num = Number(value) - if (!isNaN(num) && value !== '') return num - - return value - } - - /** - * Perform neural analysis on parsed data - */ - private async performNeuralAnalysis(data: any[], config?: any): Promise { - const detectedEntities: DetectedEntity[] = [] - const detectedRelationships: DetectedRelationship[] = [] - const insights: NeuralInsight[] = [] - - // Simple entity detection (in real implementation, would use ML) - for (const item of data) { - if (typeof item === 'object') { - // Detect entities from object properties - const entityId = item.id || item.name || item.title || `entity_${Date.now()}_${Math.random()}` - - detectedEntities.push({ - originalData: item, - nounType: await this.inferNounType(item), - confidence: 0.85, - suggestedId: String(entityId), - reasoning: 'Detected from structured data', - alternativeTypes: [] - }) - - // Detect relationships from references - await this.detectRelationships(item, entityId, detectedRelationships) - } - } - - // Generate insights - if (detectedEntities.length > 10) { - insights.push({ - type: 'pattern', - description: `Large dataset with ${detectedEntities.length} entities detected`, - confidence: 0.9, - affectedEntities: detectedEntities.slice(0, 5).map(e => e.suggestedId), - recommendation: 'Consider batch processing for optimal performance' - }) - } - - // Look for clusters - const typeGroups = this.groupByType(detectedEntities) - if (Object.keys(typeGroups).length > 1) { - insights.push({ - type: 'cluster', - description: `Multiple entity types detected: ${Object.keys(typeGroups).join(', ')}`, - confidence: 0.8, - affectedEntities: [], - recommendation: 'Data contains diverse entity types suitable for graph analysis' - }) - } - - return { - detectedEntities, - detectedRelationships, - confidence: detectedEntities.length > 0 ? 0.85 : 0.5, - insights - } - } - - /** - * Infer noun type from object structure using field heuristics - */ - private async inferNounType(obj: any): Promise { - if (typeof obj !== 'object' || obj === null) return NounType.Thing - - // Check for explicit type field - if (obj.type && typeof obj.type === 'string') { - const normalized = obj.type.charAt(0).toUpperCase() + obj.type.slice(1) - if (Object.values(NounType).includes(normalized as NounType)) { - return normalized as NounType - } - } - - if (obj.email || obj.firstName || obj.lastName || obj.username) return NounType.Person - if (obj.companyName || obj.organizationId || obj.employees) return NounType.Organization - if (obj.latitude || obj.longitude || obj.address || obj.city) return NounType.Location - if ((obj.content && (obj.title || obj.author)) || obj.pages) return NounType.Document - if (obj.startTime || obj.endTime || obj.date || obj.attendees) return NounType.Event - if (obj.price || obj.sku || obj.productId) return NounType.Product - if ((obj.status && obj.assignee) || obj.priority) return NounType.Task - if (Array.isArray(obj.data) || obj.rows || obj.columns) return NounType.Dataset - - return NounType.Thing - } - - /** - * Detect relationships from object references - */ - private async detectRelationships(obj: any, sourceId: string, relationships: DetectedRelationship[]): Promise { - // Look for reference patterns - for (const [key, value] of Object.entries(obj)) { - if (key.endsWith('Id') || key.endsWith('_id') || key === 'parentId' || key === 'userId') { - relationships.push({ - sourceId, - targetId: String(value), - verbType: await this.inferVerbType(key, obj, { id: value }), - confidence: 0.75, - weight: 1, - reasoning: `Reference detected in field: ${key}`, - context: key - }) - } - - // Array of IDs - if (Array.isArray(value) && value.length > 0 && typeof value[0] === 'string') { - if (key.endsWith('Ids') || key.endsWith('_ids')) { - for (const targetId of value) { - relationships.push({ - sourceId, - targetId: String(targetId), - verbType: await this.inferVerbType(key, obj, { id: targetId }), - confidence: 0.7, - weight: 1, - reasoning: `Array reference in field: ${key}`, - context: key - }) - } - } - } - } - } - - /** - * Infer verb type from field name using common patterns - */ - private async inferVerbType(fieldName: string, _sourceObj?: any, _targetObj?: any): Promise { - const field = fieldName.toLowerCase() - - if (field.includes('parent') || field.includes('child') || field.includes('contain')) { - return VerbType.Contains - } - if (field.includes('owner') || field.includes('created') || field.includes('author')) { - return VerbType.Creates - } - if (field.includes('member') || field.includes('belong')) { - return VerbType.MemberOf - } - if (field.includes('depend') || field.includes('require')) { - return VerbType.DependsOn - } - if (field.includes('ref') || field.includes('link') || field.includes('source')) { - return VerbType.References - } - - return VerbType.RelatedTo - } - - /** - * Group entities by type - */ - private groupByType(entities: DetectedEntity[]): Record { - const groups: Record = {} - - for (const entity of entities) { - if (!groups[entity.nounType]) { - groups[entity.nounType] = [] - } - groups[entity.nounType].push(entity) - } - - return groups - } - - /** - * Store neural analysis results - */ - private async storeNeuralAnalysis(analysis: NeuralAnalysisResult): Promise { - // Cache the analysis for potential later use - const key = `analysis_${Date.now()}` - this.analysisCache.set(key, analysis) - - // Limit cache size - if (this.analysisCache.size > 100) { - const firstKey = this.analysisCache.keys().next().value - if (firstKey) { - this.analysisCache.delete(firstKey) - } - } - } - - /** - * Helper to get data type from file path - */ - private getDataTypeFromPath(filePath: string): string { - const ext = path.extname(filePath).toLowerCase() - switch (ext) { - case '.json': return 'json' - case '.csv': return 'csv' - case '.txt': return 'text' - case '.yaml': - case '.yml': return 'yaml' - default: return 'text' - } - } - - /** - * PUBLIC API: Process raw data (for external use, like Synapses) - * This maintains compatibility with code that wants to use Neural Import directly - */ - async processRawData( - rawData: Buffer | string, - dataType: string, - options?: Record - ): Promise<{ - success: boolean - data: { - nouns: string[] - verbs: string[] - confidence?: number - insights?: Array<{ - type: string - description: string - confidence: number - }> - metadata?: Record - } - error?: string - }> { - try { - const analysis = await this.getNeuralAnalysis(rawData, dataType) - - // Convert to legacy format for compatibility - const nouns = analysis.detectedEntities.map(e => e.suggestedId) - const verbs = analysis.detectedRelationships.map(r => - `${r.sourceId}->${r.verbType}->${r.targetId}` - ) - - return { - success: true, - data: { - nouns, - verbs, - confidence: analysis.confidence, - insights: analysis.insights.map(i => ({ - type: i.type, - description: i.description, - confidence: i.confidence - })), - metadata: { - detectedEntities: analysis.detectedEntities.length, - detectedRelationships: analysis.detectedRelationships.length, - timestamp: new Date().toISOString() - } - } - } - } catch (error) { - return { - success: false, - data: { nouns: [], verbs: [] }, - error: error instanceof Error ? error.message : 'Neural analysis failed' - } - } - } -} \ No newline at end of file diff --git a/src/neural/presets.ts b/src/neural/presets.ts deleted file mode 100644 index 6d309aed..00000000 --- a/src/neural/presets.ts +++ /dev/null @@ -1,479 +0,0 @@ -/** - * Smart Import Presets - Zero-Configuration Auto-Detection - * - * Automatically selects optimal import strategy based on: - * - File type (Excel, CSV, PDF, Markdown, JSON) - * - File size and row count - * - Column structure (explicit relationships vs narrative) - * - Available memory and performance requirements - * - * Production-ready: Handles billions of entities with optimal performance - */ - -import { NounType, VerbType } from '../types/graphTypes.js' - -/** - * Signal types used for entity classification - */ -export type SignalType = 'embedding' | 'exact' | 'pattern' | 'context' - -/** - * Strategy types used for relationship extraction - */ -export type StrategyType = 'explicit' | 'pattern' | 'embedding' - -/** - * Import context for preset auto-detection - */ -export interface ImportContext { - fileType?: 'excel' | 'csv' | 'json' | 'pdf' | 'markdown' | 'unknown' - fileSize?: number // bytes - rowCount?: number - hasExplicitColumns?: boolean // Has "Related Terms" or similar columns - hasNarrativeContent?: boolean // Has long-form text/descriptions - avgDefinitionLength?: number // Average length of definitions - memoryAvailable?: number // bytes -} - -/** - * Signal configuration with weights - */ -export interface SignalConfig { - enabled: SignalType[] - weights: Record - timeout: number // milliseconds -} - -/** - * Strategy configuration with priorities - */ -export interface StrategyConfig { - enabled: StrategyType[] - timeout: number // milliseconds - earlyTermination: boolean - minConfidence: number -} - -/** - * Complete preset configuration - */ -export interface PresetConfig { - name: string - description: string - signals: SignalConfig - strategies: StrategyConfig - streaming: boolean - batchSize: number -} - -/** - * Fast Preset - For large imports (>10K rows) - * - * Optimized for speed over accuracy: - * - Only exact match and pattern signals - * - Only explicit strategy (O(1) lookups) - * - Streaming enabled for memory efficiency - * - Early termination on first high-confidence match - * - * Use case: Bulk imports, data migrations - * Performance: ~10ms per row - * Accuracy: ~85% - */ -export const FAST_PRESET: PresetConfig = { - name: 'fast', - description: 'Fast bulk import for large datasets', - signals: { - enabled: ['exact', 'pattern'], - weights: { - exact: 0.70, - pattern: 0.30, - embedding: 0, - context: 0 - }, - timeout: 50 - }, - strategies: { - enabled: ['explicit'], - timeout: 100, - earlyTermination: true, - minConfidence: 0.70 - }, - streaming: true, - batchSize: 1000 -} - -/** - * Balanced Preset - Default for most imports - * - * Good balance of speed and accuracy: - * - All signals except context (embedding, exact, pattern) - * - All strategies with smart ordering - * - Moderate timeouts - * - Early termination after high-confidence matches - * - * Use case: Standard imports, general glossaries - * Performance: ~30ms per row - * Accuracy: ~92% - */ -export const BALANCED_PRESET: PresetConfig = { - name: 'balanced', - description: 'Balanced speed and accuracy for most imports', - signals: { - enabled: ['exact', 'embedding', 'pattern'], - weights: { - exact: 0.40, - embedding: 0.35, - pattern: 0.25, - context: 0 - }, - timeout: 100 - }, - strategies: { - enabled: ['explicit', 'pattern', 'embedding'], - timeout: 200, - earlyTermination: true, - minConfidence: 0.65 - }, - streaming: false, - batchSize: 500 -} - -/** - * Accurate Preset - For small, critical imports - * - * Optimized for accuracy over speed: - * - All signals including context - * - All strategies, no early termination - * - Longer timeouts for thorough analysis - * - Lower confidence threshold (accept more matches) - * - * Use case: Knowledge bases, critical taxonomies - * Performance: ~100ms per row - * Accuracy: ~97% - */ -export const ACCURATE_PRESET: PresetConfig = { - name: 'accurate', - description: 'Maximum accuracy for critical imports', - signals: { - enabled: ['exact', 'embedding', 'pattern', 'context'], - weights: { - exact: 0.40, - embedding: 0.35, - pattern: 0.20, - context: 0.05 - }, - timeout: 500 - }, - strategies: { - enabled: ['explicit', 'pattern', 'embedding'], - timeout: 1000, - earlyTermination: false, - minConfidence: 0.50 - }, - streaming: false, - batchSize: 100 -} - -/** - * Explicit Preset - For glossaries with relationship columns - * - * Optimized for structured data with explicit relationships: - * - Only exact match signals (no AI needed) - * - Only explicit and pattern strategies - * - Fast, deterministic results - * - Perfect for Excel/CSV with "Related Terms" columns - * - * Use case: structured taxonomies and glossaries from spreadsheet sources - * Performance: ~5ms per row - * Accuracy: ~99% (high confidence) - */ -export const EXPLICIT_PRESET: PresetConfig = { - name: 'explicit', - description: 'For glossaries with explicit relationship columns', - signals: { - enabled: ['exact', 'pattern'], - weights: { - exact: 0.70, - pattern: 0.30, - embedding: 0, - context: 0 - }, - timeout: 50 - }, - strategies: { - enabled: ['explicit', 'pattern'], - timeout: 100, - earlyTermination: true, - minConfidence: 0.80 - }, - streaming: false, - batchSize: 500 -} - -/** - * Pattern Preset - For documents with narrative content - * - * Optimized for unstructured text with rich patterns: - * - Embedding and pattern signals (semantic understanding) - * - Pattern and embedding strategies - * - Good for PDFs, articles, documentation - * - * Use case: PDF imports, markdown docs, articles - * Performance: ~50ms per row - * Accuracy: ~90% - */ -export const PATTERN_PRESET: PresetConfig = { - name: 'pattern', - description: 'For documents with narrative content', - signals: { - enabled: ['embedding', 'pattern', 'context'], - weights: { - embedding: 0.50, - pattern: 0.40, - context: 0.10, - exact: 0 - }, - timeout: 200 - }, - strategies: { - enabled: ['pattern', 'embedding'], - timeout: 300, - earlyTermination: false, - minConfidence: 0.60 - }, - streaming: false, - batchSize: 200 -} - -/** - * All available presets - */ -export const PRESETS: Record = { - fast: FAST_PRESET, - balanced: BALANCED_PRESET, - accurate: ACCURATE_PRESET, - explicit: EXPLICIT_PRESET, - pattern: PATTERN_PRESET -} - -/** - * Auto-detect optimal preset based on import context - * - * Decision tree: - * 1. Large dataset (>10K rows or >10MB) โ†’ fast - * 2. Small dataset (<100 rows) โ†’ accurate - * 3. Excel/CSV with explicit columns โ†’ explicit - * 4. PDF/Markdown with long content โ†’ pattern - * 5. Default โ†’ balanced - * - * @param context Import context (file type, size, structure) - * @returns Optimal preset configuration - */ -export function autoDetectPreset(context: ImportContext = {}): PresetConfig { - const { - fileType = 'unknown', - fileSize = 0, - rowCount = 0, - hasExplicitColumns = false, - hasNarrativeContent = false, - avgDefinitionLength = 0 - } = context - - // Rule 1: Large imports โ†’ fast preset (prioritize speed) - if (rowCount > 10000 || fileSize > 10_000_000) { - return FAST_PRESET - } - - // Rule 2: Small critical imports โ†’ accurate preset (prioritize accuracy) - if (rowCount > 0 && rowCount < 100) { - return ACCURATE_PRESET - } - - // Rule 3: Structured data with explicit relationships โ†’ explicit preset - // (Handles spreadsheet imports where relationships are encoded in columns.) - if (hasExplicitColumns && (fileType === 'excel' || fileType === 'csv')) { - return EXPLICIT_PRESET - } - - // Rule 4: Narrative content โ†’ pattern preset - // Good for PDFs, articles, documentation - if ( - hasNarrativeContent || - fileType === 'pdf' || - fileType === 'markdown' || - avgDefinitionLength > 500 - ) { - return PATTERN_PRESET - } - - // Rule 5: JSON data โ†’ balanced preset - if (fileType === 'json') { - return BALANCED_PRESET - } - - // Default: balanced preset - return BALANCED_PRESET -} - -/** - * Get preset by name - * - * @param name Preset name (fast, balanced, accurate, explicit, pattern) - * @returns Preset configuration - * @throws Error if preset not found - */ -export function getPreset(name: string): PresetConfig { - const preset = PRESETS[name.toLowerCase()] - if (!preset) { - throw new Error(`Unknown preset: ${name}. Available: ${Object.keys(PRESETS).join(', ')}`) - } - return preset -} - -/** - * Get all available preset names - * - * @returns Array of preset names - */ -export function getPresetNames(): string[] { - return Object.keys(PRESETS) -} - -/** - * Explain why a preset was selected - * - * @param context Import context - * @returns Human-readable explanation - */ -export function explainPresetChoice(context: ImportContext = {}): string { - const { - fileType = 'unknown', - fileSize = 0, - rowCount = 0, - hasExplicitColumns = false, - hasNarrativeContent = false, - avgDefinitionLength = 0 - } = context - - if (rowCount > 10000 || fileSize > 10_000_000) { - return `Large dataset (${rowCount} rows, ${(fileSize / 1_000_000).toFixed(1)}MB) โ†’ fast preset for optimal performance` - } - - if (rowCount > 0 && rowCount < 100) { - return `Small critical dataset (${rowCount} rows) โ†’ accurate preset for maximum accuracy` - } - - if (hasExplicitColumns && (fileType === 'excel' || fileType === 'csv')) { - return `${fileType.toUpperCase()} with explicit relationship columns โ†’ explicit preset for deterministic results` - } - - if (hasNarrativeContent || fileType === 'pdf' || fileType === 'markdown') { - return `Narrative content (${fileType}) โ†’ pattern preset for semantic understanding` - } - - if (fileType === 'json') { - return `JSON data โ†’ balanced preset for structured imports` - } - - return `Standard import โ†’ balanced preset (default)` -} - -/** - * Create custom preset by merging with base preset - * - * @param baseName Base preset name - * @param overrides Custom overrides - * @returns Custom preset configuration - */ -export function createCustomPreset( - baseName: string, - overrides: Partial -): PresetConfig { - const base = getPreset(baseName) - - return { - ...base, - ...overrides, - signals: { - ...base.signals, - ...(overrides.signals || {}) - }, - strategies: { - ...base.strategies, - ...(overrides.strategies || {}) - } - } -} - -/** - * Validate preset configuration - * - * @param preset Preset to validate - * @returns True if valid, throws error otherwise - */ -export function validatePreset(preset: PresetConfig): boolean { - // Validate signals - if (preset.signals.enabled.length === 0) { - throw new Error('Preset must have at least one enabled signal') - } - - // Validate strategies - if (preset.strategies.enabled.length === 0) { - throw new Error('Preset must have at least one enabled strategy') - } - - // Validate weights sum to ~1.0 - const enabledSignals = preset.signals.enabled - const totalWeight = enabledSignals.reduce( - (sum, signal) => sum + preset.signals.weights[signal], - 0 - ) - - if (Math.abs(totalWeight - 1.0) > 0.01) { - throw new Error( - `Signal weights must sum to 1.0, got ${totalWeight.toFixed(2)}` - ) - } - - // Validate timeouts - if (preset.signals.timeout <= 0 || preset.strategies.timeout <= 0) { - throw new Error('Timeouts must be positive') - } - - // Validate batch size - if (preset.batchSize <= 0) { - throw new Error('Batch size must be positive') - } - - return true -} - -/** - * Format preset for display - * - * @param preset Preset configuration - * @returns Human-readable preset summary - */ -export function formatPreset(preset: PresetConfig): string { - const lines = [ - `Preset: ${preset.name}`, - `Description: ${preset.description}`, - '', - 'Signals:', - ...preset.signals.enabled.map( - (s) => ` - ${s}: ${(preset.signals.weights[s] * 100).toFixed(0)}%` - ), - ` Timeout: ${preset.signals.timeout}ms`, - '', - 'Strategies:', - ...preset.strategies.enabled.map((s) => ` - ${s}`), - ` Timeout: ${preset.strategies.timeout}ms`, - ` Early termination: ${preset.strategies.earlyTermination}`, - ` Min confidence: ${preset.strategies.minConfidence}`, - '', - `Streaming: ${preset.streaming}`, - `Batch size: ${preset.batchSize}` - ] - - return lines.join('\n') -} diff --git a/src/neural/relationshipConfidence.ts b/src/neural/relationshipConfidence.ts deleted file mode 100644 index df6a61c8..00000000 --- a/src/neural/relationshipConfidence.ts +++ /dev/null @@ -1,309 +0,0 @@ -/** - * Relationship Confidence Scoring - * - * Scores the confidence of detected relationships based on multiple factors: - * - Entity proximity in text - * - Entity confidence scores - * - Pattern matches - * - Structural analysis - */ - -import { ExtractedEntity } from './entityExtractor.js' -import { VerbType } from '../types/graphTypes.js' -import { RelationEvidence } from '../types/brainy.types.js' - -/** - * Detected relationship with confidence - */ -export interface DetectedRelationship { - sourceEntity: ExtractedEntity - targetEntity: ExtractedEntity - verbType: VerbType - confidence: number - evidence: RelationEvidence -} - -/** - * Configuration for relationship detection - */ -export interface RelationshipDetectionConfig { - minConfidence?: number // Minimum confidence to return (default: 0.5) - maxDistance?: number // Maximum token distance between entities (default: 50) - useProximityBoost?: boolean // Boost score based on proximity (default: true) - usePatternMatching?: boolean // Use verb pattern matching (default: true) - useStructuralAnalysis?: boolean // Analyze sentence structure (default: true) -} - -/** - * Relationship confidence scorer - */ -export class RelationshipConfidenceScorer { - private config: Required - - constructor(config: RelationshipDetectionConfig = {}) { - this.config = { - minConfidence: config.minConfidence || 0.5, - maxDistance: config.maxDistance || 50, - useProximityBoost: config.useProximityBoost !== false, - usePatternMatching: config.usePatternMatching !== false, - useStructuralAnalysis: config.useStructuralAnalysis !== false - } - } - - /** - * Score a potential relationship between two entities - */ - scoreRelationship( - source: ExtractedEntity, - target: ExtractedEntity, - verbType: VerbType, - context: string - ): { confidence: number, evidence: RelationEvidence } { - let confidence = 0.5 // Base confidence - - // Evidence tracking - const reasoningParts: string[] = [] - - // Factor 1: Proximity boost (closer entities = higher confidence) - if (this.config.useProximityBoost) { - const proximityBoost = this.calculateProximityBoost(source, target) - confidence += proximityBoost - if (proximityBoost > 0) { - reasoningParts.push( - `Entities are close together (boost: +${proximityBoost.toFixed(2)})` - ) - } - } - - // Factor 2: Entity confidence boost - const entityConfidence = (source.confidence + target.confidence) / 2 - const entityBoost = (entityConfidence - 0.5) * 0.2 // Scale to 0-0.2 - confidence *= (1 + entityBoost) - if (entityBoost > 0) { - reasoningParts.push( - `High entity confidence (boost: ${entityBoost.toFixed(2)})` - ) - } - - // Factor 3: Pattern match boost - if (this.config.usePatternMatching) { - const patternBoost = this.checkVerbPattern(source, target, verbType, context) - confidence += patternBoost - if (patternBoost > 0) { - reasoningParts.push( - `Matches relationship pattern (boost: +${patternBoost.toFixed(2)})` - ) - } - } - - // Factor 4: Structural boost (same sentence, clause, etc.) - if (this.config.useStructuralAnalysis) { - const structuralBoost = this.analyzeStructure(source, target, context) - confidence += structuralBoost - if (structuralBoost > 0) { - reasoningParts.push( - `Structural relationship (boost: +${structuralBoost.toFixed(2)})` - ) - } - } - - // Cap confidence at 1.0 - confidence = Math.min(confidence, 1.0) - - // Extract source text evidence - const start = Math.min(source.position.start, target.position.start) - const end = Math.max(source.position.end, target.position.end) - - const evidence: RelationEvidence = { - sourceText: context.substring(start, end), - position: { start, end }, - method: 'neural', - reasoning: reasoningParts.join('; ') - } - - return { confidence, evidence } - } - - /** - * Calculate proximity boost based on distance between entities - */ - private calculateProximityBoost( - source: ExtractedEntity, - target: ExtractedEntity - ): number { - const distance = Math.abs(source.position.start - target.position.start) - - if (distance === 0) return 0 // Same position, not meaningful - - // Very close (< 20 chars): +0.2 - if (distance < 20) return 0.2 - - // Close (< 50 chars): +0.1 - if (distance < 50) return 0.1 - - // Medium (< 100 chars): +0.05 - if (distance < 100) return 0.05 - - // Far (> 100 chars): no boost - return 0 - } - - /** - * Check if entities match a verb pattern - */ - private checkVerbPattern( - source: ExtractedEntity, - target: ExtractedEntity, - verbType: VerbType, - context: string - ): number { - const contextBetween = this.getContextBetween(source, target, context) - const contextLower = contextBetween.toLowerCase() - - // Verb-specific patterns - const patterns: Record = { - [VerbType.Creates]: ['creates', 'made', 'built', 'developed', 'produces'], - [VerbType.Owns]: ['owns', 'belongs to', 'possessed by', 'has'], - [VerbType.Contains]: ['contains', 'includes', 'has', 'holds'], - [VerbType.Requires]: ['requires', 'needs', 'depends on', 'relies on'], - [VerbType.Uses]: ['uses', 'utilizes', 'employs', 'applies'], - [VerbType.ReportsTo]: ['manages', 'oversees', 'supervises', 'controls'], - [VerbType.Causes]: ['influences', 'affects', 'impacts', 'shapes', 'causes'], - [VerbType.DependsOn]: ['depends on', 'relies on', 'based on'], - [VerbType.Modifies]: ['modifies', 'changes', 'alters', 'updates'], - [VerbType.References]: ['references', 'cites', 'mentions', 'refers to'] - } - - const verbPatterns = patterns[verbType] || [] - - for (const pattern of verbPatterns) { - if (contextLower.includes(pattern)) { - return 0.2 // Strong pattern match - } - } - - return 0 // No pattern match - } - - /** - * Analyze structural relationship - */ - private analyzeStructure( - source: ExtractedEntity, - target: ExtractedEntity, - context: string - ): number { - const contextBetween = this.getContextBetween(source, target, context) - - // Same sentence (no sentence-ending punctuation between them) - if (!contextBetween.match(/[.!?]/)) { - return 0.1 - } - - // Same paragraph (single newline between them) - if (!contextBetween.match(/\n\n/)) { - return 0.05 - } - - return 0 - } - - /** - * Get context text between two entities - */ - private getContextBetween( - source: ExtractedEntity, - target: ExtractedEntity, - context: string - ): string { - const start = Math.min(source.position.end, target.position.end) - const end = Math.max(source.position.start, target.position.start) - - if (start >= end) return '' - - return context.substring(start, end) - } - - /** - * Detect relationships between a list of entities - */ - detectRelationships( - entities: ExtractedEntity[], - context: string, - verbHints?: VerbType[] - ): DetectedRelationship[] { - const relationships: DetectedRelationship[] = [] - const verbs = verbHints || [ - VerbType.Creates, - VerbType.Uses, - VerbType.Contains, - VerbType.Requires, - VerbType.RelatedTo - ] - - // Check all entity pairs - for (let i = 0; i < entities.length; i++) { - for (let j = i + 1; j < entities.length; j++) { - const source = entities[i] - const target = entities[j] - - // Check distance - const distance = Math.abs(source.position.start - target.position.start) - if (distance > this.config.maxDistance) { - continue // Too far apart - } - - // Try each verb type - for (const verbType of verbs) { - const { confidence, evidence } = this.scoreRelationship( - source, - target, - verbType, - context - ) - - if (confidence >= this.config.minConfidence) { - relationships.push({ - sourceEntity: source, - targetEntity: target, - verbType, - confidence, - evidence - }) - } - } - } - } - - // Sort by confidence (highest first) - relationships.sort((a, b) => b.confidence - a.confidence) - - return relationships - } -} - -/** - * Convenience function to score a single relationship - */ -export function scoreRelationshipConfidence( - source: ExtractedEntity, - target: ExtractedEntity, - verbType: VerbType, - context: string, - config?: RelationshipDetectionConfig -): { confidence: number, evidence: RelationEvidence } { - const scorer = new RelationshipConfidenceScorer(config) - return scorer.scoreRelationship(source, target, verbType, context) -} - -/** - * Convenience function to detect all relationships in text - */ -export function detectRelationshipsWithConfidence( - entities: ExtractedEntity[], - context: string, - config?: RelationshipDetectionConfig -): DetectedRelationship[] { - const scorer = new RelationshipConfidenceScorer(config) - return scorer.detectRelationships(entities, context) -} diff --git a/src/neural/staticPatternMatcher.ts b/src/neural/staticPatternMatcher.ts deleted file mode 100644 index b64aaf2e..00000000 --- a/src/neural/staticPatternMatcher.ts +++ /dev/null @@ -1,188 +0,0 @@ -/** - * Static Pattern Matcher - NO runtime initialization, NO Brainy needed - * - * All patterns and embeddings are pre-computed at build time - * This is pure pattern matching with zero dependencies - */ - -import { EMBEDDED_PATTERNS, getPatternEmbeddings } from './embeddedPatterns.js' -import type { Vector } from '../coreTypes.js' -import type { TripleQuery } from '../triple/TripleIntelligence.js' - -// Pre-load patterns and embeddings at module load time (happens once) -const patterns = new Map(EMBEDDED_PATTERNS.map(p => [p.id, p])) -const patternEmbeddings = getPatternEmbeddings() - -/** - * Cosine similarity between two vectors. - * Accepts any number-indexed sequence so pre-computed Float32Array pattern - * embeddings can be compared against number[] query vectors without copying. - */ -function cosineSimilarity(a: ArrayLike, b: ArrayLike): number { - if (!a || !b || a.length !== b.length) return 0 - - let dotProduct = 0 - let normA = 0 - let normB = 0 - - for (let i = 0; i < a.length; i++) { - dotProduct += a[i] * b[i] - normA += a[i] * a[i] - normB += b[i] * b[i] - } - - const denominator = Math.sqrt(normA) * Math.sqrt(normB) - return denominator === 0 ? 0 : dotProduct / denominator -} - -/** - * Extract slots from matched pattern - */ -function extractSlots(query: string, pattern: string): Record | null { - try { - const regex = new RegExp(pattern, 'i') - const match = query.match(regex) - - if (!match) return null - - const slots: Record = {} - for (let i = 1; i < match.length; i++) { - if (match[i]) { - slots[`$${i}`] = match[i] - } - } - - return Object.keys(slots).length > 0 ? slots : null - } catch { - return null - } -} - -/** - * Apply template with extracted slots - */ -function applyTemplate(template: any, slots: Record): any { - if (!template || !slots) return template - - const result = JSON.parse(JSON.stringify(template)) - const applySlots = (obj: any): any => { - if (typeof obj === 'string') { - return obj.replace(/\$\{(\d+)\}/g, (_, num) => slots[`$${num}`] || '') - } - if (Array.isArray(obj)) { - return obj.map(applySlots) - } - if (typeof obj === 'object' && obj !== null) { - const newObj: any = {} - for (const [key, value] of Object.entries(obj)) { - newObj[key] = applySlots(value) - } - return newObj - } - return obj - } - - return applySlots(result) -} - -/** - * Match query against all patterns using embeddings - */ -export function findBestPatterns( - queryEmbedding: Vector, - k: number = 3 -): Array<{ pattern: typeof EMBEDDED_PATTERNS[0]; similarity: number }> { - - const matches: Array<{ pattern: typeof EMBEDDED_PATTERNS[0]; similarity: number }> = [] - - for (const pattern of EMBEDDED_PATTERNS) { - - const patternEmbedding = patternEmbeddings.get(pattern.id) - if (!patternEmbedding) continue - - // Pass Float32Array directly, no need for Array.from()! - const similarity = cosineSimilarity(queryEmbedding, patternEmbedding) - if (similarity > 0.5) { // Threshold for relevance - matches.push({ pattern, similarity }) - } - } - - // Sort by similarity and return top k - return matches - .sort((a, b) => b.similarity - a.similarity) - .slice(0, k) -} - -/** - * Match query against patterns using regex - */ -export function matchPatternByRegex(query: string): { - pattern: typeof EMBEDDED_PATTERNS[0] - slots: Record - query: TripleQuery -} | null { - // Try direct regex matching first (fastest) - for (const pattern of EMBEDDED_PATTERNS) { - const slots = extractSlots(query, pattern.pattern) - if (slots) { - const templatedQuery = applyTemplate(pattern.template, slots) - return { - pattern, - slots, - query: templatedQuery - } - } - } - - return null -} - -/** - * Convert natural language to structured query using STATIC patterns - * NO initialization needed, NO Brainy required - */ -export function patternMatchQuery( - query: string, - queryEmbedding?: Vector -): TripleQuery { - - // ALWAYS use vector similarity when we have embeddings (which we always do!) - if (queryEmbedding && queryEmbedding.length === 384) { - const bestPatterns = findBestPatterns(queryEmbedding, 5) // Get top 5 matches - - // Try to extract slots from best matching patterns - for (const { pattern, similarity } of bestPatterns) { - // Only try patterns with good similarity - if (similarity < 0.7) break - - const slots = extractSlots(query, pattern.pattern) - if (slots) { - // Found a good match with extractable slots! - const result = applyTemplate(pattern.template, slots) - console.log('[NLP] Applied template with slots:', JSON.stringify(result)) - return result - } - } - - // If no slots extracted but we have a good match, use the template as-is - if (bestPatterns.length > 0 && bestPatterns[0].similarity > 0.75) { - console.log('[NLP] Returning template as-is:', JSON.stringify(bestPatterns[0].pattern.template)) - return bestPatterns[0].pattern.template - } - } - - // Fallback: simple vector search (should rarely happen) - console.log('[NLP] Fallback - returning simple query') - return { - like: query, - limit: 10 - } -} - -// Export pattern statistics for monitoring -export const PATTERN_STATS = { - totalPatterns: EMBEDDED_PATTERNS.length, - categories: [...new Set(EMBEDDED_PATTERNS.map(p => p.category))], - domains: [...new Set(EMBEDDED_PATTERNS.filter(p => p.domain).map(p => p.domain!))], - hasEmbeddings: patternEmbeddings.size > 0 -} \ No newline at end of file diff --git a/src/storage/cacheManager.ts b/src/storage/cacheManager.ts deleted file mode 100644 index cfeb61d7..00000000 --- a/src/storage/cacheManager.ts +++ /dev/null @@ -1,1257 +0,0 @@ -/** - * Multi-level Cache Manager - * - * Implements a three-level caching strategy: - * - Level 1: Hot cache (most accessed nodes) - RAM (automatically detecting and adjusting in each environment) - * - Level 2: Warm cache (recent nodes) - OPFS, Filesystem or S3 depending on environment - * - Level 3: Cold storage (all nodes) - OPFS, Filesystem or S3 depending on environment - */ - -import { HNSWNoun, GraphVerb, HNSWVerb } from '../coreTypes.js' -import { BrainyError } from '../errors/brainyError.js' - -// Type aliases for better readability -type HNSWNode = HNSWNoun -type Edge = GraphVerb - -// Cache entry with metadata for LRU and TTL management -interface CacheEntry { - data: T - lastAccessed: number - accessCount: number - expiresAt: number | null -} - -// Cache statistics for monitoring and tuning -interface CacheStats { - hits: number - misses: number - evictions: number - size: number - maxSize: number - hotCacheSize: number - warmCacheSize: number - hotCacheHits: number - hotCacheMisses: number - warmCacheHits: number - warmCacheMisses: number -} - -// Storage type for warm and cold caches. Brainy 8.0 only ships filesystem + -// memory tiers, but the enum is retained for diagnostic/log output. -enum StorageType { - MEMORY, - FILESYSTEM -} - -/** - * Multi-level cache manager for efficient data access - */ -export class CacheManager { - // Hot cache (RAM) - private hotCache = new Map>() - - // Cache statistics - private stats: CacheStats = { - hits: 0, - misses: 0, - evictions: 0, - size: 0, - maxSize: 0, - hotCacheSize: 0, - warmCacheSize: 0, - hotCacheHits: 0, - hotCacheMisses: 0, - warmCacheHits: 0, - warmCacheMisses: 0 - } - - // Storage configuration - private warmStorageType: StorageType - private coldStorageType: StorageType - - // Cache configuration - private hotCacheMaxSize: number - private hotCacheEvictionThreshold: number - private warmCacheTTL: number - private batchSize: number - - // Auto-tuning configuration - private autoTune: boolean - private lastAutoTuneTime: number = 0 - private autoTuneInterval: number = 5 * 60 * 1000 // 5 minutes - private storageStatistics: any = null - - // Storage adapters for warm and cold caches - private warmStorage: any - private coldStorage: any - - // Store options for later reference - private options: { - hotCacheMaxSize?: number - hotCacheEvictionThreshold?: number - warmCacheTTL?: number - batchSize?: number - autoTune?: boolean - warmStorage?: any - coldStorage?: any - readOnly?: boolean - environmentConfig?: { - node?: { - hotCacheMaxSize?: number - hotCacheEvictionThreshold?: number - warmCacheTTL?: number - batchSize?: number - } - [key: string]: { - hotCacheMaxSize?: number - hotCacheEvictionThreshold?: number - warmCacheTTL?: number - batchSize?: number - } | undefined - } - } - - /** - * Initialize the cache manager - * @param options Configuration options - */ - constructor(options: { - hotCacheMaxSize?: number - hotCacheEvictionThreshold?: number - warmCacheTTL?: number - batchSize?: number - autoTune?: boolean - warmStorage?: any - coldStorage?: any - readOnly?: boolean - environmentConfig?: { - node?: { - hotCacheMaxSize?: number - hotCacheEvictionThreshold?: number - warmCacheTTL?: number - batchSize?: number - } - [key: string]: { - hotCacheMaxSize?: number - hotCacheEvictionThreshold?: number - warmCacheTTL?: number - batchSize?: number - } | undefined - } - } = {}) { - // Store options for later reference - this.options = options - - // Set storage types (Brainy 8.0 ships filesystem only) - this.warmStorageType = this.detectWarmStorageType() - this.coldStorageType = this.detectColdStorageType() - - // Initialize storage adapters - this.warmStorage = options.warmStorage || this.initializeWarmStorage() - this.coldStorage = options.coldStorage || this.initializeColdStorage() - - // Set auto-tuning flag - this.autoTune = options.autoTune !== undefined ? options.autoTune : true - - // Brainy 8.0 only runs on Node-like runtimes, so only the `node` slot of - // environmentConfig is honored. - const envConfig = options.environmentConfig?.node - - // Set default values or use environment-specific values or global values - this.hotCacheMaxSize = envConfig?.hotCacheMaxSize || options.hotCacheMaxSize || this.detectOptimalCacheSize() - this.hotCacheEvictionThreshold = envConfig?.hotCacheEvictionThreshold || options.hotCacheEvictionThreshold || 0.8 - this.warmCacheTTL = envConfig?.warmCacheTTL || options.warmCacheTTL || 24 * 60 * 60 * 1000 // 24 hours - this.batchSize = envConfig?.batchSize || options.batchSize || 10 - - // If auto-tuning is enabled, perform initial tuning - if (this.autoTune) { - this.tuneParameters() - } - - // Log configuration - if (process.env.DEBUG) { - console.log('Cache Manager initialized with configuration:', { - environment: 'node', - hotCacheMaxSize: this.hotCacheMaxSize, - hotCacheEvictionThreshold: this.hotCacheEvictionThreshold, - warmCacheTTL: this.warmCacheTTL, - batchSize: this.batchSize, - autoTune: this.autoTune, - warmStorageType: StorageType[this.warmStorageType], - coldStorageType: StorageType[this.coldStorageType] - }) - } - } - - - /** - * Detect the optimal cache size based on available memory and operating mode - * - * Enhanced to better handle large datasets in S3 or other storage: - * - Increases cache size for read-only mode - * - Adjusts based on total dataset size when available - * - Provides more aggressive caching for large datasets - * - Optimizes memory usage based on environment - */ - private detectOptimalCacheSize(): number { - try { - // Default to a conservative value - const defaultSize = 1000 - - // Get the total dataset size if available - const totalItems = this.storageStatistics ? - (this.storageStatistics.totalNodes || 0) + (this.storageStatistics.totalEdges || 0) : 0 - - // Determine if we're dealing with a large dataset (>100K items) - const isLargeDataset = totalItems > 100000 - - // Check if we're in read-only mode (from parent Brainy instance) - const isReadOnly = this.options?.readOnly || false - - try { - // Synchronous path can't use dynamic imports, so we use conservative - // assumed defaults. The async variant (detectAvailableMemory) reads - // real values via `node:os` when possible. - const estimatedFreeMemory = 4 * 1024 * 1024 * 1024 // Assume 4GB free - - // Estimate average entry size (in bytes) - // This is a conservative estimate for complex objects with vectors - const ESTIMATED_BYTES_PER_ENTRY = 1024 // 1KB per entry - - // Base memory percentage - 10% by default - let memoryPercentage = 0.1 - - // Adjust based on operating mode and dataset size - if (isReadOnly) { - // In read-only mode, we can use more memory for caching - memoryPercentage = 0.25 // 25% of free memory - - // For large datasets in read-only mode, be even more aggressive - if (isLargeDataset) { - memoryPercentage = 0.4 // 40% of free memory - } - } else if (isLargeDataset) { - // For large datasets in normal mode, increase slightly - memoryPercentage = 0.15 // 15% of free memory - } - - // Calculate optimal size based on adjusted percentage - const optimalSize = Math.max( - Math.floor(estimatedFreeMemory * memoryPercentage / ESTIMATED_BYTES_PER_ENTRY), - 1000 - ) - - // If we know the total dataset size, cap at a reasonable percentage - if (totalItems > 0) { - // In read-only mode, we can cache a larger percentage - const maxPercentage = isReadOnly ? 0.5 : 0.3 - const maxItems = Math.ceil(totalItems * maxPercentage) - - // Return the smaller of the two to avoid excessive memory usage - return Math.min(optimalSize, maxItems) - } - - return optimalSize - } catch (error) { - console.warn('Failed to detect optimal cache size:', error) - return defaultSize - } - } catch (error) { - console.warn('Error detecting optimal cache size:', error) - return 1000 // Conservative default - } - } - - /** - * Async version of detectOptimalCacheSize that uses dynamic imports - * to access system information in Node.js environments - * - * This method provides more accurate memory detection by using - * the OS module's dynamic import in Node.js environments - */ - private async detectOptimalCacheSizeAsync(): Promise { - try { - // Default to a conservative value - const defaultSize = 1000 - - // Get the total dataset size if available - const totalItems = this.storageStatistics ? - (this.storageStatistics.totalNodes || 0) + (this.storageStatistics.totalEdges || 0) : 0 - - // Determine if we're dealing with a large dataset (>100K items) - const isLargeDataset = totalItems > 100000 - - // Check if we're in read-only mode (from parent Brainy instance) - const isReadOnly = this.options?.readOnly || false - - // Get memory information based on environment - const memoryInfo = await this.detectAvailableMemory() - - // If memory detection failed, use the synchronous method - if (!memoryInfo) { - return this.detectOptimalCacheSize() - } - - // Estimate average entry size (in bytes) - // This is a conservative estimate for complex objects with vectors - const ESTIMATED_BYTES_PER_ENTRY = 1024 // 1KB per entry - - // Base memory percentage - 10% by default - let memoryPercentage = 0.1 - - // Adjust based on operating mode and dataset size - if (isReadOnly) { - // In read-only mode, we can use more memory for caching - memoryPercentage = 0.25 // 25% of free memory - - // For large datasets in read-only mode, be even more aggressive - if (isLargeDataset) { - memoryPercentage = 0.4 // 40% of free memory - } - } else if (isLargeDataset) { - // For large datasets in normal mode, increase slightly - memoryPercentage = 0.15 // 15% of free memory - } - - // Calculate optimal size based on adjusted percentage - const optimalSize = Math.max( - Math.floor(memoryInfo.freeMemory * memoryPercentage / ESTIMATED_BYTES_PER_ENTRY), - 1000 - ) - - // If we know the total dataset size, cap at a reasonable percentage - if (totalItems > 0) { - // In read-only mode, we can cache a larger percentage - const maxPercentage = isReadOnly ? 0.5 : 0.3 - const maxItems = Math.ceil(totalItems * maxPercentage) - - // Return the smaller of the two to avoid excessive memory usage - return Math.min(optimalSize, maxItems) - } - - return optimalSize - } catch (error) { - console.warn('Error detecting optimal cache size asynchronously:', error) - return 1000 // Conservative default - } - } - - /** - * Detect available memory using Node-like runtime primitives. - * - * Brainy 8.0 only runs on Node.js, Bun, and Deno, so this method reads real - * values from `node:os` (`process.memoryUsage()` is not used here because we - * want system-wide free memory, not just the V8 heap). If the import fails, - * we fall back to conservative defaults that match the synchronous path. - * - * @returns An object with totalMemory and freeMemory in bytes, or null if detection fails - */ - private async detectAvailableMemory(): Promise<{ totalMemory: number, freeMemory: number } | null> { - try { - try { - // Use dynamic import for OS module - const os = await import('node:os') - - // Get actual system memory information - const totalMemory = os.totalmem() - const freeMemory = os.freemem() - - return { totalMemory, freeMemory } - } catch (error) { - console.warn('Failed to detect memory via node:os:', error) - } - - // If detection failed, use conservative defaults - return { - totalMemory: 8 * 1024 * 1024 * 1024, // Assume 8GB total - freeMemory: 4 * 1024 * 1024 * 1024 // Assume 4GB free - } - } catch (error) { - console.warn('Memory detection failed:', error) - return null - } - } - - /** - * Tune cache parameters based on statistics and environment - * This method is called periodically if auto-tuning is enabled - * - * The auto-tuning process: - * 1. Retrieves storage statistics if available - * 2. Tunes each parameter based on statistics and environment - * 3. Logs the tuned parameters if debug is enabled - * - * Auto-tuning helps optimize cache performance by adapting to: - * - The current environment (Node.js, browser, worker) - * - Available system resources (memory, CPU) - * - Usage patterns (read-heavy vs. write-heavy workloads) - * - Cache efficiency (hit/miss ratios) - */ - private async tuneParameters(): Promise { - // Skip if auto-tuning is disabled - if (!this.autoTune) return - - // Check if it's time to tune parameters - const now = Date.now() - if (now - this.lastAutoTuneTime < this.autoTuneInterval) return - - // Update last tune time - this.lastAutoTuneTime = now - - try { - // Get storage statistics if available - if (this.coldStorage && typeof this.coldStorage.getStatistics === 'function') { - this.storageStatistics = await this.coldStorage.getStatistics() - } - - // Get cache statistics for adaptive tuning - const cacheStats = this.getStats() - - // Use the async version of tuneHotCacheSize which uses detectOptimalCacheSizeAsync - await this.tuneHotCacheSize() - - // Tune eviction threshold based on hit/miss ratio - this.tuneEvictionThreshold(cacheStats) - - // Tune warm cache TTL based on access patterns - this.tuneWarmCacheTTL(cacheStats) - - // Tune batch size based on access patterns and storage type - this.tuneBatchSize(cacheStats) - - // Log tuned parameters if debug is enabled - if (process.env.DEBUG) { - console.log('Cache parameters auto-tuned:', { - hotCacheMaxSize: this.hotCacheMaxSize, - hotCacheEvictionThreshold: this.hotCacheEvictionThreshold, - warmCacheTTL: this.warmCacheTTL, - batchSize: this.batchSize, - cacheStats: { - hotCacheSize: cacheStats.hotCacheSize, - warmCacheSize: cacheStats.warmCacheSize, - hotCacheHits: cacheStats.hotCacheHits, - hotCacheMisses: cacheStats.hotCacheMisses, - warmCacheHits: cacheStats.warmCacheHits, - warmCacheMisses: cacheStats.warmCacheMisses - } - }) - } - } catch (error) { - console.warn('Error during cache parameter auto-tuning:', error) - } - } - - /** - * Tune hot cache size based on statistics, environment, and operating mode - * - * The hot cache size is tuned based on: - * 1. Available memory in the current environment - * 2. Total number of nodes and edges in the system - * 3. Cache hit/miss ratio - * 4. Operating mode (read-only vs. read-write) - * 5. Storage type (S3, filesystem, memory) - * - * Enhanced algorithm: - * - Start with a size based on available memory and operating mode - * - For large datasets in S3 or other remote storage, use more aggressive caching - * - Adjust based on access patterns (read-heavy vs. write-heavy) - * - For read-only mode, prioritize cache size over eviction speed - * - Dynamically adjust based on hit/miss ratio and query patterns - */ - private async tuneHotCacheSize(): Promise { - // Use the async version to get more accurate memory information - let optimalSize = await this.detectOptimalCacheSizeAsync() - - const isReadOnly = this.options?.readOnly || false - - // Adjust by total entity count if we have storage statistics. - if (this.storageStatistics) { - const totalItems = (this.storageStatistics.totalNodes || 0) + - (this.storageStatistics.totalEdges || 0) - - if (totalItems > 0) { - let percentageToCache = isReadOnly ? 0.3 : 0.2 - - if (totalItems > 1_000_000) { - percentageToCache = Math.min(percentageToCache, 0.15) - } else if (totalItems > 100_000) { - percentageToCache = Math.min(percentageToCache, 0.25) - } - - const statisticsBasedSize = Math.ceil(totalItems * percentageToCache) - optimalSize = Math.min(optimalSize, statisticsBasedSize) - } - } - - // Adjust by observed hit ratio. - const totalAccesses = this.stats.hits + this.stats.misses - if (totalAccesses > 100) { - const hitRatio = this.stats.hits / totalAccesses - - if (hitRatio < 0.5) { - const baseAdjustment = 0.5 - hitRatio - const hitRatioFactor = isReadOnly ? 1 + baseAdjustment * 1.5 : 1 + baseAdjustment - optimalSize = Math.ceil(optimalSize * hitRatioFactor) - } else if (hitRatio > 0.9 && !isReadOnly) { - optimalSize = Math.ceil(optimalSize * 0.9) - } - } - - // Read-heavy workloads warrant a slightly larger cache. - if (this.storageStatistics?.operations) { - const ops = this.storageStatistics.operations - const totalOps = ops.total || 1 - const readOps = (ops.search || 0) + (ops.get || 0) - - if (totalOps > 100 && readOps / totalOps > 0.8) { - optimalSize = Math.ceil(optimalSize * 1.2) - } - } - - const minSize = isReadOnly ? 2000 : 1000 - optimalSize = Math.max(optimalSize, minSize) - - // Update the hot cache max size - this.hotCacheMaxSize = optimalSize - this.stats.maxSize = optimalSize - } - - /** - * Tune eviction threshold based on statistics - * - * The eviction threshold determines when items start being evicted from the hot cache. - * It is tuned based on: - * 1. Cache hit/miss ratio - * 2. Operation patterns (read-heavy vs. write-heavy workloads) - * 3. Memory pressure and available resources - * - * Algorithm: - * - Start with a default threshold of 0.8 (80% of max size) - * - For high hit ratios, increase the threshold to keep more items in cache - * - For low hit ratios, decrease the threshold to evict items more aggressively - * - For read-heavy workloads, use a higher threshold - * - For write-heavy workloads, use a lower threshold - * - Under memory pressure, use a lower threshold to conserve resources - * - * @param cacheStats Optional cache statistics for more adaptive tuning - */ - private tuneEvictionThreshold(cacheStats?: CacheStats): void { - // Default threshold - let threshold = 0.8 - - // Use provided cache stats or internal stats - const stats = cacheStats || this.getStats() - - // Adjust based on hit/miss ratio if we have enough data - const totalHotAccesses = stats.hotCacheHits + stats.hotCacheMisses - if (totalHotAccesses > 100) { - const hotHitRatio = stats.hotCacheHits / totalHotAccesses - - // If hit ratio is high, we can use a higher threshold - // If hit ratio is low, we should use a lower threshold to evict more aggressively - if (hotHitRatio > 0.8) { - // High hit ratio, increase threshold (up to 0.9) - threshold = Math.min(0.9, 0.8 + (hotHitRatio - 0.8) * 0.5) - } else if (hotHitRatio < 0.5) { - // Low hit ratio, decrease threshold (down to 0.6) - threshold = Math.max(0.6, 0.8 - (0.5 - hotHitRatio) * 0.5) - } - } - - // If we have storage statistics with operation counts, adjust based on operation patterns - if (this.storageStatistics && this.storageStatistics.operations) { - const ops = this.storageStatistics.operations - const totalOps = ops.total || 1 - - // Calculate read/write ratio - const readOps = ops.search || 0 - const writeOps = (ops.add || 0) + (ops.update || 0) + (ops.delete || 0) - - if (totalOps > 100) { - const readRatio = readOps / totalOps - const writeRatio = writeOps / totalOps - - // For read-heavy workloads, use higher threshold - // For write-heavy workloads, use lower threshold - if (readRatio > 0.8) { - // Read-heavy, increase threshold slightly - threshold = Math.min(0.9, threshold + 0.05) - } else if (writeRatio > 0.5) { - // Write-heavy, decrease threshold - threshold = Math.max(0.6, threshold - 0.1) - } - } - } - - // Check memory pressure - if hot cache is growing too fast relative to hits, - // reduce the threshold to conserve memory - if (stats.hotCacheSize > 0 && totalHotAccesses > 0) { - const sizeToAccessRatio = stats.hotCacheSize / totalHotAccesses - - // If the ratio is high, it means we're caching a lot but not getting many hits - if (sizeToAccessRatio > 10) { - // Reduce threshold more aggressively under high memory pressure - threshold = Math.max(0.5, threshold - 0.1) - } - } - - // If we're in read-only mode, we can be more aggressive with caching - const isReadOnly = this.options?.readOnly || false - if (isReadOnly) { - threshold = Math.min(0.95, threshold + 0.05) - } - - // Update the eviction threshold - this.hotCacheEvictionThreshold = threshold - } - - /** - * Tune warm cache TTL based on statistics - * - * The warm cache TTL determines how long items remain in the warm cache. - * It is tuned based on: - * 1. Update frequency from operation statistics - * 2. Warm cache hit/miss ratio - * 3. Access patterns and frequency - * 4. Available storage resources - * - * Algorithm: - * - Start with a default TTL of 24 hours - * - For frequently updated data, use a shorter TTL - * - For rarely updated data, use a longer TTL - * - For frequently accessed data, use a longer TTL - * - For rarely accessed data, use a shorter TTL - * - Under storage pressure, use a shorter TTL - * - * @param cacheStats Optional cache statistics for more adaptive tuning - */ - private tuneWarmCacheTTL(cacheStats?: CacheStats): void { - // Default TTL (24 hours) - let ttl = 24 * 60 * 60 * 1000 - - // Use provided cache stats or internal stats - const stats = cacheStats || this.getStats() - - // Adjust based on warm cache hit/miss ratio if we have enough data - const totalWarmAccesses = stats.warmCacheHits + stats.warmCacheMisses - if (totalWarmAccesses > 50) { - const warmHitRatio = stats.warmCacheHits / totalWarmAccesses - - // If warm cache hit ratio is high, items in warm cache are useful - // so we should keep them longer - if (warmHitRatio > 0.7) { - // High hit ratio, increase TTL (up to 36 hours) - ttl = Math.min(36 * 60 * 60 * 1000, ttl * (1 + (warmHitRatio - 0.7))) - } else if (warmHitRatio < 0.3) { - // Low hit ratio, decrease TTL (down to 12 hours) - ttl = Math.max(12 * 60 * 60 * 1000, ttl * (0.8 - (0.3 - warmHitRatio))) - } - } - - // If we have storage statistics with operation counts, adjust based on update frequency - if (this.storageStatistics && this.storageStatistics.operations) { - const ops = this.storageStatistics.operations - const totalOps = ops.total || 1 - const updateOps = (ops.update || 0) - - if (totalOps > 100) { - const updateRatio = updateOps / totalOps - - // For frequently updated data, use shorter TTL - // For rarely updated data, use longer TTL - if (updateRatio > 0.3) { - // Frequently updated, decrease TTL (down to 6 hours) - ttl = Math.max(6 * 60 * 60 * 1000, ttl * (1 - updateRatio * 0.5)) - } else if (updateRatio < 0.1) { - // Rarely updated, increase TTL (up to 48 hours) - ttl = Math.min(48 * 60 * 60 * 1000, ttl * (1.2 - updateRatio)) - } - } - } - - // Check warm cache size relative to hot cache size - // If warm cache is much larger than hot cache, reduce TTL to prevent excessive storage use - if (stats.warmCacheSize > 0 && stats.hotCacheSize > 0) { - const warmToHotRatio = stats.warmCacheSize / stats.hotCacheSize - - if (warmToHotRatio > 5) { - // Warm cache is much larger than hot cache, reduce TTL - ttl = Math.max(6 * 60 * 60 * 1000, ttl * (0.9 - Math.min(0.3, (warmToHotRatio - 5) / 20))) - } - } - - // If we're in read-only mode, we can use a longer TTL - const isReadOnly = this.options?.readOnly || false - if (isReadOnly) { - ttl = Math.min(72 * 60 * 60 * 1000, ttl * 1.5) - } - - // Update the warm cache TTL - this.warmCacheTTL = ttl - } - - /** - * Tune batch size based on environment, statistics, and operating mode - * - * The batch size determines how many items are processed in a single batch - * for operations like prefetching. It is tuned based on: - * 1. Current environment (Node.js, browser, worker) - * 2. Available memory - * 3. Operation patterns - * 4. Cache hit/miss ratio - * 5. Operating mode (read-only vs. read-write) - * 6. Storage type (S3, filesystem, memory) - * 7. Dataset size - * 8. Cache efficiency and access patterns - * - * Enhanced algorithm: - * - Start with a default based on the environment - * - For large datasets in S3 or other remote storage, use larger batches - * - For read-only mode, use larger batches to improve throughput - * - Dynamically adjust based on network latency and throughput - * - Balance between memory usage and performance - * - Adapt to cache hit/miss patterns - * - * @param cacheStats Optional cache statistics for more adaptive tuning - */ - private tuneBatchSize(cacheStats?: CacheStats): void { - let batchSize = 10 - const stats = cacheStats || this.getStats() - const isReadOnly = this.options?.readOnly || false - - const totalItems = this.storageStatistics - ? (this.storageStatistics.totalNodes || 0) + (this.storageStatistics.totalEdges || 0) - : 0 - const isLargeDataset = totalItems > 100_000 - const isVeryLargeDataset = totalItems > 1_000_000 - - // Brainy 8.0 ships Node-like runtimes only (Bun, Deno, Node). All paths - // funnel into the same batch-size envelope. - batchSize = isReadOnly ? 30 : 20 - if (isLargeDataset) batchSize = Math.min(100, batchSize * 1.5) - if (isVeryLargeDataset) batchSize = Math.min(200, batchSize * 2) - - // Adjust by hit-ratio observations. - const totalHotAccesses = stats.hotCacheHits + stats.hotCacheMisses - const totalWarmAccesses = stats.warmCacheHits + stats.warmCacheMisses - - if (totalHotAccesses > 100) { - const hotHitRatio = stats.hotCacheHits / totalHotAccesses - if (hotHitRatio > 0.8) batchSize = Math.min(batchSize * 1.5, 150) - else if (hotHitRatio < 0.4) batchSize = Math.max(5, batchSize * 0.8) - } - - if (totalWarmAccesses > 50) { - const warmHitRatio = stats.warmCacheHits / totalWarmAccesses - if (warmHitRatio > 0.7) batchSize = Math.min(batchSize * 1.3, 120) - else if (warmHitRatio < 0.3) batchSize = Math.max(5, batchSize * 0.9) - } - - // If we have storage statistics with operation counts, adjust based on operation patterns - if (this.storageStatistics && this.storageStatistics.operations) { - const ops = this.storageStatistics.operations - const totalOps = ops.total || 1 - const searchOps = (ops.search || 0) - const getOps = (ops.get || 0) - - if (totalOps > 100) { - const searchRatio = searchOps / totalOps - const getRatio = getOps / totalOps - - if (searchRatio > 0.6) { - batchSize = Math.min(100, Math.ceil(batchSize * 1.5)) - } - if (getRatio > 0.6) { - batchSize = Math.max(10, Math.ceil(batchSize * 0.9)) - } - } - } - - // Memory-pressure trim. - if (stats.hotCacheSize > 0 && this.hotCacheMaxSize > 0) { - const cacheUtilization = stats.hotCacheSize / this.hotCacheMaxSize - if (cacheUtilization > 0.85) { - batchSize = Math.max(5, Math.floor(batchSize * 0.8)) - } - } - - // Combined hot+warm hit-ratio adjustment. - const totalAccesses = stats.hotCacheHits + stats.hotCacheMisses + stats.warmCacheHits + stats.warmCacheMisses - if (totalAccesses > 100) { - const hitRatio = (stats.hotCacheHits + stats.warmCacheHits) / totalAccesses - const increaseFactorForLowHitRatio = isReadOnly ? 1.5 : 1.2 - const decreaseFactorForHighHitRatio = isReadOnly ? 0.9 : 0.8 - - if (hitRatio > 0.8 && !isVeryLargeDataset) { - batchSize = Math.max(isReadOnly ? 10 : 5, Math.floor(batchSize * decreaseFactorForHighHitRatio)) - } else if (hitRatio < 0.5) { - const maxBatchSize = isVeryLargeDataset ? 150 : 100 - batchSize = Math.min(maxBatchSize, Math.ceil(batchSize * increaseFactorForLowHitRatio)) - } - } - - // Min/max envelope. 8.0 is Node-like only. - const minBatchSize = isReadOnly ? 10 : 5 - batchSize = Math.max(minBatchSize, batchSize) - batchSize = Math.min(150, batchSize) - - // Update the batch size with the adaptively tuned value - this.batchSize = Math.round(batchSize) - } - - /** - * Resolve the warm-tier storage type. Brainy 8.0 ships filesystem + - * memory only. - */ - private detectWarmStorageType(): StorageType { - return StorageType.FILESYSTEM - } - - /** - * Resolve the cold-tier storage type. Brainy 8.0 ships filesystem + - * memory only. - */ - private detectColdStorageType(): StorageType { - return StorageType.FILESYSTEM - } - - /** - * Initialize warm storage adapter - */ - private initializeWarmStorage(): any { - // Implementation depends on the detected storage type - // For now, return null as this will be provided by the storage adapter - return null - } - - /** - * Initialize cold storage adapter - */ - private initializeColdStorage(): any { - // Implementation depends on the detected storage type - // For now, return null as this will be provided by the storage adapter - return null - } - - /** - * Get an item from cache, trying each level in order - * @param id The item ID - * @returns The cached item or null if not found - */ - public async get(id: string): Promise { - // Check if it's time to tune parameters - await this.checkAndTuneParameters() - - // Try hot cache first (fastest) - const hotCacheEntry = this.hotCache.get(id) - if (hotCacheEntry) { - // Update access metadata - hotCacheEntry.lastAccessed = Date.now() - hotCacheEntry.accessCount++ - - // Update stats - this.stats.hits++ - - return hotCacheEntry.data - } - - // Try warm cache next - try { - const warmCacheItem = await this.getFromWarmCache(id) - if (warmCacheItem) { - // Promote to hot cache - this.addToHotCache(id, warmCacheItem) - - // Update stats - this.stats.hits++ - - return warmCacheItem - } - } catch (error) { - console.warn(`Error accessing warm cache for ${id}:`, error) - } - - // Finally, try cold storage - try { - const coldStorageItem = await this.getFromColdStorage(id) - if (coldStorageItem) { - // Promote to hot and warm caches - this.addToHotCache(id, coldStorageItem) - await this.addToWarmCache(id, coldStorageItem) - - // Update stats - this.stats.misses++ - - return coldStorageItem - } - } catch (error) { - console.warn(`Error accessing cold storage for ${id}:`, error) - } - - // Item not found in any cache level - this.stats.misses++ - return null - } - - /** - * Get an item from warm cache - * @param id The item ID - * @returns The cached item or null if not found - */ - private async getFromWarmCache(id: string): Promise { - if (!this.warmStorage) return null - - try { - return await this.warmStorage.get(id) - } catch (error) { - console.warn(`Error getting item ${id} from warm cache:`, error) - return null - } - } - - /** - * Get an item from cold storage - * @param id The item ID - * @returns The item or null if not found - */ - private async getFromColdStorage(id: string): Promise { - if (!this.coldStorage) return null - - try { - return await this.coldStorage.get(id) - } catch (error) { - console.warn(`Error getting item ${id} from cold storage:`, error) - return null - } - } - - /** - * Add an item to hot cache - * @param id The item ID - * @param item The item to cache - */ - private addToHotCache(id: string, item: T): void { - // Check if we need to evict items - if (this.hotCache.size >= this.hotCacheMaxSize * this.hotCacheEvictionThreshold) { - this.evictFromHotCache() - } - - // Add to hot cache - this.hotCache.set(id, { - data: item, - lastAccessed: Date.now(), - accessCount: 1, - expiresAt: null // Hot cache items don't expire - }) - - // Update stats - this.stats.size = this.hotCache.size - } - - /** - * Add an item to warm cache - * @param id The item ID - * @param item The item to cache - */ - private async addToWarmCache(id: string, item: T): Promise { - if (!this.warmStorage) return - - try { - // Add to warm cache with TTL - await this.warmStorage.set(id, item, { - ttl: this.warmCacheTTL - }) - } catch (error) { - console.warn(`Error adding item ${id} to warm cache:`, error) - } - } - - /** - * Evict items from hot cache based on LRU policy - */ - private evictFromHotCache(): void { - // Find the least recently used items - const entries = Array.from(this.hotCache.entries()) - - // Sort by last accessed time (oldest first) - entries.sort((a, b) => a[1].lastAccessed - b[1].lastAccessed) - - // Remove the oldest 20% of items - const itemsToRemove = Math.ceil(this.hotCache.size * 0.2) - for (let i = 0; i < itemsToRemove && i < entries.length; i++) { - this.hotCache.delete(entries[i][0]) - this.stats.evictions++ - } - - // Update stats - this.stats.size = this.hotCache.size - - if (process.env.DEBUG) { - console.log(`Evicted ${itemsToRemove} items from hot cache, new size: ${this.hotCache.size}`) - } - } - - /** - * Set an item in all cache levels - * @param id The item ID - * @param item The item to cache - */ - public async set(id: string, item: T): Promise { - // Add to hot cache - this.addToHotCache(id, item) - - // Add to warm cache - await this.addToWarmCache(id, item) - - // Add to cold storage - if (this.coldStorage) { - try { - await this.coldStorage.set(id, item) - } catch (error) { - console.warn(`Error adding item ${id} to cold storage:`, error) - } - } - } - - /** - * Delete an item from all cache levels - * @param id The item ID to delete - */ - public async delete(id: string): Promise { - // Remove from hot cache - this.hotCache.delete(id) - - // Remove from warm cache - if (this.warmStorage) { - try { - await this.warmStorage.delete(id) - } catch (error) { - console.warn(`Error deleting item ${id} from warm cache:`, error) - } - } - - // Remove from cold storage - if (this.coldStorage) { - try { - await this.coldStorage.delete(id) - } catch (error) { - console.warn(`Error deleting item ${id} from cold storage:`, error) - } - } - - // Update stats - this.stats.size = this.hotCache.size - } - - /** - * Clear all cache levels - */ - public async clear(): Promise { - // Clear hot cache - this.hotCache.clear() - - // Clear warm cache - if (this.warmStorage) { - try { - await this.warmStorage.clear() - } catch (error) { - console.warn('Error clearing warm cache:', error) - } - } - - // Clear cold storage - if (this.coldStorage) { - try { - await this.coldStorage.clear() - } catch (error) { - console.warn('Error clearing cold storage:', error) - } - } - - // Reset stats - this.stats = { - hits: 0, - misses: 0, - evictions: 0, - size: 0, - maxSize: this.hotCacheMaxSize, - hotCacheSize: 0, - warmCacheSize: 0, - hotCacheHits: 0, - hotCacheMisses: 0, - warmCacheHits: 0, - warmCacheMisses: 0 - } - } - - /** - * Get cache statistics - * @returns Cache statistics - */ - public getStats(): CacheStats { - return { ...this.stats } - } - - /** - * Prefetch items based on ID patterns or relationships - * @param ids Array of IDs to prefetch - */ - public async prefetch(ids: string[]): Promise { - // Check if it's time to tune parameters - await this.checkAndTuneParameters() - - // Prefetch in batches to avoid overwhelming the system - const batches: string[][] = [] - - // Split into batches using the configurable batch size - for (let i = 0; i < ids.length; i += this.batchSize) { - const batch = ids.slice(i, i + this.batchSize) - batches.push(batch) - } - - // Process each batch - for (const batch of batches) { - await Promise.all( - batch.map(async (id) => { - // Skip if already in hot cache - if (this.hotCache.has(id)) return - - try { - // Try to get from any cache level - await this.get(id) - } catch (error) { - // Ignore errors during prefetching - if (process.env.DEBUG) { - console.warn(`Error prefetching ${id}:`, error) - } - } - }) - ) - } - } - - /** - * Check if it's time to tune parameters and do so if needed - * This is called before operations that might benefit from tuned parameters - * - * This method serves as a checkpoint for auto-tuning, ensuring that: - * 1. Parameters are tuned periodically based on the auto-tune interval - * 2. Tuning happens before critical operations that would benefit from optimized parameters - * 3. Tuning doesn't happen too frequently, which could impact performance - * - * By calling this method before get(), getMany(), and prefetch() operations, - * we ensure that the cache parameters are optimized for the current workload - * without adding unnecessary overhead to every operation. - */ - private async checkAndTuneParameters(): Promise { - // Skip if auto-tuning is disabled - if (!this.autoTune) return - - // Check if it's time to tune parameters - const now = Date.now() - if (now - this.lastAutoTuneTime >= this.autoTuneInterval) { - await this.tuneParameters() - } - } - - /** - * Get multiple items at once, optimizing for batch retrieval - * @param ids Array of IDs to get - * @returns Map of ID to item - */ - public async getMany(ids: string[]): Promise> { - // Check if it's time to tune parameters - await this.checkAndTuneParameters() - - const result = new Map() - - // First check hot cache for all IDs - const missingIds: string[] = [] - for (const id of ids) { - const hotCacheEntry = this.hotCache.get(id) - if (hotCacheEntry) { - // Update access metadata - hotCacheEntry.lastAccessed = Date.now() - hotCacheEntry.accessCount++ - - // Add to result - result.set(id, hotCacheEntry.data) - - // Update stats - this.stats.hits++ - } else { - missingIds.push(id) - } - } - - if (missingIds.length === 0) { - return result - } - - // Try to get missing items from warm cache - if (this.warmStorage) { - try { - const warmCacheItems = await this.warmStorage.getMany(missingIds) - for (const [id, item] of warmCacheItems.entries()) { - if (item) { - // Promote to hot cache - this.addToHotCache(id, item) - - // Add to result - result.set(id, item) - - // Update stats - this.stats.hits++ - - // Remove from missing IDs - const index = missingIds.indexOf(id) - if (index !== -1) { - missingIds.splice(index, 1) - } - } - } - } catch (error) { - console.warn('Error accessing warm cache for batch:', error) - } - } - - if (missingIds.length === 0) { - return result - } - - // Try to get remaining missing items from cold storage - if (this.coldStorage) { - try { - const coldStorageItems = await this.coldStorage.getMany(missingIds) - for (const [id, item] of coldStorageItems.entries()) { - if (item) { - // Promote to hot and warm caches - this.addToHotCache(id, item) - await this.addToWarmCache(id, item) - - // Add to result - result.set(id, item) - - // Update stats - this.stats.misses++ - } - } - } catch (error) { - console.warn('Error accessing cold storage for batch:', error) - } - } - - return result - } - - /** - * Set the storage adapters for warm and cold caches - * @param warmStorage Warm cache storage adapter - * @param coldStorage Cold storage adapter - */ - public setStorageAdapters(warmStorage: any, coldStorage: any): void { - this.warmStorage = warmStorage - this.coldStorage = coldStorage - } -} diff --git a/src/transaction/index.ts b/src/transaction/index.ts deleted file mode 100644 index a09eb234..00000000 --- a/src/transaction/index.ts +++ /dev/null @@ -1,34 +0,0 @@ -/** - * Transaction System - * - * Provides atomicity for Brainy operations. - * All operations succeed or all rollback - no partial failures. - * - * @module transaction - */ - -export * from './types.js' -export * from './errors.js' -export { Transaction } from './Transaction.js' -export { TransactionManager, type TransactionStats } from './TransactionManager.js' - -// Re-export for convenience -export type { - Operation, - RollbackAction, - TransactionState, - TransactionContext, - TransactionFunction, - TransactionResult, - TransactionOptions -} from './types.js' - -export { - TransactionError, - TransactionExecutionError, - TransactionRollbackError, - InvalidTransactionStateError, - TransactionTimeoutError -} from './errors.js' - -export { RevisionConflictError } from './RevisionConflictError.js' diff --git a/src/types/apiTypes.ts b/src/types/apiTypes.ts deleted file mode 100644 index d788d60e..00000000 --- a/src/types/apiTypes.ts +++ /dev/null @@ -1,326 +0,0 @@ -/** - * Consistent API Types for Brainy - * - * These types provide a uniform interface for all public methods, - * using object parameters for consistency and extensibility. - */ - -import type { Vector } from '../coreTypes.js' -import type { NounType, VerbType } from './graphTypes.js' - -// ============= NOUN OPERATIONS ============= - -/** - * Parameters for adding a noun - */ -export interface AddNounParams { - data: any | Vector // Content or pre-computed vector - type: NounType | string // Noun type (required) - metadata?: any // Optional metadata - id?: string // Optional custom ID - service?: string // Optional service identifier -} - -/** - * Parameters for updating a noun - */ -export interface UpdateNounParams { - id: string // Noun ID to update - data?: any // New data - metadata?: any // New metadata - type?: NounType | string // New type -} - -/** - * Parameters for getting nouns - */ -export interface GetNounsParams { - ids?: string[] // Specific IDs to fetch - type?: NounType | string | string[] // Filter by type(s) - limit?: number // Maximum results - offset?: number // Pagination offset - cursor?: string // Pagination cursor - filter?: Record // Metadata filters - service?: string // Service filter -} - -// ============= VERB OPERATIONS ============= - -/** - * Parameters for adding a verb (relationship) - */ -export interface AddVerbParams { - source: string // Source noun ID - target: string // Target noun ID - type: VerbType | string // Verb type (required) - weight?: number // Relationship weight (0-1) - metadata?: any // Optional metadata - service?: string // Optional service identifier -} - -/** - * Parameters for getting verbs - */ -export interface GetVerbsParams { - source?: string // Filter by source - target?: string // Filter by target - type?: VerbType | string | string[] // Filter by type(s) - limit?: number // Maximum results - offset?: number // Pagination offset - cursor?: string // Pagination cursor - filter?: Record // Metadata filters - service?: string // Service filter -} - -// ============= SEARCH OPERATIONS ============= - -/** - * Unified search parameters - */ -export interface SearchParams { - query: string | Vector // Text query or vector - limit?: number // Maximum results (default: 10) - threshold?: number // Similarity threshold (0-1) - filter?: { - type?: NounType | string | string[] // Filter by noun type(s) - metadata?: Record // Metadata filters - service?: string // Service filter - } - includeMetadata?: boolean // Include metadata in results - includeVectors?: boolean // Include vectors in results -} - -/** - * Parameters for similarity search - */ -export interface SimilarityParams { - id?: string // Find similar to this ID - data?: any | Vector // Or find similar to this data - limit?: number // Maximum results (default: 10) - threshold?: number // Similarity threshold (0-1) - filter?: { - type?: NounType | string | string[] - metadata?: Record - service?: string - } -} - -/** - * Parameters for related items search - */ -export interface RelatedParams { - id: string // Starting noun ID - depth?: number // Traversal depth (default: 1) - limit?: number // Max results per level - types?: VerbType[] | string[] // Relationship types to follow - direction?: 'outgoing' | 'incoming' | 'both' -} - -// ============= BATCH OPERATIONS ============= - -/** - * Parameters for batch noun operations - */ -export interface BatchNounsParams { - items: AddNounParams[] // Array of nouns to add - parallel?: boolean // Process in parallel - chunkSize?: number // Batch size for processing - onProgress?: (completed: number, total: number) => void -} - -/** - * Parameters for batch verb operations - */ -export interface BatchVerbsParams { - items: AddVerbParams[] // Array of verbs to add - parallel?: boolean // Process in parallel - chunkSize?: number // Batch size for processing - onProgress?: (completed: number, total: number) => void -} - -// ============= STATISTICS & METADATA ============= - -/** - * Parameters for statistics queries - */ -export interface StatisticsParams { - detailed?: boolean // Include detailed breakdown - includeAugmentations?: boolean // Include augmentation stats - includeMemory?: boolean // Include memory usage - service?: string // Filter by service -} - -/** - * Parameters for metadata operations - */ -export interface MetadataParams { - id: string // Entity ID - metadata: any // Metadata to set/update - merge?: boolean // Merge with existing (vs replace) -} - -// ============= CONFIGURATION ============= - -/** - * Dynamic configuration update parameters - */ -export interface ConfigUpdateParams { - embeddings?: { - model?: string - precision?: 'q8' - cache?: boolean - } - augmentations?: { - [name: string]: boolean | Record - } - storage?: { - type?: string - config?: any - } - performance?: { - batchSize?: number - maxConcurrency?: number - cacheSize?: number - } -} - -// ============= TRIPLE INTELLIGENCE API ============= - -/** - * API for Triple Intelligence Engine to access Brainy internals - * This provides type-safe access without unchecked casts - */ -export interface TripleIntelligenceAPI { - // Vector operations - vectorSearch(vector: Vector | string, limit: number): Promise> - - // Graph operations - graphTraversal(options: { - start: string | string[] - type?: string | string[] - direction?: 'in' | 'out' | 'both' - maxDepth?: number - }): Promise> - - // Metadata operations - metadataQuery(where: Record): Promise> - getEntity(id: string): Promise - - // Storage operations - getVerbsBySource(sourceId: string): Promise - getVerbsByTarget(targetId: string): Promise - - // Statistics - getStatistics(): Promise<{ - totalCount: number - fieldStats: Record - }> - - // Index access - getAllNouns(): Map - hasMetadataIndex(): boolean -} - -// ============= RESULTS ============= - -/** - * Unified search result - */ -export interface SearchResult { - id: string - score: number // Similarity score (0-1) - data?: T // Original data - metadata?: any // Metadata if requested - vector?: Vector // Vector if requested - type?: string // Noun type - distance?: number // Raw distance metric -} - -/** - * Paginated result wrapper - */ -export interface PaginatedResult { - items: T[] - total?: number - hasMore: boolean - nextCursor?: string - previousCursor?: string -} - -/** - * Batch operation result - */ -export interface BatchResult { - successful: string[] // Successfully processed IDs - failed: Array<{ - index: number - error: string - item?: any - }> - total: number - duration: number // Total time in ms -} - -/** - * Statistics result - */ -export interface StatisticsResult { - nouns: { - total: number - byType: Record - } - verbs: { - total: number - byType: Record - } - storage: { - used: number - type: string - } - performance?: { - avgLatency: number - throughput: number - cacheHitRate?: number - } - augmentations?: Record - memory?: { - used: number - limit: number - } -} - -// ============= ERRORS ============= - -/** - * Structured error for API operations - */ -export class BrainyAPIError extends Error { - constructor( - message: string, - public code: string, - public statusCode: number = 400, - public details?: any - ) { - super(message) - this.name = 'BrainyAPIError' - } -} - -// Error codes -export const ErrorCodes = { - INVALID_TYPE: 'INVALID_TYPE', - NOT_FOUND: 'NOT_FOUND', - DUPLICATE_ID: 'DUPLICATE_ID', - INVALID_VECTOR: 'INVALID_VECTOR', - STORAGE_ERROR: 'STORAGE_ERROR', - EMBEDDING_ERROR: 'EMBEDDING_ERROR', - AUGMENTATION_ERROR: 'AUGMENTATION_ERROR', - VALIDATION_ERROR: 'VALIDATION_ERROR', - QUOTA_EXCEEDED: 'QUOTA_EXCEEDED', - UNAUTHORIZED: 'UNAUTHORIZED' -} as const \ No newline at end of file diff --git a/src/types/fileSystemTypes.ts b/src/types/fileSystemTypes.ts deleted file mode 100644 index 4e927516..00000000 --- a/src/types/fileSystemTypes.ts +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Type declarations for the File System Access API - * Extends the FileSystemDirectoryHandle interface to include the [Symbol.asyncIterator] method - * and FileSystemHandle to include getFile() method for TypeScript compatibility - */ - -// Extend the FileSystemDirectoryHandle interface -interface FileSystemDirectoryHandle { - [Symbol.asyncIterator](): AsyncIterableIterator<[string, FileSystemHandle]>; - - keys(): AsyncIterableIterator; - - entries(): AsyncIterableIterator<[string, FileSystemHandle]>; -} - -// Extend the FileSystemHandle interface to include getFile method -// This is needed because TypeScript doesn't recognize that a FileSystemHandle -// can be a FileSystemFileHandle which has the getFile method -interface FileSystemHandle { - getFile?(): Promise; -} - -// Export something to make this a module -export const fileSystemTypesLoaded = true diff --git a/src/types/progress.types.ts b/src/types/progress.types.ts deleted file mode 100644 index b8444d01..00000000 --- a/src/types/progress.types.ts +++ /dev/null @@ -1,288 +0,0 @@ -/** - * Standardized Progress Reporting - * - * Provides unified progress tracking across all long-running operations - * in Brainy (imports, clustering, large searches, etc.) - */ - -/** - * Progress status states - */ -export type ProgressStatus = 'pending' | 'running' | 'completed' | 'failed' | 'cancelled' - -/** - * Standardized progress report - */ -export interface BrainyProgress { - // Core status - status: ProgressStatus - - // Progress percentage (0-100) - progress: number - - // Human-readable message - message: string - - // Detailed metadata - metadata: { - itemsProcessed: number - itemsTotal: number - currentItem?: string - estimatedTimeRemaining?: number // milliseconds - startedAt: number - completedAt?: number - throughput?: number // items per second - } - - // Result when completed - result?: T - - // Error when failed - error?: Error -} - -/** - * Progress tracker with automatic time estimation - */ -export class ProgressTracker { - private status: ProgressStatus = 'pending' - private processed = 0 - private total: number - private startedAt?: number - private completedAt?: number - private currentItem?: string - private result?: T - private error?: Error - private processingTimes: number[] = [] // Track last N processing times for estimation - - constructor(total: number) { - if (total < 0) { - throw new Error('Total must be non-negative') - } - this.total = total - } - - /** - * Factory method for creating progress trackers - */ - static create(total: number): ProgressTracker { - return new ProgressTracker(total) - } - - /** - * Start tracking progress - */ - start(): BrainyProgress { - this.status = 'running' - this.startedAt = Date.now() - return this.current() - } - - /** - * Update progress - */ - update(processed: number, currentItem?: string): BrainyProgress { - if (processed < 0) { - throw new Error('Processed count must be non-negative') - } - if (processed > this.total) { - throw new Error(`Processed count (${processed}) exceeds total (${this.total})`) - } - - const previousProcessed = this.processed - this.processed = processed - this.currentItem = currentItem - - // Track processing time for estimation - if (this.startedAt && previousProcessed < processed) { - const itemsProcessed = processed - previousProcessed - const timeTaken = Date.now() - this.startedAt - const avgTimePerItem = timeTaken / processed - this.processingTimes.push(avgTimePerItem) - - // Keep only last 100 measurements for rolling average - if (this.processingTimes.length > 100) { - this.processingTimes.shift() - } - } - - return this.current() - } - - /** - * Increment progress by 1 - */ - increment(currentItem?: string): BrainyProgress { - return this.update(this.processed + 1, currentItem) - } - - /** - * Mark as completed - */ - complete(result: T): BrainyProgress { - this.status = 'completed' - this.completedAt = Date.now() - this.processed = this.total - this.result = result - return this.current() - } - - /** - * Mark as failed - */ - fail(error: Error): BrainyProgress { - this.status = 'failed' - this.completedAt = Date.now() - this.error = error - return this.current() - } - - /** - * Mark as cancelled - */ - cancel(): BrainyProgress { - this.status = 'cancelled' - this.completedAt = Date.now() - return this.current() - } - - /** - * Get current progress state - */ - current(): BrainyProgress { - const progress = this.total > 0 ? Math.round((this.processed / this.total) * 100) : 0 - - // Generate message based on status - let message: string - switch (this.status) { - case 'pending': - message = `Ready to process ${this.total} items` - break - case 'running': - message = this.currentItem - ? `Processing: ${this.currentItem} (${this.processed}/${this.total})` - : `Processing ${this.processed}/${this.total} items` - break - case 'completed': - message = `Completed ${this.total} items` - break - case 'failed': - message = `Failed after ${this.processed} items: ${this.error?.message || 'Unknown error'}` - break - case 'cancelled': - message = `Cancelled after ${this.processed} items` - break - } - - return { - status: this.status, - progress, - message, - metadata: { - itemsProcessed: this.processed, - itemsTotal: this.total, - currentItem: this.currentItem, - estimatedTimeRemaining: this.estimateTimeRemaining(), - startedAt: this.startedAt || Date.now(), - completedAt: this.completedAt, - throughput: this.calculateThroughput() - }, - result: this.result, - error: this.error - } - } - - /** - * Estimate time remaining based on processing history - */ - private estimateTimeRemaining(): number | undefined { - if (this.status !== 'running' || !this.startedAt || this.processed === 0) { - return undefined - } - - const remaining = this.total - this.processed - if (remaining === 0) { - return 0 - } - - // Use rolling average if we have enough samples - if (this.processingTimes.length > 0) { - const avgTimePerItem = this.processingTimes.reduce((a, b) => a + b, 0) / this.processingTimes.length - return Math.round(avgTimePerItem * remaining) - } - - // Fallback to simple calculation - const elapsed = Date.now() - this.startedAt - const avgTimePerItem = elapsed / this.processed - return Math.round(avgTimePerItem * remaining) - } - - /** - * Calculate current throughput (items/second) - */ - private calculateThroughput(): number | undefined { - if (!this.startedAt || this.processed === 0) { - return undefined - } - - const elapsed = Date.now() - this.startedAt - const seconds = elapsed / 1000 - return seconds > 0 ? Math.round((this.processed / seconds) * 100) / 100 : undefined - } - - /** - * Get progress statistics - */ - getStats() { - const elapsed = this.startedAt ? Date.now() - this.startedAt : 0 - return { - status: this.status, - processed: this.processed, - total: this.total, - remaining: this.total - this.processed, - progress: this.total > 0 ? this.processed / this.total : 0, - elapsed, - estimatedTotal: elapsed > 0 && this.processed > 0 - ? Math.round((elapsed / this.processed) * this.total) - : undefined, - throughput: this.calculateThroughput() - } - } -} - -/** - * Helper to format time duration - */ -export function formatDuration(ms: number): string { - const seconds = Math.floor(ms / 1000) - const minutes = Math.floor(seconds / 60) - const hours = Math.floor(minutes / 60) - - if (hours > 0) { - return `${hours}h ${minutes % 60}m` - } else if (minutes > 0) { - return `${minutes}m ${seconds % 60}s` - } else { - return `${seconds}s` - } -} - -/** - * Helper to format progress percentage - */ -export function formatProgress(progress: BrainyProgress): string { - const { status, progress: pct, metadata } = progress - const remaining = metadata.estimatedTimeRemaining - - let str = `[${status.toUpperCase()}] ${pct}% (${metadata.itemsProcessed}/${metadata.itemsTotal})` - - if (metadata.throughput) { - str += ` - ${metadata.throughput} items/s` - } - - if (remaining && remaining > 0) { - str += ` - ${formatDuration(remaining)} remaining` - } - - return str -} diff --git a/src/types/typeMigration.ts b/src/types/typeMigration.ts deleted file mode 100644 index de4d3e06..00000000 --- a/src/types/typeMigration.ts +++ /dev/null @@ -1,174 +0,0 @@ -/** - * Type Migration Utilities for Stage 3 Taxonomy - * - * Provides migration helpers for code using removed types from v5.x - * - * ## Removed Types - * - * ### Nouns (2 removed): - * - `user` โ†’ merged into `person` - * - `topic` โ†’ merged into `concept` - * - * ### Verbs (4 removed): - * - `succeeds` โ†’ use inverse of `precedes` - * - `belongsTo` โ†’ use inverse of `owns` - * - `createdBy` โ†’ use inverse of `creates` - * - `supervises` โ†’ use inverse of `reportsTo` - */ - -import { NounType, VerbType } from './graphTypes.js' - -/** - * Migration mapping for removed noun types - */ -export const REMOVED_NOUN_TYPES = { - user: NounType.Person, - topic: NounType.Concept -} as const - -/** - * Migration mapping for removed verb types - * Note: Some verbs should use inverse relationships in Stage 3 - */ -export const REMOVED_VERB_TYPES = { - succeeds: VerbType.Precedes, // Use with inverted source/target - belongsTo: VerbType.Owns, // Use with inverted source/target - createdBy: VerbType.Creates, // Use with inverted source/target - supervises: VerbType.ReportsTo // Use with inverted source/target -} as const - -/** - * Check if a type was removed in Stage 3 - */ -export function isRemovedNounType(type: string): type is keyof typeof REMOVED_NOUN_TYPES { - return type in REMOVED_NOUN_TYPES -} - -/** - * Check if a verb type was removed in Stage 3 - */ -export function isRemovedVerbType(type: string): type is keyof typeof REMOVED_VERB_TYPES { - return type in REMOVED_VERB_TYPES -} - -/** - * Migrate a noun type (Stage 3) - * Returns the migrated type or the original if no migration needed - */ -export function migrateNounType(type: string): NounType { - if (isRemovedNounType(type)) { - console.warn(`โš ๏ธ NounType "${type}" was removed in v6.0. Migrating to "${REMOVED_NOUN_TYPES[type]}"`) - return REMOVED_NOUN_TYPES[type] - } - return type as NounType -} - -/** - * Migrate a verb type (Stage 3) - * Returns the migrated type or the original if no migration needed - * - * WARNING: Some verbs require inverting source/target relationships! - * See VERB_REQUIRES_INVERSION for details. - */ -export function migrateVerbType(type: string): VerbType { - if (isRemovedVerbType(type)) { - console.warn(`โš ๏ธ VerbType "${type}" was removed in v6.0. Migrating to "${REMOVED_VERB_TYPES[type]}" (may require source/target inversion)`) - return REMOVED_VERB_TYPES[type] - } - return type as VerbType -} - -/** - * Verbs that require inverting source/target when migrating - * - * Example: - * - Old: `A createdBy B` โ†’ New: `B creates A` - * - Old: `A belongsTo B` โ†’ New: `B owns A` - * - Old: `A supervises B` โ†’ New: `B reportsTo A` - * - Old: `A succeeds B` โ†’ New: `B precedes A` - */ -export const VERB_REQUIRES_INVERSION = new Set([ - 'succeeds', - 'belongsTo', - 'createdBy', - 'supervises' -]) - -/** - * Check if a verb type requires inverting source/target during migration - */ -export function requiresInversion(oldVerbType: string): boolean { - return VERB_REQUIRES_INVERSION.has(oldVerbType) -} - -/** - * Migrate a relationship, handling source/target inversion if needed - * - * @returns Object with migrated verb and potentially inverted source/target - */ -export function migrateRelationship(params: { - verb: string - source: string - target: string -}): { - verb: VerbType - source: string - target: string - inverted: boolean -} { - const verb = migrateVerbType(params.verb) - const inverted = requiresInversion(params.verb) - - if (inverted) { - return { - verb, - source: params.target, // Swap source and target - target: params.source, - inverted: true - } - } - - return { - verb, - source: params.source, - target: params.target, - inverted: false - } -} - -/** - * Stage 3 Type Compatibility Check - * Helps developers identify code that needs updating - */ -export function checkTypeCompatibility(nounTypes: string[], verbTypes: string[]): { - valid: boolean - removedNouns: string[] - removedVerbs: string[] - warnings: string[] -} { - const removedNouns = nounTypes.filter(isRemovedNounType) - const removedVerbs = verbTypes.filter(isRemovedVerbType) - const warnings: string[] = [] - - if (removedNouns.length > 0) { - warnings.push( - `Found ${removedNouns.length} removed noun type(s): ${removedNouns.join(', ')}. ` + - `These were merged in Stage 3. Use Person instead of User, Concept instead of Topic.` - ) - } - - if (removedVerbs.length > 0) { - warnings.push( - `Found ${removedVerbs.length} removed verb type(s): ${removedVerbs.join(', ')}. ` + - `These require using inverse relationships in Stage 3. ` + - `Example: "A createdBy B" becomes "B creates A".` - ) - } - - return { - valid: removedNouns.length === 0 && removedVerbs.length === 0, - removedNouns, - removedVerbs, - warnings - } -} diff --git a/src/unified.ts b/src/unified.ts deleted file mode 100644 index 987dd867..00000000 --- a/src/unified.ts +++ /dev/null @@ -1,39 +0,0 @@ -/** - * @module unified - * @description Unified entry point for Brainy. Re-exports the public surface - * from `index.js` and surfaces a tiny runtime-environment object the rest of - * the library can read. - * - * 8.0 supports Node-like runtimes only (Node.js, Bun, Deno); the browser - * detection helpers and Worker Thread plumbing are gone. - */ - -import './setup.js' - -import { isNode } from './utils/environment.js' - -export const environment = { - get isNode() { - return isNode() - } -} - -declare global { - /** - * Runtime-environment marker published by the unified entry point so - * embedders and diagnostics can detect the active Brainy environment. - * Ambient `var` is required โ€” `let`/`const` in `declare global` do not - * attach to `globalThis`. - */ - var __ENV__: typeof environment | undefined -} - -if (typeof globalThis !== 'undefined') { - globalThis.__ENV__ = environment -} - -console.log( - `Brainy running on ${environment.isNode ? 'Node-like runtime' : 'unknown runtime'}` -) - -export * from './index.js' diff --git a/src/universal/crypto.ts b/src/universal/crypto.ts deleted file mode 100644 index 4c117c1f..00000000 --- a/src/universal/crypto.ts +++ /dev/null @@ -1,159 +0,0 @@ -/** - * Universal Crypto implementation - * Framework-friendly: Trusts that frameworks provide crypto polyfills - * Works in all environments: Browser (via framework), Node.js, Serverless - */ - -import { isNode } from '../utils/environment.js' - -let nodeCrypto: any = null - -// Dynamic import for Node.js crypto (only in Node.js environment) -if (isNode()) { - try { - // Use node: protocol to prevent bundler polyfilling (requires Node 22+) - nodeCrypto = await import('node:crypto') - } catch { - // Ignore import errors in non-Node environments - } -} - -/** - * Generate random bytes - * Framework-friendly: Assumes crypto API is available via framework polyfills - */ -export function randomBytes(size: number): Uint8Array { - if (typeof crypto !== 'undefined') { - // Use Web Crypto API (available in browsers via framework polyfills and modern Node.js) - const array = new Uint8Array(size) - crypto.getRandomValues(array) - return array - } else if (nodeCrypto) { - // Use Node.js crypto - return new Uint8Array(nodeCrypto.randomBytes(size)) - } else { - throw new Error('Crypto API not available. Framework bundlers should provide crypto polyfills.') - } -} - -/** - * Generate random UUID - * Framework-friendly: Assumes crypto.randomUUID is available via framework polyfills - */ -export function randomUUID(): string { - if (typeof crypto !== 'undefined' && crypto.randomUUID) { - return crypto.randomUUID() - } else if (nodeCrypto && nodeCrypto.randomUUID) { - return nodeCrypto.randomUUID() - } else { - throw new Error('crypto.randomUUID not available. Framework bundlers should provide crypto polyfills.') - } -} - -/** - * Create hash (simplified interface) - * Framework-friendly: Relies on Node.js crypto or framework-provided implementations - */ -export function createHash(algorithm: string): { - update: (data: string | Uint8Array) => any - digest: (encoding: string) => string -} { - if (nodeCrypto && nodeCrypto.createHash) { - return nodeCrypto.createHash(algorithm) - } else { - throw new Error(`createHash not available. For browser environments, frameworks should provide crypto polyfills or use Web Crypto API directly.`) - } -} - -/** - * Create HMAC - * Framework-friendly: Relies on Node.js crypto or framework-provided implementations - */ -export function createHmac(algorithm: string, key: string | Uint8Array): { - update: (data: string | Uint8Array) => any - digest: (encoding: string) => string -} { - if (nodeCrypto && nodeCrypto.createHmac) { - return nodeCrypto.createHmac(algorithm, key) - } else { - throw new Error(`createHmac not available. For browser environments, frameworks should provide crypto polyfills or use Web Crypto API directly.`) - } -} - -/** - * PBKDF2 synchronous - * Framework-friendly: Relies on Node.js crypto or framework-provided implementations - */ -export function pbkdf2Sync(password: string | Uint8Array, salt: string | Uint8Array, iterations: number, keylen: number, digest: string): Uint8Array { - if (nodeCrypto && nodeCrypto.pbkdf2Sync) { - return new Uint8Array(nodeCrypto.pbkdf2Sync(password, salt, iterations, keylen, digest)) - } else { - throw new Error(`pbkdf2Sync not available. For browser environments, frameworks should provide crypto polyfills or use Web Crypto API directly.`) - } -} - -/** - * Scrypt synchronous - * Framework-friendly: Relies on Node.js crypto or framework-provided implementations - */ -export function scryptSync(password: string | Uint8Array, salt: string | Uint8Array, keylen: number, options?: any): Uint8Array { - if (nodeCrypto && nodeCrypto.scryptSync) { - return new Uint8Array(nodeCrypto.scryptSync(password, salt, keylen, options)) - } else { - throw new Error(`scryptSync not available. For browser environments, frameworks should provide crypto polyfills or use Web Crypto API directly.`) - } -} - -/** - * Create cipher - * Framework-friendly: Relies on Node.js crypto or framework-provided implementations - */ -export function createCipheriv(algorithm: string, key: Uint8Array, iv: Uint8Array): { - update: (data: string, inputEncoding?: string, outputEncoding?: string) => string - final: (outputEncoding?: string) => string -} { - if (nodeCrypto && nodeCrypto.createCipheriv) { - return nodeCrypto.createCipheriv(algorithm, key, iv) - } else { - throw new Error(`createCipheriv not available. For browser environments, frameworks should provide crypto polyfills or use Web Crypto API directly.`) - } -} - -/** - * Create decipher - * Framework-friendly: Relies on Node.js crypto or framework-provided implementations - */ -export function createDecipheriv(algorithm: string, key: Uint8Array, iv: Uint8Array): { - update: (data: string, inputEncoding?: string, outputEncoding?: string) => string - final: (outputEncoding?: string) => string -} { - if (nodeCrypto && nodeCrypto.createDecipheriv) { - return nodeCrypto.createDecipheriv(algorithm, key, iv) - } else { - throw new Error(`createDecipheriv not available. For browser environments, frameworks should provide crypto polyfills or use Web Crypto API directly.`) - } -} - -/** - * Timing safe equal - * Framework-friendly: Relies on Node.js crypto or framework-provided implementations - */ -export function timingSafeEqual(a: Uint8Array, b: Uint8Array): boolean { - if (nodeCrypto && nodeCrypto.timingSafeEqual) { - return nodeCrypto.timingSafeEqual(a, b) - } else { - throw new Error(`timingSafeEqual not available. For browser environments, frameworks should provide crypto polyfills or use Web Crypto API directly.`) - } -} - -export default { - randomBytes, - randomUUID, - createHash, - createHmac, - pbkdf2Sync, - scryptSync, - createCipheriv, - createDecipheriv, - timingSafeEqual -} \ No newline at end of file diff --git a/src/utils/BoundedRegistry.ts b/src/utils/BoundedRegistry.ts deleted file mode 100644 index f8761546..00000000 --- a/src/utils/BoundedRegistry.ts +++ /dev/null @@ -1,62 +0,0 @@ -/** - * Bounded Registry with LRU eviction - * Prevents unbounded memory growth in production - */ -export class BoundedRegistry { - private items = new Map() // item -> last accessed timestamp - private readonly maxSize: number - - constructor(maxSize: number = 10000) { - this.maxSize = maxSize - } - - /** - * Add item to registry, evicting oldest if at capacity - */ - add(item: T): void { - // Update timestamp if already exists - if (this.items.has(item)) { - this.items.delete(item) // Remove to re-add at end - this.items.set(item, Date.now()) - return - } - - // Evict oldest if at capacity - if (this.items.size >= this.maxSize) { - const oldest = this.items.entries().next().value - if (oldest) { - this.items.delete(oldest[0]) - } - } - - this.items.set(item, Date.now()) - } - - /** - * Check if item exists - */ - has(item: T): boolean { - return this.items.has(item) - } - - /** - * Get all items - */ - getAll(): T[] { - return Array.from(this.items.keys()) - } - - /** - * Get size - */ - get size(): number { - return this.items.size - } - - /** - * Clear all items - */ - clear(): void { - this.items.clear() - } -} \ No newline at end of file diff --git a/src/utils/crypto.ts b/src/utils/crypto.ts deleted file mode 100644 index f74c9fc5..00000000 --- a/src/utils/crypto.ts +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Cross-platform crypto utilities - * Provides hashing functions that work in both Node.js and browser environments - */ - -/** - * Simple string hash function that works in all environments - * Uses djb2 algorithm - fast and good distribution - * @param str - String to hash - * @returns Positive integer hash - */ -export function hashString(str: string): number { - let hash = 5381 - for (let i = 0; i < str.length; i++) { - const char = str.charCodeAt(i) - hash = ((hash << 5) + hash) + char // hash * 33 + char - } - // Ensure positive number - return Math.abs(hash) -} - -/** - * Alternative: FNV-1a hash algorithm - * Good distribution and fast - * @param str - String to hash - * @returns Positive integer hash - */ -export function fnv1aHash(str: string): number { - let hash = 2166136261 - for (let i = 0; i < str.length; i++) { - hash ^= str.charCodeAt(i) - hash = (hash * 16777619) >>> 0 - } - return hash -} - -/** - * Generate a deterministic hash for partitioning - * Uses the most appropriate algorithm for the environment - * @param input - Input string to hash - * @returns Positive integer hash suitable for modulo operations - */ -export function getPartitionHash(input: string): number { - // Use djb2 by default as it's fast and has good distribution - // This ensures consistent partitioning across all environments - return hashString(input) -} \ No newline at end of file diff --git a/src/utils/deletedItemsIndex.ts b/src/utils/deletedItemsIndex.ts deleted file mode 100644 index 3f497f40..00000000 --- a/src/utils/deletedItemsIndex.ts +++ /dev/null @@ -1,106 +0,0 @@ -/** - * Dedicated index for tracking soft-deleted items - * This is MUCH more efficient than checking every item in the database - * - * Performance characteristics: - * - Add deleted item: O(1) - * - Remove deleted item: O(1) - * - Check if deleted: O(1) - * - Get all deleted: O(d) where d = number of deleted items << total items - */ - -export class DeletedItemsIndex { - private deletedIds: Set = new Set() - private deletedCount: number = 0 - - /** - * Mark an item as deleted - */ - markDeleted(id: string): void { - if (!this.deletedIds.has(id)) { - this.deletedIds.add(id) - this.deletedCount++ - } - } - - /** - * Mark an item as not deleted (restored) - */ - markRestored(id: string): void { - if (this.deletedIds.delete(id)) { - this.deletedCount-- - } - } - - /** - * Check if an item is deleted - O(1) - */ - isDeleted(id: string): boolean { - return this.deletedIds.has(id) - } - - /** - * Get all deleted item IDs - O(d) - */ - getAllDeleted(): string[] { - return Array.from(this.deletedIds) - } - - /** - * Filter out deleted items from results - O(k) where k = result count - */ - filterDeleted(items: T[]): T[] { - if (this.deletedCount === 0) { - // Fast path - no deleted items - return items - } - - return items.filter(item => { - const id = item.id - return id ? !this.deletedIds.has(id) : true - }) - } - - /** - * Get statistics - */ - getStats() { - return { - deletedCount: this.deletedCount, - memoryUsage: this.deletedCount * 100 // Rough estimate: 100 bytes per ID - } - } - - /** - * Clear all deleted items (for testing) - */ - clear(): void { - this.deletedIds.clear() - this.deletedCount = 0 - } - - /** - * Serialize for persistence - */ - serialize(): string { - return JSON.stringify(Array.from(this.deletedIds)) - } - - /** - * Deserialize from persistence - */ - deserialize(data: string): void { - try { - const ids = JSON.parse(data) - this.deletedIds = new Set(ids) - this.deletedCount = this.deletedIds.size - } catch (e) { - console.warn('Failed to deserialize deleted items index') - } - } -} - -/** - * Global singleton for deleted items tracking - */ -export const deletedItemsIndex = new DeletedItemsIndex() \ No newline at end of file diff --git a/src/utils/ensureDeleted.ts b/src/utils/ensureDeleted.ts deleted file mode 100644 index 6d9ddce7..00000000 --- a/src/utils/ensureDeleted.ts +++ /dev/null @@ -1,88 +0,0 @@ -/** - * Utility to ensure all metadata has the deleted field set properly - * This is CRITICAL for O(1) soft delete filtering performance - * - * Uses _brainy namespace to avoid conflicts with user metadata - */ - -const BRAINY_NAMESPACE = '_brainy' - -/** - * Ensure metadata has internal Brainy fields set - * @param metadata The metadata object (could be null/undefined) - * @param preserveExisting If true, preserve existing deleted value - * @returns Metadata with internal fields guaranteed - */ -export function ensureDeletedField(metadata: any, preserveExisting: boolean = true): any { - // Handle null/undefined metadata - if (!metadata) { - return { - [BRAINY_NAMESPACE]: { - deleted: false, - version: 1 - } - } - } - - // Clone to avoid mutation - const result = { ...metadata } - - // Ensure _brainy namespace exists - if (!result[BRAINY_NAMESPACE]) { - result[BRAINY_NAMESPACE] = {} - } - - // Set deleted field if not present - if (!('deleted' in result[BRAINY_NAMESPACE])) { - result[BRAINY_NAMESPACE].deleted = false - } else if (!preserveExisting) { - // Force to false if not preserving - result[BRAINY_NAMESPACE].deleted = false - } - - return result -} - -/** - * Mark an item as soft deleted - * @param metadata The metadata object - * @returns Metadata with _brainy.deleted=true - */ -export function markAsDeleted(metadata: any): any { - const result = ensureDeletedField(metadata) - result[BRAINY_NAMESPACE].deleted = true - return result -} - -/** - * Mark an item as restored (not deleted) - * @param metadata The metadata object - * @returns Metadata with _brainy.deleted=false - */ -export function markAsRestored(metadata: any): any { - const result = ensureDeletedField(metadata) - result[BRAINY_NAMESPACE].deleted = false - return result -} - -/** - * Check if an item is deleted - * @param metadata The metadata object - * @returns true if deleted, false otherwise (including if field missing) - */ -export function isDeleted(metadata: any): boolean { - return metadata?.[BRAINY_NAMESPACE]?.deleted === true -} - -/** - * Check if an item is active (not deleted) - * @param metadata The metadata object - * @returns true if not deleted (default), false if deleted - */ -export function isActive(metadata: any): boolean { - // If no deleted field or deleted=false, item is active - return !isDeleted(metadata) -} - -// Export the namespace constant for use in queries -export const BRAINY_DELETED_FIELD = `${BRAINY_NAMESPACE}.deleted` \ No newline at end of file diff --git a/src/utils/metadataNamespace.ts b/src/utils/metadataNamespace.ts deleted file mode 100644 index e14af1df..00000000 --- a/src/utils/metadataNamespace.ts +++ /dev/null @@ -1,255 +0,0 @@ -/** - * Clean Metadata Architecture for Brainy 2.2 - * No backward compatibility - doing it RIGHT from the start! - */ - -// Namespace constants -export const BRAINY_NS = '_brainy' as const -export const AUG_NS = '_augmentations' as const -export const AUDIT_NS = '_audit' as const - -// Field paths for O(1) indexing -export const DELETED_FIELD = `${BRAINY_NS}.deleted` as const -export const INDEXED_FIELD = `${BRAINY_NS}.indexed` as const -export const VERSION_FIELD = `${BRAINY_NS}.version` as const - -/** - * Internal Brainy metadata structure - * These fields are ALWAYS present and indexed for O(1) access - */ -export interface BrainyInternalMetadata { - deleted: boolean // ALWAYS boolean, enables O(1) soft delete - indexed: boolean // Whether in search index - version: number // Schema version - created: number // Unix timestamp - updated: number // Unix timestamp - - // Optional internal fields - domain?: string // Domain classification - priority?: number // Query priority hint - ttl?: number // Time to live -} - -/** - * Complete metadata structure with namespaces - */ -export interface NamespacedMetadata { - // User metadata - any fields they want - [key: string]: any - - // Internal metadata - our fields - [BRAINY_NS]: BrainyInternalMetadata - - // Augmentation metadata - isolated per augmentation - [AUG_NS]?: { - [augmentationName: string]: any - } - - // Audit trail - optional - [AUDIT_NS]?: Array<{ - timestamp: number - augmentation: string - field: string - oldValue: any - newValue: any - }> -} - -/** - * Create properly namespaced metadata - * This is called for EVERY noun/verb creation - */ -export function createNamespacedMetadata( - userMetadata?: T -): NamespacedMetadata { - const now = Date.now() - - // Start with user metadata or empty object - const result: any = userMetadata ? { ...userMetadata } : {} - - // ALWAYS add internal namespace with required fields - result[BRAINY_NS] = { - deleted: false, // CRITICAL: Always false for new items - indexed: true, // New items are indexed - version: 1, // Current schema version - created: now, - updated: now - } - - return result -} - -/** - * Update metadata while preserving namespaces - */ -export function updateNamespacedMetadata( - existing: NamespacedMetadata, - updates: Partial -): NamespacedMetadata { - const now = Date.now() - - // Merge user fields - const result: any = { - ...existing, - ...updates - } - - // Preserve internal namespace but update timestamp - result[BRAINY_NS] = { - ...existing[BRAINY_NS], - updated: now - } - - // Preserve augmentation namespace - if (existing[AUG_NS]) { - result[AUG_NS] = existing[AUG_NS] - } - - // Preserve audit trail - if (existing[AUDIT_NS]) { - result[AUDIT_NS] = existing[AUDIT_NS] - } - - return result -} - -/** - * Soft delete a noun (O(1) operation) - */ -export function markDeleted( - metadata: NamespacedMetadata -): NamespacedMetadata { - return { - ...metadata, - [BRAINY_NS]: { - ...metadata[BRAINY_NS], - deleted: true, - updated: Date.now() - } - } -} - -/** - * Restore a soft-deleted noun (O(1) operation) - */ -export function markRestored( - metadata: NamespacedMetadata -): NamespacedMetadata { - return { - ...metadata, - [BRAINY_NS]: { - ...metadata[BRAINY_NS], - deleted: false, - updated: Date.now() - } - } -} - -/** - * Check if a noun is deleted (O(1) check) - */ -export function isDeleted( - metadata: NamespacedMetadata -): boolean { - return metadata[BRAINY_NS]?.deleted === true -} - -/** - * Get user metadata without internal fields - * Used by augmentations to get clean user data - */ -export function getUserMetadata( - metadata: NamespacedMetadata -): T { - const { [BRAINY_NS]: _, [AUG_NS]: __, [AUDIT_NS]: ___, ...userMeta } = metadata - return userMeta as T -} - -/** - * Set augmentation data in isolated namespace - */ -export function setAugmentationData( - metadata: NamespacedMetadata, - augmentationName: string, - data: any -): NamespacedMetadata { - const result = { ...metadata } - - if (!result[AUG_NS]) { - result[AUG_NS] = {} - } - - result[AUG_NS][augmentationName] = data - - return result -} - -/** - * Add audit entry for tracking - */ -export function addAuditEntry( - metadata: NamespacedMetadata, - entry: { - augmentation: string - field: string - oldValue: any - newValue: any - } -): NamespacedMetadata { - const result = { ...metadata } - - if (!result[AUDIT_NS]) { - result[AUDIT_NS] = [] - } - - result[AUDIT_NS].push({ - ...entry, - timestamp: Date.now() - }) - - return result -} - -/** - * INDEXING EXPLANATION: - * - * The MetadataIndex flattens nested objects into dot-notation keys: - * - * Input metadata: - * { - * name: "Django", - * _brainy: { - * deleted: false, - * indexed: true - * } - * } - * - * Creates index entries: - * - "name" -> "django" -> Set([id1, id2...]) - * - "_brainy.deleted" -> "false" -> Set([id1, id2...]) // O(1) lookup! - * - "_brainy.indexed" -> "true" -> Set([id1, id2...]) - * - * Query: { "_brainy.deleted": false } - * Lookup: index["_brainy.deleted"]["false"] -> Set of IDs in O(1) - * - * This is why namespacing doesn't hurt performance - it's all flattened! - */ - -/** - * Fields that should ALWAYS be indexed for O(1) access - */ -export const ALWAYS_INDEXED_FIELDS = [ - DELETED_FIELD, // For soft delete filtering - INDEXED_FIELD, // For index management - VERSION_FIELD // For schema versioning -] - -/** - * Fields that should use sorted index for O(log n) range queries - */ -export const SORTED_INDEX_FIELDS = [ - `${BRAINY_NS}.created`, - `${BRAINY_NS}.updated`, - `${BRAINY_NS}.priority`, - `${BRAINY_NS}.ttl` -] \ No newline at end of file diff --git a/src/utils/structuredLogger.ts b/src/utils/structuredLogger.ts deleted file mode 100644 index a5baa039..00000000 --- a/src/utils/structuredLogger.ts +++ /dev/null @@ -1,545 +0,0 @@ -/** - * Enhanced Structured Logging System for Brainy - * Provides production-ready logging with structured output, context preservation, - * performance tracking, and multiple transport support - */ - -import { performance } from 'perf_hooks' -import { randomUUID } from '../universal/crypto.js' - -export enum LogLevel { - SILENT = -1, - FATAL = 0, - ERROR = 1, - WARN = 2, - INFO = 3, - DEBUG = 4, - TRACE = 5 -} - -export interface LogContext { - requestId?: string - userId?: string - operation?: string - entityId?: string - entityType?: string - [key: string]: any -} - -export interface LogEntry { - timestamp: string - level: string - levelNumeric: number - module: string - message: string - context?: LogContext - data?: any - error?: { - name: string - message: string - stack?: string - code?: string - } - performance?: { - duration?: number - memory?: { - used: number - total: number - } - } - host?: string - pid: number - version?: string -} - -export interface LogTransport { - name: string - log(entry: LogEntry): void | Promise - flush?(): Promise -} - -export interface StructuredLoggerConfig { - level: LogLevel - modules?: Record - format: 'json' | 'pretty' | 'simple' - transports: LogTransport[] - context?: LogContext - includeHost?: boolean - includeMemory?: boolean - bufferSize?: number - flushInterval?: number - version?: string -} - -class ConsoleTransport implements LogTransport { - name = 'console' - private format: 'json' | 'pretty' | 'simple' - - constructor(format: 'json' | 'pretty' | 'simple' = 'json') { - this.format = format - } - - log(entry: LogEntry): void { - const method = this.getConsoleMethod(entry.levelNumeric) - - if (this.format === 'json') { - method(JSON.stringify(entry)) - } else if (this.format === 'pretty') { - const color = this.getColor(entry.levelNumeric) - const prefix = `${entry.timestamp} ${color}[${entry.level}]\\x1b[0m [${entry.module}]` - const message = entry.message - - if (entry.error) { - method(`${prefix} ${message}`, entry.error) - } else if (entry.data) { - method(`${prefix} ${message}`, entry.data) - } else { - method(`${prefix} ${message}`) - } - } else { - // Simple format - method(`[${entry.level}] ${entry.message}`) - } - } - - private getConsoleMethod(level: number): (...args: any[]) => void { - switch (level) { - case LogLevel.FATAL: - case LogLevel.ERROR: - return console.error - case LogLevel.WARN: - return console.warn - case LogLevel.INFO: - return console.info - default: - return console.log - } - } - - private getColor(level: number): string { - switch (level) { - case LogLevel.FATAL: - return '\\x1b[35m' // Magenta - case LogLevel.ERROR: - return '\\x1b[31m' // Red - case LogLevel.WARN: - return '\\x1b[33m' // Yellow - case LogLevel.INFO: - return '\\x1b[36m' // Cyan - case LogLevel.DEBUG: - return '\\x1b[32m' // Green - case LogLevel.TRACE: - return '\\x1b[90m' // Gray - default: - return '\\x1b[0m' // Reset - } - } -} - -class BufferedTransport implements LogTransport { - name = 'buffered' - private buffer: LogEntry[] = [] - private innerTransport: LogTransport - private bufferSize: number - private flushTimer?: NodeJS.Timeout - - constructor(innerTransport: LogTransport, bufferSize: number = 100, flushInterval: number = 5000) { - this.innerTransport = innerTransport - this.bufferSize = bufferSize - - if (flushInterval > 0) { - this.flushTimer = setInterval(() => this.flush(), flushInterval) - } - } - - log(entry: LogEntry): void { - this.buffer.push(entry) - - if (this.buffer.length >= this.bufferSize) { - this.flush() - } - } - - async flush(): Promise { - const entries = this.buffer.splice(0) - for (const entry of entries) { - await this.innerTransport.log(entry) - } - - if (this.innerTransport.flush) { - await this.innerTransport.flush() - } - } - - destroy(): void { - if (this.flushTimer) { - clearInterval(this.flushTimer) - } - this.flush() - } -} - -export class StructuredLogger { - private static instance: StructuredLogger - private config: StructuredLoggerConfig - private defaultContext: LogContext = {} - private performanceMarks = new Map() - - private constructor() { - const isDevelopment = process.env.NODE_ENV !== 'production' - const format = isDevelopment ? 'pretty' : 'json' - - this.config = { - level: isDevelopment ? LogLevel.DEBUG : LogLevel.INFO, - format, - transports: [new ConsoleTransport(format)], - includeHost: !isDevelopment, - includeMemory: false, - bufferSize: 100, - flushInterval: 5000, - version: process.env.npm_package_version - } - - // Load from environment - this.loadEnvironmentConfig() - } - - private loadEnvironmentConfig(): void { - const envLevel = process.env.BRAINY_LOG_LEVEL - if (envLevel) { - const level = LogLevel[envLevel.toUpperCase() as keyof typeof LogLevel] - if (level !== undefined) { - this.config.level = level - } - } - - const envFormat = process.env.BRAINY_LOG_FORMAT - if (envFormat && ['json', 'pretty', 'simple'].includes(envFormat)) { - this.config.format = envFormat as 'json' | 'pretty' | 'simple' - } - - const moduleConfig = process.env.BRAINY_MODULE_LOG_LEVELS - if (moduleConfig) { - try { - this.config.modules = JSON.parse(moduleConfig) - } catch { - // Ignore parse errors - } - } - } - - static getInstance(): StructuredLogger { - if (!StructuredLogger.instance) { - StructuredLogger.instance = new StructuredLogger() - } - return StructuredLogger.instance - } - - configure(config: Partial): void { - this.config = { ...this.config, ...config } - } - - setContext(context: LogContext): void { - this.defaultContext = { ...this.defaultContext, ...context } - } - - clearContext(): void { - this.defaultContext = {} - } - - withContext(context: LogContext): StructuredLogger { - const contextualLogger = Object.create(this) - contextualLogger.defaultContext = { ...this.defaultContext, ...context } - return contextualLogger - } - - startTimer(label: string): void { - this.performanceMarks.set(label, performance.now()) - } - - endTimer(label: string): number | undefined { - const start = this.performanceMarks.get(label) - if (start === undefined) return undefined - - const duration = performance.now() - start - this.performanceMarks.delete(label) - return duration - } - - private shouldLog(level: LogLevel, module: string): boolean { - if (this.config.modules?.[module] !== undefined) { - return level <= this.config.modules[module] - } - return level <= this.config.level - } - - private createLogEntry( - level: LogLevel, - module: string, - message: string, - context?: LogContext, - data?: any, - error?: Error - ): LogEntry { - const entry: LogEntry = { - timestamp: new Date().toISOString(), - level: LogLevel[level], - levelNumeric: level, - module, - message, - pid: process.pid, - version: this.config.version - } - - // Merge contexts - const mergedContext = { ...this.defaultContext, ...context } - if (Object.keys(mergedContext).length > 0) { - entry.context = mergedContext - } - - if (data !== undefined) { - entry.data = data - } - - if (error) { - entry.error = { - name: error.name, - message: error.message, - stack: error.stack, - code: (error as Error & { code?: string }).code - } - } - - if (this.config.includeHost) { - try { - const os = require('node:os') - entry.host = os.hostname() - } catch { - entry.host = 'unknown' - } - } - - if (this.config.includeMemory && typeof process !== 'undefined' && process.memoryUsage) { - const mem = process.memoryUsage() - entry.performance = { - memory: { - used: Math.round(mem.heapUsed / 1024 / 1024), - total: Math.round(mem.heapTotal / 1024 / 1024) - } - } - } - - return entry - } - - private log( - level: LogLevel, - module: string, - message: string, - contextOrData?: LogContext | any, - data?: any - ): void { - if (!this.shouldLog(level, module)) { - return - } - - // Handle overloaded parameters - let context: LogContext | undefined - let logData: any - - if (contextOrData && typeof contextOrData === 'object' && !Array.isArray(contextOrData)) { - // Check if it looks like a context object - const hasContextKeys = ['requestId', 'userId', 'operation', 'entityId', 'entityType'] - .some(key => key in contextOrData) - - if (hasContextKeys) { - context = contextOrData - logData = data - } else { - logData = contextOrData - } - } else { - logData = contextOrData - } - - // Extract error if present - let error: Error | undefined - if (logData instanceof Error) { - error = logData - logData = undefined - } else if (logData?.error instanceof Error) { - error = logData.error - delete logData.error - } - - const entry = this.createLogEntry(level, module, message, context, logData, error) - - // Send to all transports - for (const transport of this.config.transports) { - try { - transport.log(entry) - } catch (err) { - // Fallback to console.error if transport fails - console.error('Logger transport error:', err) - } - } - } - - fatal(module: string, message: string, contextOrData?: LogContext | any, data?: any): void { - this.log(LogLevel.FATAL, module, message, contextOrData, data) - } - - error(module: string, message: string, contextOrData?: LogContext | any, data?: any): void { - this.log(LogLevel.ERROR, module, message, contextOrData, data) - } - - warn(module: string, message: string, contextOrData?: LogContext | any, data?: any): void { - this.log(LogLevel.WARN, module, message, contextOrData, data) - } - - info(module: string, message: string, contextOrData?: LogContext | any, data?: any): void { - this.log(LogLevel.INFO, module, message, contextOrData, data) - } - - debug(module: string, message: string, contextOrData?: LogContext | any, data?: any): void { - this.log(LogLevel.DEBUG, module, message, contextOrData, data) - } - - trace(module: string, message: string, contextOrData?: LogContext | any, data?: any): void { - this.log(LogLevel.TRACE, module, message, contextOrData, data) - } - - createModuleLogger(module: string) { - const self = this - return { - fatal: (message: string, contextOrData?: LogContext | any, data?: any) => - self.fatal(module, message, contextOrData, data), - error: (message: string, contextOrData?: LogContext | any, data?: any) => - self.error(module, message, contextOrData, data), - warn: (message: string, contextOrData?: LogContext | any, data?: any) => - self.warn(module, message, contextOrData, data), - info: (message: string, contextOrData?: LogContext | any, data?: any) => - self.info(module, message, contextOrData, data), - debug: (message: string, contextOrData?: LogContext | any, data?: any) => - self.debug(module, message, contextOrData, data), - trace: (message: string, contextOrData?: LogContext | any, data?: any) => - self.trace(module, message, contextOrData, data), - withContext: (context: LogContext) => { - const contextual = self.withContext(context) - return contextual.createModuleLogger(module) - }, - startTimer: (label: string) => self.startTimer(`${module}:${label}`), - endTimer: (label: string) => self.endTimer(`${module}:${label}`) - } - } - - async flush(): Promise { - const flushPromises = this.config.transports - .filter(t => t.flush) - .map(t => t.flush!()) - - await Promise.all(flushPromises) - } - - addTransport(transport: LogTransport): void { - this.config.transports.push(transport) - } - - removeTransport(name: string): void { - this.config.transports = this.config.transports.filter(t => t.name !== name) - } - - child(context: LogContext): StructuredLogger { - return this.withContext(context) - } -} - -// Singleton instance -export const structuredLogger = StructuredLogger.getInstance() - -// Convenience functions -export function createModuleLogger(module: string) { - return structuredLogger.createModuleLogger(module) -} - -export function setLogContext(context: LogContext) { - structuredLogger.setContext(context) -} - -export function withLogContext(context: LogContext) { - return structuredLogger.withContext(context) -} - -// Correlation ID middleware helper -export function createCorrelationId(): string { - return randomUUID() -} - -// Performance logging helper -export function logPerformance( - logger: ReturnType, - operation: string, - fn: () => any -): any { - logger.startTimer(operation) - try { - const result = fn() - if (result && typeof result.then === 'function') { - return result.finally(() => { - const duration = logger.endTimer(operation) - logger.debug(`${operation} completed`, { duration }) - }) - } - const duration = logger.endTimer(operation) - logger.debug(`${operation} completed`, { duration }) - return result - } catch (error) { - const duration = logger.endTimer(operation) - logger.error(`${operation} failed`, { duration, error }) - throw error - } -} - -// Backward compatibility wrapper for existing logger -export class LoggerCompatibilityWrapper { - private moduleLogger: ReturnType - - constructor(module: string = 'legacy') { - this.moduleLogger = createModuleLogger(module) - } - - error(module: string, message: string, ...args: any[]): void { - this.moduleLogger.error(message, { module, data: args }) - } - - warn(module: string, message: string, ...args: any[]): void { - this.moduleLogger.warn(message, { module, data: args }) - } - - info(module: string, message: string, ...args: any[]): void { - this.moduleLogger.info(message, { module, data: args }) - } - - debug(module: string, message: string, ...args: any[]): void { - this.moduleLogger.debug(message, { module, data: args }) - } - - trace(module: string, message: string, ...args: any[]): void { - this.moduleLogger.trace(message, { module, data: args }) - } - - createModuleLogger(module: string) { - const logger = createModuleLogger(module) - return { - error: (message: string, ...args: any[]) => logger.error(message, { data: args }), - warn: (message: string, ...args: any[]) => logger.warn(message, { data: args }), - info: (message: string, ...args: any[]) => logger.info(message, { data: args }), - debug: (message: string, ...args: any[]) => logger.debug(message, { data: args }), - trace: (message: string, ...args: any[]) => logger.trace(message, { data: args }) - } - } -} - -// Export types for external use -export type ModuleLogger = ReturnType -// Types are already exported above, no need to re-export \ No newline at end of file diff --git a/src/utils/writeBuffer.ts b/src/utils/writeBuffer.ts deleted file mode 100644 index b62f98d8..00000000 --- a/src/utils/writeBuffer.ts +++ /dev/null @@ -1,411 +0,0 @@ -/** - * Write Buffer - * Accumulates writes and flushes them in bulk to reduce S3 operations - * Implements intelligent deduplication and compression - */ - -import { HNSWNoun, HNSWVerb } from '../coreTypes.js' -import { createModuleLogger } from './logger.js' -import { getGlobalBackpressure } from './adaptiveBackpressure.js' - -interface BufferedWrite { - id: string - data: T - timestamp: number - type: 'noun' | 'verb' | 'metadata' - retryCount: number -} - -interface FlushResult { - successful: number - failed: number - duration: number -} - -/** - * High-performance write buffer for bulk operations - */ -export class WriteBuffer { - private logger = createModuleLogger('WriteBuffer') - - // Buffer storage - private buffer = new Map>() - - // Configuration - More aggressive for high volume - private maxBufferSize = 2000 // Allow larger buffers - private flushInterval = 500 // Flush more frequently (0.5 seconds) - private minFlushSize = 50 // Lower minimum to flush sooner - private maxRetries = 3 // Maximum retry attempts - - // State - private flushTimer: NodeJS.Timeout | null = null - private isFlushing = false - private lastFlush = Date.now() - private pendingFlush: Promise | null = null - - // Statistics - private totalWrites = 0 - private totalFlushes = 0 - private failedWrites = 0 - private duplicatesRemoved = 0 - - // Write function - private writeFunction: (items: Map) => Promise - private type: 'noun' | 'verb' | 'metadata' - - // Backpressure integration - private backpressure = getGlobalBackpressure() - - constructor( - type: 'noun' | 'verb' | 'metadata', - writeFunction: (items: Map) => Promise, - options?: { - maxBufferSize?: number - flushInterval?: number - minFlushSize?: number - } - ) { - this.type = type - this.writeFunction = writeFunction - - if (options) { - this.maxBufferSize = options.maxBufferSize || this.maxBufferSize - this.flushInterval = options.flushInterval || this.flushInterval - this.minFlushSize = options.minFlushSize || this.minFlushSize - } - - // Start periodic flush - this.startPeriodicFlush() - } - - /** - * Add item to buffer - */ - public async add(id: string, data: T): Promise { - // Check if we're already at capacity - if (this.buffer.size >= this.maxBufferSize) { - // Wait for current flush to complete - if (this.pendingFlush) { - await this.pendingFlush - } - - // Force flush if still at capacity - if (this.buffer.size >= this.maxBufferSize) { - await this.flush('capacity') - } - } - - // Check for duplicate and update if newer - const existing = this.buffer.get(id) - if (existing) { - // Update with newer data - existing.data = data - existing.timestamp = Date.now() - this.duplicatesRemoved++ - } else { - // Add new item - this.buffer.set(id, { - id, - data, - timestamp: Date.now(), - type: this.type, - retryCount: 0 - }) - } - - this.totalWrites++ - - // Log buffer growth periodically - if (this.totalWrites % 100 === 0) { - this.logger.info(`๐Ÿ“ˆ BUFFER GROWTH: ${this.buffer.size} ${this.type} items buffered (${this.totalWrites} total writes, ${this.duplicatesRemoved} deduplicated)`) - } - - // Check if we should flush - this.checkFlush() - } - - /** - * Check if we should flush - */ - private checkFlush(): void { - const bufferSize = this.buffer.size - const timeSinceFlush = Date.now() - this.lastFlush - - // Immediate flush conditions - if (bufferSize >= this.maxBufferSize) { - this.flush('size') - return - } - - // Time-based flush with minimum size - if (timeSinceFlush >= this.flushInterval && bufferSize >= this.minFlushSize) { - this.flush('time') - return - } - - // Adaptive flush based on system load - const backpressureStatus = this.backpressure.getStatus() - if (backpressureStatus.queueLength > 1000 && bufferSize > 10) { - // System under pressure - flush smaller batches more frequently - this.flush('pressure') - } - } - - /** - * Flush buffer to storage - */ - public async flush(reason: string = 'manual'): Promise { - // Prevent concurrent flushes - if (this.isFlushing) { - if (this.pendingFlush) { - return this.pendingFlush - } - return { successful: 0, failed: 0, duration: 0 } - } - - // Nothing to flush - if (this.buffer.size === 0) { - return { successful: 0, failed: 0, duration: 0 } - } - - this.isFlushing = true - const startTime = Date.now() - - // Create flush promise - this.pendingFlush = this.doFlush(reason, startTime) - - try { - const result = await this.pendingFlush - return result - } finally { - this.isFlushing = false - this.pendingFlush = null - } - } - - /** - * Perform the actual flush - */ - private async doFlush(reason: string, startTime: number): Promise { - const itemsToFlush = new Map() - const flushingItems = new Map>() - - // Take items from buffer - let count = 0 - for (const [id, item] of this.buffer.entries()) { - itemsToFlush.set(id, item.data) - flushingItems.set(id, item) - count++ - - // Limit batch size for better performance - if (count >= 500) { - break - } - } - - // Remove from buffer - for (const id of itemsToFlush.keys()) { - this.buffer.delete(id) - } - - this.logger.warn(`๐Ÿ”„ BUFFERING: Flushing ${itemsToFlush.size} ${this.type} items (buffer had ${this.buffer.size + itemsToFlush.size}) - reason: ${reason}`) - - try { - // Request permission from backpressure system - const opId = `flush-${Date.now()}` - await this.backpressure.requestPermission(opId, 2) // Higher priority - - try { - // Perform bulk write - await this.writeFunction(itemsToFlush) - - // Success - this.backpressure.releasePermission(opId, true) - this.totalFlushes++ - this.lastFlush = Date.now() - - const duration = Date.now() - startTime - this.logger.warn(`๐Ÿš€ BATCH FLUSH: ${itemsToFlush.size} ${this.type} items โ†’ 1 bulk S3 operation (${duration}ms, reason: ${reason})`) - - return { - successful: itemsToFlush.size, - failed: 0, - duration - } - } catch (error) { - // Release with error - this.backpressure.releasePermission(opId, false) - throw error - } - } catch (error) { - this.logger.error(`Flush failed: ${error}`) - - // Put items back with retry count - for (const [id, item] of flushingItems.entries()) { - item.retryCount++ - - if (item.retryCount < this.maxRetries) { - // Put back for retry - this.buffer.set(id, item) - } else { - // Max retries exceeded - this.failedWrites++ - this.logger.error(`Max retries exceeded for ${this.type} ${id}`) - } - } - - const duration = Date.now() - startTime - - return { - successful: 0, - failed: itemsToFlush.size, - duration - } - } - } - - /** - * Start periodic flush timer - */ - private startPeriodicFlush(): void { - if (this.flushTimer) { - return - } - - this.flushTimer = setInterval(() => { - if (this.buffer.size > 0) { - const timeSinceFlush = Date.now() - this.lastFlush - - // Flush if we have items and enough time has passed - if (timeSinceFlush >= this.flushInterval) { - this.flush('periodic').catch(error => { - this.logger.error('Periodic flush failed:', error) - }) - } - } - }, Math.min(100, this.flushInterval / 2)) - } - - /** - * Stop periodic flush timer - */ - public stop(): void { - if (this.flushTimer) { - clearInterval(this.flushTimer) - this.flushTimer = null - } - } - - /** - * Force flush all pending writes - */ - public async forceFlush(): Promise { - // Flush everything regardless of size - const oldMinSize = this.minFlushSize - this.minFlushSize = 0 - - try { - const result = await this.flush('force') - - // Flush any remaining items - while (this.buffer.size > 0) { - const additionalResult = await this.flush('force-remaining') - result.successful += additionalResult.successful - result.failed += additionalResult.failed - result.duration += additionalResult.duration - } - - return result - } finally { - this.minFlushSize = oldMinSize - } - } - - /** - * Get buffer statistics - */ - public getStats(): { - bufferSize: number - totalWrites: number - totalFlushes: number - failedWrites: number - duplicatesRemoved: number - avgFlushSize: number - } { - return { - bufferSize: this.buffer.size, - totalWrites: this.totalWrites, - totalFlushes: this.totalFlushes, - failedWrites: this.failedWrites, - duplicatesRemoved: this.duplicatesRemoved, - avgFlushSize: this.totalFlushes > 0 ? this.totalWrites / this.totalFlushes : 0 - } - } - - /** - * Adjust parameters based on load - */ - public adjustForLoad(pendingRequests: number): void { - if (pendingRequests > 10000) { - // Extreme load - buffer more aggressively - this.maxBufferSize = 5000 - this.flushInterval = 500 - this.minFlushSize = 500 - } else if (pendingRequests > 1000) { - // High load - this.maxBufferSize = 2000 - this.flushInterval = 1000 - this.minFlushSize = 200 - } else if (pendingRequests > 100) { - // Moderate load - this.maxBufferSize = 1000 - this.flushInterval = 2000 - this.minFlushSize = 100 - } else { - // Low load - optimize for latency - this.maxBufferSize = 500 - this.flushInterval = 5000 - this.minFlushSize = 50 - } - } -} - -// Global write buffers -const writeBuffers = new Map>() - -/** - * Get or create a write buffer - */ -export function getWriteBuffer( - id: string, - type: 'noun' | 'verb' | 'metadata', - writeFunction: (items: Map) => Promise -): WriteBuffer { - if (!writeBuffers.has(id)) { - writeBuffers.set(id, new WriteBuffer(type, writeFunction)) - } - return writeBuffers.get(id)! -} - -/** - * Flush all write buffers - */ -export async function flushAllBuffers(): Promise { - const promises: Promise[] = [] - - for (const buffer of writeBuffers.values()) { - promises.push(buffer.forceFlush()) - } - - await Promise.all(promises) -} - -/** - * Clear all write buffers - */ -export function clearWriteBuffers(): void { - for (const buffer of writeBuffers.values()) { - buffer.stop() - } - writeBuffers.clear() -} \ No newline at end of file diff --git a/src/vfs/FSCompat.ts b/src/vfs/FSCompat.ts deleted file mode 100644 index 4715aa79..00000000 --- a/src/vfs/FSCompat.ts +++ /dev/null @@ -1,333 +0,0 @@ -/** - * fs-Compatible Interface for VFS - * - * Provides a drop-in replacement for Node's fs module - * that uses VFS for storage instead of the real filesystem. - * - * Usage: - * import { FSCompat } from '@soulcraft/brainy/vfs' - * const fs = new FSCompat(brain.vfs) - * - * // Now use like Node's fs - * await fs.promises.readFile('/path') - * fs.createReadStream('/path').pipe(output) - */ - -import { VirtualFileSystem } from './VirtualFileSystem.js' -import type { - ReadOptions, - WriteOptions, - MkdirOptions, - VFSStats, - VFSDirent -} from './types.js' - -export class FSCompat { - /** - * Promise-based API (like fs.promises) - */ - public readonly promises: FSPromises - - constructor(private vfs: VirtualFileSystem) { - this.promises = new FSPromises(vfs) - } - - // ============= Callback-style methods (for compatibility) ============= - - readFile(path: string, callback: (err: Error | null, data?: Buffer) => void): void - readFile(path: string, encoding: BufferEncoding, callback: (err: Error | null, data?: string) => void): void - readFile(path: string, options: any, callback?: any): void { - if (typeof options === 'function') { - callback = options - options = {} - } - - this.vfs.readFile(path, options) - .then(data => { - if (options?.encoding) { - callback(null, data.toString(options.encoding)) - } else { - callback(null, data) - } - }) - .catch(err => callback(err)) - } - - writeFile(path: string, data: Buffer | string, callback: (err: Error | null) => void): void - writeFile(path: string, data: Buffer | string, options: any, callback?: any): void { - if (typeof options === 'function') { - callback = options - options = {} - } - - const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, options?.encoding || 'utf8') - - this.vfs.writeFile(path, buffer, options) - .then(() => callback(null)) - .catch(err => callback(err)) - } - - mkdir(path: string, callback: (err: Error | null) => void): void - mkdir(path: string, options: any, callback?: any): void { - if (typeof options === 'function') { - callback = options - options = {} - } - - this.vfs.mkdir(path, options) - .then(() => callback(null)) - .catch(err => callback(err)) - } - - rmdir(path: string, callback: (err: Error | null) => void): void - rmdir(path: string, options: any, callback?: any): void { - if (typeof options === 'function') { - callback = options - options = {} - } - - this.vfs.rmdir(path, options) - .then(() => callback(null)) - .catch(err => callback(err)) - } - - readdir(path: string, callback: (err: Error | null, files?: string[]) => void): void - readdir(path: string, options: any, callback?: any): void { - if (typeof options === 'function') { - callback = options - options = {} - } - - this.vfs.readdir(path, options) - .then(files => callback(null, files)) - .catch(err => callback(err)) - } - - stat(path: string, callback: (err: Error | null, stats?: VFSStats) => void): void { - this.vfs.stat(path) - .then(stats => callback(null, stats)) - .catch(err => callback(err)) - } - - lstat(path: string, callback: (err: Error | null, stats?: VFSStats) => void): void { - this.vfs.lstat(path) - .then(stats => callback(null, stats)) - .catch(err => callback(err)) - } - - unlink(path: string, callback: (err: Error | null) => void): void { - this.vfs.unlink(path) - .then(() => callback(null)) - .catch(err => callback(err)) - } - - rename(oldPath: string, newPath: string, callback: (err: Error | null) => void): void { - this.vfs.rename(oldPath, newPath) - .then(() => callback(null)) - .catch(err => callback(err)) - } - - copyFile(src: string, dest: string, callback: (err: Error | null) => void): void - copyFile(src: string, dest: string, flags: number, callback: (err: Error | null) => void): void - copyFile(src: string, dest: string, flagsOrCallback: any, callback?: any): void { - if (typeof flagsOrCallback === 'function') { - callback = flagsOrCallback - } else { - // flags provided but not used - } - - this.vfs.copy(src, dest) - .then(() => callback(null)) - .catch(err => callback(err)) - } - - exists(path: string, callback: (exists: boolean) => void): void { - this.vfs.exists(path) - .then(exists => callback(exists)) - .catch(() => callback(false)) - } - - access(path: string, callback: (err: Error | null) => void): void - access(path: string, mode: number, callback: (err: Error | null) => void): void - access(path: string, modeOrCallback: any, callback?: any): void { - if (typeof modeOrCallback === 'function') { - callback = modeOrCallback - } else { - // mode provided but not used - } - - this.vfs.exists(path) - .then(exists => { - if (exists) { - callback(null) - } else { - const err: any = new Error('ENOENT: no such file or directory') - err.code = 'ENOENT' - callback(err) - } - }) - .catch(err => callback(err)) - } - - appendFile(path: string, data: Buffer | string, callback: (err: Error | null) => void): void - appendFile(path: string, data: Buffer | string, options: any, callback?: any): void { - if (typeof options === 'function') { - callback = options - options = {} - } - - const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, options?.encoding || 'utf8') - - this.vfs.appendFile(path, buffer, options) - .then(() => callback(null)) - .catch(err => callback(err)) - } - - // ============= Stream methods ============= - - createReadStream(path: string, options?: any) { - return this.vfs.createReadStream(path, options) - } - - createWriteStream(path: string, options?: any) { - return this.vfs.createWriteStream(path, options) - } - - // ============= Watch methods ============= - - watch(path: string, listener?: any) { - return this.vfs.watch(path, listener) - } - - watchFile(path: string, listener: any) { - return this.vfs.watchFile(path, listener) - } - - unwatchFile(path: string) { - return this.vfs.unwatchFile(path) - } - - // ============= Additional methods ============= - - /** - * Import a directory from real filesystem (VFS extension) - */ - async importDirectory(sourcePath: string, options?: any) { - return this.vfs.importDirectory(sourcePath, options) - } - - /** - * Search files semantically (VFS extension) - */ - async search(query: string, options?: any) { - return this.vfs.search(query, options) - } -} - -/** - * Promise-based fs API (like fs.promises) - */ -class FSPromises { - constructor(private vfs: VirtualFileSystem) {} - - async readFile(path: string, options?: any): Promise { - const buffer = await this.vfs.readFile(path, options) - if (options?.encoding) { - return buffer.toString(options.encoding) - } - return buffer - } - - async writeFile(path: string, data: Buffer | string, options?: any): Promise { - const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, options?.encoding || 'utf8') - return this.vfs.writeFile(path, buffer, options) - } - - async mkdir(path: string, options?: any): Promise { - return this.vfs.mkdir(path, options) - } - - async rmdir(path: string, options?: any): Promise { - return this.vfs.rmdir(path, options) - } - - async readdir(path: string, options?: any): Promise { - return this.vfs.readdir(path, options) - } - - async stat(path: string): Promise { - return this.vfs.stat(path) - } - - async lstat(path: string): Promise { - return this.vfs.lstat(path) - } - - async unlink(path: string): Promise { - return this.vfs.unlink(path) - } - - async rename(oldPath: string, newPath: string): Promise { - return this.vfs.rename(oldPath, newPath) - } - - async copyFile(src: string, dest: string, flags?: number): Promise { - return this.vfs.copy(src, dest) - } - - async access(path: string, mode?: number): Promise { - const exists = await this.vfs.exists(path) - if (!exists) { - const err: any = new Error('ENOENT: no such file or directory') - err.code = 'ENOENT' - throw err - } - } - - async appendFile(path: string, data: Buffer | string, options?: any): Promise { - const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, options?.encoding || 'utf8') - return this.vfs.appendFile(path, buffer, options) - } - - async realpath(path: string): Promise { - return this.vfs.realpath(path) - } - - async chmod(path: string, mode: number): Promise { - return this.vfs.chmod(path, mode) - } - - async chown(path: string, uid: number, gid: number): Promise { - return this.vfs.chown(path, uid, gid) - } - - async utimes(path: string, atime: Date, mtime: Date): Promise { - return this.vfs.utimes(path, atime, mtime) - } - - async symlink(target: string, path: string): Promise { - return this.vfs.symlink(target, path) - } - - async readlink(path: string): Promise { - return this.vfs.readlink(path) - } - - // VFS Extensions - async search(query: string, options?: any) { - return this.vfs.search(query, options) - } - - async findSimilar(path: string, options?: any) { - return this.vfs.findSimilar(path, options) - } - - async importDirectory(sourcePath: string, options?: any) { - return this.vfs.importDirectory(sourcePath, options) - } -} - -// Export a convenience function to create fs replacement -export function createFS(vfs: VirtualFileSystem): FSCompat { - return new FSCompat(vfs) -} \ No newline at end of file diff --git a/src/vfs/index.ts b/src/vfs/index.ts deleted file mode 100644 index 16684214..00000000 --- a/src/vfs/index.ts +++ /dev/null @@ -1,27 +0,0 @@ -/** - * Brainy Virtual Filesystem - * - * A simplified fs-compatible filesystem that stores data in Brainy - * Works across all storage adapters and scales to millions of files - */ - -// Core VFS -export { VirtualFileSystem } from './VirtualFileSystem.js' -export { PathResolver } from './PathResolver.js' -export * from './types.js' - -// MIME Type Detection -export { MimeTypeDetector, mimeDetector } from './MimeTypeDetector.js' - -// fs compatibility layer -export { FSCompat, createFS } from './FSCompat.js' - -// Directory import -export { DirectoryImporter } from './importers/DirectoryImporter.js' - -// Streaming -export { VFSReadStream } from './streams/VFSReadStream.js' -export { VFSWriteStream } from './streams/VFSWriteStream.js' - -// Convenience alias -export { VirtualFileSystem as VFS } from './VirtualFileSystem.js' \ No newline at end of file diff --git a/tests/unit/import/InstancePool.test.ts b/tests/unit/import/InstancePool.test.ts deleted file mode 100644 index 740be5d7..00000000 --- a/tests/unit/import/InstancePool.test.ts +++ /dev/null @@ -1,317 +0,0 @@ -import { describe, it, expect, beforeEach } from 'vitest' -import { Brainy } from '../../../src/brainy.js' -import { InstancePool, createInstancePool } from '../../../src/import/InstancePool.js' - -describe('InstancePool', () => { - let brain: Brainy - let pool: InstancePool - - beforeEach(async () => { - brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) - await brain.init() - pool = new InstancePool(brain) - }) - - describe('lazy initialization', () => { - it('should not create instances until requested', () => { - const stats = pool.getStats() - expect(stats.nlpCreated).toBe(false) - expect(stats.extractorCreated).toBe(false) - }) - - it('should create NLP instance on first access', async () => { - const nlp = await pool.getNLP() - expect(nlp).toBeDefined() - - const stats = pool.getStats() - expect(stats.nlpCreated).toBe(true) - expect(stats.nlpReuses).toBe(1) - }) - - it('should create extractor instance on first access', () => { - const extractor = pool.getExtractor() - expect(extractor).toBeDefined() - - const stats = pool.getStats() - expect(stats.extractorCreated).toBe(true) - expect(stats.extractorReuses).toBe(1) - }) - }) - - describe('instance reuse', () => { - it('should return same NLP instance on multiple calls', async () => { - const nlp1 = await pool.getNLP() - const nlp2 = await pool.getNLP() - const nlp3 = await pool.getNLP() - - expect(nlp1).toBe(nlp2) - expect(nlp2).toBe(nlp3) - - const stats = pool.getStats() - expect(stats.nlpReuses).toBe(3) - }) - - it('should return same extractor instance on multiple calls', () => { - const extractor1 = pool.getExtractor() - const extractor2 = pool.getExtractor() - const extractor3 = pool.getExtractor() - - expect(extractor1).toBe(extractor2) - expect(extractor2).toBe(extractor3) - - const stats = pool.getStats() - expect(stats.extractorReuses).toBe(3) - }) - - it('should track reuse counts correctly', async () => { - await pool.getNLP() - await pool.getNLP() - pool.getExtractor() - pool.getExtractor() - pool.getExtractor() - - const stats = pool.getStats() - expect(stats.nlpReuses).toBe(2) - expect(stats.extractorReuses).toBe(3) - }) - }) - - describe('initialization', () => { - it('should initialize all instances with init()', async () => { - await pool.init() - - expect(pool.isInitialized()).toBe(true) - - const stats = pool.getStats() - expect(stats.nlpCreated).toBe(true) - expect(stats.extractorCreated).toBe(true) - expect(stats.initialized).toBe(true) - }) - - it('should handle concurrent init calls safely', async () => { - // Call init multiple times concurrently - const promises = [ - pool.init(), - pool.init(), - pool.init() - ] - - await Promise.all(promises) - - // Should only initialize once - expect(pool.isInitialized()).toBe(true) - }) - - it('should auto-initialize NLP when accessed', async () => { - const nlp = await pool.getNLP() - - // NLP is lazy-initialized but extractor might not be - const stats = pool.getStats() - expect(stats.nlpCreated).toBe(true) - expect(stats.initialized).toBe(true) - }) - - it('should provide sync access to NLP', () => { - const nlp = pool.getNLPSync() - expect(nlp).toBeDefined() - - const stats = pool.getStats() - expect(stats.nlpCreated).toBe(true) - }) - }) - - describe('statistics', () => { - it('should track creation time', async () => { - await pool.init() - - const stats = pool.getStats() - expect(stats.creationTime).toBeGreaterThanOrEqual(0) - }) - - it('should calculate memory saved', async () => { - // Use instances multiple times - await pool.getNLP() - await pool.getNLP() - await pool.getNLP() - pool.getExtractor() - pool.getExtractor() - - const stats = pool.getStats() - expect(stats.memorySaved).toBeGreaterThan(0) - }) - - it('should reset statistics', async () => { - await pool.getNLP() - pool.getExtractor() - - pool.resetStats() - - const stats = pool.getStats() - expect(stats.nlpReuses).toBe(0) - expect(stats.extractorReuses).toBe(0) - expect(stats.creationTime).toBe(0) - }) - - it('should provide string representation', async () => { - await pool.init() - - const str = pool.toString() - expect(str).toContain('InstancePool') - expect(str).toContain('nlp=true') - expect(str).toContain('extractor=true') - }) - }) - - describe('memory efficiency', () => { - it('should reuse instances in loop (no memory leak)', async () => { - const initialStats = pool.getStats() - - // Simulate import loop - for (let i = 0; i < 1000; i++) { - const nlp = await pool.getNLP() - const extractor = pool.getExtractor() - - // All iterations should get same instances - expect(nlp).toBeDefined() - expect(extractor).toBeDefined() - } - - const finalStats = pool.getStats() - expect(finalStats.nlpReuses).toBe(1000) - expect(finalStats.extractorReuses).toBe(1000) - - // Should have saved ~60GB of memory (1000 iterations ร— ~60MB) - expect(finalStats.memorySaved).toBeGreaterThan(50 * 1024 * 1024 * 1000) // > 50GB - }) - - it('should handle rapid concurrent access', async () => { - // Simulate concurrent row processing - const promises = [] - for (let i = 0; i < 100; i++) { - promises.push(pool.getNLP()) - promises.push(Promise.resolve(pool.getExtractor())) - } - - await Promise.all(promises) - - const stats = pool.getStats() - expect(stats.nlpReuses).toBe(100) - expect(stats.extractorReuses).toBe(100) - }) - }) - - describe('cleanup', () => { - it('should cleanup instances', async () => { - await pool.init() - expect(pool.isInitialized()).toBe(true) - - pool.cleanup() - - expect(pool.isInitialized()).toBe(false) - const stats = pool.getStats() - expect(stats.nlpCreated).toBe(false) - expect(stats.extractorCreated).toBe(false) - }) - - it('should allow reinitialization after cleanup', async () => { - await pool.init() - pool.cleanup() - - await pool.init() - expect(pool.isInitialized()).toBe(true) - }) - }) - - describe('factory function', () => { - it('should create pool with auto-init', async () => { - const newPool = await createInstancePool(brain, true) - - expect(newPool.isInitialized()).toBe(true) - }) - - it('should create pool without auto-init', async () => { - const newPool = await createInstancePool(brain, false) - - expect(newPool.isInitialized()).toBe(false) - }) - - it('should default to auto-init', async () => { - const newPool = await createInstancePool(brain) - - expect(newPool.isInitialized()).toBe(true) - }) - }) - - describe('error handling', () => { - it('should handle missing NLP instance gracefully', async () => { - const emptyPool = new InstancePool(brain) - - // Should create NLP on first access - const nlp = await emptyPool.getNLP() - expect(nlp).toBeDefined() - }) - - it('should handle missing extractor instance gracefully', () => { - const emptyPool = new InstancePool(brain) - - // Should create extractor on first access - const extractor = emptyPool.getExtractor() - expect(extractor).toBeDefined() - }) - }) - - describe('real-world usage', () => { - it('should work with actual NLP operations', async () => { - const nlp = await pool.getNLP() - - // Should be initialized and ready to use - expect(nlp).toBeDefined() - - // NLP should have init method - expect(typeof nlp.init).toBe('function') - }) - - it('should work with actual entity extraction', async () => { - const extractor = pool.getExtractor() - - // Should be ready to use - expect(extractor).toBeDefined() - - // Can call extractor methods - const entities = await extractor.extract('Paris is a beautiful city', { - confidence: 0.5 - }) - - expect(Array.isArray(entities)).toBe(true) - }) - - it('should handle full import workflow', async () => { - // Initialize pool - await pool.init() - - // Simulate processing multiple rows - const rows = [ - { text: 'Paris is beautiful' }, - { text: 'London is historic' }, - { text: 'Tokyo is modern' } - ] - - for (const row of rows) { - const nlp = await pool.getNLP() - const extractor = pool.getExtractor() - - // Process row - extract entities - const entities = await extractor.extract(row.text, { confidence: 0.5 }) - - expect(nlp).toBeDefined() - expect(extractor).toBeDefined() - expect(entities).toBeDefined() - } - - // Verify instances were reused - const stats = pool.getStats() - expect(stats.nlpReuses).toBe(3) - expect(stats.extractorReuses).toBe(3) - }) - }) -}) diff --git a/tests/unit/neural/presets.test.ts b/tests/unit/neural/presets.test.ts deleted file mode 100644 index d27add8f..00000000 --- a/tests/unit/neural/presets.test.ts +++ /dev/null @@ -1,561 +0,0 @@ -import { describe, it, expect } from 'vitest' -import { - autoDetectPreset, - getPreset, - getPresetNames, - explainPresetChoice, - createCustomPreset, - validatePreset, - formatPreset, - FAST_PRESET, - BALANCED_PRESET, - ACCURATE_PRESET, - EXPLICIT_PRESET, - PATTERN_PRESET, - PRESETS, - type ImportContext, - type PresetConfig -} from '../../../src/neural/presets.js' - -describe('Presets', () => { - describe('preset definitions', () => { - it('should have all 5 presets defined', () => { - expect(PRESETS).toBeDefined() - expect(Object.keys(PRESETS)).toHaveLength(5) - expect(PRESETS.fast).toBe(FAST_PRESET) - expect(PRESETS.balanced).toBe(BALANCED_PRESET) - expect(PRESETS.accurate).toBe(ACCURATE_PRESET) - expect(PRESETS.explicit).toBe(EXPLICIT_PRESET) - expect(PRESETS.pattern).toBe(PATTERN_PRESET) - }) - - it('should have valid fast preset', () => { - expect(FAST_PRESET.name).toBe('fast') - expect(FAST_PRESET.signals.enabled).toEqual(['exact', 'pattern']) - expect(FAST_PRESET.strategies.enabled).toEqual(['explicit']) - expect(FAST_PRESET.streaming).toBe(true) - expect(FAST_PRESET.strategies.earlyTermination).toBe(true) - }) - - it('should have valid balanced preset', () => { - expect(BALANCED_PRESET.name).toBe('balanced') - expect(BALANCED_PRESET.signals.enabled).toEqual(['exact', 'embedding', 'pattern']) - expect(BALANCED_PRESET.strategies.enabled).toEqual(['explicit', 'pattern', 'embedding']) - expect(BALANCED_PRESET.streaming).toBe(false) - }) - - it('should have valid accurate preset', () => { - expect(ACCURATE_PRESET.name).toBe('accurate') - expect(ACCURATE_PRESET.signals.enabled).toEqual(['exact', 'embedding', 'pattern', 'context']) - expect(ACCURATE_PRESET.strategies.enabled).toEqual(['explicit', 'pattern', 'embedding']) - expect(ACCURATE_PRESET.strategies.earlyTermination).toBe(false) - }) - - it('should have valid explicit preset', () => { - expect(EXPLICIT_PRESET.name).toBe('explicit') - expect(EXPLICIT_PRESET.signals.enabled).toEqual(['exact', 'pattern']) - expect(EXPLICIT_PRESET.strategies.enabled).toEqual(['explicit', 'pattern']) - expect(EXPLICIT_PRESET.strategies.minConfidence).toBeGreaterThanOrEqual(0.80) - }) - - it('should have valid pattern preset', () => { - expect(PATTERN_PRESET.name).toBe('pattern') - expect(PATTERN_PRESET.signals.enabled).toEqual(['embedding', 'pattern', 'context']) - expect(PATTERN_PRESET.strategies.enabled).toEqual(['pattern', 'embedding']) - }) - }) - - describe('autoDetectPreset', () => { - it('should return fast preset for large datasets', () => { - const context: ImportContext = { - rowCount: 15000, - fileSize: 5_000_000 - } - - const preset = autoDetectPreset(context) - expect(preset.name).toBe('fast') - }) - - it('should return fast preset for large files', () => { - const context: ImportContext = { - fileSize: 15_000_000 - } - - const preset = autoDetectPreset(context) - expect(preset.name).toBe('fast') - }) - - it('should return accurate preset for small datasets', () => { - const context: ImportContext = { - rowCount: 50 - } - - const preset = autoDetectPreset(context) - expect(preset.name).toBe('accurate') - }) - - it('should return explicit preset for Excel with explicit columns', () => { - const context: ImportContext = { - fileType: 'excel', - hasExplicitColumns: true, - rowCount: 500 - } - - const preset = autoDetectPreset(context) - expect(preset.name).toBe('explicit') - }) - - it('should return explicit preset for CSV with explicit columns', () => { - const context: ImportContext = { - fileType: 'csv', - hasExplicitColumns: true - } - - const preset = autoDetectPreset(context) - expect(preset.name).toBe('explicit') - }) - - it('should return pattern preset for PDF files', () => { - const context: ImportContext = { - fileType: 'pdf', - rowCount: 200 - } - - const preset = autoDetectPreset(context) - expect(preset.name).toBe('pattern') - }) - - it('should return pattern preset for Markdown files', () => { - const context: ImportContext = { - fileType: 'markdown' - } - - const preset = autoDetectPreset(context) - expect(preset.name).toBe('pattern') - }) - - it('should return pattern preset for narrative content', () => { - const context: ImportContext = { - hasNarrativeContent: true, - rowCount: 300 - } - - const preset = autoDetectPreset(context) - expect(preset.name).toBe('pattern') - }) - - it('should return pattern preset for long definitions', () => { - const context: ImportContext = { - avgDefinitionLength: 800, - fileType: 'csv' - } - - const preset = autoDetectPreset(context) - expect(preset.name).toBe('pattern') - }) - - it('should return balanced preset for JSON', () => { - const context: ImportContext = { - fileType: 'json', - rowCount: 500 - } - - const preset = autoDetectPreset(context) - expect(preset.name).toBe('balanced') - }) - - it('should return balanced preset for medium datasets', () => { - const context: ImportContext = { - fileType: 'excel', - rowCount: 2000 - } - - const preset = autoDetectPreset(context) - expect(preset.name).toBe('balanced') - }) - - it('should return balanced preset for empty context', () => { - const preset = autoDetectPreset() - expect(preset.name).toBe('balanced') - }) - - it('should return balanced preset for unknown file type', () => { - const context: ImportContext = { - fileType: 'unknown', - rowCount: 500 - } - - const preset = autoDetectPreset(context) - expect(preset.name).toBe('balanced') - }) - }) - - describe('getPreset', () => { - it('should get preset by name', () => { - expect(getPreset('fast')).toBe(FAST_PRESET) - expect(getPreset('balanced')).toBe(BALANCED_PRESET) - expect(getPreset('accurate')).toBe(ACCURATE_PRESET) - expect(getPreset('explicit')).toBe(EXPLICIT_PRESET) - expect(getPreset('pattern')).toBe(PATTERN_PRESET) - }) - - it('should be case-insensitive', () => { - expect(getPreset('FAST')).toBe(FAST_PRESET) - expect(getPreset('Balanced')).toBe(BALANCED_PRESET) - expect(getPreset('EXPLICIT')).toBe(EXPLICIT_PRESET) - }) - - it('should throw error for unknown preset', () => { - expect(() => getPreset('unknown')).toThrow('Unknown preset: unknown') - }) - }) - - describe('getPresetNames', () => { - it('should return all preset names', () => { - const names = getPresetNames() - expect(names).toHaveLength(5) - expect(names).toContain('fast') - expect(names).toContain('balanced') - expect(names).toContain('accurate') - expect(names).toContain('explicit') - expect(names).toContain('pattern') - }) - }) - - describe('explainPresetChoice', () => { - it('should explain large dataset choice', () => { - const context: ImportContext = { - rowCount: 15000, - fileSize: 12_000_000 - } - - const explanation = explainPresetChoice(context) - expect(explanation).toContain('Large dataset') - expect(explanation).toContain('15000 rows') - expect(explanation).toContain('fast preset') - }) - - it('should explain small dataset choice', () => { - const context: ImportContext = { - rowCount: 50 - } - - const explanation = explainPresetChoice(context) - expect(explanation).toContain('Small critical dataset') - expect(explanation).toContain('50 rows') - expect(explanation).toContain('accurate preset') - }) - - it('should explain explicit columns choice', () => { - const context: ImportContext = { - fileType: 'excel', - hasExplicitColumns: true - } - - const explanation = explainPresetChoice(context) - expect(explanation).toContain('EXCEL') - expect(explanation).toContain('explicit relationship columns') - expect(explanation).toContain('explicit preset') - }) - - it('should explain narrative content choice', () => { - const context: ImportContext = { - fileType: 'pdf', - hasNarrativeContent: true - } - - const explanation = explainPresetChoice(context) - expect(explanation).toContain('Narrative content') - expect(explanation).toContain('pattern preset') - }) - - it('should explain default choice', () => { - const context: ImportContext = { - rowCount: 500 - } - - const explanation = explainPresetChoice(context) - expect(explanation).toContain('balanced preset') - }) - }) - - describe('createCustomPreset', () => { - it('should create custom preset from base', () => { - const custom = createCustomPreset('balanced', { - name: 'my-custom', - batchSize: 2000 - }) - - expect(custom.name).toBe('my-custom') - expect(custom.batchSize).toBe(2000) - expect(custom.signals).toEqual(BALANCED_PRESET.signals) - expect(custom.strategies).toEqual(BALANCED_PRESET.strategies) - }) - - it('should override signals', () => { - const custom = createCustomPreset('fast', { - signals: { - enabled: ['embedding'], - weights: { embedding: 1.0, exact: 0, pattern: 0, context: 0 }, - timeout: 200 - } - }) - - expect(custom.signals.enabled).toEqual(['embedding']) - expect(custom.signals.timeout).toBe(200) - }) - - it('should override strategies', () => { - const custom = createCustomPreset('balanced', { - strategies: { - enabled: ['pattern'], - timeout: 500, - earlyTermination: false, - minConfidence: 0.75 - } - }) - - expect(custom.strategies.enabled).toEqual(['pattern']) - expect(custom.strategies.timeout).toBe(500) - expect(custom.strategies.earlyTermination).toBe(false) - }) - - it('should merge partial signal overrides', () => { - const custom = createCustomPreset('balanced', { - signals: { - timeout: 300 - } as any - }) - - expect(custom.signals.enabled).toEqual(BALANCED_PRESET.signals.enabled) - expect(custom.signals.timeout).toBe(300) - }) - }) - - describe('validatePreset', () => { - it('should validate all built-in presets', () => { - expect(() => validatePreset(FAST_PRESET)).not.toThrow() - expect(() => validatePreset(BALANCED_PRESET)).not.toThrow() - expect(() => validatePreset(ACCURATE_PRESET)).not.toThrow() - expect(() => validatePreset(EXPLICIT_PRESET)).not.toThrow() - expect(() => validatePreset(PATTERN_PRESET)).not.toThrow() - }) - - it('should reject preset with no signals', () => { - const invalid: PresetConfig = { - ...BALANCED_PRESET, - signals: { - ...BALANCED_PRESET.signals, - enabled: [] - } - } - - expect(() => validatePreset(invalid)).toThrow('at least one enabled signal') - }) - - it('should reject preset with no strategies', () => { - const invalid: PresetConfig = { - ...BALANCED_PRESET, - strategies: { - ...BALANCED_PRESET.strategies, - enabled: [] - } - } - - expect(() => validatePreset(invalid)).toThrow('at least one enabled strategy') - }) - - it('should reject preset with invalid weight sum', () => { - const invalid: PresetConfig = { - ...BALANCED_PRESET, - signals: { - enabled: ['exact', 'embedding'], - weights: { - exact: 0.3, - embedding: 0.5, - pattern: 0, - context: 0 - }, - timeout: 100 - } - } - - expect(() => validatePreset(invalid)).toThrow('weights must sum to 1.0') - }) - - it('should reject preset with negative timeout', () => { - const invalid: PresetConfig = { - ...BALANCED_PRESET, - signals: { - ...BALANCED_PRESET.signals, - timeout: -100 - } - } - - expect(() => validatePreset(invalid)).toThrow('Timeouts must be positive') - }) - - it('should reject preset with invalid batch size', () => { - const invalid: PresetConfig = { - ...BALANCED_PRESET, - batchSize: 0 - } - - expect(() => validatePreset(invalid)).toThrow('Batch size must be positive') - }) - }) - - describe('formatPreset', () => { - it('should format preset for display', () => { - const formatted = formatPreset(BALANCED_PRESET) - - expect(formatted).toContain('Preset: balanced') - expect(formatted).toContain('Description:') - expect(formatted).toContain('Signals:') - expect(formatted).toContain('exact: 40%') - expect(formatted).toContain('embedding: 35%') - expect(formatted).toContain('Strategies:') - expect(formatted).toContain('explicit') - expect(formatted).toContain('pattern') - expect(formatted).toContain('embedding') - expect(formatted).toContain('Streaming: false') - expect(formatted).toContain('Batch size: 500') - }) - - it('should format fast preset correctly', () => { - const formatted = formatPreset(FAST_PRESET) - - expect(formatted).toContain('fast') - expect(formatted).toContain('Streaming: true') - expect(formatted).toContain('Early termination: true') - }) - }) - - describe('preset priorities', () => { - it('should prioritize size over explicit columns for large datasets', () => { - const context: ImportContext = { - rowCount: 20000, - fileType: 'excel', - hasExplicitColumns: true - } - - const preset = autoDetectPreset(context) - expect(preset.name).toBe('fast') // Size trumps explicit - }) - - it('should prioritize small size over other factors', () => { - const context: ImportContext = { - rowCount: 50, - fileType: 'pdf', - hasNarrativeContent: true - } - - const preset = autoDetectPreset(context) - expect(preset.name).toBe('accurate') // Small size trumps pattern - }) - - it('should prioritize explicit columns over narrative for Excel', () => { - const context: ImportContext = { - rowCount: 500, - fileType: 'excel', - hasExplicitColumns: true, - hasNarrativeContent: true - } - - const preset = autoDetectPreset(context) - expect(preset.name).toBe('explicit') // Explicit trumps narrative - }) - }) - - describe('edge cases', () => { - it('should handle zero row count', () => { - const context: ImportContext = { - rowCount: 0, - fileType: 'csv' - } - - const preset = autoDetectPreset(context) - expect(preset.name).toBe('balanced') - }) - - it('should handle boundary row count (exactly 100)', () => { - const context: ImportContext = { - rowCount: 100 - } - - const preset = autoDetectPreset(context) - expect(preset.name).toBe('balanced') // Not accurate (< 100) - }) - - it('should handle boundary row count (exactly 10000)', () => { - const context: ImportContext = { - rowCount: 10000 - } - - const preset = autoDetectPreset(context) - expect(preset.name).toBe('balanced') // Not fast (> 10000) - }) - - it('should handle missing hasExplicitColumns flag', () => { - const context: ImportContext = { - fileType: 'excel', - rowCount: 500 - } - - const preset = autoDetectPreset(context) - expect(preset.name).toBe('balanced') - }) - }) - - describe('real-world scenarios', () => { - it('should handle glossary correctly', () => { - const context: ImportContext = { - fileType: 'excel', - rowCount: 567, - hasExplicitColumns: true, // Has "Related Terms" column - fileSize: 50_000 - } - - const preset = autoDetectPreset(context) - expect(preset.name).toBe('explicit') - - const explanation = explainPresetChoice(context) - expect(explanation).toContain('explicit') - }) - - it('should handle large CSV import', () => { - const context: ImportContext = { - fileType: 'csv', - rowCount: 50000, - fileSize: 25_000_000 - } - - const preset = autoDetectPreset(context) - expect(preset.name).toBe('fast') - expect(preset.streaming).toBe(true) - }) - - it('should handle PDF documentation', () => { - const context: ImportContext = { - fileType: 'pdf', - rowCount: 150, - hasNarrativeContent: true, - avgDefinitionLength: 600 - } - - const preset = autoDetectPreset(context) - expect(preset.name).toBe('pattern') - }) - - it('should handle JSON API import', () => { - const context: ImportContext = { - fileType: 'json', - rowCount: 1000, - fileSize: 500_000 - } - - const preset = autoDetectPreset(context) - expect(preset.name).toBe('balanced') - }) - }) -}) diff --git a/tests/vfs/vfs-initialization.unit.test.ts b/tests/vfs/vfs-initialization.unit.test.ts index 944e2200..e4c8d3e3 100644 --- a/tests/vfs/vfs-initialization.unit.test.ts +++ b/tests/vfs/vfs-initialization.unit.test.ts @@ -5,7 +5,7 @@ */ import { describe, it, expect } from 'vitest' -import { VirtualFileSystem } from '../../src/vfs/index.js' +import { VirtualFileSystem } from '../../src/vfs/VirtualFileSystem.js' import { Brainy } from '../../src/brainy.js' describe('VFS Initialization', () => { diff --git a/tests/vfs/vfs.unit.test.ts b/tests/vfs/vfs.unit.test.ts index cc201de5..5ea79377 100644 --- a/tests/vfs/vfs.unit.test.ts +++ b/tests/vfs/vfs.unit.test.ts @@ -6,7 +6,7 @@ */ import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import { VirtualFileSystem } from '../../src/vfs/index.js' +import { VirtualFileSystem } from '../../src/vfs/VirtualFileSystem.js' import { Brainy } from '../../src/brainy.js' import { VFSErrorCode } from '../../src/vfs/types.js'