refactor(8.0): remove dead, unreachable, and unwired modules
Pre-GA dead-code sweep. A deterministic import-reachability walk from the
package's real entry points (the exports map, the CLI bin, the conversation
surface) found 34 source modules that nothing reachable imports — they
compile and ship as dist output that no consumer or internal path can ever
reach. All were superseded duplicates, abandoned parallel implementations,
or built-but-never-wired features:
- superseded duplicates of live modules: an older "unified" entry, a
standalone neural-import variant, a static NLP processor + its matcher,
a parallel API-types module, a duplicate progress-types module
- an abandoned import path (orchestrator + entity deduplicator + barrels)
left behind when ingestion moved to the coordinator
- unwired feature modules (instance pool, import presets, cached
embeddings, relationship-confidence scorer) reachable only from tests
- dead leaf utilities (write buffer, deleted-items index, bounded
registry, two crypto shims, a cache manager, a structured logger, a
stale v5 type-migration helper, a browser-only FS type shim) and
several dead re-export barrels
- a CLI catalog command wired into no command
Also removed two tests that only exercised deleted modules, repointed two
VFS tests off a deleted barrel onto the implementation module, and deleted
one example that demoed removed features.
Verified: clean build (both tsc passes), unit 1505 + integration 607 green,
and a re-run of the reachability walk reports zero remaining orphans.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
parent
03d654061f
commit
bf0afe8563
39 changed files with 2 additions and 10497 deletions
|
|
@ -1,237 +0,0 @@
|
||||||
/**
|
|
||||||
* Directory Import with Entity Extraction Caching Example
|
|
||||||
*
|
|
||||||
* Demonstrates:
|
|
||||||
* - Importing directories with progress tracking
|
|
||||||
* - Entity extraction caching for performance
|
|
||||||
* - Relationship detection with confidence scores
|
|
||||||
* - Cache statistics monitoring
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { Brainy, NounType, VerbType } from '../src/brainy.js'
|
|
||||||
import { DirectoryImporter } from '../src/vfs/importers/DirectoryImporter.js'
|
|
||||||
import { ProgressTracker, formatProgress } from '../src/types/progress.types.js'
|
|
||||||
import { detectRelationshipsWithConfidence } from '../src/neural/relationshipConfidence.js'
|
|
||||||
import { NeuralEntityExtractor } from '../src/neural/entityExtractor.js'
|
|
||||||
|
|
||||||
async function main() {
|
|
||||||
console.log('🧠 Brainy 3.21.0 - Directory Import with Caching Example\n')
|
|
||||||
|
|
||||||
// Initialize Brainy
|
|
||||||
const brain = new Brainy({ verbose: false })
|
|
||||||
await brain.init()
|
|
||||||
|
|
||||||
console.log('✅ Brainy initialized\n')
|
|
||||||
|
|
||||||
// The entity extractor (and its extraction cache) is constructed directly.
|
|
||||||
const extractor = new NeuralEntityExtractor(brain)
|
|
||||||
|
|
||||||
// Example 1: Import directory with entity extraction caching
|
|
||||||
console.log('📁 Example 1: Import Directory with Caching\n')
|
|
||||||
|
|
||||||
const vfs = brain.vfs
|
|
||||||
const importer = new DirectoryImporter(vfs, brain)
|
|
||||||
|
|
||||||
// Progress tracking
|
|
||||||
const tracker = ProgressTracker.create(100)
|
|
||||||
tracker.start()
|
|
||||||
|
|
||||||
try {
|
|
||||||
// Import with progress (using async generator)
|
|
||||||
console.log('Importing directory...')
|
|
||||||
|
|
||||||
let filesProcessed = 0
|
|
||||||
for await (const progress of importer.importStream('./examples', {
|
|
||||||
batchSize: 10,
|
|
||||||
recursive: true,
|
|
||||||
generateEmbeddings: true,
|
|
||||||
extractMetadata: true
|
|
||||||
})) {
|
|
||||||
if (progress.type === 'progress') {
|
|
||||||
filesProcessed = progress.processed
|
|
||||||
const trackedProgress = tracker.update(progress.processed, progress.current)
|
|
||||||
console.log(` ${formatProgress(trackedProgress)}`)
|
|
||||||
} else if (progress.type === 'complete') {
|
|
||||||
console.log(`\n✅ Import complete! Processed ${progress.processed} files\n`)
|
|
||||||
} else if (progress.type === 'error') {
|
|
||||||
console.error(`❌ Error: ${progress.error?.message}`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
tracker.complete({ filesProcessed })
|
|
||||||
|
|
||||||
} catch (error) {
|
|
||||||
console.error('Import failed:', error)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Example 2: Entity extraction with caching
|
|
||||||
console.log('\n📝 Example 2: Entity Extraction with Caching\n')
|
|
||||||
|
|
||||||
const sampleText = `
|
|
||||||
John Smith created the user authentication system for the application.
|
|
||||||
The authentication system uses JWT tokens and bcrypt for password hashing.
|
|
||||||
Mary Johnson manages the backend team that maintains the system.
|
|
||||||
The system was built using Node.js and PostgreSQL database.
|
|
||||||
`
|
|
||||||
|
|
||||||
console.log('First extraction (cache miss):')
|
|
||||||
const startTime1 = Date.now()
|
|
||||||
const entities1 = await extractor.extract(sampleText, {
|
|
||||||
types: [NounType.Person, NounType.Service, NounType.Technology],
|
|
||||||
confidence: 0.7,
|
|
||||||
cache: {
|
|
||||||
enabled: true,
|
|
||||||
ttl: 7 * 24 * 60 * 60 * 1000, // 7 days
|
|
||||||
invalidateOn: 'hash'
|
|
||||||
}
|
|
||||||
})
|
|
||||||
const time1 = Date.now() - startTime1
|
|
||||||
console.log(` Extracted ${entities1.length} entities in ${time1}ms`)
|
|
||||||
console.log(` Entities: ${entities1.map(e => e.text).join(', ')}\n`)
|
|
||||||
|
|
||||||
console.log('Second extraction (cache hit):')
|
|
||||||
const startTime2 = Date.now()
|
|
||||||
const entities2 = await extractor.extract(sampleText, {
|
|
||||||
types: [NounType.Person, NounType.Service, NounType.Technology],
|
|
||||||
confidence: 0.7,
|
|
||||||
cache: {
|
|
||||||
enabled: true,
|
|
||||||
invalidateOn: 'hash'
|
|
||||||
}
|
|
||||||
})
|
|
||||||
const time2 = Date.now() - startTime2
|
|
||||||
console.log(` Extracted ${entities2.length} entities in ${time2}ms`)
|
|
||||||
console.log(` Speedup: ${Math.round(time1 / time2)}x faster!\n`)
|
|
||||||
|
|
||||||
// Show cache statistics
|
|
||||||
const cacheStats = extractor.getCacheStats()
|
|
||||||
console.log('📊 Cache Statistics:')
|
|
||||||
console.log(` Hits: ${cacheStats.hits}`)
|
|
||||||
console.log(` Misses: ${cacheStats.misses}`)
|
|
||||||
console.log(` Hit Rate: ${(cacheStats.hitRate * 100).toFixed(1)}%`)
|
|
||||||
console.log(` Total Entries: ${cacheStats.totalEntries}`)
|
|
||||||
console.log(` Avg Entities per Entry: ${cacheStats.averageEntitiesPerEntry}\n`)
|
|
||||||
|
|
||||||
// Example 3: Relationship detection with confidence
|
|
||||||
console.log('🔗 Example 3: Relationship Detection with Confidence\n')
|
|
||||||
|
|
||||||
const relationships = detectRelationshipsWithConfidence(
|
|
||||||
entities1,
|
|
||||||
sampleText,
|
|
||||||
{
|
|
||||||
minConfidence: 0.6,
|
|
||||||
maxDistance: 100,
|
|
||||||
useProximityBoost: true,
|
|
||||||
usePatternMatching: true,
|
|
||||||
useStructuralAnalysis: true
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
console.log(`Detected ${relationships.length} relationships:\n`)
|
|
||||||
for (const rel of relationships.slice(0, 5)) { // Show top 5
|
|
||||||
console.log(` ${rel.sourceEntity.text} --[${rel.verbType}]--> ${rel.targetEntity.text}`)
|
|
||||||
console.log(` Confidence: ${(rel.confidence * 100).toFixed(1)}%`)
|
|
||||||
console.log(` Evidence: ${rel.evidence.reasoning}`)
|
|
||||||
console.log(` Method: ${rel.evidence.method}`)
|
|
||||||
console.log(` Source: "${rel.evidence.sourceText?.substring(0, 60)}..."\n`)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Example 4: Create relationships in graph with confidence
|
|
||||||
console.log('📊 Example 4: Creating Relationships in Graph\n')
|
|
||||||
|
|
||||||
const createdRelations = []
|
|
||||||
for (const rel of relationships.slice(0, 3)) { // Create top 3
|
|
||||||
try {
|
|
||||||
// Add entities to brain
|
|
||||||
const sourceId = await brain.add({
|
|
||||||
data: rel.sourceEntity.text,
|
|
||||||
type: rel.sourceEntity.type,
|
|
||||||
metadata: {
|
|
||||||
confidence: rel.sourceEntity.confidence,
|
|
||||||
extractedFrom: 'sample text'
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
const targetId = await brain.add({
|
|
||||||
data: rel.targetEntity.text,
|
|
||||||
type: rel.targetEntity.type,
|
|
||||||
metadata: {
|
|
||||||
confidence: rel.targetEntity.confidence,
|
|
||||||
extractedFrom: 'sample text'
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// Create relationship with confidence
|
|
||||||
const relationId = await brain.relate({
|
|
||||||
from: sourceId,
|
|
||||||
to: targetId,
|
|
||||||
type: rel.verbType,
|
|
||||||
confidence: rel.confidence,
|
|
||||||
evidence: rel.evidence,
|
|
||||||
metadata: {
|
|
||||||
autoDetected: true,
|
|
||||||
detectedAt: new Date().toISOString()
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
createdRelations.push(relationId)
|
|
||||||
console.log(` ✅ Created: ${rel.sourceEntity.text} → ${rel.targetEntity.text}`)
|
|
||||||
} catch (error) {
|
|
||||||
console.error(` ❌ Failed to create relationship:`, error)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log(`\n✅ Created ${createdRelations.length} relationships in knowledge graph`)
|
|
||||||
|
|
||||||
// Example 5: Query relationships by confidence
|
|
||||||
console.log('\n🔍 Example 5: Query High-Confidence Relationships\n')
|
|
||||||
|
|
||||||
const allRelations = await brain.getRelations({
|
|
||||||
limit: 100
|
|
||||||
})
|
|
||||||
|
|
||||||
const highConfidence = allRelations.filter(r => (r.confidence || 0) >= 0.7)
|
|
||||||
console.log(`Found ${highConfidence.length} high-confidence relationships (≥70%):\n`)
|
|
||||||
|
|
||||||
for (const rel of highConfidence.slice(0, 5)) {
|
|
||||||
console.log(` ${rel.from} → ${rel.to} (${rel.type})`)
|
|
||||||
console.log(` Confidence: ${((rel.confidence || 0) * 100).toFixed(1)}%`)
|
|
||||||
if (rel.evidence) {
|
|
||||||
console.log(` Method: ${rel.evidence.method}`)
|
|
||||||
console.log(` Reasoning: ${rel.evidence.reasoning}\n`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Example 6: Cache management
|
|
||||||
console.log('🧹 Example 6: Cache Management\n')
|
|
||||||
|
|
||||||
console.log('Cache operations:')
|
|
||||||
|
|
||||||
// Cleanup expired entries
|
|
||||||
const cleaned = extractor.cleanupCache()
|
|
||||||
console.log(` Cleaned ${cleaned} expired entries`)
|
|
||||||
|
|
||||||
// Invalidate specific cache entry
|
|
||||||
const invalidated = extractor.invalidateCache('hash:abc123')
|
|
||||||
console.log(` Invalidated entry: ${invalidated}`)
|
|
||||||
|
|
||||||
// Get final stats
|
|
||||||
const finalStats = extractor.getCacheStats()
|
|
||||||
console.log(` Final cache size: ${finalStats.totalEntries} entries`)
|
|
||||||
console.log(` Memory used: ~${Math.round(finalStats.cacheSize / 1024)}KB`)
|
|
||||||
|
|
||||||
// Clear all cache (optional)
|
|
||||||
// extractor.clearCache()
|
|
||||||
// console.log(' Cleared entire cache')
|
|
||||||
|
|
||||||
console.log('\n✨ Example complete!')
|
|
||||||
console.log('\n📚 Key Takeaways:')
|
|
||||||
console.log(' • Entity extraction caching provides 10-100x speedup on repeated content')
|
|
||||||
console.log(' • Progress tracking gives real-time feedback for long operations')
|
|
||||||
console.log(' • Relationship confidence helps filter low-quality connections')
|
|
||||||
console.log(' • Evidence tracking makes relationships explainable and debuggable')
|
|
||||||
console.log(' • All features are opt-in and backward compatible')
|
|
||||||
}
|
|
||||||
|
|
||||||
// Run example
|
|
||||||
main().catch(console.error)
|
|
||||||
|
|
@ -1,440 +0,0 @@
|
||||||
/**
|
|
||||||
* Augmentation Catalog for CLI
|
|
||||||
*
|
|
||||||
* Displays available augmentations catalog
|
|
||||||
* Local catalog with caching support
|
|
||||||
*/
|
|
||||||
|
|
||||||
import chalk from 'chalk'
|
|
||||||
import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'node:fs'
|
|
||||||
import { join } from 'node:path'
|
|
||||||
import { homedir } from 'node:os'
|
|
||||||
|
|
||||||
const CATALOG_API = process.env.BRAINY_CATALOG_URL || null
|
|
||||||
const CACHE_PATH = join(homedir(), '.brainy', 'catalog-cache.json')
|
|
||||||
const CACHE_TTL = 24 * 60 * 60 * 1000 // 24 hours
|
|
||||||
|
|
||||||
interface Augmentation {
|
|
||||||
id: string
|
|
||||||
name: string
|
|
||||||
description: string
|
|
||||||
category: string
|
|
||||||
status: 'available' | 'coming_soon' | 'deprecated'
|
|
||||||
popular?: boolean
|
|
||||||
eta?: string
|
|
||||||
}
|
|
||||||
|
|
||||||
interface Category {
|
|
||||||
id: string
|
|
||||||
name: string
|
|
||||||
icon: string
|
|
||||||
description: string
|
|
||||||
}
|
|
||||||
|
|
||||||
interface Catalog {
|
|
||||||
version: string
|
|
||||||
categories: Category[]
|
|
||||||
augmentations: Augmentation[]
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Fetch catalog from API with caching
|
|
||||||
*/
|
|
||||||
export async function fetchCatalog(): Promise<Catalog | null> {
|
|
||||||
try {
|
|
||||||
// Check cache first
|
|
||||||
const cached = loadCache()
|
|
||||||
if (cached) return cached
|
|
||||||
|
|
||||||
// If external catalog API is configured, try to fetch
|
|
||||||
if (CATALOG_API) {
|
|
||||||
const response = await fetch(`${CATALOG_API}/api/catalog/cli`)
|
|
||||||
if (!response.ok) throw new Error('API unavailable')
|
|
||||||
|
|
||||||
const catalog = await response.json()
|
|
||||||
|
|
||||||
// Save to cache
|
|
||||||
saveCache(catalog)
|
|
||||||
|
|
||||||
return catalog
|
|
||||||
}
|
|
||||||
|
|
||||||
// Fall back to local catalog
|
|
||||||
return getDefaultCatalog()
|
|
||||||
} catch (error) {
|
|
||||||
// Try loading from cache even if expired
|
|
||||||
const cached = loadCache(true)
|
|
||||||
if (cached) {
|
|
||||||
console.log(chalk.yellow('📡 Using cached catalog'))
|
|
||||||
return cached
|
|
||||||
}
|
|
||||||
|
|
||||||
// Fall back to hardcoded catalog
|
|
||||||
return getDefaultCatalog()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Display catalog in CLI
|
|
||||||
*/
|
|
||||||
export async function showCatalog(options: {
|
|
||||||
category?: string
|
|
||||||
search?: string
|
|
||||||
detailed?: boolean
|
|
||||||
}) {
|
|
||||||
const catalog = await fetchCatalog()
|
|
||||||
if (!catalog) {
|
|
||||||
console.log(chalk.red('❌ Could not load augmentation catalog'))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log(chalk.cyan.bold('🧠 Brainy Augmentation Catalog'))
|
|
||||||
console.log(chalk.gray(`Version ${catalog.version}`))
|
|
||||||
console.log('')
|
|
||||||
|
|
||||||
// Filter augmentations
|
|
||||||
let augmentations = catalog.augmentations
|
|
||||||
|
|
||||||
if (options.category) {
|
|
||||||
augmentations = augmentations.filter(a => a.category === options.category)
|
|
||||||
}
|
|
||||||
|
|
||||||
if (options.search) {
|
|
||||||
const query = options.search.toLowerCase()
|
|
||||||
augmentations = augmentations.filter(a =>
|
|
||||||
a.name.toLowerCase().includes(query) ||
|
|
||||||
a.description.toLowerCase().includes(query)
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Group by category
|
|
||||||
const grouped = groupByCategory(augmentations, catalog.categories)
|
|
||||||
|
|
||||||
// Display
|
|
||||||
for (const [category, augs] of Object.entries(grouped)) {
|
|
||||||
if (augs.length === 0) continue
|
|
||||||
|
|
||||||
const cat = catalog.categories.find(c => c.id === category)
|
|
||||||
console.log(chalk.bold(`${cat?.icon || '📦'} ${cat?.name || category}`))
|
|
||||||
|
|
||||||
for (const aug of augs) {
|
|
||||||
const status = getStatusIcon(aug.status)
|
|
||||||
const popular = aug.popular ? chalk.yellow(' ⭐') : ''
|
|
||||||
const eta = aug.eta ? chalk.gray(` (${aug.eta})`) : ''
|
|
||||||
|
|
||||||
console.log(` ${status} ${aug.name}${popular}${eta}`)
|
|
||||||
if (options.detailed) {
|
|
||||||
console.log(chalk.gray(` ${aug.description}`))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
console.log('')
|
|
||||||
}
|
|
||||||
|
|
||||||
// Show summary
|
|
||||||
const available = augmentations.filter(a => a.status === 'available').length
|
|
||||||
const coming = augmentations.filter(a => a.status === 'coming_soon').length
|
|
||||||
|
|
||||||
console.log(chalk.gray('─'.repeat(50)))
|
|
||||||
console.log(chalk.green(`✅ ${available} available`) + chalk.gray(` • `) +
|
|
||||||
chalk.yellow(`🔜 ${coming} coming soon`))
|
|
||||||
console.log('')
|
|
||||||
console.log(chalk.dim('Configure augmentations with "brainy augment"'))
|
|
||||||
console.log(chalk.dim('Run "brainy augment info <name>" for details'))
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Show detailed info about an augmentation
|
|
||||||
*/
|
|
||||||
export async function showAugmentationInfo(id: string) {
|
|
||||||
const catalog = await fetchCatalog()
|
|
||||||
if (!catalog) {
|
|
||||||
console.log(chalk.red('❌ Could not load augmentation catalog'))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
const aug = catalog.augmentations.find(a => a.id === id)
|
|
||||||
if (!aug) {
|
|
||||||
console.log(chalk.red(`❌ Augmentation not found: ${id}`))
|
|
||||||
console.log('')
|
|
||||||
console.log('Available augmentations:')
|
|
||||||
catalog.augmentations.forEach(a => {
|
|
||||||
console.log(` • ${a.id}`)
|
|
||||||
})
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Fetch full details from API if available
|
|
||||||
try {
|
|
||||||
if (!CATALOG_API) throw new Error('No external catalog configured')
|
|
||||||
|
|
||||||
const response = await fetch(`${CATALOG_API}/api/catalog/augmentation/${id}`)
|
|
||||||
const details = await response.json()
|
|
||||||
|
|
||||||
console.log(chalk.cyan.bold(`📦 ${details.name}`))
|
|
||||||
if (details.popular) console.log(chalk.yellow('⭐ Popular'))
|
|
||||||
console.log('')
|
|
||||||
|
|
||||||
console.log(chalk.bold('Category:'), getCategoryName(details.category, catalog.categories))
|
|
||||||
console.log(chalk.bold('Status:'), getStatusText(details.status))
|
|
||||||
if (details.eta) console.log(chalk.bold('Expected:'), details.eta)
|
|
||||||
console.log('')
|
|
||||||
|
|
||||||
console.log(chalk.bold('Description:'))
|
|
||||||
console.log(details.longDescription || details.description)
|
|
||||||
console.log('')
|
|
||||||
|
|
||||||
if (details.features) {
|
|
||||||
console.log(chalk.bold('Features:'))
|
|
||||||
details.features.forEach((f: string) => console.log(` ✓ ${f}`))
|
|
||||||
console.log('')
|
|
||||||
}
|
|
||||||
|
|
||||||
if (details.example) {
|
|
||||||
console.log(chalk.bold('Example:'))
|
|
||||||
console.log(chalk.gray('─'.repeat(50)))
|
|
||||||
console.log(details.example.code)
|
|
||||||
console.log(chalk.gray('─'.repeat(50)))
|
|
||||||
console.log('')
|
|
||||||
}
|
|
||||||
|
|
||||||
if (details.requirements?.config) {
|
|
||||||
console.log(chalk.bold('Required Configuration:'))
|
|
||||||
details.requirements.config.forEach((c: string) => console.log(` • ${c}`))
|
|
||||||
console.log('')
|
|
||||||
}
|
|
||||||
|
|
||||||
if (details.pricing) {
|
|
||||||
console.log(chalk.bold('Available in:'))
|
|
||||||
details.pricing.tiers.forEach((t: string) => console.log(` • ${t}`))
|
|
||||||
console.log('')
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log(chalk.dim('To activate: brainy augment activate'))
|
|
||||||
} catch (error) {
|
|
||||||
// Show basic info if API fails
|
|
||||||
console.log(chalk.cyan.bold(`📦 ${aug.name}`))
|
|
||||||
console.log(aug.description)
|
|
||||||
console.log('')
|
|
||||||
console.log(chalk.dim('Full details unavailable (no external catalog configured)'))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Show user's available augmentations
|
|
||||||
*/
|
|
||||||
export async function showAvailable(licenseKey?: string) {
|
|
||||||
// Show local catalog as default
|
|
||||||
const catalog = await fetchCatalog()
|
|
||||||
if (!catalog) {
|
|
||||||
console.log(chalk.red('❌ Could not load augmentation catalog'))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log(chalk.cyan.bold('🧠 Available Augmentations'))
|
|
||||||
console.log('')
|
|
||||||
|
|
||||||
const available = catalog.augmentations.filter(a => a.status === 'available')
|
|
||||||
const grouped = groupByCategory(available, catalog.categories)
|
|
||||||
|
|
||||||
for (const [category, augs] of Object.entries(grouped)) {
|
|
||||||
if (augs.length === 0) continue
|
|
||||||
|
|
||||||
const cat = catalog.categories.find(c => c.id === category)
|
|
||||||
console.log(chalk.bold(`${cat?.icon || '📦'} ${cat?.name || category}`))
|
|
||||||
augs.forEach(aug => {
|
|
||||||
console.log(` ✅ ${aug.name}`)
|
|
||||||
console.log(chalk.gray(` ${aug.description}`))
|
|
||||||
})
|
|
||||||
console.log('')
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log(chalk.green(`✅ ${available.length} augmentations available`))
|
|
||||||
|
|
||||||
// If external API is configured and license key provided, try to fetch personalized data
|
|
||||||
if (CATALOG_API && licenseKey) {
|
|
||||||
try {
|
|
||||||
const response = await fetch(`${CATALOG_API}/api/catalog/available`, {
|
|
||||||
headers: { 'x-license-key': licenseKey }
|
|
||||||
})
|
|
||||||
|
|
||||||
if (response.ok) {
|
|
||||||
const data = await response.json()
|
|
||||||
console.log(chalk.gray(`Plan: ${data.plan || 'Standard'}`))
|
|
||||||
|
|
||||||
if (data.operations) {
|
|
||||||
const used = data.operations.used || 0
|
|
||||||
const limit = data.operations.limit
|
|
||||||
const percent = limit === 'unlimited' ? 0 : Math.round((used / limit) * 100)
|
|
||||||
|
|
||||||
console.log(chalk.bold('Usage:'))
|
|
||||||
if (limit === 'unlimited') {
|
|
||||||
console.log(` Unlimited operations`)
|
|
||||||
} else {
|
|
||||||
console.log(` ${used.toLocaleString()} / ${limit.toLocaleString()} operations (${percent}%)`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
// Ignore external API errors - local catalog is sufficient
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Helper functions
|
|
||||||
|
|
||||||
function loadCache(ignoreExpiry = false): Catalog | null {
|
|
||||||
try {
|
|
||||||
if (!existsSync(CACHE_PATH)) return null
|
|
||||||
|
|
||||||
const data = JSON.parse(readFileSync(CACHE_PATH, 'utf8'))
|
|
||||||
|
|
||||||
if (!ignoreExpiry && Date.now() - data.timestamp > CACHE_TTL) {
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
|
|
||||||
return data.catalog
|
|
||||||
} catch {
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function saveCache(catalog: Catalog): void {
|
|
||||||
try {
|
|
||||||
const dir = join(homedir(), '.brainy')
|
|
||||||
if (!existsSync(dir)) {
|
|
||||||
mkdirSync(dir, { recursive: true })
|
|
||||||
}
|
|
||||||
|
|
||||||
writeFileSync(CACHE_PATH, JSON.stringify({
|
|
||||||
catalog,
|
|
||||||
timestamp: Date.now()
|
|
||||||
}))
|
|
||||||
} catch {
|
|
||||||
// Ignore cache save errors
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function groupByCategory(augmentations: Augmentation[], categories: Category[]) {
|
|
||||||
const grouped: Record<string, Augmentation[]> = {}
|
|
||||||
|
|
||||||
for (const aug of augmentations) {
|
|
||||||
if (!grouped[aug.category]) {
|
|
||||||
grouped[aug.category] = []
|
|
||||||
}
|
|
||||||
grouped[aug.category].push(aug)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Sort by category order
|
|
||||||
const ordered: Record<string, Augmentation[]> = {}
|
|
||||||
const categoryOrder = ['memory', 'coordination', 'enterprise', 'perception', 'dialog', 'activation', 'cognition', 'websocket']
|
|
||||||
|
|
||||||
for (const cat of categoryOrder) {
|
|
||||||
if (grouped[cat]) {
|
|
||||||
ordered[cat] = grouped[cat]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return ordered
|
|
||||||
}
|
|
||||||
|
|
||||||
function getStatusIcon(status: string): string {
|
|
||||||
switch (status) {
|
|
||||||
case 'available': return chalk.green('✅')
|
|
||||||
case 'coming_soon': return chalk.yellow('🔜')
|
|
||||||
case 'deprecated': return chalk.red('⚠️')
|
|
||||||
default: return '❓'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function getStatusText(status: string): string {
|
|
||||||
switch (status) {
|
|
||||||
case 'available': return chalk.green('Available')
|
|
||||||
case 'coming_soon': return chalk.yellow('Coming Soon')
|
|
||||||
case 'deprecated': return chalk.red('Deprecated')
|
|
||||||
default: return 'Unknown'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function getCategoryName(categoryId: string, categories: Category[]): string {
|
|
||||||
const cat = categories.find(c => c.id === categoryId)
|
|
||||||
return cat ? `${cat.icon} ${cat.name}` : categoryId
|
|
||||||
}
|
|
||||||
|
|
||||||
function readLicenseFile(): string | null {
|
|
||||||
try {
|
|
||||||
const licensePath = join(homedir(), '.brainy', 'license')
|
|
||||||
if (existsSync(licensePath)) {
|
|
||||||
return readFileSync(licensePath, 'utf8').trim()
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
// License file read failed, return null
|
|
||||||
console.debug('Failed to read license file:', error)
|
|
||||||
}
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
|
|
||||||
function getDefaultCatalog(): Catalog {
|
|
||||||
// Local catalog with current features
|
|
||||||
return {
|
|
||||||
version: '1.5.0',
|
|
||||||
categories: [
|
|
||||||
{ id: 'core', name: 'Core Features', icon: '🧠', description: 'Essential brainy functionality' },
|
|
||||||
{ id: 'neural', name: 'Neural API', icon: '🔗', description: 'Semantic similarity and clustering' },
|
|
||||||
{ id: 'enterprise', name: 'Enterprise', icon: '🏢', description: 'Business integrations' },
|
|
||||||
{ id: 'storage', name: 'Storage', icon: '💾', description: 'Data persistence and caching' }
|
|
||||||
],
|
|
||||||
augmentations: [
|
|
||||||
{
|
|
||||||
id: 'vector-search',
|
|
||||||
name: 'Vector Search',
|
|
||||||
category: 'core',
|
|
||||||
description: 'High-performance semantic search with HNSW indexing',
|
|
||||||
status: 'available',
|
|
||||||
popular: true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
id: 'neural-similarity',
|
|
||||||
name: 'Neural Similarity API',
|
|
||||||
category: 'neural',
|
|
||||||
description: 'Advanced semantic similarity, clustering, and hierarchy detection',
|
|
||||||
status: 'available',
|
|
||||||
popular: true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
id: 'intelligent-verb-scoring',
|
|
||||||
name: 'Intelligent Verb Scoring',
|
|
||||||
category: 'neural',
|
|
||||||
description: 'Smart relationship scoring with taxonomy understanding',
|
|
||||||
status: 'available'
|
|
||||||
},
|
|
||||||
{
|
|
||||||
id: 'connection-pooling',
|
|
||||||
name: 'Connection Pooling',
|
|
||||||
category: 'enterprise',
|
|
||||||
description: 'Efficient database connection management',
|
|
||||||
status: 'available'
|
|
||||||
},
|
|
||||||
{
|
|
||||||
id: 'batch-processing',
|
|
||||||
name: 'Batch Processing',
|
|
||||||
category: 'enterprise',
|
|
||||||
description: 'High-throughput batch operations with deduplication',
|
|
||||||
status: 'available'
|
|
||||||
},
|
|
||||||
{
|
|
||||||
id: 's3-storage',
|
|
||||||
name: 'S3 Compatible Storage',
|
|
||||||
category: 'storage',
|
|
||||||
description: 'Cloud storage with optimized batch operations',
|
|
||||||
status: 'available'
|
|
||||||
},
|
|
||||||
{
|
|
||||||
id: 'opfs-storage',
|
|
||||||
name: 'OPFS Storage',
|
|
||||||
category: 'storage',
|
|
||||||
description: 'Browser-based persistent storage',
|
|
||||||
status: 'available'
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,163 +0,0 @@
|
||||||
/**
|
|
||||||
* Cached Embeddings - Performance Optimization Layer
|
|
||||||
*
|
|
||||||
* Provides pre-computed embeddings for common terms to avoid
|
|
||||||
* unnecessary model calls. Falls back to EmbeddingManager for
|
|
||||||
* unknown terms.
|
|
||||||
*
|
|
||||||
* This is purely a performance optimization - it doesn't affect
|
|
||||||
* the consistency or accuracy of embeddings.
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { Vector } from '../coreTypes.js'
|
|
||||||
import { embeddingManager } from './EmbeddingManager.js'
|
|
||||||
|
|
||||||
// Pre-computed embeddings for top common terms
|
|
||||||
// In production, this could be loaded from a file or expanded significantly
|
|
||||||
const PRECOMPUTED_EMBEDDINGS: Record<string, Vector> = {
|
|
||||||
// Programming languages
|
|
||||||
'javascript': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.1)),
|
|
||||||
'python': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.1)),
|
|
||||||
'typescript': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.15)),
|
|
||||||
'java': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.15)),
|
|
||||||
'rust': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.2)),
|
|
||||||
'go': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.2)),
|
|
||||||
'c++': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.22)),
|
|
||||||
'c#': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.22)),
|
|
||||||
|
|
||||||
// Web frameworks
|
|
||||||
'react': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.25)),
|
|
||||||
'vue': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.25)),
|
|
||||||
'angular': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.3)),
|
|
||||||
'svelte': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.3)),
|
|
||||||
'nextjs': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.32)),
|
|
||||||
'nuxt': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.32)),
|
|
||||||
|
|
||||||
// Databases
|
|
||||||
'postgresql': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.35)),
|
|
||||||
'mysql': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.35)),
|
|
||||||
'mongodb': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.4)),
|
|
||||||
'redis': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.4)),
|
|
||||||
'elasticsearch': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.42)),
|
|
||||||
|
|
||||||
// Common tech terms
|
|
||||||
'database': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.45)),
|
|
||||||
'api': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.45)),
|
|
||||||
'server': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.5)),
|
|
||||||
'client': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.5)),
|
|
||||||
'frontend': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.55)),
|
|
||||||
'backend': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.55)),
|
|
||||||
'fullstack': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.57)),
|
|
||||||
'devops': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.57)),
|
|
||||||
'cloud': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.6)),
|
|
||||||
'docker': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.6)),
|
|
||||||
'kubernetes': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.62)),
|
|
||||||
'microservices': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.62)),
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Simple character n-gram based embedding for short text
|
|
||||||
* This is much faster than using the model for simple terms
|
|
||||||
*/
|
|
||||||
function computeSimpleEmbedding(text: string): Vector {
|
|
||||||
const normalized = text.toLowerCase().trim()
|
|
||||||
const vector = new Array(384).fill(0)
|
|
||||||
|
|
||||||
// Character trigrams for simple semantic similarity
|
|
||||||
for (let i = 0; i < normalized.length - 2; i++) {
|
|
||||||
const trigram = normalized.slice(i, i + 3)
|
|
||||||
const hash = trigram.charCodeAt(0) * 31 +
|
|
||||||
trigram.charCodeAt(1) * 7 +
|
|
||||||
trigram.charCodeAt(2)
|
|
||||||
const index = Math.abs(hash) % 384
|
|
||||||
vector[index] += 1 / (normalized.length - 2)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Normalize vector
|
|
||||||
const magnitude = Math.sqrt(vector.reduce((sum, val) => sum + val * val, 0))
|
|
||||||
if (magnitude > 0) {
|
|
||||||
for (let i = 0; i < vector.length; i++) {
|
|
||||||
vector[i] /= magnitude
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return vector
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Cached Embeddings with fallback to EmbeddingManager
|
|
||||||
*/
|
|
||||||
export class CachedEmbeddings {
|
|
||||||
private stats = {
|
|
||||||
cacheHits: 0,
|
|
||||||
simpleComputes: 0,
|
|
||||||
modelCalls: 0
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Generate embedding with caching
|
|
||||||
*/
|
|
||||||
async embed(text: string | string[]): Promise<Vector | Vector[]> {
|
|
||||||
if (Array.isArray(text)) {
|
|
||||||
return Promise.all(text.map(t => this.embedSingle(t)))
|
|
||||||
}
|
|
||||||
return this.embedSingle(text)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Embed single text with cache lookup
|
|
||||||
*/
|
|
||||||
private async embedSingle(text: string): Promise<Vector> {
|
|
||||||
const normalized = text.toLowerCase().trim()
|
|
||||||
|
|
||||||
// 1. Check pre-computed cache (instant, zero cost)
|
|
||||||
if (PRECOMPUTED_EMBEDDINGS[normalized]) {
|
|
||||||
this.stats.cacheHits++
|
|
||||||
return PRECOMPUTED_EMBEDDINGS[normalized]
|
|
||||||
}
|
|
||||||
|
|
||||||
// 2. Check for partial matches in cache
|
|
||||||
for (const [term, embedding] of Object.entries(PRECOMPUTED_EMBEDDINGS)) {
|
|
||||||
if (normalized.includes(term) || term.includes(normalized)) {
|
|
||||||
this.stats.cacheHits++
|
|
||||||
// Return slightly modified version to maintain uniqueness
|
|
||||||
return embedding.map(v => v * 0.95)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 3. For short text, use simple embedding (fast, low cost)
|
|
||||||
if (normalized.length < 50 && normalized.split(' ').length < 5) {
|
|
||||||
this.stats.simpleComputes++
|
|
||||||
return computeSimpleEmbedding(normalized)
|
|
||||||
}
|
|
||||||
|
|
||||||
// 4. Fall back to EmbeddingManager for complex text
|
|
||||||
this.stats.modelCalls++
|
|
||||||
return await embeddingManager.embed(text)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get cache statistics
|
|
||||||
*/
|
|
||||||
getStats() {
|
|
||||||
return {
|
|
||||||
...this.stats,
|
|
||||||
totalEmbeddings: this.stats.cacheHits + this.stats.simpleComputes + this.stats.modelCalls,
|
|
||||||
cacheHitRate: this.stats.cacheHits /
|
|
||||||
(this.stats.cacheHits + this.stats.simpleComputes + this.stats.modelCalls) || 0
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Add custom pre-computed embeddings
|
|
||||||
*/
|
|
||||||
addPrecomputed(term: string, embedding: Vector) {
|
|
||||||
if (embedding.length !== 384) {
|
|
||||||
throw new Error('Embedding must have 384 dimensions')
|
|
||||||
}
|
|
||||||
PRECOMPUTED_EMBEDDINGS[term.toLowerCase()] = embedding
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Export singleton instance
|
|
||||||
export const cachedEmbeddings = new CachedEmbeddings()
|
|
||||||
|
|
@ -1,28 +0,0 @@
|
||||||
/**
|
|
||||||
* Embeddings Module - Clean, Unified Architecture
|
|
||||||
*
|
|
||||||
* This module provides all embedding functionality for Brainy.
|
|
||||||
*
|
|
||||||
* Main Components:
|
|
||||||
* - EmbeddingManager: Core embedding generation with Q8/FP32 support
|
|
||||||
* - CachedEmbeddings: Performance optimization layer with pre-computed embeddings
|
|
||||||
*/
|
|
||||||
|
|
||||||
// Core embedding functionality
|
|
||||||
export {
|
|
||||||
EmbeddingManager,
|
|
||||||
embeddingManager,
|
|
||||||
embed,
|
|
||||||
getEmbeddingFunction,
|
|
||||||
getEmbeddingStats,
|
|
||||||
type ModelPrecision
|
|
||||||
} from './EmbeddingManager.js'
|
|
||||||
|
|
||||||
// Cached embeddings for performance
|
|
||||||
export {
|
|
||||||
CachedEmbeddings,
|
|
||||||
cachedEmbeddings
|
|
||||||
} from './CachedEmbeddings.js'
|
|
||||||
|
|
||||||
// Default export is the singleton manager
|
|
||||||
export { embeddingManager as default } from './EmbeddingManager.js'
|
|
||||||
|
|
@ -1,523 +0,0 @@
|
||||||
/**
|
|
||||||
* Advanced Graph Pathfinding Algorithms
|
|
||||||
* Provides shortest path, multi-hop traversal, and path ranking
|
|
||||||
*/
|
|
||||||
|
|
||||||
// Graph pathfinding doesn't need to import from coreTypes
|
|
||||||
|
|
||||||
export interface GraphNode {
|
|
||||||
id: string
|
|
||||||
[key: string]: any
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface GraphEdge {
|
|
||||||
source: string
|
|
||||||
target: string
|
|
||||||
type: string
|
|
||||||
weight: number
|
|
||||||
metadata?: any
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface Path {
|
|
||||||
nodes: string[]
|
|
||||||
edges: GraphEdge[]
|
|
||||||
totalWeight: number
|
|
||||||
length: number
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface PathfindingOptions {
|
|
||||||
maxDepth?: number
|
|
||||||
maxPaths?: number
|
|
||||||
bidirectional?: boolean
|
|
||||||
weightField?: string
|
|
||||||
relationshipTypes?: string[]
|
|
||||||
nodeFilter?: (node: GraphNode) => boolean
|
|
||||||
edgeFilter?: (edge: GraphEdge) => boolean
|
|
||||||
}
|
|
||||||
|
|
||||||
export class GraphPathfinding {
|
|
||||||
private adjacencyList: Map<string, Map<string, GraphEdge[]>> = new Map()
|
|
||||||
private nodes: Map<string, GraphNode> = new Map()
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Add a node to the graph
|
|
||||||
*/
|
|
||||||
public addNode(node: GraphNode): void {
|
|
||||||
this.nodes.set(node.id, node)
|
|
||||||
if (!this.adjacencyList.has(node.id)) {
|
|
||||||
this.adjacencyList.set(node.id, new Map())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Add an edge to the graph
|
|
||||||
*/
|
|
||||||
public addEdge(edge: GraphEdge): void {
|
|
||||||
// Ensure nodes exist
|
|
||||||
if (!this.adjacencyList.has(edge.source)) {
|
|
||||||
this.adjacencyList.set(edge.source, new Map())
|
|
||||||
}
|
|
||||||
if (!this.adjacencyList.has(edge.target)) {
|
|
||||||
this.adjacencyList.set(edge.target, new Map())
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add edge to adjacency list
|
|
||||||
const sourceEdges = this.adjacencyList.get(edge.source)!
|
|
||||||
if (!sourceEdges.has(edge.target)) {
|
|
||||||
sourceEdges.set(edge.target, [])
|
|
||||||
}
|
|
||||||
sourceEdges.get(edge.target)!.push(edge)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Find shortest path using Dijkstra's algorithm
|
|
||||||
* O((V + E) log V) with binary heap
|
|
||||||
*/
|
|
||||||
public shortestPath(
|
|
||||||
start: string,
|
|
||||||
end: string,
|
|
||||||
options: PathfindingOptions = {}
|
|
||||||
): Path | null {
|
|
||||||
const {
|
|
||||||
maxDepth = Infinity,
|
|
||||||
relationshipTypes,
|
|
||||||
edgeFilter
|
|
||||||
} = options
|
|
||||||
|
|
||||||
// Priority queue: [nodeId, distance, path]
|
|
||||||
const pq: Array<[string, number, string[], GraphEdge[]]> = [[start, 0, [start], []]]
|
|
||||||
const visited = new Set<string>()
|
|
||||||
const distances = new Map<string, number>([[start, 0]])
|
|
||||||
|
|
||||||
while (pq.length > 0) {
|
|
||||||
// Sort by distance (simple array, could optimize with heap)
|
|
||||||
pq.sort((a, b) => a[1] - b[1])
|
|
||||||
const [current, distance, path, edges] = pq.shift()!
|
|
||||||
|
|
||||||
if (visited.has(current)) continue
|
|
||||||
visited.add(current)
|
|
||||||
|
|
||||||
// Found target
|
|
||||||
if (current === end) {
|
|
||||||
return {
|
|
||||||
nodes: path,
|
|
||||||
edges,
|
|
||||||
totalWeight: distance,
|
|
||||||
length: path.length - 1
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Max depth reached
|
|
||||||
if (path.length > maxDepth) continue
|
|
||||||
|
|
||||||
// Explore neighbors
|
|
||||||
const neighbors = this.adjacencyList.get(current)
|
|
||||||
if (!neighbors) continue
|
|
||||||
|
|
||||||
for (const [neighbor, edgeList] of neighbors) {
|
|
||||||
if (visited.has(neighbor)) continue
|
|
||||||
|
|
||||||
// Find best edge to neighbor
|
|
||||||
let bestEdge: GraphEdge | null = null
|
|
||||||
let bestWeight = Infinity
|
|
||||||
|
|
||||||
for (const edge of edgeList) {
|
|
||||||
// Apply filters
|
|
||||||
if (relationshipTypes && !relationshipTypes.includes(edge.type)) continue
|
|
||||||
if (edgeFilter && !edgeFilter(edge)) continue
|
|
||||||
|
|
||||||
if (edge.weight < bestWeight) {
|
|
||||||
bestWeight = edge.weight
|
|
||||||
bestEdge = edge
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!bestEdge) continue
|
|
||||||
|
|
||||||
const newDistance = distance + bestWeight
|
|
||||||
const currentBest = distances.get(neighbor) ?? Infinity
|
|
||||||
|
|
||||||
if (newDistance < currentBest) {
|
|
||||||
distances.set(neighbor, newDistance)
|
|
||||||
pq.push([
|
|
||||||
neighbor,
|
|
||||||
newDistance,
|
|
||||||
[...path, neighbor],
|
|
||||||
[...edges, bestEdge]
|
|
||||||
])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return null // No path found
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Find all paths between two nodes
|
|
||||||
* Uses DFS with cycle detection
|
|
||||||
*/
|
|
||||||
public allPaths(
|
|
||||||
start: string,
|
|
||||||
end: string,
|
|
||||||
options: PathfindingOptions = {}
|
|
||||||
): Path[] {
|
|
||||||
const {
|
|
||||||
maxDepth = 10,
|
|
||||||
maxPaths = 100,
|
|
||||||
relationshipTypes,
|
|
||||||
edgeFilter
|
|
||||||
} = options
|
|
||||||
|
|
||||||
const paths: Path[] = []
|
|
||||||
const visited = new Set<string>()
|
|
||||||
|
|
||||||
const dfs = (
|
|
||||||
current: string,
|
|
||||||
path: string[],
|
|
||||||
edges: GraphEdge[],
|
|
||||||
weight: number
|
|
||||||
): void => {
|
|
||||||
if (paths.length >= maxPaths) return
|
|
||||||
if (path.length > maxDepth) return
|
|
||||||
|
|
||||||
if (current === end && path.length > 1) {
|
|
||||||
paths.push({
|
|
||||||
nodes: [...path],
|
|
||||||
edges: [...edges],
|
|
||||||
totalWeight: weight,
|
|
||||||
length: path.length - 1
|
|
||||||
})
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
visited.add(current)
|
|
||||||
|
|
||||||
const neighbors = this.adjacencyList.get(current)
|
|
||||||
if (neighbors) {
|
|
||||||
for (const [neighbor, edgeList] of neighbors) {
|
|
||||||
if (visited.has(neighbor)) continue
|
|
||||||
|
|
||||||
for (const edge of edgeList) {
|
|
||||||
// Apply filters
|
|
||||||
if (relationshipTypes && !relationshipTypes.includes(edge.type)) continue
|
|
||||||
if (edgeFilter && !edgeFilter(edge)) continue
|
|
||||||
|
|
||||||
dfs(
|
|
||||||
neighbor,
|
|
||||||
[...path, neighbor],
|
|
||||||
[...edges, edge],
|
|
||||||
weight + edge.weight
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
visited.delete(current)
|
|
||||||
}
|
|
||||||
|
|
||||||
dfs(start, [start], [], 0)
|
|
||||||
|
|
||||||
// Sort paths by weight
|
|
||||||
paths.sort((a, b) => a.totalWeight - b.totalWeight)
|
|
||||||
|
|
||||||
return paths
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Bidirectional search for faster pathfinding
|
|
||||||
* Searches from both start and end simultaneously
|
|
||||||
*/
|
|
||||||
public bidirectionalSearch(
|
|
||||||
start: string,
|
|
||||||
end: string,
|
|
||||||
options: PathfindingOptions = {}
|
|
||||||
): Path | null {
|
|
||||||
const { maxDepth = 10 } = options
|
|
||||||
|
|
||||||
// Two search frontiers
|
|
||||||
const forwardVisited = new Map<string, { path: string[], edges: GraphEdge[], weight: number }>()
|
|
||||||
const backwardVisited = new Map<string, { path: string[], edges: GraphEdge[], weight: number }>()
|
|
||||||
|
|
||||||
forwardVisited.set(start, { path: [start], edges: [], weight: 0 })
|
|
||||||
backwardVisited.set(end, { path: [end], edges: [], weight: 0 })
|
|
||||||
|
|
||||||
const forwardQueue = [start]
|
|
||||||
const backwardQueue = [end]
|
|
||||||
|
|
||||||
let depth = 0
|
|
||||||
|
|
||||||
while (
|
|
||||||
(forwardQueue.length > 0 || backwardQueue.length > 0) &&
|
|
||||||
depth < maxDepth
|
|
||||||
) {
|
|
||||||
// Expand forward frontier
|
|
||||||
const forwardNext: string[] = []
|
|
||||||
for (const current of forwardQueue) {
|
|
||||||
const currentData = forwardVisited.get(current)!
|
|
||||||
const neighbors = this.adjacencyList.get(current)
|
|
||||||
|
|
||||||
if (neighbors) {
|
|
||||||
for (const [neighbor, edges] of neighbors) {
|
|
||||||
if (forwardVisited.has(neighbor)) continue
|
|
||||||
|
|
||||||
// Select edge with lowest weight for optimal path
|
|
||||||
const bestEdge = edges.reduce((best, edge) =>
|
|
||||||
edge.weight < best.weight ? edge : best, edges[0])
|
|
||||||
forwardVisited.set(neighbor, {
|
|
||||||
path: [...currentData.path, neighbor],
|
|
||||||
edges: [...currentData.edges, bestEdge],
|
|
||||||
weight: currentData.weight + bestEdge.weight
|
|
||||||
})
|
|
||||||
|
|
||||||
// Check if we met the backward search
|
|
||||||
if (backwardVisited.has(neighbor)) {
|
|
||||||
const forward = forwardVisited.get(neighbor)!
|
|
||||||
const backward = backwardVisited.get(neighbor)!
|
|
||||||
|
|
||||||
// Combine paths
|
|
||||||
const fullPath = [
|
|
||||||
...forward.path,
|
|
||||||
...backward.path.slice(1).reverse()
|
|
||||||
]
|
|
||||||
|
|
||||||
// Reverse backward edges and combine
|
|
||||||
const backwardEdgesReversed = backward.edges
|
|
||||||
.map(e => ({
|
|
||||||
...e,
|
|
||||||
source: e.target,
|
|
||||||
target: e.source
|
|
||||||
}))
|
|
||||||
.reverse()
|
|
||||||
|
|
||||||
return {
|
|
||||||
nodes: fullPath,
|
|
||||||
edges: [...forward.edges, ...backwardEdgesReversed],
|
|
||||||
totalWeight: forward.weight + backward.weight,
|
|
||||||
length: fullPath.length - 1
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
forwardNext.push(neighbor)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Expand backward frontier
|
|
||||||
const backwardNext: string[] = []
|
|
||||||
for (const current of backwardQueue) {
|
|
||||||
const currentData = backwardVisited.get(current)!
|
|
||||||
|
|
||||||
// For backward search, we need to look at incoming edges
|
|
||||||
for (const [nodeId, neighbors] of this.adjacencyList) {
|
|
||||||
const edges = neighbors.get(current)
|
|
||||||
if (!edges) continue
|
|
||||||
|
|
||||||
if (backwardVisited.has(nodeId)) continue
|
|
||||||
|
|
||||||
// Select edge with lowest weight for optimal path
|
|
||||||
const bestEdge = edges.reduce((best, edge) =>
|
|
||||||
edge.weight < best.weight ? edge : best, edges[0])
|
|
||||||
backwardVisited.set(nodeId, {
|
|
||||||
path: [...currentData.path, nodeId],
|
|
||||||
edges: [...currentData.edges, bestEdge],
|
|
||||||
weight: currentData.weight + bestEdge.weight
|
|
||||||
})
|
|
||||||
|
|
||||||
// Check if we met the forward search
|
|
||||||
if (forwardVisited.has(nodeId)) {
|
|
||||||
const forward = forwardVisited.get(nodeId)!
|
|
||||||
const backward = backwardVisited.get(nodeId)!
|
|
||||||
|
|
||||||
// Combine paths
|
|
||||||
const fullPath = [
|
|
||||||
...forward.path,
|
|
||||||
...backward.path.slice(1).reverse()
|
|
||||||
]
|
|
||||||
|
|
||||||
// Reverse backward edges and combine
|
|
||||||
const backwardEdgesReversed = backward.edges
|
|
||||||
.map(e => ({
|
|
||||||
...e,
|
|
||||||
source: e.target,
|
|
||||||
target: e.source
|
|
||||||
}))
|
|
||||||
.reverse()
|
|
||||||
|
|
||||||
return {
|
|
||||||
nodes: fullPath,
|
|
||||||
edges: [...forward.edges, ...backwardEdgesReversed],
|
|
||||||
totalWeight: forward.weight + backward.weight,
|
|
||||||
length: fullPath.length - 1
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
backwardNext.push(nodeId)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
forwardQueue.splice(0, forwardQueue.length, ...forwardNext)
|
|
||||||
backwardQueue.splice(0, backwardQueue.length, ...backwardNext)
|
|
||||||
depth++
|
|
||||||
}
|
|
||||||
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Multi-hop traversal (e.g., friends of friends)
|
|
||||||
* Returns all nodes within N hops
|
|
||||||
*/
|
|
||||||
public multiHopTraversal(
|
|
||||||
start: string,
|
|
||||||
hops: number,
|
|
||||||
options: PathfindingOptions = {}
|
|
||||||
): Map<string, { distance: number, paths: Path[] }> {
|
|
||||||
const { relationshipTypes, nodeFilter, edgeFilter } = options
|
|
||||||
|
|
||||||
const results = new Map<string, { distance: number, paths: Path[] }>()
|
|
||||||
const visited = new Set<string>()
|
|
||||||
const queue: Array<{ node: string, distance: number, path: string[], edges: GraphEdge[] }> = [
|
|
||||||
{ node: start, distance: 0, path: [start], edges: [] }
|
|
||||||
]
|
|
||||||
|
|
||||||
while (queue.length > 0) {
|
|
||||||
const { node, distance, path, edges } = queue.shift()!
|
|
||||||
|
|
||||||
if (distance > hops) continue
|
|
||||||
|
|
||||||
// Record this node
|
|
||||||
if (!results.has(node)) {
|
|
||||||
results.set(node, { distance, paths: [] })
|
|
||||||
}
|
|
||||||
results.get(node)!.paths.push({
|
|
||||||
nodes: path,
|
|
||||||
edges,
|
|
||||||
totalWeight: edges.reduce((sum, e) => sum + e.weight, 0),
|
|
||||||
length: path.length - 1
|
|
||||||
})
|
|
||||||
|
|
||||||
if (distance === hops) continue
|
|
||||||
|
|
||||||
// Explore neighbors
|
|
||||||
const neighbors = this.adjacencyList.get(node)
|
|
||||||
if (neighbors) {
|
|
||||||
for (const [neighbor, edgeList] of neighbors) {
|
|
||||||
// Apply node filter
|
|
||||||
if (nodeFilter) {
|
|
||||||
const neighborNode = this.nodes.get(neighbor)
|
|
||||||
if (neighborNode && !nodeFilter(neighborNode)) continue
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const edge of edgeList) {
|
|
||||||
// Apply filters
|
|
||||||
if (relationshipTypes && !relationshipTypes.includes(edge.type)) continue
|
|
||||||
if (edgeFilter && !edgeFilter(edge)) continue
|
|
||||||
|
|
||||||
queue.push({
|
|
||||||
node: neighbor,
|
|
||||||
distance: distance + 1,
|
|
||||||
path: [...path, neighbor],
|
|
||||||
edges: [...edges, edge]
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return results
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Find connected components using DFS
|
|
||||||
*/
|
|
||||||
public connectedComponents(): Array<Set<string>> {
|
|
||||||
const visited = new Set<string>()
|
|
||||||
const components: Array<Set<string>> = []
|
|
||||||
|
|
||||||
const dfs = (node: string, component: Set<string>): void => {
|
|
||||||
visited.add(node)
|
|
||||||
component.add(node)
|
|
||||||
|
|
||||||
const neighbors = this.adjacencyList.get(node)
|
|
||||||
if (neighbors) {
|
|
||||||
for (const neighbor of neighbors.keys()) {
|
|
||||||
if (!visited.has(neighbor)) {
|
|
||||||
dfs(neighbor, component)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const node of this.adjacencyList.keys()) {
|
|
||||||
if (!visited.has(node)) {
|
|
||||||
const component = new Set<string>()
|
|
||||||
dfs(node, component)
|
|
||||||
components.push(component)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return components
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Calculate PageRank for all nodes
|
|
||||||
* Useful for ranking importance in the graph
|
|
||||||
*/
|
|
||||||
public pageRank(iterations: number = 100, damping: number = 0.85): Map<string, number> {
|
|
||||||
const nodes = Array.from(this.adjacencyList.keys())
|
|
||||||
const n = nodes.length
|
|
||||||
|
|
||||||
if (n === 0) return new Map()
|
|
||||||
|
|
||||||
// Initialize ranks
|
|
||||||
const ranks = new Map<string, number>()
|
|
||||||
for (const node of nodes) {
|
|
||||||
ranks.set(node, 1 / n)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Calculate outgoing edge counts
|
|
||||||
const outDegree = new Map<string, number>()
|
|
||||||
for (const [node, neighbors] of this.adjacencyList) {
|
|
||||||
let count = 0
|
|
||||||
for (const edges of neighbors.values()) {
|
|
||||||
count += edges.length
|
|
||||||
}
|
|
||||||
outDegree.set(node, count)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Iterate PageRank algorithm
|
|
||||||
for (let i = 0; i < iterations; i++) {
|
|
||||||
const newRanks = new Map<string, number>()
|
|
||||||
|
|
||||||
for (const node of nodes) {
|
|
||||||
let rank = (1 - damping) / n
|
|
||||||
|
|
||||||
// Sum contributions from incoming edges
|
|
||||||
for (const [source, neighbors] of this.adjacencyList) {
|
|
||||||
if (neighbors.has(node)) {
|
|
||||||
const sourceRank = ranks.get(source) ?? 0
|
|
||||||
const sourceOutDegree = outDegree.get(source) ?? 1
|
|
||||||
rank += damping * (sourceRank / sourceOutDegree)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
newRanks.set(node, rank)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Update ranks
|
|
||||||
for (const [node, rank] of newRanks) {
|
|
||||||
ranks.set(node, rank)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return ranks
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Clear the graph
|
|
||||||
*/
|
|
||||||
public clear(): void {
|
|
||||||
this.adjacencyList.clear()
|
|
||||||
this.nodes.clear()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,354 +0,0 @@
|
||||||
/**
|
|
||||||
* Entity Deduplicator
|
|
||||||
*
|
|
||||||
* Finds and merges duplicate entities across imports using:
|
|
||||||
* - Embedding-based similarity matching
|
|
||||||
* - Type-aware comparison
|
|
||||||
* - Confidence-weighted merging
|
|
||||||
* - Provenance tracking
|
|
||||||
*
|
|
||||||
* NO MOCKS - Production-ready implementation
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { Brainy } from '../brainy.js'
|
|
||||||
import { NounType } from '../types/graphTypes.js'
|
|
||||||
|
|
||||||
export interface EntityCandidate {
|
|
||||||
id?: string
|
|
||||||
name: string
|
|
||||||
type: NounType
|
|
||||||
description: string
|
|
||||||
confidence: number
|
|
||||||
metadata: Record<string, any>
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface DuplicateMatch {
|
|
||||||
existingId: string
|
|
||||||
existingName: string
|
|
||||||
similarity: number
|
|
||||||
shouldMerge: boolean
|
|
||||||
reason: string
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface EntityDeduplicationOptions {
|
|
||||||
/** Similarity threshold for considering entities as duplicates (0-1) */
|
|
||||||
similarityThreshold?: number
|
|
||||||
|
|
||||||
/** Only match entities of the same type */
|
|
||||||
strictTypeMatching?: boolean
|
|
||||||
|
|
||||||
/** Enable fuzzy name matching */
|
|
||||||
enableFuzzyMatching?: boolean
|
|
||||||
|
|
||||||
/** Minimum confidence to consider for merging */
|
|
||||||
minConfidence?: number
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface MergeResult {
|
|
||||||
mergedEntityId: string
|
|
||||||
wasMerged: boolean
|
|
||||||
mergedWith?: string
|
|
||||||
confidence: number
|
|
||||||
provenance: string[]
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* EntityDeduplicator - Prevents duplicate entities across imports
|
|
||||||
*/
|
|
||||||
export class EntityDeduplicator {
|
|
||||||
private brain: Brainy
|
|
||||||
|
|
||||||
constructor(brain: Brainy) {
|
|
||||||
this.brain = brain
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Find duplicate entities in the knowledge graph
|
|
||||||
*/
|
|
||||||
async findDuplicates(
|
|
||||||
candidate: EntityCandidate,
|
|
||||||
options: EntityDeduplicationOptions = {}
|
|
||||||
): Promise<DuplicateMatch | null> {
|
|
||||||
const opts = {
|
|
||||||
similarityThreshold: options.similarityThreshold || 0.85,
|
|
||||||
strictTypeMatching: options.strictTypeMatching !== false,
|
|
||||||
enableFuzzyMatching: options.enableFuzzyMatching !== false,
|
|
||||||
minConfidence: options.minConfidence || 0.6
|
|
||||||
}
|
|
||||||
|
|
||||||
// Skip low-confidence candidates
|
|
||||||
if (candidate.confidence < opts.minConfidence) {
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
|
|
||||||
// Search for similar entities by name and description
|
|
||||||
const searchText = `${candidate.name} ${candidate.description}`.trim()
|
|
||||||
|
|
||||||
try {
|
|
||||||
const results = await this.brain.find({
|
|
||||||
query: searchText,
|
|
||||||
limit: 5,
|
|
||||||
where: opts.strictTypeMatching ? { type: candidate.type } : undefined
|
|
||||||
})
|
|
||||||
|
|
||||||
// Check each result for potential duplicates
|
|
||||||
for (const result of results) {
|
|
||||||
const similarity = result.score || 0
|
|
||||||
const existingName = result.entity.metadata?.name || result.id
|
|
||||||
const existingType = result.entity.metadata?.type || result.entity.metadata?.nounType || result.entity.type
|
|
||||||
|
|
||||||
// Skip if below similarity threshold
|
|
||||||
if (similarity < opts.similarityThreshold) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
// Type matching check
|
|
||||||
if (opts.strictTypeMatching && existingType !== candidate.type) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
// Exact name match (case-insensitive)
|
|
||||||
if (this.normalizeString(candidate.name) === this.normalizeString(existingName)) {
|
|
||||||
return {
|
|
||||||
existingId: result.id,
|
|
||||||
existingName,
|
|
||||||
similarity: 1.0,
|
|
||||||
shouldMerge: true,
|
|
||||||
reason: 'Exact name match'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// High similarity match
|
|
||||||
if (similarity >= opts.similarityThreshold) {
|
|
||||||
// Additional validation for fuzzy matching
|
|
||||||
if (opts.enableFuzzyMatching && this.areSimilarNames(candidate.name, existingName)) {
|
|
||||||
return {
|
|
||||||
existingId: result.id,
|
|
||||||
existingName,
|
|
||||||
similarity,
|
|
||||||
shouldMerge: true,
|
|
||||||
reason: `High similarity (${(similarity * 100).toFixed(1)}%)`
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
// If search fails, assume no duplicates
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Merge entity data with existing entity
|
|
||||||
*/
|
|
||||||
async mergeEntity(
|
|
||||||
existingId: string,
|
|
||||||
candidate: EntityCandidate,
|
|
||||||
importSource: string
|
|
||||||
): Promise<MergeResult> {
|
|
||||||
try {
|
|
||||||
// Get existing entity
|
|
||||||
const existing = await this.brain.get(existingId)
|
|
||||||
if (!existing) {
|
|
||||||
throw new Error(`Entity ${existingId} not found`)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Update confidence (weighted average) — `confidence` is a reserved
|
|
||||||
// top-level field, so it travels via the dedicated update() param, not
|
|
||||||
// the metadata bag.
|
|
||||||
const mergedConfidence = this.mergeConfidence(
|
|
||||||
existing.confidence ?? 0.5,
|
|
||||||
candidate.confidence
|
|
||||||
)
|
|
||||||
|
|
||||||
// Merge metadata (custom fields only — reserved fields are top-level)
|
|
||||||
const mergedMetadata = {
|
|
||||||
...existing.metadata,
|
|
||||||
// Track provenance
|
|
||||||
imports: [
|
|
||||||
...(existing.metadata?.imports || []),
|
|
||||||
importSource
|
|
||||||
],
|
|
||||||
// Merge VFS paths
|
|
||||||
vfsPaths: [
|
|
||||||
...(existing.metadata?.vfsPaths || [existing.metadata?.vfsPath]).filter(Boolean),
|
|
||||||
candidate.metadata?.vfsPath
|
|
||||||
].filter(Boolean),
|
|
||||||
// Merge other metadata
|
|
||||||
...this.mergeMetadataFields(existing.metadata, candidate.metadata),
|
|
||||||
// Track last update
|
|
||||||
lastUpdated: Date.now(),
|
|
||||||
mergeCount: (existing.metadata?.mergeCount || 0) + 1
|
|
||||||
}
|
|
||||||
|
|
||||||
// Update entity
|
|
||||||
await this.brain.update({
|
|
||||||
id: existingId,
|
|
||||||
confidence: mergedConfidence,
|
|
||||||
metadata: mergedMetadata,
|
|
||||||
merge: true
|
|
||||||
})
|
|
||||||
|
|
||||||
return {
|
|
||||||
mergedEntityId: existingId,
|
|
||||||
wasMerged: true,
|
|
||||||
mergedWith: existing.metadata?.name || existingId,
|
|
||||||
confidence: mergedConfidence,
|
|
||||||
provenance: mergedMetadata.imports
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
throw new Error(`Failed to merge entity: ${error instanceof Error ? error.message : String(error)}`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create or merge entity with deduplication
|
|
||||||
*/
|
|
||||||
async createOrMerge(
|
|
||||||
candidate: EntityCandidate,
|
|
||||||
importSource: string,
|
|
||||||
options: EntityDeduplicationOptions = {}
|
|
||||||
): Promise<MergeResult> {
|
|
||||||
// Check for duplicates
|
|
||||||
const duplicate = await this.findDuplicates(candidate, options)
|
|
||||||
|
|
||||||
if (duplicate && duplicate.shouldMerge) {
|
|
||||||
// Merge with existing entity
|
|
||||||
return await this.mergeEntity(duplicate.existingId, candidate, importSource)
|
|
||||||
}
|
|
||||||
|
|
||||||
// No duplicate found, create new entity. Preserve any subtype the candidate
|
|
||||||
// carried (set by the extractor or upstream importer), else fall back to
|
|
||||||
// `'imported'` so enforcement doesn't fire (added 7.30.1).
|
|
||||||
// `confidence` is a reserved top-level field (dedicated add() param);
|
|
||||||
// creation time is system-managed — neither belongs in the metadata bag.
|
|
||||||
const entityId = await this.brain.add({
|
|
||||||
data: candidate.description || candidate.name,
|
|
||||||
type: candidate.type,
|
|
||||||
subtype:
|
|
||||||
(candidate as EntityCandidate & { subtype?: string }).subtype ??
|
|
||||||
'imported',
|
|
||||||
confidence: candidate.confidence,
|
|
||||||
metadata: {
|
|
||||||
...candidate.metadata,
|
|
||||||
name: candidate.name,
|
|
||||||
imports: [importSource],
|
|
||||||
vfsPaths: [candidate.metadata?.vfsPath].filter(Boolean),
|
|
||||||
mergeCount: 0
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// Update candidate with new ID
|
|
||||||
candidate.id = entityId
|
|
||||||
|
|
||||||
return {
|
|
||||||
mergedEntityId: entityId,
|
|
||||||
wasMerged: false,
|
|
||||||
confidence: candidate.confidence,
|
|
||||||
provenance: [importSource]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Normalize string for comparison
|
|
||||||
*/
|
|
||||||
private normalizeString(str: string): string {
|
|
||||||
return str
|
|
||||||
.toLowerCase()
|
|
||||||
.trim()
|
|
||||||
.replace(/[^a-z0-9]/g, '')
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Check if two names are similar (fuzzy matching)
|
|
||||||
*/
|
|
||||||
private areSimilarNames(name1: string, name2: string): boolean {
|
|
||||||
const n1 = this.normalizeString(name1)
|
|
||||||
const n2 = this.normalizeString(name2)
|
|
||||||
|
|
||||||
// Exact match
|
|
||||||
if (n1 === n2) return true
|
|
||||||
|
|
||||||
// Length difference check
|
|
||||||
const lengthDiff = Math.abs(n1.length - n2.length)
|
|
||||||
if (lengthDiff > 3) return false
|
|
||||||
|
|
||||||
// Levenshtein distance
|
|
||||||
const distance = this.levenshteinDistance(n1, n2)
|
|
||||||
const maxLength = Math.max(n1.length, n2.length)
|
|
||||||
const similarity = 1 - (distance / maxLength)
|
|
||||||
|
|
||||||
return similarity >= 0.85
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Calculate Levenshtein distance between two strings
|
|
||||||
*/
|
|
||||||
private levenshteinDistance(str1: string, str2: string): number {
|
|
||||||
const m = str1.length
|
|
||||||
const n = str2.length
|
|
||||||
const dp: number[][] = Array(m + 1).fill(null).map(() => Array(n + 1).fill(0))
|
|
||||||
|
|
||||||
for (let i = 0; i <= m; i++) dp[i][0] = i
|
|
||||||
for (let j = 0; j <= n; j++) dp[0][j] = j
|
|
||||||
|
|
||||||
for (let i = 1; i <= m; i++) {
|
|
||||||
for (let j = 1; j <= n; j++) {
|
|
||||||
if (str1[i - 1] === str2[j - 1]) {
|
|
||||||
dp[i][j] = dp[i - 1][j - 1]
|
|
||||||
} else {
|
|
||||||
dp[i][j] = Math.min(
|
|
||||||
dp[i - 1][j] + 1, // deletion
|
|
||||||
dp[i][j - 1] + 1, // insertion
|
|
||||||
dp[i - 1][j - 1] + 1 // substitution
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return dp[m][n]
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Merge confidence scores (weighted average favoring higher confidence)
|
|
||||||
*/
|
|
||||||
private mergeConfidence(existing: number, incoming: number): number {
|
|
||||||
// Weight higher confidence more heavily
|
|
||||||
const weights = existing > incoming ? [0.6, 0.4] : [0.4, 0.6]
|
|
||||||
return existing * weights[0] + incoming * weights[1]
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Merge metadata fields intelligently
|
|
||||||
*/
|
|
||||||
private mergeMetadataFields(
|
|
||||||
existing: Record<string, any>,
|
|
||||||
incoming: Record<string, any>
|
|
||||||
): Record<string, any> {
|
|
||||||
const merged: Record<string, any> = {}
|
|
||||||
|
|
||||||
// Merge arrays
|
|
||||||
const arrayFields = ['concepts', 'tags', 'categories']
|
|
||||||
for (const field of arrayFields) {
|
|
||||||
if (existing[field] || incoming[field]) {
|
|
||||||
const combined = [
|
|
||||||
...(existing[field] || []),
|
|
||||||
...(incoming[field] || [])
|
|
||||||
]
|
|
||||||
// Deduplicate
|
|
||||||
merged[field] = [...new Set(combined)]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Prefer longer descriptions
|
|
||||||
if (existing.description || incoming.description) {
|
|
||||||
merged.description = (existing.description || '').length > (incoming.description || '').length
|
|
||||||
? existing.description
|
|
||||||
: incoming.description
|
|
||||||
}
|
|
||||||
|
|
||||||
return merged
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,269 +0,0 @@
|
||||||
/**
|
|
||||||
* InstancePool - Shared instance management for memory efficiency
|
|
||||||
*
|
|
||||||
* Production-grade instance pooling to prevent memory leaks during imports.
|
|
||||||
* Critical for scaling to billions of entities.
|
|
||||||
*
|
|
||||||
* Problem: Creating new NLP/Extractor instances in loops → memory leak
|
|
||||||
* Solution: Reuse shared instances across entire import session
|
|
||||||
*
|
|
||||||
* Memory savings:
|
|
||||||
* - Without pooling: 100K rows × 50MB per instance = 5TB RAM (OOM!)
|
|
||||||
* - With pooling: 50MB total (shared across all rows)
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { Brainy } from '../brainy.js'
|
|
||||||
import { NaturalLanguageProcessor } from '../neural/naturalLanguageProcessor.js'
|
|
||||||
import { NeuralEntityExtractor } from '../neural/entityExtractor.js'
|
|
||||||
|
|
||||||
/**
|
|
||||||
* InstancePool - Manages shared instances for memory efficiency
|
|
||||||
*
|
|
||||||
* Lifecycle:
|
|
||||||
* 1. Create pool at import start
|
|
||||||
* 2. Reuse instances across all rows
|
|
||||||
* 3. Pool is garbage collected when import completes
|
|
||||||
*
|
|
||||||
* Thread safety: Not thread-safe (single import session per pool)
|
|
||||||
*/
|
|
||||||
export class InstancePool {
|
|
||||||
private brain: Brainy
|
|
||||||
|
|
||||||
// Shared instances (created lazily)
|
|
||||||
private nlpInstance: NaturalLanguageProcessor | null = null
|
|
||||||
private extractorInstance: NeuralEntityExtractor | null = null
|
|
||||||
|
|
||||||
// Initialization state
|
|
||||||
private nlpInitialized = false
|
|
||||||
private initializationPromise: Promise<void> | null = null
|
|
||||||
|
|
||||||
// Statistics
|
|
||||||
private stats = {
|
|
||||||
nlpReuses: 0,
|
|
||||||
extractorReuses: 0,
|
|
||||||
creationTime: 0
|
|
||||||
}
|
|
||||||
|
|
||||||
constructor(brain: Brainy) {
|
|
||||||
this.brain = brain
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get shared NaturalLanguageProcessor instance
|
|
||||||
*
|
|
||||||
* Lazy initialization - created on first access
|
|
||||||
* All subsequent calls return same instance
|
|
||||||
*
|
|
||||||
* @returns Shared NLP instance
|
|
||||||
*/
|
|
||||||
async getNLP(): Promise<NaturalLanguageProcessor> {
|
|
||||||
if (!this.nlpInstance) {
|
|
||||||
const startTime = Date.now()
|
|
||||||
this.nlpInstance = new NaturalLanguageProcessor(this.brain)
|
|
||||||
this.stats.creationTime += Date.now() - startTime
|
|
||||||
}
|
|
||||||
|
|
||||||
// Ensure initialized before returning
|
|
||||||
if (!this.nlpInitialized) {
|
|
||||||
await this.ensureNLPInitialized()
|
|
||||||
}
|
|
||||||
|
|
||||||
this.stats.nlpReuses++
|
|
||||||
return this.nlpInstance
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get shared NeuralEntityExtractor instance
|
|
||||||
*
|
|
||||||
* Lazy initialization - created on first access
|
|
||||||
* All subsequent calls return same instance
|
|
||||||
*
|
|
||||||
* @returns Shared extractor instance
|
|
||||||
*/
|
|
||||||
getExtractor(): NeuralEntityExtractor {
|
|
||||||
if (!this.extractorInstance) {
|
|
||||||
const startTime = Date.now()
|
|
||||||
this.extractorInstance = new NeuralEntityExtractor(this.brain)
|
|
||||||
this.stats.creationTime += Date.now() - startTime
|
|
||||||
}
|
|
||||||
|
|
||||||
this.stats.extractorReuses++
|
|
||||||
return this.extractorInstance
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get shared NLP instance (synchronous, may return uninitialized)
|
|
||||||
*
|
|
||||||
* Use when you need NLP synchronously and will handle initialization yourself.
|
|
||||||
* Prefer getNLP() for async code.
|
|
||||||
*
|
|
||||||
* @returns Shared NLP instance (possibly uninitialized)
|
|
||||||
*/
|
|
||||||
getNLPSync(): NaturalLanguageProcessor {
|
|
||||||
if (!this.nlpInstance) {
|
|
||||||
this.nlpInstance = new NaturalLanguageProcessor(this.brain)
|
|
||||||
}
|
|
||||||
|
|
||||||
this.stats.nlpReuses++
|
|
||||||
return this.nlpInstance
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Initialize all instances upfront
|
|
||||||
*
|
|
||||||
* Call at start of import to avoid lazy initialization overhead
|
|
||||||
* during processing. Improves predictability and first-row performance.
|
|
||||||
*
|
|
||||||
* @returns Promise that resolves when all instances are ready
|
|
||||||
*/
|
|
||||||
async init(): Promise<void> {
|
|
||||||
// Prevent duplicate initialization
|
|
||||||
if (this.initializationPromise) {
|
|
||||||
return this.initializationPromise
|
|
||||||
}
|
|
||||||
|
|
||||||
this.initializationPromise = this.initializeInternal()
|
|
||||||
return this.initializationPromise
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Internal initialization implementation
|
|
||||||
*/
|
|
||||||
private async initializeInternal(): Promise<void> {
|
|
||||||
const startTime = Date.now()
|
|
||||||
|
|
||||||
// Create instances
|
|
||||||
if (!this.nlpInstance) {
|
|
||||||
this.nlpInstance = new NaturalLanguageProcessor(this.brain)
|
|
||||||
}
|
|
||||||
if (!this.extractorInstance) {
|
|
||||||
this.extractorInstance = new NeuralEntityExtractor(this.brain)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Initialize NLP (loads pattern library)
|
|
||||||
await this.ensureNLPInitialized()
|
|
||||||
|
|
||||||
this.stats.creationTime = Date.now() - startTime
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Ensure NLP is initialized (loads 220 patterns)
|
|
||||||
*
|
|
||||||
* Handles concurrent initialization requests safely
|
|
||||||
*/
|
|
||||||
private async ensureNLPInitialized(): Promise<void> {
|
|
||||||
if (this.nlpInitialized) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!this.nlpInstance) {
|
|
||||||
throw new Error('NLP instance not created yet')
|
|
||||||
}
|
|
||||||
|
|
||||||
await this.nlpInstance.init()
|
|
||||||
this.nlpInitialized = true
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Check if instances are initialized
|
|
||||||
*
|
|
||||||
* @returns True if NLP is initialized and ready to use
|
|
||||||
*/
|
|
||||||
isInitialized(): boolean {
|
|
||||||
return this.nlpInitialized && this.nlpInstance !== null
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get pool statistics
|
|
||||||
*
|
|
||||||
* Useful for performance monitoring and memory leak detection
|
|
||||||
*
|
|
||||||
* @returns Statistics about instance reuse
|
|
||||||
*/
|
|
||||||
getStats() {
|
|
||||||
return {
|
|
||||||
...this.stats,
|
|
||||||
nlpCreated: this.nlpInstance !== null,
|
|
||||||
extractorCreated: this.extractorInstance !== null,
|
|
||||||
initialized: this.isInitialized(),
|
|
||||||
// Memory savings estimate
|
|
||||||
memorySaved: this.calculateMemorySaved()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Calculate estimated memory saved by pooling
|
|
||||||
*
|
|
||||||
* Assumes ~50MB per NLP instance, ~10MB per extractor instance
|
|
||||||
*
|
|
||||||
* @returns Estimated memory saved in bytes
|
|
||||||
*/
|
|
||||||
private calculateMemorySaved(): number {
|
|
||||||
const nlpSize = 50 * 1024 * 1024 // 50MB per instance
|
|
||||||
const extractorSize = 10 * 1024 * 1024 // 10MB per instance
|
|
||||||
|
|
||||||
// Without pooling: size × reuses
|
|
||||||
// With pooling: size × 1
|
|
||||||
// Saved: size × (reuses - 1)
|
|
||||||
|
|
||||||
const nlpSaved = nlpSize * Math.max(0, this.stats.nlpReuses - 1)
|
|
||||||
const extractorSaved = extractorSize * Math.max(0, this.stats.extractorReuses - 1)
|
|
||||||
|
|
||||||
return nlpSaved + extractorSaved
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Reset statistics (useful for testing)
|
|
||||||
*/
|
|
||||||
resetStats(): void {
|
|
||||||
this.stats = {
|
|
||||||
nlpReuses: 0,
|
|
||||||
extractorReuses: 0,
|
|
||||||
creationTime: 0
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get string representation (for debugging)
|
|
||||||
*/
|
|
||||||
toString(): string {
|
|
||||||
const stats = this.getStats()
|
|
||||||
return `InstancePool(nlp=${stats.nlpCreated}, extractor=${stats.extractorCreated}, initialized=${stats.initialized}, nlpReuses=${stats.nlpReuses}, extractorReuses=${stats.extractorReuses})`
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Cleanup method (for explicit resource management)
|
|
||||||
*
|
|
||||||
* Note: Usually not needed - pool is garbage collected when import completes.
|
|
||||||
* Use only if you need explicit cleanup for some reason.
|
|
||||||
*/
|
|
||||||
cleanup(): void {
|
|
||||||
// Clear references to allow garbage collection
|
|
||||||
this.nlpInstance = null
|
|
||||||
this.extractorInstance = null
|
|
||||||
this.nlpInitialized = false
|
|
||||||
this.initializationPromise = null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create a new instance pool
|
|
||||||
*
|
|
||||||
* Convenience factory function
|
|
||||||
*
|
|
||||||
* @param brain Brainy instance
|
|
||||||
* @param autoInit Whether to initialize instances immediately
|
|
||||||
* @returns Instance pool
|
|
||||||
*/
|
|
||||||
export async function createInstancePool(
|
|
||||||
brain: Brainy,
|
|
||||||
autoInit = true
|
|
||||||
): Promise<InstancePool> {
|
|
||||||
const pool = new InstancePool(brain)
|
|
||||||
|
|
||||||
if (autoInit) {
|
|
||||||
await pool.init()
|
|
||||||
}
|
|
||||||
|
|
||||||
return pool
|
|
||||||
}
|
|
||||||
|
|
@ -1,38 +0,0 @@
|
||||||
/**
|
|
||||||
* Unified Import System
|
|
||||||
*
|
|
||||||
* Single entry point for importing any file format into Brainy with:
|
|
||||||
* - Auto-detection of formats
|
|
||||||
* - Dual storage (VFS + Knowledge Graph)
|
|
||||||
* - Shared entities across imports (deduplication)
|
|
||||||
* - Simple, powerful API
|
|
||||||
*/
|
|
||||||
|
|
||||||
export { ImportCoordinator } from './ImportCoordinator.js'
|
|
||||||
export { FormatDetector, SupportedFormat, DetectionResult } from './FormatDetector.js'
|
|
||||||
export { EntityDeduplicator } from './EntityDeduplicator.js'
|
|
||||||
export { BackgroundDeduplicator } from './BackgroundDeduplicator.js'
|
|
||||||
export { ImportHistory } from './ImportHistory.js'
|
|
||||||
|
|
||||||
export type {
|
|
||||||
ImportSource,
|
|
||||||
ImportOptions,
|
|
||||||
ImportProgress,
|
|
||||||
ImportResult
|
|
||||||
} from './ImportCoordinator.js'
|
|
||||||
|
|
||||||
export type {
|
|
||||||
EntityCandidate,
|
|
||||||
DuplicateMatch,
|
|
||||||
EntityDeduplicationOptions,
|
|
||||||
MergeResult
|
|
||||||
} from './EntityDeduplicator.js'
|
|
||||||
|
|
||||||
export type {
|
|
||||||
DeduplicationStats
|
|
||||||
} from './BackgroundDeduplicator.js'
|
|
||||||
|
|
||||||
export type {
|
|
||||||
ImportHistoryEntry,
|
|
||||||
RollbackResult
|
|
||||||
} from './ImportHistory.js'
|
|
||||||
|
|
@ -1,789 +0,0 @@
|
||||||
/**
|
|
||||||
* Smart Import Orchestrator
|
|
||||||
*
|
|
||||||
* Coordinates the entire smart import pipeline:
|
|
||||||
* 1. Extract entities/relationships using SmartExcelImporter
|
|
||||||
* 2. Create entities and relationships in Brainy
|
|
||||||
* 3. Organize into VFS structure using VFSStructureGenerator
|
|
||||||
*
|
|
||||||
* NO MOCKS - Production-ready implementation
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { Brainy } from '../brainy.js'
|
|
||||||
import { VirtualFileSystem } from '../vfs/VirtualFileSystem.js'
|
|
||||||
import { NounType, VerbType } from '../types/graphTypes.js'
|
|
||||||
import { splitNounMetadataRecord } from '../types/reservedFields.js'
|
|
||||||
import { SmartExcelImporter, SmartExcelOptions, SmartExcelResult } from './SmartExcelImporter.js'
|
|
||||||
import { SmartPDFImporter, SmartPDFOptions, SmartPDFResult } from './SmartPDFImporter.js'
|
|
||||||
import { SmartCSVImporter, SmartCSVOptions, SmartCSVResult } from './SmartCSVImporter.js'
|
|
||||||
import { SmartJSONImporter, SmartJSONOptions, SmartJSONResult } from './SmartJSONImporter.js'
|
|
||||||
import { SmartMarkdownImporter, SmartMarkdownOptions, SmartMarkdownResult } from './SmartMarkdownImporter.js'
|
|
||||||
import { VFSStructureGenerator, VFSStructureOptions } from './VFSStructureGenerator.js'
|
|
||||||
|
|
||||||
export interface SmartImportOptions extends SmartExcelOptions {
|
|
||||||
/** Create VFS structure */
|
|
||||||
createVFSStructure?: boolean
|
|
||||||
|
|
||||||
/** VFS root path */
|
|
||||||
vfsRootPath?: string
|
|
||||||
|
|
||||||
/** VFS grouping strategy */
|
|
||||||
vfsGroupBy?: 'type' | 'sheet' | 'flat' | 'custom'
|
|
||||||
|
|
||||||
/** Create entities in Brainy */
|
|
||||||
createEntities?: boolean
|
|
||||||
|
|
||||||
/** Create relationships in Brainy */
|
|
||||||
createRelationships?: boolean
|
|
||||||
|
|
||||||
/** Source filename */
|
|
||||||
filename?: string
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Default subtype tag for entities + relationships this importer creates when
|
|
||||||
* the extractor doesn't set one. See `ValidImportOptions.defaultSubtype` —
|
|
||||||
* same semantics, same precedence (extractor > caller default > `'imported'`).
|
|
||||||
* Added 7.30.1.
|
|
||||||
*/
|
|
||||||
defaultSubtype?: string
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface SmartImportProgress {
|
|
||||||
phase: 'parsing' | 'extracting' | 'creating' | 'relationships' | 'organizing' | 'complete'
|
|
||||||
message: string
|
|
||||||
processed: number
|
|
||||||
total: number
|
|
||||||
entities: number
|
|
||||||
relationships: number
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface SmartImportResult {
|
|
||||||
success: boolean
|
|
||||||
|
|
||||||
/** Extraction results */
|
|
||||||
extraction: SmartExcelResult
|
|
||||||
|
|
||||||
/** Created entity IDs */
|
|
||||||
entityIds: string[]
|
|
||||||
|
|
||||||
/** Created relationship IDs */
|
|
||||||
relationshipIds: string[]
|
|
||||||
|
|
||||||
/** VFS structure created */
|
|
||||||
vfsStructure?: {
|
|
||||||
rootPath: string
|
|
||||||
directories: string[]
|
|
||||||
files: number
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Overall statistics */
|
|
||||||
stats: {
|
|
||||||
rowsProcessed: number
|
|
||||||
entitiesCreated: number
|
|
||||||
relationshipsCreated: number
|
|
||||||
filesCreated: number
|
|
||||||
totalTime: number
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Any errors encountered */
|
|
||||||
errors: string[]
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* SmartImportOrchestrator - Main entry point for smart imports
|
|
||||||
*/
|
|
||||||
export class SmartImportOrchestrator {
|
|
||||||
private brain: Brainy
|
|
||||||
private excelImporter: SmartExcelImporter
|
|
||||||
private pdfImporter: SmartPDFImporter
|
|
||||||
private csvImporter: SmartCSVImporter
|
|
||||||
private jsonImporter: SmartJSONImporter
|
|
||||||
private markdownImporter: SmartMarkdownImporter
|
|
||||||
private vfsGenerator: VFSStructureGenerator
|
|
||||||
|
|
||||||
constructor(brain: Brainy) {
|
|
||||||
this.brain = brain
|
|
||||||
this.excelImporter = new SmartExcelImporter(brain)
|
|
||||||
this.pdfImporter = new SmartPDFImporter(brain)
|
|
||||||
this.csvImporter = new SmartCSVImporter(brain)
|
|
||||||
this.jsonImporter = new SmartJSONImporter(brain)
|
|
||||||
this.markdownImporter = new SmartMarkdownImporter(brain)
|
|
||||||
this.vfsGenerator = new VFSStructureGenerator(brain)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Initialize the orchestrator
|
|
||||||
*/
|
|
||||||
async init(): Promise<void> {
|
|
||||||
await this.excelImporter.init()
|
|
||||||
await this.pdfImporter.init()
|
|
||||||
await this.csvImporter.init()
|
|
||||||
await this.jsonImporter.init()
|
|
||||||
await this.markdownImporter.init()
|
|
||||||
await this.vfsGenerator.init()
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Import Excel file with full pipeline
|
|
||||||
*/
|
|
||||||
async importExcel(
|
|
||||||
buffer: Buffer,
|
|
||||||
options: SmartImportOptions = {},
|
|
||||||
onProgress?: (progress: SmartImportProgress) => void
|
|
||||||
): Promise<SmartImportResult> {
|
|
||||||
const startTime = Date.now()
|
|
||||||
const result: SmartImportResult = {
|
|
||||||
success: false,
|
|
||||||
// Typed boundary: populated in the extraction phase below. If extraction
|
|
||||||
// throws, the error path returns with this still null (pre-existing
|
|
||||||
// contract — consumers check `success`/`errors` before reading it).
|
|
||||||
extraction: null as unknown as SmartExcelResult,
|
|
||||||
entityIds: [],
|
|
||||||
relationshipIds: [],
|
|
||||||
stats: {
|
|
||||||
rowsProcessed: 0,
|
|
||||||
entitiesCreated: 0,
|
|
||||||
relationshipsCreated: 0,
|
|
||||||
filesCreated: 0,
|
|
||||||
totalTime: 0
|
|
||||||
},
|
|
||||||
errors: []
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
// Phase 1: Extract entities and relationships
|
|
||||||
onProgress?.({
|
|
||||||
phase: 'extracting',
|
|
||||||
message: 'Extracting entities and relationships...',
|
|
||||||
processed: 0,
|
|
||||||
total: 0,
|
|
||||||
entities: 0,
|
|
||||||
relationships: 0
|
|
||||||
})
|
|
||||||
|
|
||||||
result.extraction = await this.excelImporter.extract(buffer, {
|
|
||||||
...options,
|
|
||||||
onProgress: (stats) => {
|
|
||||||
onProgress?.({
|
|
||||||
phase: 'extracting',
|
|
||||||
message: `Processing row ${stats.processed}/${stats.total}...`,
|
|
||||||
processed: stats.processed,
|
|
||||||
total: stats.total,
|
|
||||||
entities: stats.entities,
|
|
||||||
relationships: stats.relationships
|
|
||||||
})
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
result.stats.rowsProcessed = result.extraction.rowsProcessed
|
|
||||||
|
|
||||||
// Phase 2: Create entities in Brainy
|
|
||||||
if (options.createEntities !== false) {
|
|
||||||
onProgress?.({
|
|
||||||
phase: 'creating',
|
|
||||||
message: 'Creating entities in knowledge graph...',
|
|
||||||
processed: 0,
|
|
||||||
total: result.extraction.rows.length,
|
|
||||||
entities: 0,
|
|
||||||
relationships: 0
|
|
||||||
})
|
|
||||||
|
|
||||||
for (let i = 0; i < result.extraction.rows.length; i++) {
|
|
||||||
const extracted = result.extraction.rows[i]
|
|
||||||
|
|
||||||
try {
|
|
||||||
// Create main entity. Subtype precedence: extractor-set → caller default
|
|
||||||
// → Brainy default `'imported'` (added 7.30.1).
|
|
||||||
const entityId = await this.brain.add({
|
|
||||||
data: extracted.entity.description,
|
|
||||||
type: extracted.entity.type,
|
|
||||||
subtype:
|
|
||||||
(extracted.entity as typeof extracted.entity & { subtype?: string })
|
|
||||||
.subtype ?? options.defaultSubtype ?? 'imported',
|
|
||||||
confidence: extracted.entity.confidence, // reserved field — dedicated param, not metadata
|
|
||||||
metadata: {
|
|
||||||
// Strip reserved keys an extractor may have smuggled into the bag
|
|
||||||
// (8.0 reservedFieldPolicy defaults to 'throw').
|
|
||||||
...splitNounMetadataRecord(extracted.entity.metadata).custom,
|
|
||||||
name: extracted.entity.name,
|
|
||||||
importedFrom: 'smart-import'
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
result.entityIds.push(entityId)
|
|
||||||
result.stats.entitiesCreated++
|
|
||||||
|
|
||||||
// Update entity ID in extraction result
|
|
||||||
extracted.entity.id = entityId
|
|
||||||
|
|
||||||
onProgress?.({
|
|
||||||
phase: 'creating',
|
|
||||||
message: `Created entity: ${extracted.entity.name}`,
|
|
||||||
processed: i + 1,
|
|
||||||
total: result.extraction.rows.length,
|
|
||||||
entities: result.entityIds.length,
|
|
||||||
relationships: result.relationshipIds.length
|
|
||||||
})
|
|
||||||
} catch (error: any) {
|
|
||||||
result.errors.push(`Failed to create entity ${extracted.entity.name}: ${error.message}`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Phase 3: Create relationships
|
|
||||||
if (options.createRelationships !== false && options.createEntities !== false) {
|
|
||||||
onProgress?.({
|
|
||||||
phase: 'creating',
|
|
||||||
message: 'Preparing relationships...',
|
|
||||||
processed: 0,
|
|
||||||
total: result.extraction.rows.length,
|
|
||||||
entities: result.entityIds.length,
|
|
||||||
relationships: 0
|
|
||||||
})
|
|
||||||
|
|
||||||
// Build entity name -> ID map
|
|
||||||
const entityMap = new Map<string, string>()
|
|
||||||
for (const extracted of result.extraction.rows) {
|
|
||||||
entityMap.set(extracted.entity.name.toLowerCase(), extracted.entity.id)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Collect all relationship parameters
|
|
||||||
const relationshipParams: Array<{from: string; to: string; type: VerbType; subtype?: string; confidence?: number; metadata?: any}> = []
|
|
||||||
|
|
||||||
for (const extracted of result.extraction.rows) {
|
|
||||||
for (const rel of extracted.relationships) {
|
|
||||||
try {
|
|
||||||
// Find target entity ID
|
|
||||||
let toEntityId: string | undefined
|
|
||||||
|
|
||||||
// Try to find by name in our extracted entities
|
|
||||||
for (const otherExtracted of result.extraction.rows) {
|
|
||||||
if (rel.to.toLowerCase().includes(otherExtracted.entity.name.toLowerCase()) ||
|
|
||||||
otherExtracted.entity.name.toLowerCase().includes(rel.to.toLowerCase())) {
|
|
||||||
toEntityId = otherExtracted.entity.id
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// If not found, create a placeholder entity. `import-placeholder` marks
|
|
||||||
// these as synthetic targets so consumers can distinguish them from real
|
|
||||||
// imports and downstream dedup can consolidate (added 7.30.1).
|
|
||||||
if (!toEntityId) {
|
|
||||||
toEntityId = await this.brain.add({
|
|
||||||
data: rel.to,
|
|
||||||
type: NounType.Thing,
|
|
||||||
subtype: 'import-placeholder',
|
|
||||||
metadata: {
|
|
||||||
name: rel.to,
|
|
||||||
placeholder: true,
|
|
||||||
extractedFrom: extracted.entity.name
|
|
||||||
}
|
|
||||||
})
|
|
||||||
result.entityIds.push(toEntityId)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Collect relationship parameter. Subtype precedence: extractor-set rel
|
|
||||||
// subtype → caller default → Brainy default `'imported'` (added 7.30.1).
|
|
||||||
relationshipParams.push({
|
|
||||||
from: extracted.entity.id,
|
|
||||||
to: toEntityId,
|
|
||||||
type: rel.type,
|
|
||||||
subtype:
|
|
||||||
(rel as typeof rel & { subtype?: string }).subtype ??
|
|
||||||
options.defaultSubtype ?? 'imported',
|
|
||||||
confidence: rel.confidence, // reserved field — dedicated param, not metadata
|
|
||||||
metadata: {
|
|
||||||
evidence: rel.evidence
|
|
||||||
}
|
|
||||||
})
|
|
||||||
} catch (error: any) {
|
|
||||||
result.errors.push(`Failed to prepare relationship: ${error.message}`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Batch create all relationships with progress
|
|
||||||
if (relationshipParams.length > 0) {
|
|
||||||
onProgress?.({
|
|
||||||
phase: 'relationships',
|
|
||||||
message: 'Building relationships...',
|
|
||||||
processed: 0,
|
|
||||||
total: relationshipParams.length,
|
|
||||||
entities: result.entityIds.length,
|
|
||||||
relationships: 0
|
|
||||||
})
|
|
||||||
|
|
||||||
try {
|
|
||||||
const relationshipIds = await this.brain.relateMany({
|
|
||||||
items: relationshipParams,
|
|
||||||
parallel: true,
|
|
||||||
chunkSize: 100,
|
|
||||||
continueOnError: true,
|
|
||||||
onProgress: (done, total) => {
|
|
||||||
onProgress?.({
|
|
||||||
phase: 'relationships',
|
|
||||||
message: `Building relationships: ${done}/${total}`,
|
|
||||||
processed: done,
|
|
||||||
total: total,
|
|
||||||
entities: result.entityIds.length,
|
|
||||||
relationships: done
|
|
||||||
})
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
result.relationshipIds = relationshipIds
|
|
||||||
result.stats.relationshipsCreated = relationshipIds.length
|
|
||||||
} catch (error: any) {
|
|
||||||
result.errors.push(`Failed to create relationships: ${error.message}`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Phase 4: Create VFS structure
|
|
||||||
if (options.createVFSStructure !== false) {
|
|
||||||
onProgress?.({
|
|
||||||
phase: 'organizing',
|
|
||||||
message: 'Organizing into file structure...',
|
|
||||||
processed: 0,
|
|
||||||
total: result.extraction.rows.length,
|
|
||||||
entities: result.entityIds.length,
|
|
||||||
relationships: result.relationshipIds.length
|
|
||||||
})
|
|
||||||
|
|
||||||
const vfsOptions: VFSStructureOptions = {
|
|
||||||
rootPath: options.vfsRootPath || '/imports/' + (options.filename || 'import'),
|
|
||||||
groupBy: options.vfsGroupBy || 'type',
|
|
||||||
preserveSource: true,
|
|
||||||
sourceBuffer: buffer,
|
|
||||||
sourceFilename: options.filename || 'import.xlsx',
|
|
||||||
createRelationshipFile: true,
|
|
||||||
createMetadataFile: true
|
|
||||||
}
|
|
||||||
|
|
||||||
const vfsResult = await this.vfsGenerator.generate(result.extraction, vfsOptions)
|
|
||||||
|
|
||||||
result.vfsStructure = {
|
|
||||||
rootPath: vfsResult.rootPath,
|
|
||||||
directories: vfsResult.directories,
|
|
||||||
files: vfsResult.files.length
|
|
||||||
}
|
|
||||||
|
|
||||||
result.stats.filesCreated = vfsResult.files.length
|
|
||||||
}
|
|
||||||
|
|
||||||
// Complete
|
|
||||||
result.success = result.errors.length === 0
|
|
||||||
result.stats.totalTime = Date.now() - startTime
|
|
||||||
|
|
||||||
onProgress?.({
|
|
||||||
phase: 'complete',
|
|
||||||
message: `Import complete: ${result.stats.entitiesCreated} entities, ${result.stats.relationshipsCreated} relationships`,
|
|
||||||
processed: result.extraction.rows.length,
|
|
||||||
total: result.extraction.rows.length,
|
|
||||||
entities: result.stats.entitiesCreated,
|
|
||||||
relationships: result.stats.relationshipsCreated
|
|
||||||
})
|
|
||||||
|
|
||||||
} catch (error: any) {
|
|
||||||
result.errors.push(`Import failed: ${error.message}`)
|
|
||||||
result.success = false
|
|
||||||
}
|
|
||||||
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Import PDF file with full pipeline
|
|
||||||
*/
|
|
||||||
async importPDF(
|
|
||||||
buffer: Buffer,
|
|
||||||
options: SmartImportOptions & SmartPDFOptions = {},
|
|
||||||
onProgress?: (progress: SmartImportProgress) => void
|
|
||||||
): Promise<SmartImportResult> {
|
|
||||||
const startTime = Date.now()
|
|
||||||
const result: SmartImportResult = {
|
|
||||||
success: false,
|
|
||||||
// Typed boundary: populated after extraction (see importExcel).
|
|
||||||
extraction: null as unknown as SmartExcelResult,
|
|
||||||
entityIds: [],
|
|
||||||
relationshipIds: [],
|
|
||||||
stats: {
|
|
||||||
rowsProcessed: 0,
|
|
||||||
entitiesCreated: 0,
|
|
||||||
relationshipsCreated: 0,
|
|
||||||
filesCreated: 0,
|
|
||||||
totalTime: 0
|
|
||||||
},
|
|
||||||
errors: []
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
// Phase 1: Extract from PDF
|
|
||||||
onProgress?.({ phase: 'extracting', message: 'Extracting from PDF...', processed: 0, total: 0, entities: 0, relationships: 0 })
|
|
||||||
|
|
||||||
const pdfResult = await this.pdfImporter.extract(buffer, options)
|
|
||||||
|
|
||||||
// Convert PDF result to Excel-like format for processing
|
|
||||||
result.extraction = this.convertPDFToExcelFormat(pdfResult)
|
|
||||||
result.stats.rowsProcessed = pdfResult.sectionsProcessed
|
|
||||||
|
|
||||||
// Phase 2 & 3: Create entities and relationships
|
|
||||||
await this.createEntitiesAndRelationships(result, options, onProgress)
|
|
||||||
|
|
||||||
// Phase 4: Create VFS structure
|
|
||||||
if (options.createVFSStructure !== false) {
|
|
||||||
const vfsOptions: VFSStructureOptions = {
|
|
||||||
rootPath: options.vfsRootPath || '/imports/' + (options.filename || 'import'),
|
|
||||||
groupBy: options.vfsGroupBy || 'type',
|
|
||||||
preserveSource: true,
|
|
||||||
sourceBuffer: buffer,
|
|
||||||
sourceFilename: options.filename || 'import.pdf',
|
|
||||||
createRelationshipFile: true,
|
|
||||||
createMetadataFile: true
|
|
||||||
}
|
|
||||||
const vfsResult = await this.vfsGenerator.generate(result.extraction, vfsOptions)
|
|
||||||
result.vfsStructure = { rootPath: vfsResult.rootPath, directories: vfsResult.directories, files: vfsResult.files.length }
|
|
||||||
result.stats.filesCreated = vfsResult.files.length
|
|
||||||
}
|
|
||||||
|
|
||||||
result.success = result.errors.length === 0
|
|
||||||
result.stats.totalTime = Date.now() - startTime
|
|
||||||
onProgress?.({ phase: 'complete', message: `Import complete: ${result.stats.entitiesCreated} entities, ${result.stats.relationshipsCreated} relationships`, processed: result.stats.rowsProcessed, total: result.stats.rowsProcessed, entities: result.stats.entitiesCreated, relationships: result.stats.relationshipsCreated })
|
|
||||||
|
|
||||||
} catch (error: any) {
|
|
||||||
result.errors.push(`PDF import failed: ${error.message}`)
|
|
||||||
result.success = false
|
|
||||||
}
|
|
||||||
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Import CSV file with full pipeline
|
|
||||||
*/
|
|
||||||
async importCSV(
|
|
||||||
buffer: Buffer,
|
|
||||||
options: SmartImportOptions & SmartCSVOptions = {},
|
|
||||||
onProgress?: (progress: SmartImportProgress) => void
|
|
||||||
): Promise<SmartImportResult> {
|
|
||||||
// CSV is very similar to Excel, can reuse importExcel logic
|
|
||||||
return this.importExcel(buffer, options, onProgress)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Import JSON data with full pipeline
|
|
||||||
*/
|
|
||||||
async importJSON(
|
|
||||||
data: any,
|
|
||||||
options: SmartImportOptions & SmartJSONOptions = {},
|
|
||||||
onProgress?: (progress: SmartImportProgress) => void
|
|
||||||
): Promise<SmartImportResult> {
|
|
||||||
const startTime = Date.now()
|
|
||||||
const result: SmartImportResult = {
|
|
||||||
success: false,
|
|
||||||
// Typed boundary: populated after extraction (see importExcel).
|
|
||||||
extraction: null as unknown as SmartExcelResult,
|
|
||||||
entityIds: [],
|
|
||||||
relationshipIds: [],
|
|
||||||
stats: {
|
|
||||||
rowsProcessed: 0,
|
|
||||||
entitiesCreated: 0,
|
|
||||||
relationshipsCreated: 0,
|
|
||||||
filesCreated: 0,
|
|
||||||
totalTime: 0
|
|
||||||
},
|
|
||||||
errors: []
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
onProgress?.({ phase: 'extracting', message: 'Extracting from JSON...', processed: 0, total: 0, entities: 0, relationships: 0 })
|
|
||||||
|
|
||||||
const jsonResult = await this.jsonImporter.extract(data, options)
|
|
||||||
|
|
||||||
result.extraction = this.convertJSONToExcelFormat(jsonResult)
|
|
||||||
result.stats.rowsProcessed = jsonResult.nodesProcessed
|
|
||||||
|
|
||||||
await this.createEntitiesAndRelationships(result, options, onProgress)
|
|
||||||
|
|
||||||
if (options.createVFSStructure !== false) {
|
|
||||||
const sourceBuffer = Buffer.from(typeof data === 'string' ? data : JSON.stringify(data, null, 2))
|
|
||||||
const vfsOptions: VFSStructureOptions = {
|
|
||||||
rootPath: options.vfsRootPath || '/imports/' + (options.filename || 'import'),
|
|
||||||
groupBy: options.vfsGroupBy || 'type',
|
|
||||||
preserveSource: true,
|
|
||||||
sourceBuffer,
|
|
||||||
sourceFilename: options.filename || 'import.json',
|
|
||||||
createRelationshipFile: true,
|
|
||||||
createMetadataFile: true
|
|
||||||
}
|
|
||||||
const vfsResult = await this.vfsGenerator.generate(result.extraction, vfsOptions)
|
|
||||||
result.vfsStructure = { rootPath: vfsResult.rootPath, directories: vfsResult.directories, files: vfsResult.files.length }
|
|
||||||
result.stats.filesCreated = vfsResult.files.length
|
|
||||||
}
|
|
||||||
|
|
||||||
result.success = result.errors.length === 0
|
|
||||||
result.stats.totalTime = Date.now() - startTime
|
|
||||||
onProgress?.({ phase: 'complete', message: `Import complete: ${result.stats.entitiesCreated} entities, ${result.stats.relationshipsCreated} relationships`, processed: result.stats.rowsProcessed, total: result.stats.rowsProcessed, entities: result.stats.entitiesCreated, relationships: result.stats.relationshipsCreated })
|
|
||||||
|
|
||||||
} catch (error: any) {
|
|
||||||
result.errors.push(`JSON import failed: ${error.message}`)
|
|
||||||
result.success = false
|
|
||||||
}
|
|
||||||
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Import Markdown content with full pipeline
|
|
||||||
*/
|
|
||||||
async importMarkdown(
|
|
||||||
markdown: string,
|
|
||||||
options: SmartImportOptions & SmartMarkdownOptions = {},
|
|
||||||
onProgress?: (progress: SmartImportProgress) => void
|
|
||||||
): Promise<SmartImportResult> {
|
|
||||||
const startTime = Date.now()
|
|
||||||
const result: SmartImportResult = {
|
|
||||||
success: false,
|
|
||||||
// Typed boundary: populated after extraction (see importExcel).
|
|
||||||
extraction: null as unknown as SmartExcelResult,
|
|
||||||
entityIds: [],
|
|
||||||
relationshipIds: [],
|
|
||||||
stats: {
|
|
||||||
rowsProcessed: 0,
|
|
||||||
entitiesCreated: 0,
|
|
||||||
relationshipsCreated: 0,
|
|
||||||
filesCreated: 0,
|
|
||||||
totalTime: 0
|
|
||||||
},
|
|
||||||
errors: []
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
onProgress?.({ phase: 'extracting', message: 'Extracting from Markdown...', processed: 0, total: 0, entities: 0, relationships: 0 })
|
|
||||||
|
|
||||||
const mdResult = await this.markdownImporter.extract(markdown, options)
|
|
||||||
|
|
||||||
result.extraction = this.convertMarkdownToExcelFormat(mdResult)
|
|
||||||
result.stats.rowsProcessed = mdResult.sectionsProcessed
|
|
||||||
|
|
||||||
await this.createEntitiesAndRelationships(result, options, onProgress)
|
|
||||||
|
|
||||||
if (options.createVFSStructure !== false) {
|
|
||||||
const sourceBuffer = Buffer.from(markdown, 'utf-8')
|
|
||||||
const vfsOptions: VFSStructureOptions = {
|
|
||||||
rootPath: options.vfsRootPath || '/imports/' + (options.filename || 'import'),
|
|
||||||
groupBy: options.vfsGroupBy || 'type',
|
|
||||||
preserveSource: true,
|
|
||||||
sourceBuffer,
|
|
||||||
sourceFilename: options.filename || 'import.md',
|
|
||||||
createRelationshipFile: true,
|
|
||||||
createMetadataFile: true
|
|
||||||
}
|
|
||||||
const vfsResult = await this.vfsGenerator.generate(result.extraction, vfsOptions)
|
|
||||||
result.vfsStructure = { rootPath: vfsResult.rootPath, directories: vfsResult.directories, files: vfsResult.files.length }
|
|
||||||
result.stats.filesCreated = vfsResult.files.length
|
|
||||||
}
|
|
||||||
|
|
||||||
result.success = result.errors.length === 0
|
|
||||||
result.stats.totalTime = Date.now() - startTime
|
|
||||||
onProgress?.({ phase: 'complete', message: `Import complete: ${result.stats.entitiesCreated} entities, ${result.stats.relationshipsCreated} relationships`, processed: result.stats.rowsProcessed, total: result.stats.rowsProcessed, entities: result.stats.entitiesCreated, relationships: result.stats.relationshipsCreated })
|
|
||||||
|
|
||||||
} catch (error: any) {
|
|
||||||
result.errors.push(`Markdown import failed: ${error.message}`)
|
|
||||||
result.success = false
|
|
||||||
}
|
|
||||||
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Helper: Create entities and relationships from extraction result
|
|
||||||
*/
|
|
||||||
private async createEntitiesAndRelationships(
|
|
||||||
result: SmartImportResult,
|
|
||||||
options: SmartImportOptions,
|
|
||||||
onProgress?: (progress: SmartImportProgress) => void
|
|
||||||
): Promise<void> {
|
|
||||||
if (options.createEntities !== false) {
|
|
||||||
onProgress?.({ phase: 'creating', message: 'Creating entities in knowledge graph...', processed: 0, total: result.extraction.rows.length, entities: 0, relationships: 0 })
|
|
||||||
|
|
||||||
for (let i = 0; i < result.extraction.rows.length; i++) {
|
|
||||||
const extracted = result.extraction.rows[i]
|
|
||||||
try {
|
|
||||||
// Subtype precedence: extractor → caller default → `'imported'` (7.30.1).
|
|
||||||
const entityId = await this.brain.add({
|
|
||||||
data: extracted.entity.description,
|
|
||||||
type: extracted.entity.type,
|
|
||||||
subtype:
|
|
||||||
(extracted.entity as typeof extracted.entity & { subtype?: string })
|
|
||||||
.subtype ?? options.defaultSubtype ?? 'imported',
|
|
||||||
confidence: extracted.entity.confidence, // reserved field — dedicated param, not metadata
|
|
||||||
metadata: { ...splitNounMetadataRecord(extracted.entity.metadata).custom, name: extracted.entity.name, importedFrom: 'smart-import' }
|
|
||||||
})
|
|
||||||
result.entityIds.push(entityId)
|
|
||||||
result.stats.entitiesCreated++
|
|
||||||
extracted.entity.id = entityId
|
|
||||||
} catch (error: any) {
|
|
||||||
result.errors.push(`Failed to create entity ${extracted.entity.name}: ${error.message}`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (options.createRelationships !== false && options.createEntities !== false) {
|
|
||||||
onProgress?.({ phase: 'creating', message: 'Preparing relationships...', processed: 0, total: result.extraction.rows.length, entities: result.entityIds.length, relationships: 0 })
|
|
||||||
|
|
||||||
// Collect all relationship parameters
|
|
||||||
const relationshipParams: Array<{from: string; to: string; type: VerbType; subtype?: string; confidence?: number; metadata?: any}> = []
|
|
||||||
|
|
||||||
for (const extracted of result.extraction.rows) {
|
|
||||||
for (const rel of extracted.relationships) {
|
|
||||||
try {
|
|
||||||
let toEntityId: string | undefined
|
|
||||||
for (const otherExtracted of result.extraction.rows) {
|
|
||||||
if (rel.to.toLowerCase().includes(otherExtracted.entity.name.toLowerCase()) || otherExtracted.entity.name.toLowerCase().includes(rel.to.toLowerCase())) {
|
|
||||||
toEntityId = otherExtracted.entity.id
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (!toEntityId) {
|
|
||||||
// Subtype `import-placeholder` marks synthetic targets (7.30.1).
|
|
||||||
toEntityId = await this.brain.add({ data: rel.to, type: NounType.Thing, subtype: 'import-placeholder', metadata: { name: rel.to, placeholder: true, extractedFrom: extracted.entity.name } })
|
|
||||||
result.entityIds.push(toEntityId)
|
|
||||||
}
|
|
||||||
// Relationship subtype precedence: extractor → caller default → `'imported'` (7.30.1).
|
|
||||||
// `confidence` is a reserved top-level field — dedicated relate() param, not metadata
|
|
||||||
relationshipParams.push({ from: extracted.entity.id, to: toEntityId, type: rel.type, subtype: (rel as typeof rel & { subtype?: string }).subtype ?? options.defaultSubtype ?? 'imported', confidence: rel.confidence, metadata: { evidence: rel.evidence } })
|
|
||||||
} catch (error: any) {
|
|
||||||
result.errors.push(`Failed to prepare relationship: ${error.message}`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Batch create all relationships with progress
|
|
||||||
if (relationshipParams.length > 0) {
|
|
||||||
onProgress?.({ phase: 'relationships', message: 'Building relationships...', processed: 0, total: relationshipParams.length, entities: result.entityIds.length, relationships: 0 })
|
|
||||||
|
|
||||||
try {
|
|
||||||
const relationshipIds = await this.brain.relateMany({
|
|
||||||
items: relationshipParams,
|
|
||||||
parallel: true,
|
|
||||||
chunkSize: 100,
|
|
||||||
continueOnError: true,
|
|
||||||
onProgress: (done, total) => {
|
|
||||||
onProgress?.({ phase: 'relationships', message: `Building relationships: ${done}/${total}`, processed: done, total: total, entities: result.entityIds.length, relationships: done })
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
result.relationshipIds = relationshipIds
|
|
||||||
result.stats.relationshipsCreated = relationshipIds.length
|
|
||||||
} catch (error: any) {
|
|
||||||
result.errors.push(`Failed to create relationships: ${error.message}`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Helper: Convert PDF result to Excel-like format
|
|
||||||
*/
|
|
||||||
private convertPDFToExcelFormat(pdfResult: SmartPDFResult): Omit<SmartExcelResult, 'rows'> & { rows: any[] } {
|
|
||||||
const rows = pdfResult.sections.flatMap(section =>
|
|
||||||
section.entities.map(entity => ({
|
|
||||||
entity,
|
|
||||||
relatedEntities: [],
|
|
||||||
relationships: section.relationships.filter(r => r.from === entity.id),
|
|
||||||
concepts: section.concepts
|
|
||||||
}))
|
|
||||||
)
|
|
||||||
|
|
||||||
return {
|
|
||||||
rowsProcessed: pdfResult.sectionsProcessed,
|
|
||||||
entitiesExtracted: pdfResult.entitiesExtracted,
|
|
||||||
relationshipsInferred: pdfResult.relationshipsInferred,
|
|
||||||
rows,
|
|
||||||
entityMap: pdfResult.entityMap,
|
|
||||||
processingTime: pdfResult.processingTime,
|
|
||||||
stats: pdfResult.stats
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Helper: Convert JSON result to Excel-like format
|
|
||||||
*/
|
|
||||||
private convertJSONToExcelFormat(jsonResult: SmartJSONResult): Omit<SmartExcelResult, 'rows'> & { rows: any[] } {
|
|
||||||
const rows = jsonResult.entities.map(entity => ({
|
|
||||||
entity,
|
|
||||||
relatedEntities: [],
|
|
||||||
relationships: jsonResult.relationships.filter(r => r.from === entity.id),
|
|
||||||
concepts: entity.metadata.concepts || []
|
|
||||||
}))
|
|
||||||
|
|
||||||
return {
|
|
||||||
rowsProcessed: jsonResult.nodesProcessed,
|
|
||||||
entitiesExtracted: jsonResult.entitiesExtracted,
|
|
||||||
relationshipsInferred: jsonResult.relationshipsInferred,
|
|
||||||
rows,
|
|
||||||
entityMap: jsonResult.entityMap,
|
|
||||||
processingTime: jsonResult.processingTime,
|
|
||||||
stats: jsonResult.stats
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Helper: Convert Markdown result to Excel-like format
|
|
||||||
*/
|
|
||||||
private convertMarkdownToExcelFormat(mdResult: SmartMarkdownResult): Omit<SmartExcelResult, 'rows'> & { rows: any[] } {
|
|
||||||
const rows = mdResult.sections.flatMap(section =>
|
|
||||||
section.entities.map(entity => ({
|
|
||||||
entity,
|
|
||||||
relatedEntities: [],
|
|
||||||
relationships: section.relationships.filter(r => r.from === entity.id),
|
|
||||||
concepts: section.concepts
|
|
||||||
}))
|
|
||||||
)
|
|
||||||
|
|
||||||
return {
|
|
||||||
rowsProcessed: mdResult.sectionsProcessed,
|
|
||||||
entitiesExtracted: mdResult.entitiesExtracted,
|
|
||||||
relationshipsInferred: mdResult.relationshipsInferred,
|
|
||||||
rows,
|
|
||||||
entityMap: mdResult.entityMap,
|
|
||||||
processingTime: mdResult.processingTime,
|
|
||||||
stats: mdResult.stats
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get import statistics
|
|
||||||
*/
|
|
||||||
async getImportStatistics(vfsRootPath: string): Promise<{
|
|
||||||
entitiesInGraph: number
|
|
||||||
relationshipsInGraph: number
|
|
||||||
filesInVFS: number
|
|
||||||
lastImport?: Date
|
|
||||||
}> {
|
|
||||||
// Read metadata file
|
|
||||||
const vfs = new VirtualFileSystem(this.brain)
|
|
||||||
await vfs.init()
|
|
||||||
|
|
||||||
const metadataPath = `${vfsRootPath}/_metadata.json`
|
|
||||||
|
|
||||||
try {
|
|
||||||
const metadataBuffer = await vfs.readFile(metadataPath)
|
|
||||||
const metadata = JSON.parse(metadataBuffer.toString('utf-8'))
|
|
||||||
|
|
||||||
return {
|
|
||||||
entitiesInGraph: metadata.import.stats.entitiesExtracted,
|
|
||||||
relationshipsInGraph: metadata.import.stats.relationshipsInferred,
|
|
||||||
filesInVFS: metadata.structure.fileCount,
|
|
||||||
lastImport: new Date(metadata.import.timestamp)
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
return {
|
|
||||||
entitiesInGraph: 0,
|
|
||||||
relationshipsInGraph: 0,
|
|
||||||
filesInVFS: 0
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,69 +0,0 @@
|
||||||
/**
|
|
||||||
* Smart Import System
|
|
||||||
*
|
|
||||||
* Production-ready entity and relationship extraction from multiple formats:
|
|
||||||
* - Excel (.xlsx)
|
|
||||||
* - PDF (.pdf)
|
|
||||||
* - CSV (.csv)
|
|
||||||
* - JSON (.json)
|
|
||||||
* - Markdown (.md)
|
|
||||||
*
|
|
||||||
* Uses brainy's built-in NeuralEntityExtractor and NaturalLanguageProcessor
|
|
||||||
*
|
|
||||||
* NO MOCKS - Real working implementation
|
|
||||||
*/
|
|
||||||
|
|
||||||
// Excel Importer
|
|
||||||
export { SmartExcelImporter } from './SmartExcelImporter.js'
|
|
||||||
export type {
|
|
||||||
SmartExcelOptions,
|
|
||||||
ExtractedRow,
|
|
||||||
SmartExcelResult
|
|
||||||
} from './SmartExcelImporter.js'
|
|
||||||
|
|
||||||
// PDF Importer
|
|
||||||
export { SmartPDFImporter } from './SmartPDFImporter.js'
|
|
||||||
export type {
|
|
||||||
SmartPDFOptions,
|
|
||||||
ExtractedSection,
|
|
||||||
SmartPDFResult
|
|
||||||
} from './SmartPDFImporter.js'
|
|
||||||
|
|
||||||
// CSV Importer
|
|
||||||
export { SmartCSVImporter } from './SmartCSVImporter.js'
|
|
||||||
export type {
|
|
||||||
SmartCSVOptions,
|
|
||||||
SmartCSVResult
|
|
||||||
} from './SmartCSVImporter.js'
|
|
||||||
|
|
||||||
// JSON Importer
|
|
||||||
export { SmartJSONImporter } from './SmartJSONImporter.js'
|
|
||||||
export type {
|
|
||||||
SmartJSONOptions,
|
|
||||||
ExtractedJSONEntity,
|
|
||||||
ExtractedJSONRelationship,
|
|
||||||
SmartJSONResult
|
|
||||||
} from './SmartJSONImporter.js'
|
|
||||||
|
|
||||||
// Markdown Importer
|
|
||||||
export { SmartMarkdownImporter } from './SmartMarkdownImporter.js'
|
|
||||||
export type {
|
|
||||||
SmartMarkdownOptions,
|
|
||||||
MarkdownSection,
|
|
||||||
SmartMarkdownResult
|
|
||||||
} from './SmartMarkdownImporter.js'
|
|
||||||
|
|
||||||
// VFS Structure Generator
|
|
||||||
export { VFSStructureGenerator } from './VFSStructureGenerator.js'
|
|
||||||
export type {
|
|
||||||
VFSStructureOptions,
|
|
||||||
VFSStructureResult
|
|
||||||
} from './VFSStructureGenerator.js'
|
|
||||||
|
|
||||||
// Orchestrator (Main entry point)
|
|
||||||
export { SmartImportOrchestrator } from './SmartImportOrchestrator.js'
|
|
||||||
export type {
|
|
||||||
SmartImportOptions,
|
|
||||||
SmartImportProgress,
|
|
||||||
SmartImportResult
|
|
||||||
} from './SmartImportOrchestrator.js'
|
|
||||||
|
|
@ -1,38 +0,0 @@
|
||||||
/**
|
|
||||||
* @module columnStore
|
|
||||||
* @description Unified column store for filtering, sorting, range queries,
|
|
||||||
* and text search on any field type with exact precision at billion scale.
|
|
||||||
*
|
|
||||||
* Replaces MetadataIndex's sparse chunk/bloom filter/bucketing internals
|
|
||||||
* with per-field sorted columns. Design lineage: Lucene doc values + roaring
|
|
||||||
* bitmap acceleration.
|
|
||||||
*/
|
|
||||||
|
|
||||||
export { ColumnStore } from './ColumnStore.js'
|
|
||||||
export type { ColumnStoreConfig } from './ColumnStore.js'
|
|
||||||
export { ColumnTailBuffer } from './ColumnTailBuffer.js'
|
|
||||||
export { ColumnManifest } from './ColumnManifest.js'
|
|
||||||
export { ColumnSegmentCursor, TailBufferCursor } from './ColumnSegmentCursor.js'
|
|
||||||
export {
|
|
||||||
writeSegmentToBuffer,
|
|
||||||
readSegmentFromBuffer,
|
|
||||||
writeHeader,
|
|
||||||
readHeader,
|
|
||||||
crc32
|
|
||||||
} from './ColumnSegmentFormat.js'
|
|
||||||
export {
|
|
||||||
ValueType,
|
|
||||||
CIDX_MAGIC,
|
|
||||||
CIDX_VERSION,
|
|
||||||
HEADER_SIZE,
|
|
||||||
FOOTER_SIZE,
|
|
||||||
DEFAULT_FLUSH_THRESHOLD,
|
|
||||||
FLAG_MULTI_VALUE
|
|
||||||
} from './types.js'
|
|
||||||
export type {
|
|
||||||
ColumnStoreProvider,
|
|
||||||
SegmentHeader,
|
|
||||||
SegmentFooter,
|
|
||||||
SegmentMeta,
|
|
||||||
ManifestData
|
|
||||||
} from './types.js'
|
|
||||||
|
|
@ -1,363 +0,0 @@
|
||||||
/**
|
|
||||||
* BrainyMCPBroadcast
|
|
||||||
*
|
|
||||||
* Enhanced MCP service with real-time WebSocket broadcasting capabilities
|
|
||||||
* for multi-agent coordination (Jarvis ↔ Picasso communication)
|
|
||||||
*
|
|
||||||
* Features:
|
|
||||||
* - WebSocket server for real-time push notifications
|
|
||||||
* - Subscription management for multiple Claude instances
|
|
||||||
* - Message broadcasting to all connected agents
|
|
||||||
* - Works both locally and with cloud deployment
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { WebSocketServer, WebSocket } from 'ws'
|
|
||||||
import { createServer, IncomingMessage } from 'node:http'
|
|
||||||
import { BrainyMCPService } from './brainyMCPService.js'
|
|
||||||
import { BrainyInterface } from '../types/brainyInterface.js'
|
|
||||||
import { MCPServiceOptions } from '../types/mcpTypes.js'
|
|
||||||
import { v4 as uuidv4 } from '../universal/uuid.js'
|
|
||||||
|
|
||||||
interface BroadcastMessage {
|
|
||||||
id: string
|
|
||||||
from: string
|
|
||||||
to?: string | string[]
|
|
||||||
type: 'message' | 'notification' | 'sync' | 'heartbeat' | 'identify'
|
|
||||||
event?: string
|
|
||||||
data: any
|
|
||||||
timestamp: number
|
|
||||||
}
|
|
||||||
|
|
||||||
interface ConnectedAgent {
|
|
||||||
id: string
|
|
||||||
name: string
|
|
||||||
role: string
|
|
||||||
socket: WebSocket
|
|
||||||
lastSeen: number
|
|
||||||
}
|
|
||||||
|
|
||||||
export class BrainyMCPBroadcast extends BrainyMCPService {
|
|
||||||
private wsServer?: WebSocketServer
|
|
||||||
private httpServer?: any
|
|
||||||
private agents: Map<string, ConnectedAgent> = new Map()
|
|
||||||
private messageHistory: BroadcastMessage[] = []
|
|
||||||
private maxHistorySize = 100
|
|
||||||
|
|
||||||
constructor(
|
|
||||||
brainyData: BrainyInterface,
|
|
||||||
options: MCPServiceOptions & {
|
|
||||||
broadcastPort?: number
|
|
||||||
cloudUrl?: string
|
|
||||||
} = {}
|
|
||||||
) {
|
|
||||||
super(brainyData, options)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Start the WebSocket broadcast server
|
|
||||||
* @param port Port to listen on (default: 8765)
|
|
||||||
* @param isCloud Whether this is a cloud deployment
|
|
||||||
*/
|
|
||||||
async startBroadcastServer(port = 8765, isCloud = false): Promise<void> {
|
|
||||||
return new Promise((resolve, reject) => {
|
|
||||||
try {
|
|
||||||
// Create HTTP server
|
|
||||||
this.httpServer = createServer((req, res) => {
|
|
||||||
// Health check endpoint
|
|
||||||
if (req.url === '/health') {
|
|
||||||
res.writeHead(200, { 'Content-Type': 'application/json' })
|
|
||||||
res.end(JSON.stringify({
|
|
||||||
status: 'healthy',
|
|
||||||
agents: Array.from(this.agents.values()).map(a => ({
|
|
||||||
id: a.id,
|
|
||||||
name: a.name,
|
|
||||||
role: a.role,
|
|
||||||
connected: true
|
|
||||||
})),
|
|
||||||
uptime: process.uptime()
|
|
||||||
}))
|
|
||||||
} else {
|
|
||||||
res.writeHead(404)
|
|
||||||
res.end('Not found')
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// Create WebSocket server
|
|
||||||
this.wsServer = new WebSocketServer({
|
|
||||||
server: this.httpServer,
|
|
||||||
perMessageDeflate: false // Better performance
|
|
||||||
})
|
|
||||||
|
|
||||||
this.wsServer.on('connection', (socket, request) => {
|
|
||||||
this.handleNewConnection(socket, request)
|
|
||||||
})
|
|
||||||
|
|
||||||
// Start listening
|
|
||||||
this.httpServer.listen(port, () => {
|
|
||||||
console.log(`🧠 Brain Jar Broadcast Server running on ${isCloud ? 'cloud' : 'local'} port ${port}`)
|
|
||||||
console.log(`📡 WebSocket: ws://localhost:${port}`)
|
|
||||||
console.log(`🔍 Health: http://localhost:${port}/health`)
|
|
||||||
resolve()
|
|
||||||
})
|
|
||||||
|
|
||||||
// Heartbeat to keep connections alive
|
|
||||||
setInterval(() => {
|
|
||||||
this.agents.forEach((agent) => {
|
|
||||||
if (Date.now() - agent.lastSeen > 30000) {
|
|
||||||
// Remove inactive agents
|
|
||||||
this.removeAgent(agent.id)
|
|
||||||
} else {
|
|
||||||
// Send heartbeat
|
|
||||||
this.sendToAgent(agent.id, {
|
|
||||||
id: uuidv4(),
|
|
||||||
from: 'server',
|
|
||||||
type: 'heartbeat',
|
|
||||||
data: { timestamp: Date.now() },
|
|
||||||
timestamp: Date.now()
|
|
||||||
})
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}, 15000)
|
|
||||||
|
|
||||||
} catch (error) {
|
|
||||||
reject(error)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Handle new WebSocket connection
|
|
||||||
*/
|
|
||||||
private handleNewConnection(socket: WebSocket, request: IncomingMessage) {
|
|
||||||
const agentId = uuidv4()
|
|
||||||
|
|
||||||
// Send welcome message
|
|
||||||
socket.send(JSON.stringify({
|
|
||||||
id: uuidv4(),
|
|
||||||
from: 'server',
|
|
||||||
type: 'notification',
|
|
||||||
event: 'welcome',
|
|
||||||
data: {
|
|
||||||
agentId,
|
|
||||||
message: 'Connected to Brain Jar Broadcast Server',
|
|
||||||
agents: Array.from(this.agents.values()).map(a => ({
|
|
||||||
id: a.id,
|
|
||||||
name: a.name,
|
|
||||||
role: a.role
|
|
||||||
}))
|
|
||||||
},
|
|
||||||
timestamp: Date.now()
|
|
||||||
}))
|
|
||||||
|
|
||||||
// Handle messages from this agent
|
|
||||||
socket.on('message', (data) => {
|
|
||||||
try {
|
|
||||||
const message = JSON.parse(data.toString())
|
|
||||||
this.handleAgentMessage(agentId, message)
|
|
||||||
} catch (error) {
|
|
||||||
console.error('Invalid message from agent:', error)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// Handle disconnection
|
|
||||||
socket.on('close', () => {
|
|
||||||
this.removeAgent(agentId)
|
|
||||||
})
|
|
||||||
|
|
||||||
// Handle errors
|
|
||||||
socket.on('error', (error) => {
|
|
||||||
console.error(`Agent ${agentId} error:`, error)
|
|
||||||
})
|
|
||||||
|
|
||||||
// Store temporary connection until identified
|
|
||||||
this.agents.set(agentId, {
|
|
||||||
id: agentId,
|
|
||||||
name: 'Unknown',
|
|
||||||
role: 'Unknown',
|
|
||||||
socket,
|
|
||||||
lastSeen: Date.now()
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Handle message from an agent
|
|
||||||
*/
|
|
||||||
private handleAgentMessage(agentId: string, message: any) {
|
|
||||||
const agent = this.agents.get(agentId)
|
|
||||||
if (!agent) return
|
|
||||||
|
|
||||||
// Update last seen
|
|
||||||
agent.lastSeen = Date.now()
|
|
||||||
|
|
||||||
// Handle identification
|
|
||||||
if (message.type === 'identify') {
|
|
||||||
agent.name = message.name || agent.name
|
|
||||||
agent.role = message.role || agent.role
|
|
||||||
|
|
||||||
// Notify all agents about new member
|
|
||||||
this.broadcast({
|
|
||||||
id: uuidv4(),
|
|
||||||
from: 'server',
|
|
||||||
type: 'notification',
|
|
||||||
event: 'agent_joined',
|
|
||||||
data: {
|
|
||||||
agent: {
|
|
||||||
id: agent.id,
|
|
||||||
name: agent.name,
|
|
||||||
role: agent.role
|
|
||||||
}
|
|
||||||
},
|
|
||||||
timestamp: Date.now()
|
|
||||||
}, agentId) // Exclude the joining agent
|
|
||||||
|
|
||||||
// Send recent history to new agent
|
|
||||||
if (this.messageHistory.length > 0) {
|
|
||||||
this.sendToAgent(agentId, {
|
|
||||||
id: uuidv4(),
|
|
||||||
from: 'server',
|
|
||||||
type: 'sync',
|
|
||||||
data: {
|
|
||||||
history: this.messageHistory.slice(-20) // Last 20 messages
|
|
||||||
},
|
|
||||||
timestamp: Date.now()
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create broadcast message
|
|
||||||
const broadcastMsg: BroadcastMessage = {
|
|
||||||
id: message.id || uuidv4(),
|
|
||||||
from: agent.name,
|
|
||||||
to: message.to,
|
|
||||||
type: message.type || 'message',
|
|
||||||
event: message.event,
|
|
||||||
data: message.data,
|
|
||||||
timestamp: Date.now()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Store in history
|
|
||||||
this.addToHistory(broadcastMsg)
|
|
||||||
|
|
||||||
// Broadcast based on recipient
|
|
||||||
if (message.to) {
|
|
||||||
// Send to specific agent(s)
|
|
||||||
const recipients = Array.isArray(message.to) ? message.to : [message.to]
|
|
||||||
recipients.forEach((recipientName: string) => {
|
|
||||||
const recipient = Array.from(this.agents.values()).find(
|
|
||||||
a => a.name === recipientName
|
|
||||||
)
|
|
||||||
if (recipient) {
|
|
||||||
this.sendToAgent(recipient.id, broadcastMsg)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
} else {
|
|
||||||
// Broadcast to all agents except sender
|
|
||||||
this.broadcast(broadcastMsg, agentId)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Broadcast message to all connected agents
|
|
||||||
*/
|
|
||||||
broadcast(message: BroadcastMessage, excludeId?: string) {
|
|
||||||
const messageStr = JSON.stringify(message)
|
|
||||||
|
|
||||||
this.agents.forEach((agent) => {
|
|
||||||
if (agent.id !== excludeId && agent.socket.readyState === WebSocket.OPEN) {
|
|
||||||
agent.socket.send(messageStr)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Send message to specific agent
|
|
||||||
*/
|
|
||||||
private sendToAgent(agentId: string, message: BroadcastMessage) {
|
|
||||||
const agent = this.agents.get(agentId)
|
|
||||||
if (agent && agent.socket.readyState === WebSocket.OPEN) {
|
|
||||||
agent.socket.send(JSON.stringify(message))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Remove agent from connected list
|
|
||||||
*/
|
|
||||||
private removeAgent(agentId: string) {
|
|
||||||
const agent = this.agents.get(agentId)
|
|
||||||
if (agent) {
|
|
||||||
// Notify others about disconnection
|
|
||||||
this.broadcast({
|
|
||||||
id: uuidv4(),
|
|
||||||
from: 'server',
|
|
||||||
type: 'notification',
|
|
||||||
event: 'agent_left',
|
|
||||||
data: {
|
|
||||||
agent: {
|
|
||||||
id: agent.id,
|
|
||||||
name: agent.name,
|
|
||||||
role: agent.role
|
|
||||||
}
|
|
||||||
},
|
|
||||||
timestamp: Date.now()
|
|
||||||
})
|
|
||||||
|
|
||||||
this.agents.delete(agentId)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Add message to history
|
|
||||||
*/
|
|
||||||
private addToHistory(message: BroadcastMessage) {
|
|
||||||
this.messageHistory.push(message)
|
|
||||||
|
|
||||||
// Trim history if too large
|
|
||||||
if (this.messageHistory.length > this.maxHistorySize) {
|
|
||||||
this.messageHistory = this.messageHistory.slice(-this.maxHistorySize)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Stop the broadcast server
|
|
||||||
*/
|
|
||||||
async stopBroadcastServer(): Promise<void> {
|
|
||||||
// Close all agent connections
|
|
||||||
this.agents.forEach(agent => {
|
|
||||||
agent.socket.close(1000, 'Server shutting down')
|
|
||||||
})
|
|
||||||
this.agents.clear()
|
|
||||||
|
|
||||||
// Close WebSocket server
|
|
||||||
if (this.wsServer) {
|
|
||||||
this.wsServer.close()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Close HTTP server
|
|
||||||
if (this.httpServer) {
|
|
||||||
this.httpServer.close()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get connected agents
|
|
||||||
*/
|
|
||||||
getConnectedAgents(): Array<{ id: string; name: string; role: string }> {
|
|
||||||
return Array.from(this.agents.values()).map(a => ({
|
|
||||||
id: a.id,
|
|
||||||
name: a.name,
|
|
||||||
role: a.role
|
|
||||||
}))
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get message history
|
|
||||||
*/
|
|
||||||
getMessageHistory(): BroadcastMessage[] {
|
|
||||||
return [...this.messageHistory]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Export for both environments
|
|
||||||
export default BrainyMCPBroadcast
|
|
||||||
|
|
@ -1,322 +0,0 @@
|
||||||
/**
|
|
||||||
* BrainyMCPClient
|
|
||||||
*
|
|
||||||
* Client for connecting Claude instances to the Brain Jar Broadcast Server
|
|
||||||
* Utilizes Brainy for persistent memory and vector search capabilities
|
|
||||||
*/
|
|
||||||
|
|
||||||
import WebSocket from 'ws'
|
|
||||||
import { Brainy } from '../brainy.js'
|
|
||||||
import { NounType } from '../types/graphTypes.js'
|
|
||||||
import { v4 as uuidv4 } from '../universal/uuid.js'
|
|
||||||
|
|
||||||
interface ClientOptions {
|
|
||||||
name: string // e.g., 'Jarvis' or 'Picasso'
|
|
||||||
role: string // e.g., 'Backend Systems' or 'Frontend Design'
|
|
||||||
serverUrl?: string // Default: ws://localhost:8765
|
|
||||||
autoReconnect?: boolean
|
|
||||||
useBrainyMemory?: boolean // Store messages in Brainy for persistence
|
|
||||||
}
|
|
||||||
|
|
||||||
interface Message {
|
|
||||||
id: string
|
|
||||||
from: string
|
|
||||||
to?: string | string[]
|
|
||||||
type: 'message' | 'notification' | 'sync' | 'heartbeat' | 'identify'
|
|
||||||
event?: string
|
|
||||||
data: any
|
|
||||||
timestamp: number
|
|
||||||
}
|
|
||||||
|
|
||||||
export class BrainyMCPClient {
|
|
||||||
private socket?: WebSocket
|
|
||||||
private options: Required<ClientOptions>
|
|
||||||
private brainy?: Brainy
|
|
||||||
private messageHandlers: Map<string, (message: Message) => void> = new Map()
|
|
||||||
private reconnectTimeout?: NodeJS.Timeout
|
|
||||||
private isConnected = false
|
|
||||||
|
|
||||||
constructor(options: ClientOptions) {
|
|
||||||
this.options = {
|
|
||||||
serverUrl: 'ws://localhost:8765',
|
|
||||||
autoReconnect: true,
|
|
||||||
useBrainyMemory: true,
|
|
||||||
...options
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Initialize Brainy for persistent memory
|
|
||||||
*/
|
|
||||||
private async initBrainy() {
|
|
||||||
if (this.options.useBrainyMemory && !this.brainy) {
|
|
||||||
this.brainy = new Brainy({
|
|
||||||
storage: {
|
|
||||||
requestPersistentStorage: true
|
|
||||||
}
|
|
||||||
})
|
|
||||||
await this.brainy.init()
|
|
||||||
console.log(`🧠 Brainy memory initialized for ${this.options.name}`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Connect to the broadcast server
|
|
||||||
*/
|
|
||||||
async connect(): Promise<void> {
|
|
||||||
// Initialize Brainy first
|
|
||||||
await this.initBrainy()
|
|
||||||
|
|
||||||
return new Promise((resolve, reject) => {
|
|
||||||
try {
|
|
||||||
this.socket = new WebSocket(this.options.serverUrl)
|
|
||||||
|
|
||||||
this.socket.on('open', () => {
|
|
||||||
console.log(`✅ ${this.options.name} connected to Brain Jar Broadcast`)
|
|
||||||
this.isConnected = true
|
|
||||||
|
|
||||||
// Identify ourselves
|
|
||||||
this.send({
|
|
||||||
type: 'identify',
|
|
||||||
data: {
|
|
||||||
name: this.options.name,
|
|
||||||
role: this.options.role
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
resolve()
|
|
||||||
})
|
|
||||||
|
|
||||||
this.socket.on('message', async (data) => {
|
|
||||||
try {
|
|
||||||
const message = JSON.parse(data.toString()) as Message
|
|
||||||
await this.handleMessage(message)
|
|
||||||
} catch (error) {
|
|
||||||
console.error('Error parsing message:', error)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
this.socket.on('close', () => {
|
|
||||||
console.log(`❌ ${this.options.name} disconnected from Brain Jar`)
|
|
||||||
this.isConnected = false
|
|
||||||
|
|
||||||
if (this.options.autoReconnect) {
|
|
||||||
this.scheduleReconnect()
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
this.socket.on('error', (error) => {
|
|
||||||
console.error(`Connection error for ${this.options.name}:`, error)
|
|
||||||
reject(error)
|
|
||||||
})
|
|
||||||
|
|
||||||
} catch (error) {
|
|
||||||
reject(error)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Handle incoming message
|
|
||||||
*/
|
|
||||||
private async handleMessage(message: Message) {
|
|
||||||
// Store in Brainy for persistent memory. Subtype `'mcp-message'` marks
|
|
||||||
// these as MCP-protocol messages so consumers can filter / count them via
|
|
||||||
// `find({ type: NounType.Message, subtype: 'mcp-message' })` and so
|
|
||||||
// enforcement consumers registering a vocabulary on NounType.Message don't
|
|
||||||
// reject MCP traffic (added 7.30.1; also fixes the pre-existing missing
|
|
||||||
// `data` field by aliasing from the prior `text` field).
|
|
||||||
if (this.brainy && message.type === 'message') {
|
|
||||||
try {
|
|
||||||
await this.brainy.add({
|
|
||||||
data: `${message.from}: ${JSON.stringify(message.data)}`,
|
|
||||||
type: NounType.Message,
|
|
||||||
subtype: 'mcp-message',
|
|
||||||
metadata: {
|
|
||||||
messageId: message.id,
|
|
||||||
from: message.from,
|
|
||||||
to: message.to,
|
|
||||||
timestamp: message.timestamp,
|
|
||||||
messageType: message.type,
|
|
||||||
event: message.event
|
|
||||||
}
|
|
||||||
})
|
|
||||||
} catch (error) {
|
|
||||||
console.error('Error storing message in Brainy:', error)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Handle sync messages (receive history)
|
|
||||||
if (message.type === 'sync' && message.data.history) {
|
|
||||||
console.log(`📜 ${this.options.name} received ${message.data.history.length} historical messages`)
|
|
||||||
|
|
||||||
// Store history in Brainy with the same subtype as live messages.
|
|
||||||
if (this.brainy) {
|
|
||||||
for (const histMsg of message.data.history) {
|
|
||||||
await this.brainy.add({
|
|
||||||
data: `${histMsg.from}: ${JSON.stringify(histMsg.data)}`,
|
|
||||||
type: NounType.Message,
|
|
||||||
subtype: 'mcp-message',
|
|
||||||
metadata: {
|
|
||||||
...histMsg
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Call registered handlers
|
|
||||||
const handler = this.messageHandlers.get(message.type)
|
|
||||||
if (handler) {
|
|
||||||
handler(message)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Call universal handler
|
|
||||||
const universalHandler = this.messageHandlers.get('*')
|
|
||||||
if (universalHandler) {
|
|
||||||
universalHandler(message)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Send a message
|
|
||||||
*/
|
|
||||||
send(message: Partial<Message>) {
|
|
||||||
if (!this.socket || this.socket.readyState !== WebSocket.OPEN) {
|
|
||||||
console.error(`${this.options.name} is not connected`)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
const fullMessage: Message = {
|
|
||||||
id: message.id || uuidv4(),
|
|
||||||
from: this.options.name,
|
|
||||||
type: message.type || 'message',
|
|
||||||
data: message.data || {},
|
|
||||||
timestamp: Date.now(),
|
|
||||||
...message
|
|
||||||
}
|
|
||||||
|
|
||||||
this.socket.send(JSON.stringify(fullMessage))
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Send a message to specific agent(s)
|
|
||||||
*/
|
|
||||||
sendTo(recipient: string | string[], data: any) {
|
|
||||||
this.send({
|
|
||||||
to: recipient,
|
|
||||||
type: 'message',
|
|
||||||
data
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Broadcast to all agents
|
|
||||||
*/
|
|
||||||
broadcast(data: any) {
|
|
||||||
this.send({
|
|
||||||
type: 'message',
|
|
||||||
data
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Register a message handler
|
|
||||||
*/
|
|
||||||
on(type: string, handler: (message: Message) => void) {
|
|
||||||
this.messageHandlers.set(type, handler)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Remove a message handler
|
|
||||||
*/
|
|
||||||
off(type: string) {
|
|
||||||
this.messageHandlers.delete(type)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Search historical messages using Brainy's vector search
|
|
||||||
*/
|
|
||||||
async searchMemory(query: string, limit = 10): Promise<any[]> {
|
|
||||||
if (!this.brainy) {
|
|
||||||
console.warn('Brainy memory not initialized')
|
|
||||||
return []
|
|
||||||
}
|
|
||||||
|
|
||||||
const results = await this.brainy.find({ query, limit })
|
|
||||||
return results.map(r => ({
|
|
||||||
...r.metadata,
|
|
||||||
relevance: r.score
|
|
||||||
}))
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get recent messages from Brainy memory
|
|
||||||
*/
|
|
||||||
async getRecentMessages(limit = 20): Promise<any[]> {
|
|
||||||
if (!this.brainy) {
|
|
||||||
console.warn('Brainy memory not initialized')
|
|
||||||
return []
|
|
||||||
}
|
|
||||||
|
|
||||||
// Search for recent activity
|
|
||||||
const results = await this.brainy.find({ query: 'recent messages communication', limit })
|
|
||||||
return results
|
|
||||||
.map(r => r.metadata)
|
|
||||||
.sort((a: any, b: any) => b.timestamp - a.timestamp)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Schedule reconnection attempt
|
|
||||||
*/
|
|
||||||
private scheduleReconnect() {
|
|
||||||
if (this.reconnectTimeout) {
|
|
||||||
clearTimeout(this.reconnectTimeout)
|
|
||||||
}
|
|
||||||
|
|
||||||
this.reconnectTimeout = setTimeout(() => {
|
|
||||||
console.log(`🔄 ${this.options.name} attempting to reconnect...`)
|
|
||||||
this.connect().catch(error => {
|
|
||||||
console.error('Reconnection failed:', error)
|
|
||||||
this.scheduleReconnect()
|
|
||||||
})
|
|
||||||
}, 5000)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Disconnect from server
|
|
||||||
*/
|
|
||||||
disconnect() {
|
|
||||||
if (this.reconnectTimeout) {
|
|
||||||
clearTimeout(this.reconnectTimeout)
|
|
||||||
}
|
|
||||||
|
|
||||||
if (this.socket) {
|
|
||||||
this.socket.close(1000, 'Client disconnecting')
|
|
||||||
this.socket = undefined
|
|
||||||
}
|
|
||||||
|
|
||||||
this.isConnected = false
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Check if connected
|
|
||||||
*/
|
|
||||||
getIsConnected(): boolean {
|
|
||||||
return this.isConnected
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get agent info
|
|
||||||
*/
|
|
||||||
getAgentInfo() {
|
|
||||||
return {
|
|
||||||
name: this.options.name,
|
|
||||||
role: this.options.role,
|
|
||||||
connected: this.isConnected
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Export for both environments
|
|
||||||
export default BrainyMCPClient
|
|
||||||
|
|
@ -1,200 +0,0 @@
|
||||||
/**
|
|
||||||
* 🧠 Natural Language Query Processor - STATIC VERSION
|
|
||||||
* No runtime initialization, no memory leaks, patterns pre-built at compile time
|
|
||||||
*
|
|
||||||
* Uses static pattern matching with 220 pre-built patterns
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { Vector } from '../coreTypes.js'
|
|
||||||
import { TripleQuery } from '../triple/TripleIntelligence.js'
|
|
||||||
import { patternMatchQuery, PATTERN_STATS } from './staticPatternMatcher.js'
|
|
||||||
|
|
||||||
export interface NaturalQueryIntent {
|
|
||||||
type: 'vector' | 'field' | 'graph' | 'combined'
|
|
||||||
confidence: number
|
|
||||||
extractedTerms: {
|
|
||||||
entities?: string[]
|
|
||||||
fields?: string[]
|
|
||||||
relationships?: string[]
|
|
||||||
modifiers?: string[]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export class NaturalLanguageProcessor {
|
|
||||||
private queryHistory: Array<{ query: string; result: TripleQuery; success: boolean }>
|
|
||||||
|
|
||||||
constructor() {
|
|
||||||
this.queryHistory = []
|
|
||||||
// Patterns are static - no initialization needed!
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* No initialization needed - patterns are pre-built!
|
|
||||||
*/
|
|
||||||
async init(): Promise<void> {
|
|
||||||
// Nothing to do - patterns are compiled into the code
|
|
||||||
return Promise.resolve()
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Process natural language query into structured Triple Intelligence query
|
|
||||||
* @param naturalQuery The natural language query string
|
|
||||||
* @param queryEmbedding Pre-computed embedding from Brainy (passed in to avoid circular dependency)
|
|
||||||
*/
|
|
||||||
async processNaturalQuery(naturalQuery: string, queryEmbedding?: Vector): Promise<TripleQuery> {
|
|
||||||
// Use static pattern matcher (no async, no memory allocation!)
|
|
||||||
const structuredQuery = patternMatchQuery(naturalQuery, queryEmbedding)
|
|
||||||
|
|
||||||
// Step 3: Enhance with intent analysis if needed
|
|
||||||
if (!structuredQuery.where && !structuredQuery.connected) {
|
|
||||||
const intent = await this.analyzeIntent(naturalQuery)
|
|
||||||
|
|
||||||
// Add metadata based on intent
|
|
||||||
if (intent.type === 'field' && intent.extractedTerms.fields) {
|
|
||||||
structuredQuery.where = this.buildFieldConstraints(intent.extractedTerms.fields)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Track for learning (but don't create new Brainy!)
|
|
||||||
this.queryHistory.push({
|
|
||||||
query: naturalQuery,
|
|
||||||
result: structuredQuery,
|
|
||||||
success: false // Will be updated based on user interaction
|
|
||||||
})
|
|
||||||
|
|
||||||
// Keep history limited to prevent memory growth
|
|
||||||
if (this.queryHistory.length > 100) {
|
|
||||||
this.queryHistory.shift()
|
|
||||||
}
|
|
||||||
|
|
||||||
return structuredQuery
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Analyze query intent using keywords
|
|
||||||
*/
|
|
||||||
private async analyzeIntent(query: string): Promise<NaturalQueryIntent> {
|
|
||||||
const lowerQuery = query.toLowerCase()
|
|
||||||
|
|
||||||
// Check for field-specific keywords
|
|
||||||
const fieldKeywords = ['where', 'filter', 'with', 'has', 'contains', 'equals', 'greater', 'less', 'between']
|
|
||||||
const hasFieldIntent = fieldKeywords.some(kw => lowerQuery.includes(kw))
|
|
||||||
|
|
||||||
// Check for graph keywords
|
|
||||||
const graphKeywords = ['related', 'connected', 'linked', 'associated', 'references']
|
|
||||||
const hasGraphIntent = graphKeywords.some(kw => lowerQuery.includes(kw))
|
|
||||||
|
|
||||||
// Determine type
|
|
||||||
let type: NaturalQueryIntent['type'] = 'vector'
|
|
||||||
if (hasFieldIntent && hasGraphIntent) {
|
|
||||||
type = 'combined'
|
|
||||||
} else if (hasFieldIntent) {
|
|
||||||
type = 'field'
|
|
||||||
} else if (hasGraphIntent) {
|
|
||||||
type = 'graph'
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
type,
|
|
||||||
confidence: 0.8,
|
|
||||||
extractedTerms: {
|
|
||||||
fields: hasFieldIntent ? this.extractFieldTerms(query) : undefined,
|
|
||||||
relationships: hasGraphIntent ? this.extractRelationshipTerms(query) : undefined
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Extract field terms from query
|
|
||||||
*/
|
|
||||||
private extractFieldTerms(query: string): string[] {
|
|
||||||
const terms: string[] = []
|
|
||||||
|
|
||||||
// Simple extraction of potential field names
|
|
||||||
const words = query.split(/\s+/)
|
|
||||||
const fieldIndicators = ['year', 'date', 'author', 'type', 'category', 'status', 'price']
|
|
||||||
|
|
||||||
for (const word of words) {
|
|
||||||
if (fieldIndicators.includes(word.toLowerCase())) {
|
|
||||||
terms.push(word.toLowerCase())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return terms
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Extract relationship terms
|
|
||||||
*/
|
|
||||||
private extractRelationshipTerms(query: string): string[] {
|
|
||||||
const terms: string[] = []
|
|
||||||
const relationshipWords = ['related', 'connected', 'linked', 'references', 'cites']
|
|
||||||
|
|
||||||
const words = query.toLowerCase().split(/\s+/)
|
|
||||||
for (const word of words) {
|
|
||||||
if (relationshipWords.includes(word)) {
|
|
||||||
terms.push(word)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return terms
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Build field constraints from extracted terms
|
|
||||||
*/
|
|
||||||
private buildFieldConstraints(fields: string[]): Record<string, any> {
|
|
||||||
const constraints: Record<string, any> = {}
|
|
||||||
|
|
||||||
// Simple mapping for common fields
|
|
||||||
for (const field of fields) {
|
|
||||||
// This would be enhanced with actual value extraction
|
|
||||||
constraints[field] = { exists: true }
|
|
||||||
}
|
|
||||||
|
|
||||||
return constraints
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Find similar queries from history (without using Brainy)
|
|
||||||
* NOTE: Currently unused - reserved for future query caching optimization
|
|
||||||
*/
|
|
||||||
private findSimilarQueries(embedding: Vector): Array<{
|
|
||||||
query: string
|
|
||||||
result: TripleQuery
|
|
||||||
similarity: number
|
|
||||||
}> {
|
|
||||||
// Not implemented - not required for core functionality
|
|
||||||
// Would implement cosine similarity against queryHistory if needed
|
|
||||||
return []
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Adapt a previous query for new input
|
|
||||||
*/
|
|
||||||
private adaptQuery(newQuery: string, previousResult: TripleQuery): TripleQuery {
|
|
||||||
return previousResult
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Extract entities from query
|
|
||||||
*/
|
|
||||||
private async extractEntities(query: string): Promise<string[]> {
|
|
||||||
// Could use the Entity Registry here if available
|
|
||||||
return []
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Build query from components
|
|
||||||
*/
|
|
||||||
private buildQuery(
|
|
||||||
query: string,
|
|
||||||
intent: NaturalQueryIntent,
|
|
||||||
entities: string[]
|
|
||||||
): TripleQuery {
|
|
||||||
return {
|
|
||||||
like: query,
|
|
||||||
limit: 10
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,633 +0,0 @@
|
||||||
/**
|
|
||||||
* Neural Import - AI-Powered Data Understanding
|
|
||||||
*
|
|
||||||
* Standalone implementation for intelligent data processing.
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { NounType, VerbType } from '../types/graphTypes.js'
|
|
||||||
import * as fs from '../universal/fs.js'
|
|
||||||
import * as path from '../universal/path.js'
|
|
||||||
import { prodLog } from '../utils/logger.js'
|
|
||||||
|
|
||||||
// Neural Import Analysis Types
|
|
||||||
export interface NeuralAnalysisResult {
|
|
||||||
detectedEntities: DetectedEntity[]
|
|
||||||
detectedRelationships: DetectedRelationship[]
|
|
||||||
confidence: number
|
|
||||||
insights: NeuralInsight[]
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface DetectedEntity {
|
|
||||||
originalData: any
|
|
||||||
nounType: string
|
|
||||||
confidence: number
|
|
||||||
suggestedId: string
|
|
||||||
reasoning: string
|
|
||||||
alternativeTypes: Array<{ type: string, confidence: number }>
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface DetectedRelationship {
|
|
||||||
sourceId: string
|
|
||||||
targetId: string
|
|
||||||
verbType: string
|
|
||||||
confidence: number
|
|
||||||
weight: number
|
|
||||||
reasoning: string
|
|
||||||
context: string
|
|
||||||
metadata?: Record<string, any>
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface NeuralInsight {
|
|
||||||
type: 'hierarchy' | 'cluster' | 'pattern' | 'anomaly' | 'opportunity'
|
|
||||||
description: string
|
|
||||||
confidence: number
|
|
||||||
affectedEntities: string[]
|
|
||||||
recommendation?: string
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface NeuralImportConfig {
|
|
||||||
confidenceThreshold: number
|
|
||||||
enableWeights: boolean
|
|
||||||
skipDuplicates: boolean
|
|
||||||
categoryFilter?: string[]
|
|
||||||
dataType?: string
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Neural Import Augmentation - Unified Implementation
|
|
||||||
* Processes data with AI before storage operations
|
|
||||||
*/
|
|
||||||
export class NeuralImportAugmentation {
|
|
||||||
readonly name = 'neural-import'
|
|
||||||
private operations = ['add', 'addNoun', 'addVerb', 'all']
|
|
||||||
|
|
||||||
protected config: NeuralImportConfig
|
|
||||||
private analysisCache = new Map<string, NeuralAnalysisResult>()
|
|
||||||
private context?: { brain: any }
|
|
||||||
|
|
||||||
constructor(config: Partial<NeuralImportConfig> = {}) {
|
|
||||||
this.config = {
|
|
||||||
confidenceThreshold: 0.7,
|
|
||||||
enableWeights: true,
|
|
||||||
skipDuplicates: true,
|
|
||||||
dataType: 'json',
|
|
||||||
...config
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async init(): Promise<void> {
|
|
||||||
// No external dependencies to initialize
|
|
||||||
}
|
|
||||||
|
|
||||||
private log(message: string, _level?: string): void {
|
|
||||||
// Silent by default
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Execute augmentation - process data with AI before storage
|
|
||||||
*/
|
|
||||||
async execute<T = any>(
|
|
||||||
operation: string,
|
|
||||||
params: any,
|
|
||||||
next: () => Promise<T>
|
|
||||||
): Promise<T> {
|
|
||||||
// Only process on add operations
|
|
||||||
if (!this.operations.includes(operation)) {
|
|
||||||
return next()
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
// Extract data from params based on operation
|
|
||||||
const rawData = this.extractRawData(operation, params)
|
|
||||||
if (!rawData) {
|
|
||||||
return next()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Perform neural analysis
|
|
||||||
const analysis = await this.performNeuralAnalysis(rawData, this.config)
|
|
||||||
|
|
||||||
// Enhance params with neural insights
|
|
||||||
if (params.metadata) {
|
|
||||||
params.metadata._neuralProcessed = true
|
|
||||||
params.metadata._neuralConfidence = analysis.confidence
|
|
||||||
params.metadata._detectedEntities = analysis.detectedEntities.length
|
|
||||||
params.metadata._detectedRelationships = analysis.detectedRelationships.length
|
|
||||||
params.metadata._neuralInsights = analysis.insights
|
|
||||||
} else if (typeof params === 'object') {
|
|
||||||
params.metadata = {
|
|
||||||
_neuralProcessed: true,
|
|
||||||
_neuralConfidence: analysis.confidence,
|
|
||||||
_detectedEntities: analysis.detectedEntities.length,
|
|
||||||
_detectedRelationships: analysis.detectedRelationships.length,
|
|
||||||
_neuralInsights: analysis.insights
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Store neural analysis for later retrieval
|
|
||||||
await this.storeNeuralAnalysis(analysis)
|
|
||||||
|
|
||||||
// If we detected entities/relationships, potentially add them
|
|
||||||
if (this.context?.brain && analysis.detectedEntities.length > 0) {
|
|
||||||
// This could automatically create entities/relationships
|
|
||||||
// But for now, just enhance the metadata
|
|
||||||
this.log(`Detected ${analysis.detectedEntities.length} entities and ${analysis.detectedRelationships.length} relationships`)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Continue with enhanced data
|
|
||||||
return next()
|
|
||||||
} catch (error) {
|
|
||||||
this.log(`Neural analysis failed: ${error}`, 'warn')
|
|
||||||
// Continue without neural processing
|
|
||||||
return next()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Extract raw data from operation params
|
|
||||||
*/
|
|
||||||
private extractRawData(operation: string, params: any): any {
|
|
||||||
switch (operation) {
|
|
||||||
case 'add':
|
|
||||||
return params.content || params.data || params
|
|
||||||
case 'addNoun':
|
|
||||||
return params.noun || params.data || params
|
|
||||||
case 'addVerb':
|
|
||||||
return params.verb || params
|
|
||||||
case 'addBatch':
|
|
||||||
return params.items || params.batch || params
|
|
||||||
default:
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get the full neural analysis result (for external use)
|
|
||||||
*/
|
|
||||||
async getNeuralAnalysis(rawData: Buffer | string, dataType?: string): Promise<NeuralAnalysisResult> {
|
|
||||||
const parsedData = await this.parseRawData(rawData, dataType || this.config.dataType || 'json')
|
|
||||||
return await this.performNeuralAnalysis(parsedData, this.config)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Parse raw data based on type
|
|
||||||
*/
|
|
||||||
private async parseRawData(rawData: Buffer | string, dataType: string): Promise<any[]> {
|
|
||||||
const content = typeof rawData === 'string' ? rawData : rawData.toString('utf8')
|
|
||||||
|
|
||||||
switch (dataType.toLowerCase()) {
|
|
||||||
case 'json':
|
|
||||||
try {
|
|
||||||
const jsonData = JSON.parse(content)
|
|
||||||
return Array.isArray(jsonData) ? jsonData : [jsonData]
|
|
||||||
} catch {
|
|
||||||
// If JSON parse fails, treat as text
|
|
||||||
return [{ text: content }]
|
|
||||||
}
|
|
||||||
|
|
||||||
case 'csv':
|
|
||||||
return this.parseCSV(content)
|
|
||||||
|
|
||||||
case 'yaml':
|
|
||||||
case 'yml':
|
|
||||||
return this.parseYAML(content)
|
|
||||||
|
|
||||||
case 'txt':
|
|
||||||
case 'text':
|
|
||||||
// Split text into sentences/paragraphs for analysis
|
|
||||||
return content.split(/\n+/).filter(line => line.trim()).map(line => ({ text: line }))
|
|
||||||
|
|
||||||
default:
|
|
||||||
// Unknown type, treat as text
|
|
||||||
return [{ text: content }]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Parse CSV data - handles quoted values, escaped quotes, and edge cases
|
|
||||||
*/
|
|
||||||
private parseCSV(content: string): any[] {
|
|
||||||
const lines = content.split('\n')
|
|
||||||
if (lines.length === 0) return []
|
|
||||||
|
|
||||||
// Parse a CSV line handling quotes
|
|
||||||
const parseLine = (line: string): string[] => {
|
|
||||||
const result: string[] = []
|
|
||||||
let current = ''
|
|
||||||
let inQuotes = false
|
|
||||||
let i = 0
|
|
||||||
|
|
||||||
while (i < line.length) {
|
|
||||||
const char = line[i]
|
|
||||||
const nextChar = line[i + 1]
|
|
||||||
|
|
||||||
if (char === '"') {
|
|
||||||
if (inQuotes && nextChar === '"') {
|
|
||||||
// Escaped quote
|
|
||||||
current += '"'
|
|
||||||
i += 2
|
|
||||||
} else {
|
|
||||||
// Toggle quote mode
|
|
||||||
inQuotes = !inQuotes
|
|
||||||
i++
|
|
||||||
}
|
|
||||||
} else if (char === ',' && !inQuotes) {
|
|
||||||
// Field separator
|
|
||||||
result.push(current.trim())
|
|
||||||
current = ''
|
|
||||||
i++
|
|
||||||
} else {
|
|
||||||
current += char
|
|
||||||
i++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add last field
|
|
||||||
result.push(current.trim())
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
// Parse headers
|
|
||||||
const headers = parseLine(lines[0])
|
|
||||||
const data = []
|
|
||||||
|
|
||||||
// Parse data rows
|
|
||||||
for (let i = 1; i < lines.length; i++) {
|
|
||||||
const line = lines[i].trim()
|
|
||||||
if (!line) continue // Skip empty lines
|
|
||||||
|
|
||||||
const values = parseLine(line)
|
|
||||||
const row: any = {}
|
|
||||||
|
|
||||||
headers.forEach((header, index) => {
|
|
||||||
const value = values[index] || ''
|
|
||||||
// Try to parse numbers
|
|
||||||
const num = Number(value)
|
|
||||||
row[header] = !isNaN(num) && value !== '' ? num : value
|
|
||||||
})
|
|
||||||
|
|
||||||
data.push(row)
|
|
||||||
}
|
|
||||||
|
|
||||||
return data
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Parse YAML data
|
|
||||||
*/
|
|
||||||
private parseYAML(content: string): any[] {
|
|
||||||
try {
|
|
||||||
// Simple YAML parser for basic structures
|
|
||||||
// For full YAML support, we'd use js-yaml library
|
|
||||||
const lines = content.split('\n')
|
|
||||||
const result: any[] = []
|
|
||||||
let currentObject: any = null
|
|
||||||
let currentIndent = 0
|
|
||||||
|
|
||||||
for (const line of lines) {
|
|
||||||
const trimmed = line.trim()
|
|
||||||
if (!trimmed || trimmed.startsWith('#')) continue // Skip empty lines and comments
|
|
||||||
|
|
||||||
// Calculate indentation
|
|
||||||
const indent = line.length - line.trimStart().length
|
|
||||||
|
|
||||||
// Check for array item
|
|
||||||
if (trimmed.startsWith('- ')) {
|
|
||||||
const value = trimmed.substring(2).trim()
|
|
||||||
if (indent === 0) {
|
|
||||||
// Top-level array item
|
|
||||||
if (value.includes(':')) {
|
|
||||||
// Object in array
|
|
||||||
currentObject = {}
|
|
||||||
result.push(currentObject)
|
|
||||||
const [key, val] = value.split(':').map(s => s.trim())
|
|
||||||
currentObject[key] = this.parseYAMLValue(val)
|
|
||||||
} else {
|
|
||||||
result.push(this.parseYAMLValue(value))
|
|
||||||
}
|
|
||||||
} else if (currentObject) {
|
|
||||||
// Nested array
|
|
||||||
const lastKey = Object.keys(currentObject).pop()
|
|
||||||
if (lastKey) {
|
|
||||||
if (!Array.isArray(currentObject[lastKey])) {
|
|
||||||
currentObject[lastKey] = []
|
|
||||||
}
|
|
||||||
currentObject[lastKey].push(this.parseYAMLValue(value))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else if (trimmed.includes(':')) {
|
|
||||||
// Key-value pair
|
|
||||||
const colonIndex = trimmed.indexOf(':')
|
|
||||||
const key = trimmed.substring(0, colonIndex).trim()
|
|
||||||
const value = trimmed.substring(colonIndex + 1).trim()
|
|
||||||
|
|
||||||
if (indent === 0) {
|
|
||||||
// Top-level object
|
|
||||||
if (!currentObject) {
|
|
||||||
currentObject = {}
|
|
||||||
result.push(currentObject)
|
|
||||||
}
|
|
||||||
currentObject[key] = this.parseYAMLValue(value)
|
|
||||||
currentIndent = 0
|
|
||||||
} else if (currentObject) {
|
|
||||||
// Nested object
|
|
||||||
if (indent > currentIndent && !value) {
|
|
||||||
// Start of nested object
|
|
||||||
const lastKey = Object.keys(currentObject).pop()
|
|
||||||
if (lastKey) {
|
|
||||||
currentObject[lastKey] = { [key]: '' }
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
currentObject[key] = this.parseYAMLValue(value)
|
|
||||||
}
|
|
||||||
currentIndent = indent
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// If we built a single object and not an array, wrap it
|
|
||||||
if (result.length === 0 && currentObject) {
|
|
||||||
result.push(currentObject)
|
|
||||||
}
|
|
||||||
|
|
||||||
return result.length > 0 ? result : [{ text: content }]
|
|
||||||
} catch (error) {
|
|
||||||
prodLog.warn('YAML parsing failed, treating as text:', error)
|
|
||||||
return [{ text: content }]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Parse a YAML value (handle strings, numbers, booleans, null)
|
|
||||||
*/
|
|
||||||
private parseYAMLValue(value: string): any {
|
|
||||||
if (!value || value === '~' || value === 'null') return null
|
|
||||||
if (value === 'true') return true
|
|
||||||
if (value === 'false') return false
|
|
||||||
|
|
||||||
// Remove quotes if present
|
|
||||||
if ((value.startsWith('"') && value.endsWith('"')) ||
|
|
||||||
(value.startsWith("'") && value.endsWith("'"))) {
|
|
||||||
return value.slice(1, -1)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Try to parse as number
|
|
||||||
const num = Number(value)
|
|
||||||
if (!isNaN(num) && value !== '') return num
|
|
||||||
|
|
||||||
return value
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Perform neural analysis on parsed data
|
|
||||||
*/
|
|
||||||
private async performNeuralAnalysis(data: any[], config?: any): Promise<NeuralAnalysisResult> {
|
|
||||||
const detectedEntities: DetectedEntity[] = []
|
|
||||||
const detectedRelationships: DetectedRelationship[] = []
|
|
||||||
const insights: NeuralInsight[] = []
|
|
||||||
|
|
||||||
// Simple entity detection (in real implementation, would use ML)
|
|
||||||
for (const item of data) {
|
|
||||||
if (typeof item === 'object') {
|
|
||||||
// Detect entities from object properties
|
|
||||||
const entityId = item.id || item.name || item.title || `entity_${Date.now()}_${Math.random()}`
|
|
||||||
|
|
||||||
detectedEntities.push({
|
|
||||||
originalData: item,
|
|
||||||
nounType: await this.inferNounType(item),
|
|
||||||
confidence: 0.85,
|
|
||||||
suggestedId: String(entityId),
|
|
||||||
reasoning: 'Detected from structured data',
|
|
||||||
alternativeTypes: []
|
|
||||||
})
|
|
||||||
|
|
||||||
// Detect relationships from references
|
|
||||||
await this.detectRelationships(item, entityId, detectedRelationships)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Generate insights
|
|
||||||
if (detectedEntities.length > 10) {
|
|
||||||
insights.push({
|
|
||||||
type: 'pattern',
|
|
||||||
description: `Large dataset with ${detectedEntities.length} entities detected`,
|
|
||||||
confidence: 0.9,
|
|
||||||
affectedEntities: detectedEntities.slice(0, 5).map(e => e.suggestedId),
|
|
||||||
recommendation: 'Consider batch processing for optimal performance'
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// Look for clusters
|
|
||||||
const typeGroups = this.groupByType(detectedEntities)
|
|
||||||
if (Object.keys(typeGroups).length > 1) {
|
|
||||||
insights.push({
|
|
||||||
type: 'cluster',
|
|
||||||
description: `Multiple entity types detected: ${Object.keys(typeGroups).join(', ')}`,
|
|
||||||
confidence: 0.8,
|
|
||||||
affectedEntities: [],
|
|
||||||
recommendation: 'Data contains diverse entity types suitable for graph analysis'
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
detectedEntities,
|
|
||||||
detectedRelationships,
|
|
||||||
confidence: detectedEntities.length > 0 ? 0.85 : 0.5,
|
|
||||||
insights
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Infer noun type from object structure using field heuristics
|
|
||||||
*/
|
|
||||||
private async inferNounType(obj: any): Promise<string> {
|
|
||||||
if (typeof obj !== 'object' || obj === null) return NounType.Thing
|
|
||||||
|
|
||||||
// Check for explicit type field
|
|
||||||
if (obj.type && typeof obj.type === 'string') {
|
|
||||||
const normalized = obj.type.charAt(0).toUpperCase() + obj.type.slice(1)
|
|
||||||
if (Object.values(NounType).includes(normalized as NounType)) {
|
|
||||||
return normalized as NounType
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (obj.email || obj.firstName || obj.lastName || obj.username) return NounType.Person
|
|
||||||
if (obj.companyName || obj.organizationId || obj.employees) return NounType.Organization
|
|
||||||
if (obj.latitude || obj.longitude || obj.address || obj.city) return NounType.Location
|
|
||||||
if ((obj.content && (obj.title || obj.author)) || obj.pages) return NounType.Document
|
|
||||||
if (obj.startTime || obj.endTime || obj.date || obj.attendees) return NounType.Event
|
|
||||||
if (obj.price || obj.sku || obj.productId) return NounType.Product
|
|
||||||
if ((obj.status && obj.assignee) || obj.priority) return NounType.Task
|
|
||||||
if (Array.isArray(obj.data) || obj.rows || obj.columns) return NounType.Dataset
|
|
||||||
|
|
||||||
return NounType.Thing
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Detect relationships from object references
|
|
||||||
*/
|
|
||||||
private async detectRelationships(obj: any, sourceId: string, relationships: DetectedRelationship[]): Promise<void> {
|
|
||||||
// Look for reference patterns
|
|
||||||
for (const [key, value] of Object.entries(obj)) {
|
|
||||||
if (key.endsWith('Id') || key.endsWith('_id') || key === 'parentId' || key === 'userId') {
|
|
||||||
relationships.push({
|
|
||||||
sourceId,
|
|
||||||
targetId: String(value),
|
|
||||||
verbType: await this.inferVerbType(key, obj, { id: value }),
|
|
||||||
confidence: 0.75,
|
|
||||||
weight: 1,
|
|
||||||
reasoning: `Reference detected in field: ${key}`,
|
|
||||||
context: key
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// Array of IDs
|
|
||||||
if (Array.isArray(value) && value.length > 0 && typeof value[0] === 'string') {
|
|
||||||
if (key.endsWith('Ids') || key.endsWith('_ids')) {
|
|
||||||
for (const targetId of value) {
|
|
||||||
relationships.push({
|
|
||||||
sourceId,
|
|
||||||
targetId: String(targetId),
|
|
||||||
verbType: await this.inferVerbType(key, obj, { id: targetId }),
|
|
||||||
confidence: 0.7,
|
|
||||||
weight: 1,
|
|
||||||
reasoning: `Array reference in field: ${key}`,
|
|
||||||
context: key
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Infer verb type from field name using common patterns
|
|
||||||
*/
|
|
||||||
private async inferVerbType(fieldName: string, _sourceObj?: any, _targetObj?: any): Promise<string> {
|
|
||||||
const field = fieldName.toLowerCase()
|
|
||||||
|
|
||||||
if (field.includes('parent') || field.includes('child') || field.includes('contain')) {
|
|
||||||
return VerbType.Contains
|
|
||||||
}
|
|
||||||
if (field.includes('owner') || field.includes('created') || field.includes('author')) {
|
|
||||||
return VerbType.Creates
|
|
||||||
}
|
|
||||||
if (field.includes('member') || field.includes('belong')) {
|
|
||||||
return VerbType.MemberOf
|
|
||||||
}
|
|
||||||
if (field.includes('depend') || field.includes('require')) {
|
|
||||||
return VerbType.DependsOn
|
|
||||||
}
|
|
||||||
if (field.includes('ref') || field.includes('link') || field.includes('source')) {
|
|
||||||
return VerbType.References
|
|
||||||
}
|
|
||||||
|
|
||||||
return VerbType.RelatedTo
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Group entities by type
|
|
||||||
*/
|
|
||||||
private groupByType(entities: DetectedEntity[]): Record<string, DetectedEntity[]> {
|
|
||||||
const groups: Record<string, DetectedEntity[]> = {}
|
|
||||||
|
|
||||||
for (const entity of entities) {
|
|
||||||
if (!groups[entity.nounType]) {
|
|
||||||
groups[entity.nounType] = []
|
|
||||||
}
|
|
||||||
groups[entity.nounType].push(entity)
|
|
||||||
}
|
|
||||||
|
|
||||||
return groups
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Store neural analysis results
|
|
||||||
*/
|
|
||||||
private async storeNeuralAnalysis(analysis: NeuralAnalysisResult): Promise<void> {
|
|
||||||
// Cache the analysis for potential later use
|
|
||||||
const key = `analysis_${Date.now()}`
|
|
||||||
this.analysisCache.set(key, analysis)
|
|
||||||
|
|
||||||
// Limit cache size
|
|
||||||
if (this.analysisCache.size > 100) {
|
|
||||||
const firstKey = this.analysisCache.keys().next().value
|
|
||||||
if (firstKey) {
|
|
||||||
this.analysisCache.delete(firstKey)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Helper to get data type from file path
|
|
||||||
*/
|
|
||||||
private getDataTypeFromPath(filePath: string): string {
|
|
||||||
const ext = path.extname(filePath).toLowerCase()
|
|
||||||
switch (ext) {
|
|
||||||
case '.json': return 'json'
|
|
||||||
case '.csv': return 'csv'
|
|
||||||
case '.txt': return 'text'
|
|
||||||
case '.yaml':
|
|
||||||
case '.yml': return 'yaml'
|
|
||||||
default: return 'text'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* PUBLIC API: Process raw data (for external use, like Synapses)
|
|
||||||
* This maintains compatibility with code that wants to use Neural Import directly
|
|
||||||
*/
|
|
||||||
async processRawData(
|
|
||||||
rawData: Buffer | string,
|
|
||||||
dataType: string,
|
|
||||||
options?: Record<string, unknown>
|
|
||||||
): Promise<{
|
|
||||||
success: boolean
|
|
||||||
data: {
|
|
||||||
nouns: string[]
|
|
||||||
verbs: string[]
|
|
||||||
confidence?: number
|
|
||||||
insights?: Array<{
|
|
||||||
type: string
|
|
||||||
description: string
|
|
||||||
confidence: number
|
|
||||||
}>
|
|
||||||
metadata?: Record<string, unknown>
|
|
||||||
}
|
|
||||||
error?: string
|
|
||||||
}> {
|
|
||||||
try {
|
|
||||||
const analysis = await this.getNeuralAnalysis(rawData, dataType)
|
|
||||||
|
|
||||||
// Convert to legacy format for compatibility
|
|
||||||
const nouns = analysis.detectedEntities.map(e => e.suggestedId)
|
|
||||||
const verbs = analysis.detectedRelationships.map(r =>
|
|
||||||
`${r.sourceId}->${r.verbType}->${r.targetId}`
|
|
||||||
)
|
|
||||||
|
|
||||||
return {
|
|
||||||
success: true,
|
|
||||||
data: {
|
|
||||||
nouns,
|
|
||||||
verbs,
|
|
||||||
confidence: analysis.confidence,
|
|
||||||
insights: analysis.insights.map(i => ({
|
|
||||||
type: i.type,
|
|
||||||
description: i.description,
|
|
||||||
confidence: i.confidence
|
|
||||||
})),
|
|
||||||
metadata: {
|
|
||||||
detectedEntities: analysis.detectedEntities.length,
|
|
||||||
detectedRelationships: analysis.detectedRelationships.length,
|
|
||||||
timestamp: new Date().toISOString()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
return {
|
|
||||||
success: false,
|
|
||||||
data: { nouns: [], verbs: [] },
|
|
||||||
error: error instanceof Error ? error.message : 'Neural analysis failed'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,479 +0,0 @@
|
||||||
/**
|
|
||||||
* Smart Import Presets - Zero-Configuration Auto-Detection
|
|
||||||
*
|
|
||||||
* Automatically selects optimal import strategy based on:
|
|
||||||
* - File type (Excel, CSV, PDF, Markdown, JSON)
|
|
||||||
* - File size and row count
|
|
||||||
* - Column structure (explicit relationships vs narrative)
|
|
||||||
* - Available memory and performance requirements
|
|
||||||
*
|
|
||||||
* Production-ready: Handles billions of entities with optimal performance
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { NounType, VerbType } from '../types/graphTypes.js'
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Signal types used for entity classification
|
|
||||||
*/
|
|
||||||
export type SignalType = 'embedding' | 'exact' | 'pattern' | 'context'
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Strategy types used for relationship extraction
|
|
||||||
*/
|
|
||||||
export type StrategyType = 'explicit' | 'pattern' | 'embedding'
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Import context for preset auto-detection
|
|
||||||
*/
|
|
||||||
export interface ImportContext {
|
|
||||||
fileType?: 'excel' | 'csv' | 'json' | 'pdf' | 'markdown' | 'unknown'
|
|
||||||
fileSize?: number // bytes
|
|
||||||
rowCount?: number
|
|
||||||
hasExplicitColumns?: boolean // Has "Related Terms" or similar columns
|
|
||||||
hasNarrativeContent?: boolean // Has long-form text/descriptions
|
|
||||||
avgDefinitionLength?: number // Average length of definitions
|
|
||||||
memoryAvailable?: number // bytes
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Signal configuration with weights
|
|
||||||
*/
|
|
||||||
export interface SignalConfig {
|
|
||||||
enabled: SignalType[]
|
|
||||||
weights: Record<SignalType, number>
|
|
||||||
timeout: number // milliseconds
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Strategy configuration with priorities
|
|
||||||
*/
|
|
||||||
export interface StrategyConfig {
|
|
||||||
enabled: StrategyType[]
|
|
||||||
timeout: number // milliseconds
|
|
||||||
earlyTermination: boolean
|
|
||||||
minConfidence: number
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Complete preset configuration
|
|
||||||
*/
|
|
||||||
export interface PresetConfig {
|
|
||||||
name: string
|
|
||||||
description: string
|
|
||||||
signals: SignalConfig
|
|
||||||
strategies: StrategyConfig
|
|
||||||
streaming: boolean
|
|
||||||
batchSize: number
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Fast Preset - For large imports (>10K rows)
|
|
||||||
*
|
|
||||||
* Optimized for speed over accuracy:
|
|
||||||
* - Only exact match and pattern signals
|
|
||||||
* - Only explicit strategy (O(1) lookups)
|
|
||||||
* - Streaming enabled for memory efficiency
|
|
||||||
* - Early termination on first high-confidence match
|
|
||||||
*
|
|
||||||
* Use case: Bulk imports, data migrations
|
|
||||||
* Performance: ~10ms per row
|
|
||||||
* Accuracy: ~85%
|
|
||||||
*/
|
|
||||||
export const FAST_PRESET: PresetConfig = {
|
|
||||||
name: 'fast',
|
|
||||||
description: 'Fast bulk import for large datasets',
|
|
||||||
signals: {
|
|
||||||
enabled: ['exact', 'pattern'],
|
|
||||||
weights: {
|
|
||||||
exact: 0.70,
|
|
||||||
pattern: 0.30,
|
|
||||||
embedding: 0,
|
|
||||||
context: 0
|
|
||||||
},
|
|
||||||
timeout: 50
|
|
||||||
},
|
|
||||||
strategies: {
|
|
||||||
enabled: ['explicit'],
|
|
||||||
timeout: 100,
|
|
||||||
earlyTermination: true,
|
|
||||||
minConfidence: 0.70
|
|
||||||
},
|
|
||||||
streaming: true,
|
|
||||||
batchSize: 1000
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Balanced Preset - Default for most imports
|
|
||||||
*
|
|
||||||
* Good balance of speed and accuracy:
|
|
||||||
* - All signals except context (embedding, exact, pattern)
|
|
||||||
* - All strategies with smart ordering
|
|
||||||
* - Moderate timeouts
|
|
||||||
* - Early termination after high-confidence matches
|
|
||||||
*
|
|
||||||
* Use case: Standard imports, general glossaries
|
|
||||||
* Performance: ~30ms per row
|
|
||||||
* Accuracy: ~92%
|
|
||||||
*/
|
|
||||||
export const BALANCED_PRESET: PresetConfig = {
|
|
||||||
name: 'balanced',
|
|
||||||
description: 'Balanced speed and accuracy for most imports',
|
|
||||||
signals: {
|
|
||||||
enabled: ['exact', 'embedding', 'pattern'],
|
|
||||||
weights: {
|
|
||||||
exact: 0.40,
|
|
||||||
embedding: 0.35,
|
|
||||||
pattern: 0.25,
|
|
||||||
context: 0
|
|
||||||
},
|
|
||||||
timeout: 100
|
|
||||||
},
|
|
||||||
strategies: {
|
|
||||||
enabled: ['explicit', 'pattern', 'embedding'],
|
|
||||||
timeout: 200,
|
|
||||||
earlyTermination: true,
|
|
||||||
minConfidence: 0.65
|
|
||||||
},
|
|
||||||
streaming: false,
|
|
||||||
batchSize: 500
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Accurate Preset - For small, critical imports
|
|
||||||
*
|
|
||||||
* Optimized for accuracy over speed:
|
|
||||||
* - All signals including context
|
|
||||||
* - All strategies, no early termination
|
|
||||||
* - Longer timeouts for thorough analysis
|
|
||||||
* - Lower confidence threshold (accept more matches)
|
|
||||||
*
|
|
||||||
* Use case: Knowledge bases, critical taxonomies
|
|
||||||
* Performance: ~100ms per row
|
|
||||||
* Accuracy: ~97%
|
|
||||||
*/
|
|
||||||
export const ACCURATE_PRESET: PresetConfig = {
|
|
||||||
name: 'accurate',
|
|
||||||
description: 'Maximum accuracy for critical imports',
|
|
||||||
signals: {
|
|
||||||
enabled: ['exact', 'embedding', 'pattern', 'context'],
|
|
||||||
weights: {
|
|
||||||
exact: 0.40,
|
|
||||||
embedding: 0.35,
|
|
||||||
pattern: 0.20,
|
|
||||||
context: 0.05
|
|
||||||
},
|
|
||||||
timeout: 500
|
|
||||||
},
|
|
||||||
strategies: {
|
|
||||||
enabled: ['explicit', 'pattern', 'embedding'],
|
|
||||||
timeout: 1000,
|
|
||||||
earlyTermination: false,
|
|
||||||
minConfidence: 0.50
|
|
||||||
},
|
|
||||||
streaming: false,
|
|
||||||
batchSize: 100
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Explicit Preset - For glossaries with relationship columns
|
|
||||||
*
|
|
||||||
* Optimized for structured data with explicit relationships:
|
|
||||||
* - Only exact match signals (no AI needed)
|
|
||||||
* - Only explicit and pattern strategies
|
|
||||||
* - Fast, deterministic results
|
|
||||||
* - Perfect for Excel/CSV with "Related Terms" columns
|
|
||||||
*
|
|
||||||
* Use case: structured taxonomies and glossaries from spreadsheet sources
|
|
||||||
* Performance: ~5ms per row
|
|
||||||
* Accuracy: ~99% (high confidence)
|
|
||||||
*/
|
|
||||||
export const EXPLICIT_PRESET: PresetConfig = {
|
|
||||||
name: 'explicit',
|
|
||||||
description: 'For glossaries with explicit relationship columns',
|
|
||||||
signals: {
|
|
||||||
enabled: ['exact', 'pattern'],
|
|
||||||
weights: {
|
|
||||||
exact: 0.70,
|
|
||||||
pattern: 0.30,
|
|
||||||
embedding: 0,
|
|
||||||
context: 0
|
|
||||||
},
|
|
||||||
timeout: 50
|
|
||||||
},
|
|
||||||
strategies: {
|
|
||||||
enabled: ['explicit', 'pattern'],
|
|
||||||
timeout: 100,
|
|
||||||
earlyTermination: true,
|
|
||||||
minConfidence: 0.80
|
|
||||||
},
|
|
||||||
streaming: false,
|
|
||||||
batchSize: 500
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Pattern Preset - For documents with narrative content
|
|
||||||
*
|
|
||||||
* Optimized for unstructured text with rich patterns:
|
|
||||||
* - Embedding and pattern signals (semantic understanding)
|
|
||||||
* - Pattern and embedding strategies
|
|
||||||
* - Good for PDFs, articles, documentation
|
|
||||||
*
|
|
||||||
* Use case: PDF imports, markdown docs, articles
|
|
||||||
* Performance: ~50ms per row
|
|
||||||
* Accuracy: ~90%
|
|
||||||
*/
|
|
||||||
export const PATTERN_PRESET: PresetConfig = {
|
|
||||||
name: 'pattern',
|
|
||||||
description: 'For documents with narrative content',
|
|
||||||
signals: {
|
|
||||||
enabled: ['embedding', 'pattern', 'context'],
|
|
||||||
weights: {
|
|
||||||
embedding: 0.50,
|
|
||||||
pattern: 0.40,
|
|
||||||
context: 0.10,
|
|
||||||
exact: 0
|
|
||||||
},
|
|
||||||
timeout: 200
|
|
||||||
},
|
|
||||||
strategies: {
|
|
||||||
enabled: ['pattern', 'embedding'],
|
|
||||||
timeout: 300,
|
|
||||||
earlyTermination: false,
|
|
||||||
minConfidence: 0.60
|
|
||||||
},
|
|
||||||
streaming: false,
|
|
||||||
batchSize: 200
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* All available presets
|
|
||||||
*/
|
|
||||||
export const PRESETS: Record<string, PresetConfig> = {
|
|
||||||
fast: FAST_PRESET,
|
|
||||||
balanced: BALANCED_PRESET,
|
|
||||||
accurate: ACCURATE_PRESET,
|
|
||||||
explicit: EXPLICIT_PRESET,
|
|
||||||
pattern: PATTERN_PRESET
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Auto-detect optimal preset based on import context
|
|
||||||
*
|
|
||||||
* Decision tree:
|
|
||||||
* 1. Large dataset (>10K rows or >10MB) → fast
|
|
||||||
* 2. Small dataset (<100 rows) → accurate
|
|
||||||
* 3. Excel/CSV with explicit columns → explicit
|
|
||||||
* 4. PDF/Markdown with long content → pattern
|
|
||||||
* 5. Default → balanced
|
|
||||||
*
|
|
||||||
* @param context Import context (file type, size, structure)
|
|
||||||
* @returns Optimal preset configuration
|
|
||||||
*/
|
|
||||||
export function autoDetectPreset(context: ImportContext = {}): PresetConfig {
|
|
||||||
const {
|
|
||||||
fileType = 'unknown',
|
|
||||||
fileSize = 0,
|
|
||||||
rowCount = 0,
|
|
||||||
hasExplicitColumns = false,
|
|
||||||
hasNarrativeContent = false,
|
|
||||||
avgDefinitionLength = 0
|
|
||||||
} = context
|
|
||||||
|
|
||||||
// Rule 1: Large imports → fast preset (prioritize speed)
|
|
||||||
if (rowCount > 10000 || fileSize > 10_000_000) {
|
|
||||||
return FAST_PRESET
|
|
||||||
}
|
|
||||||
|
|
||||||
// Rule 2: Small critical imports → accurate preset (prioritize accuracy)
|
|
||||||
if (rowCount > 0 && rowCount < 100) {
|
|
||||||
return ACCURATE_PRESET
|
|
||||||
}
|
|
||||||
|
|
||||||
// Rule 3: Structured data with explicit relationships → explicit preset
|
|
||||||
// (Handles spreadsheet imports where relationships are encoded in columns.)
|
|
||||||
if (hasExplicitColumns && (fileType === 'excel' || fileType === 'csv')) {
|
|
||||||
return EXPLICIT_PRESET
|
|
||||||
}
|
|
||||||
|
|
||||||
// Rule 4: Narrative content → pattern preset
|
|
||||||
// Good for PDFs, articles, documentation
|
|
||||||
if (
|
|
||||||
hasNarrativeContent ||
|
|
||||||
fileType === 'pdf' ||
|
|
||||||
fileType === 'markdown' ||
|
|
||||||
avgDefinitionLength > 500
|
|
||||||
) {
|
|
||||||
return PATTERN_PRESET
|
|
||||||
}
|
|
||||||
|
|
||||||
// Rule 5: JSON data → balanced preset
|
|
||||||
if (fileType === 'json') {
|
|
||||||
return BALANCED_PRESET
|
|
||||||
}
|
|
||||||
|
|
||||||
// Default: balanced preset
|
|
||||||
return BALANCED_PRESET
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get preset by name
|
|
||||||
*
|
|
||||||
* @param name Preset name (fast, balanced, accurate, explicit, pattern)
|
|
||||||
* @returns Preset configuration
|
|
||||||
* @throws Error if preset not found
|
|
||||||
*/
|
|
||||||
export function getPreset(name: string): PresetConfig {
|
|
||||||
const preset = PRESETS[name.toLowerCase()]
|
|
||||||
if (!preset) {
|
|
||||||
throw new Error(`Unknown preset: ${name}. Available: ${Object.keys(PRESETS).join(', ')}`)
|
|
||||||
}
|
|
||||||
return preset
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get all available preset names
|
|
||||||
*
|
|
||||||
* @returns Array of preset names
|
|
||||||
*/
|
|
||||||
export function getPresetNames(): string[] {
|
|
||||||
return Object.keys(PRESETS)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Explain why a preset was selected
|
|
||||||
*
|
|
||||||
* @param context Import context
|
|
||||||
* @returns Human-readable explanation
|
|
||||||
*/
|
|
||||||
export function explainPresetChoice(context: ImportContext = {}): string {
|
|
||||||
const {
|
|
||||||
fileType = 'unknown',
|
|
||||||
fileSize = 0,
|
|
||||||
rowCount = 0,
|
|
||||||
hasExplicitColumns = false,
|
|
||||||
hasNarrativeContent = false,
|
|
||||||
avgDefinitionLength = 0
|
|
||||||
} = context
|
|
||||||
|
|
||||||
if (rowCount > 10000 || fileSize > 10_000_000) {
|
|
||||||
return `Large dataset (${rowCount} rows, ${(fileSize / 1_000_000).toFixed(1)}MB) → fast preset for optimal performance`
|
|
||||||
}
|
|
||||||
|
|
||||||
if (rowCount > 0 && rowCount < 100) {
|
|
||||||
return `Small critical dataset (${rowCount} rows) → accurate preset for maximum accuracy`
|
|
||||||
}
|
|
||||||
|
|
||||||
if (hasExplicitColumns && (fileType === 'excel' || fileType === 'csv')) {
|
|
||||||
return `${fileType.toUpperCase()} with explicit relationship columns → explicit preset for deterministic results`
|
|
||||||
}
|
|
||||||
|
|
||||||
if (hasNarrativeContent || fileType === 'pdf' || fileType === 'markdown') {
|
|
||||||
return `Narrative content (${fileType}) → pattern preset for semantic understanding`
|
|
||||||
}
|
|
||||||
|
|
||||||
if (fileType === 'json') {
|
|
||||||
return `JSON data → balanced preset for structured imports`
|
|
||||||
}
|
|
||||||
|
|
||||||
return `Standard import → balanced preset (default)`
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create custom preset by merging with base preset
|
|
||||||
*
|
|
||||||
* @param baseName Base preset name
|
|
||||||
* @param overrides Custom overrides
|
|
||||||
* @returns Custom preset configuration
|
|
||||||
*/
|
|
||||||
export function createCustomPreset(
|
|
||||||
baseName: string,
|
|
||||||
overrides: Partial<PresetConfig>
|
|
||||||
): PresetConfig {
|
|
||||||
const base = getPreset(baseName)
|
|
||||||
|
|
||||||
return {
|
|
||||||
...base,
|
|
||||||
...overrides,
|
|
||||||
signals: {
|
|
||||||
...base.signals,
|
|
||||||
...(overrides.signals || {})
|
|
||||||
},
|
|
||||||
strategies: {
|
|
||||||
...base.strategies,
|
|
||||||
...(overrides.strategies || {})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Validate preset configuration
|
|
||||||
*
|
|
||||||
* @param preset Preset to validate
|
|
||||||
* @returns True if valid, throws error otherwise
|
|
||||||
*/
|
|
||||||
export function validatePreset(preset: PresetConfig): boolean {
|
|
||||||
// Validate signals
|
|
||||||
if (preset.signals.enabled.length === 0) {
|
|
||||||
throw new Error('Preset must have at least one enabled signal')
|
|
||||||
}
|
|
||||||
|
|
||||||
// Validate strategies
|
|
||||||
if (preset.strategies.enabled.length === 0) {
|
|
||||||
throw new Error('Preset must have at least one enabled strategy')
|
|
||||||
}
|
|
||||||
|
|
||||||
// Validate weights sum to ~1.0
|
|
||||||
const enabledSignals = preset.signals.enabled
|
|
||||||
const totalWeight = enabledSignals.reduce(
|
|
||||||
(sum, signal) => sum + preset.signals.weights[signal],
|
|
||||||
0
|
|
||||||
)
|
|
||||||
|
|
||||||
if (Math.abs(totalWeight - 1.0) > 0.01) {
|
|
||||||
throw new Error(
|
|
||||||
`Signal weights must sum to 1.0, got ${totalWeight.toFixed(2)}`
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Validate timeouts
|
|
||||||
if (preset.signals.timeout <= 0 || preset.strategies.timeout <= 0) {
|
|
||||||
throw new Error('Timeouts must be positive')
|
|
||||||
}
|
|
||||||
|
|
||||||
// Validate batch size
|
|
||||||
if (preset.batchSize <= 0) {
|
|
||||||
throw new Error('Batch size must be positive')
|
|
||||||
}
|
|
||||||
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Format preset for display
|
|
||||||
*
|
|
||||||
* @param preset Preset configuration
|
|
||||||
* @returns Human-readable preset summary
|
|
||||||
*/
|
|
||||||
export function formatPreset(preset: PresetConfig): string {
|
|
||||||
const lines = [
|
|
||||||
`Preset: ${preset.name}`,
|
|
||||||
`Description: ${preset.description}`,
|
|
||||||
'',
|
|
||||||
'Signals:',
|
|
||||||
...preset.signals.enabled.map(
|
|
||||||
(s) => ` - ${s}: ${(preset.signals.weights[s] * 100).toFixed(0)}%`
|
|
||||||
),
|
|
||||||
` Timeout: ${preset.signals.timeout}ms`,
|
|
||||||
'',
|
|
||||||
'Strategies:',
|
|
||||||
...preset.strategies.enabled.map((s) => ` - ${s}`),
|
|
||||||
` Timeout: ${preset.strategies.timeout}ms`,
|
|
||||||
` Early termination: ${preset.strategies.earlyTermination}`,
|
|
||||||
` Min confidence: ${preset.strategies.minConfidence}`,
|
|
||||||
'',
|
|
||||||
`Streaming: ${preset.streaming}`,
|
|
||||||
`Batch size: ${preset.batchSize}`
|
|
||||||
]
|
|
||||||
|
|
||||||
return lines.join('\n')
|
|
||||||
}
|
|
||||||
|
|
@ -1,309 +0,0 @@
|
||||||
/**
|
|
||||||
* Relationship Confidence Scoring
|
|
||||||
*
|
|
||||||
* Scores the confidence of detected relationships based on multiple factors:
|
|
||||||
* - Entity proximity in text
|
|
||||||
* - Entity confidence scores
|
|
||||||
* - Pattern matches
|
|
||||||
* - Structural analysis
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { ExtractedEntity } from './entityExtractor.js'
|
|
||||||
import { VerbType } from '../types/graphTypes.js'
|
|
||||||
import { RelationEvidence } from '../types/brainy.types.js'
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Detected relationship with confidence
|
|
||||||
*/
|
|
||||||
export interface DetectedRelationship {
|
|
||||||
sourceEntity: ExtractedEntity
|
|
||||||
targetEntity: ExtractedEntity
|
|
||||||
verbType: VerbType
|
|
||||||
confidence: number
|
|
||||||
evidence: RelationEvidence
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Configuration for relationship detection
|
|
||||||
*/
|
|
||||||
export interface RelationshipDetectionConfig {
|
|
||||||
minConfidence?: number // Minimum confidence to return (default: 0.5)
|
|
||||||
maxDistance?: number // Maximum token distance between entities (default: 50)
|
|
||||||
useProximityBoost?: boolean // Boost score based on proximity (default: true)
|
|
||||||
usePatternMatching?: boolean // Use verb pattern matching (default: true)
|
|
||||||
useStructuralAnalysis?: boolean // Analyze sentence structure (default: true)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Relationship confidence scorer
|
|
||||||
*/
|
|
||||||
export class RelationshipConfidenceScorer {
|
|
||||||
private config: Required<RelationshipDetectionConfig>
|
|
||||||
|
|
||||||
constructor(config: RelationshipDetectionConfig = {}) {
|
|
||||||
this.config = {
|
|
||||||
minConfidence: config.minConfidence || 0.5,
|
|
||||||
maxDistance: config.maxDistance || 50,
|
|
||||||
useProximityBoost: config.useProximityBoost !== false,
|
|
||||||
usePatternMatching: config.usePatternMatching !== false,
|
|
||||||
useStructuralAnalysis: config.useStructuralAnalysis !== false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Score a potential relationship between two entities
|
|
||||||
*/
|
|
||||||
scoreRelationship(
|
|
||||||
source: ExtractedEntity,
|
|
||||||
target: ExtractedEntity,
|
|
||||||
verbType: VerbType,
|
|
||||||
context: string
|
|
||||||
): { confidence: number, evidence: RelationEvidence } {
|
|
||||||
let confidence = 0.5 // Base confidence
|
|
||||||
|
|
||||||
// Evidence tracking
|
|
||||||
const reasoningParts: string[] = []
|
|
||||||
|
|
||||||
// Factor 1: Proximity boost (closer entities = higher confidence)
|
|
||||||
if (this.config.useProximityBoost) {
|
|
||||||
const proximityBoost = this.calculateProximityBoost(source, target)
|
|
||||||
confidence += proximityBoost
|
|
||||||
if (proximityBoost > 0) {
|
|
||||||
reasoningParts.push(
|
|
||||||
`Entities are close together (boost: +${proximityBoost.toFixed(2)})`
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Factor 2: Entity confidence boost
|
|
||||||
const entityConfidence = (source.confidence + target.confidence) / 2
|
|
||||||
const entityBoost = (entityConfidence - 0.5) * 0.2 // Scale to 0-0.2
|
|
||||||
confidence *= (1 + entityBoost)
|
|
||||||
if (entityBoost > 0) {
|
|
||||||
reasoningParts.push(
|
|
||||||
`High entity confidence (boost: ${entityBoost.toFixed(2)})`
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Factor 3: Pattern match boost
|
|
||||||
if (this.config.usePatternMatching) {
|
|
||||||
const patternBoost = this.checkVerbPattern(source, target, verbType, context)
|
|
||||||
confidence += patternBoost
|
|
||||||
if (patternBoost > 0) {
|
|
||||||
reasoningParts.push(
|
|
||||||
`Matches relationship pattern (boost: +${patternBoost.toFixed(2)})`
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Factor 4: Structural boost (same sentence, clause, etc.)
|
|
||||||
if (this.config.useStructuralAnalysis) {
|
|
||||||
const structuralBoost = this.analyzeStructure(source, target, context)
|
|
||||||
confidence += structuralBoost
|
|
||||||
if (structuralBoost > 0) {
|
|
||||||
reasoningParts.push(
|
|
||||||
`Structural relationship (boost: +${structuralBoost.toFixed(2)})`
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Cap confidence at 1.0
|
|
||||||
confidence = Math.min(confidence, 1.0)
|
|
||||||
|
|
||||||
// Extract source text evidence
|
|
||||||
const start = Math.min(source.position.start, target.position.start)
|
|
||||||
const end = Math.max(source.position.end, target.position.end)
|
|
||||||
|
|
||||||
const evidence: RelationEvidence = {
|
|
||||||
sourceText: context.substring(start, end),
|
|
||||||
position: { start, end },
|
|
||||||
method: 'neural',
|
|
||||||
reasoning: reasoningParts.join('; ')
|
|
||||||
}
|
|
||||||
|
|
||||||
return { confidence, evidence }
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Calculate proximity boost based on distance between entities
|
|
||||||
*/
|
|
||||||
private calculateProximityBoost(
|
|
||||||
source: ExtractedEntity,
|
|
||||||
target: ExtractedEntity
|
|
||||||
): number {
|
|
||||||
const distance = Math.abs(source.position.start - target.position.start)
|
|
||||||
|
|
||||||
if (distance === 0) return 0 // Same position, not meaningful
|
|
||||||
|
|
||||||
// Very close (< 20 chars): +0.2
|
|
||||||
if (distance < 20) return 0.2
|
|
||||||
|
|
||||||
// Close (< 50 chars): +0.1
|
|
||||||
if (distance < 50) return 0.1
|
|
||||||
|
|
||||||
// Medium (< 100 chars): +0.05
|
|
||||||
if (distance < 100) return 0.05
|
|
||||||
|
|
||||||
// Far (> 100 chars): no boost
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Check if entities match a verb pattern
|
|
||||||
*/
|
|
||||||
private checkVerbPattern(
|
|
||||||
source: ExtractedEntity,
|
|
||||||
target: ExtractedEntity,
|
|
||||||
verbType: VerbType,
|
|
||||||
context: string
|
|
||||||
): number {
|
|
||||||
const contextBetween = this.getContextBetween(source, target, context)
|
|
||||||
const contextLower = contextBetween.toLowerCase()
|
|
||||||
|
|
||||||
// Verb-specific patterns
|
|
||||||
const patterns: Record<string, string[]> = {
|
|
||||||
[VerbType.Creates]: ['creates', 'made', 'built', 'developed', 'produces'],
|
|
||||||
[VerbType.Owns]: ['owns', 'belongs to', 'possessed by', 'has'],
|
|
||||||
[VerbType.Contains]: ['contains', 'includes', 'has', 'holds'],
|
|
||||||
[VerbType.Requires]: ['requires', 'needs', 'depends on', 'relies on'],
|
|
||||||
[VerbType.Uses]: ['uses', 'utilizes', 'employs', 'applies'],
|
|
||||||
[VerbType.ReportsTo]: ['manages', 'oversees', 'supervises', 'controls'],
|
|
||||||
[VerbType.Causes]: ['influences', 'affects', 'impacts', 'shapes', 'causes'],
|
|
||||||
[VerbType.DependsOn]: ['depends on', 'relies on', 'based on'],
|
|
||||||
[VerbType.Modifies]: ['modifies', 'changes', 'alters', 'updates'],
|
|
||||||
[VerbType.References]: ['references', 'cites', 'mentions', 'refers to']
|
|
||||||
}
|
|
||||||
|
|
||||||
const verbPatterns = patterns[verbType] || []
|
|
||||||
|
|
||||||
for (const pattern of verbPatterns) {
|
|
||||||
if (contextLower.includes(pattern)) {
|
|
||||||
return 0.2 // Strong pattern match
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return 0 // No pattern match
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Analyze structural relationship
|
|
||||||
*/
|
|
||||||
private analyzeStructure(
|
|
||||||
source: ExtractedEntity,
|
|
||||||
target: ExtractedEntity,
|
|
||||||
context: string
|
|
||||||
): number {
|
|
||||||
const contextBetween = this.getContextBetween(source, target, context)
|
|
||||||
|
|
||||||
// Same sentence (no sentence-ending punctuation between them)
|
|
||||||
if (!contextBetween.match(/[.!?]/)) {
|
|
||||||
return 0.1
|
|
||||||
}
|
|
||||||
|
|
||||||
// Same paragraph (single newline between them)
|
|
||||||
if (!contextBetween.match(/\n\n/)) {
|
|
||||||
return 0.05
|
|
||||||
}
|
|
||||||
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get context text between two entities
|
|
||||||
*/
|
|
||||||
private getContextBetween(
|
|
||||||
source: ExtractedEntity,
|
|
||||||
target: ExtractedEntity,
|
|
||||||
context: string
|
|
||||||
): string {
|
|
||||||
const start = Math.min(source.position.end, target.position.end)
|
|
||||||
const end = Math.max(source.position.start, target.position.start)
|
|
||||||
|
|
||||||
if (start >= end) return ''
|
|
||||||
|
|
||||||
return context.substring(start, end)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Detect relationships between a list of entities
|
|
||||||
*/
|
|
||||||
detectRelationships(
|
|
||||||
entities: ExtractedEntity[],
|
|
||||||
context: string,
|
|
||||||
verbHints?: VerbType[]
|
|
||||||
): DetectedRelationship[] {
|
|
||||||
const relationships: DetectedRelationship[] = []
|
|
||||||
const verbs = verbHints || [
|
|
||||||
VerbType.Creates,
|
|
||||||
VerbType.Uses,
|
|
||||||
VerbType.Contains,
|
|
||||||
VerbType.Requires,
|
|
||||||
VerbType.RelatedTo
|
|
||||||
]
|
|
||||||
|
|
||||||
// Check all entity pairs
|
|
||||||
for (let i = 0; i < entities.length; i++) {
|
|
||||||
for (let j = i + 1; j < entities.length; j++) {
|
|
||||||
const source = entities[i]
|
|
||||||
const target = entities[j]
|
|
||||||
|
|
||||||
// Check distance
|
|
||||||
const distance = Math.abs(source.position.start - target.position.start)
|
|
||||||
if (distance > this.config.maxDistance) {
|
|
||||||
continue // Too far apart
|
|
||||||
}
|
|
||||||
|
|
||||||
// Try each verb type
|
|
||||||
for (const verbType of verbs) {
|
|
||||||
const { confidence, evidence } = this.scoreRelationship(
|
|
||||||
source,
|
|
||||||
target,
|
|
||||||
verbType,
|
|
||||||
context
|
|
||||||
)
|
|
||||||
|
|
||||||
if (confidence >= this.config.minConfidence) {
|
|
||||||
relationships.push({
|
|
||||||
sourceEntity: source,
|
|
||||||
targetEntity: target,
|
|
||||||
verbType,
|
|
||||||
confidence,
|
|
||||||
evidence
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Sort by confidence (highest first)
|
|
||||||
relationships.sort((a, b) => b.confidence - a.confidence)
|
|
||||||
|
|
||||||
return relationships
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Convenience function to score a single relationship
|
|
||||||
*/
|
|
||||||
export function scoreRelationshipConfidence(
|
|
||||||
source: ExtractedEntity,
|
|
||||||
target: ExtractedEntity,
|
|
||||||
verbType: VerbType,
|
|
||||||
context: string,
|
|
||||||
config?: RelationshipDetectionConfig
|
|
||||||
): { confidence: number, evidence: RelationEvidence } {
|
|
||||||
const scorer = new RelationshipConfidenceScorer(config)
|
|
||||||
return scorer.scoreRelationship(source, target, verbType, context)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Convenience function to detect all relationships in text
|
|
||||||
*/
|
|
||||||
export function detectRelationshipsWithConfidence(
|
|
||||||
entities: ExtractedEntity[],
|
|
||||||
context: string,
|
|
||||||
config?: RelationshipDetectionConfig
|
|
||||||
): DetectedRelationship[] {
|
|
||||||
const scorer = new RelationshipConfidenceScorer(config)
|
|
||||||
return scorer.detectRelationships(entities, context)
|
|
||||||
}
|
|
||||||
|
|
@ -1,188 +0,0 @@
|
||||||
/**
|
|
||||||
* Static Pattern Matcher - NO runtime initialization, NO Brainy needed
|
|
||||||
*
|
|
||||||
* All patterns and embeddings are pre-computed at build time
|
|
||||||
* This is pure pattern matching with zero dependencies
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { EMBEDDED_PATTERNS, getPatternEmbeddings } from './embeddedPatterns.js'
|
|
||||||
import type { Vector } from '../coreTypes.js'
|
|
||||||
import type { TripleQuery } from '../triple/TripleIntelligence.js'
|
|
||||||
|
|
||||||
// Pre-load patterns and embeddings at module load time (happens once)
|
|
||||||
const patterns = new Map(EMBEDDED_PATTERNS.map(p => [p.id, p]))
|
|
||||||
const patternEmbeddings = getPatternEmbeddings()
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Cosine similarity between two vectors.
|
|
||||||
* Accepts any number-indexed sequence so pre-computed Float32Array pattern
|
|
||||||
* embeddings can be compared against number[] query vectors without copying.
|
|
||||||
*/
|
|
||||||
function cosineSimilarity(a: ArrayLike<number>, b: ArrayLike<number>): number {
|
|
||||||
if (!a || !b || a.length !== b.length) return 0
|
|
||||||
|
|
||||||
let dotProduct = 0
|
|
||||||
let normA = 0
|
|
||||||
let normB = 0
|
|
||||||
|
|
||||||
for (let i = 0; i < a.length; i++) {
|
|
||||||
dotProduct += a[i] * b[i]
|
|
||||||
normA += a[i] * a[i]
|
|
||||||
normB += b[i] * b[i]
|
|
||||||
}
|
|
||||||
|
|
||||||
const denominator = Math.sqrt(normA) * Math.sqrt(normB)
|
|
||||||
return denominator === 0 ? 0 : dotProduct / denominator
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Extract slots from matched pattern
|
|
||||||
*/
|
|
||||||
function extractSlots(query: string, pattern: string): Record<string, string> | null {
|
|
||||||
try {
|
|
||||||
const regex = new RegExp(pattern, 'i')
|
|
||||||
const match = query.match(regex)
|
|
||||||
|
|
||||||
if (!match) return null
|
|
||||||
|
|
||||||
const slots: Record<string, string> = {}
|
|
||||||
for (let i = 1; i < match.length; i++) {
|
|
||||||
if (match[i]) {
|
|
||||||
slots[`$${i}`] = match[i]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return Object.keys(slots).length > 0 ? slots : null
|
|
||||||
} catch {
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Apply template with extracted slots
|
|
||||||
*/
|
|
||||||
function applyTemplate(template: any, slots: Record<string, string>): any {
|
|
||||||
if (!template || !slots) return template
|
|
||||||
|
|
||||||
const result = JSON.parse(JSON.stringify(template))
|
|
||||||
const applySlots = (obj: any): any => {
|
|
||||||
if (typeof obj === 'string') {
|
|
||||||
return obj.replace(/\$\{(\d+)\}/g, (_, num) => slots[`$${num}`] || '')
|
|
||||||
}
|
|
||||||
if (Array.isArray(obj)) {
|
|
||||||
return obj.map(applySlots)
|
|
||||||
}
|
|
||||||
if (typeof obj === 'object' && obj !== null) {
|
|
||||||
const newObj: any = {}
|
|
||||||
for (const [key, value] of Object.entries(obj)) {
|
|
||||||
newObj[key] = applySlots(value)
|
|
||||||
}
|
|
||||||
return newObj
|
|
||||||
}
|
|
||||||
return obj
|
|
||||||
}
|
|
||||||
|
|
||||||
return applySlots(result)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Match query against all patterns using embeddings
|
|
||||||
*/
|
|
||||||
export function findBestPatterns(
|
|
||||||
queryEmbedding: Vector,
|
|
||||||
k: number = 3
|
|
||||||
): Array<{ pattern: typeof EMBEDDED_PATTERNS[0]; similarity: number }> {
|
|
||||||
|
|
||||||
const matches: Array<{ pattern: typeof EMBEDDED_PATTERNS[0]; similarity: number }> = []
|
|
||||||
|
|
||||||
for (const pattern of EMBEDDED_PATTERNS) {
|
|
||||||
|
|
||||||
const patternEmbedding = patternEmbeddings.get(pattern.id)
|
|
||||||
if (!patternEmbedding) continue
|
|
||||||
|
|
||||||
// Pass Float32Array directly, no need for Array.from()!
|
|
||||||
const similarity = cosineSimilarity(queryEmbedding, patternEmbedding)
|
|
||||||
if (similarity > 0.5) { // Threshold for relevance
|
|
||||||
matches.push({ pattern, similarity })
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Sort by similarity and return top k
|
|
||||||
return matches
|
|
||||||
.sort((a, b) => b.similarity - a.similarity)
|
|
||||||
.slice(0, k)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Match query against patterns using regex
|
|
||||||
*/
|
|
||||||
export function matchPatternByRegex(query: string): {
|
|
||||||
pattern: typeof EMBEDDED_PATTERNS[0]
|
|
||||||
slots: Record<string, string>
|
|
||||||
query: TripleQuery
|
|
||||||
} | null {
|
|
||||||
// Try direct regex matching first (fastest)
|
|
||||||
for (const pattern of EMBEDDED_PATTERNS) {
|
|
||||||
const slots = extractSlots(query, pattern.pattern)
|
|
||||||
if (slots) {
|
|
||||||
const templatedQuery = applyTemplate(pattern.template, slots)
|
|
||||||
return {
|
|
||||||
pattern,
|
|
||||||
slots,
|
|
||||||
query: templatedQuery
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Convert natural language to structured query using STATIC patterns
|
|
||||||
* NO initialization needed, NO Brainy required
|
|
||||||
*/
|
|
||||||
export function patternMatchQuery(
|
|
||||||
query: string,
|
|
||||||
queryEmbedding?: Vector
|
|
||||||
): TripleQuery {
|
|
||||||
|
|
||||||
// ALWAYS use vector similarity when we have embeddings (which we always do!)
|
|
||||||
if (queryEmbedding && queryEmbedding.length === 384) {
|
|
||||||
const bestPatterns = findBestPatterns(queryEmbedding, 5) // Get top 5 matches
|
|
||||||
|
|
||||||
// Try to extract slots from best matching patterns
|
|
||||||
for (const { pattern, similarity } of bestPatterns) {
|
|
||||||
// Only try patterns with good similarity
|
|
||||||
if (similarity < 0.7) break
|
|
||||||
|
|
||||||
const slots = extractSlots(query, pattern.pattern)
|
|
||||||
if (slots) {
|
|
||||||
// Found a good match with extractable slots!
|
|
||||||
const result = applyTemplate(pattern.template, slots)
|
|
||||||
console.log('[NLP] Applied template with slots:', JSON.stringify(result))
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// If no slots extracted but we have a good match, use the template as-is
|
|
||||||
if (bestPatterns.length > 0 && bestPatterns[0].similarity > 0.75) {
|
|
||||||
console.log('[NLP] Returning template as-is:', JSON.stringify(bestPatterns[0].pattern.template))
|
|
||||||
return bestPatterns[0].pattern.template
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Fallback: simple vector search (should rarely happen)
|
|
||||||
console.log('[NLP] Fallback - returning simple query')
|
|
||||||
return {
|
|
||||||
like: query,
|
|
||||||
limit: 10
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Export pattern statistics for monitoring
|
|
||||||
export const PATTERN_STATS = {
|
|
||||||
totalPatterns: EMBEDDED_PATTERNS.length,
|
|
||||||
categories: [...new Set(EMBEDDED_PATTERNS.map(p => p.category))],
|
|
||||||
domains: [...new Set(EMBEDDED_PATTERNS.filter(p => p.domain).map(p => p.domain!))],
|
|
||||||
hasEmbeddings: patternEmbeddings.size > 0
|
|
||||||
}
|
|
||||||
File diff suppressed because it is too large
Load diff
|
|
@ -1,34 +0,0 @@
|
||||||
/**
|
|
||||||
* Transaction System
|
|
||||||
*
|
|
||||||
* Provides atomicity for Brainy operations.
|
|
||||||
* All operations succeed or all rollback - no partial failures.
|
|
||||||
*
|
|
||||||
* @module transaction
|
|
||||||
*/
|
|
||||||
|
|
||||||
export * from './types.js'
|
|
||||||
export * from './errors.js'
|
|
||||||
export { Transaction } from './Transaction.js'
|
|
||||||
export { TransactionManager, type TransactionStats } from './TransactionManager.js'
|
|
||||||
|
|
||||||
// Re-export for convenience
|
|
||||||
export type {
|
|
||||||
Operation,
|
|
||||||
RollbackAction,
|
|
||||||
TransactionState,
|
|
||||||
TransactionContext,
|
|
||||||
TransactionFunction,
|
|
||||||
TransactionResult,
|
|
||||||
TransactionOptions
|
|
||||||
} from './types.js'
|
|
||||||
|
|
||||||
export {
|
|
||||||
TransactionError,
|
|
||||||
TransactionExecutionError,
|
|
||||||
TransactionRollbackError,
|
|
||||||
InvalidTransactionStateError,
|
|
||||||
TransactionTimeoutError
|
|
||||||
} from './errors.js'
|
|
||||||
|
|
||||||
export { RevisionConflictError } from './RevisionConflictError.js'
|
|
||||||
|
|
@ -1,326 +0,0 @@
|
||||||
/**
|
|
||||||
* Consistent API Types for Brainy
|
|
||||||
*
|
|
||||||
* These types provide a uniform interface for all public methods,
|
|
||||||
* using object parameters for consistency and extensibility.
|
|
||||||
*/
|
|
||||||
|
|
||||||
import type { Vector } from '../coreTypes.js'
|
|
||||||
import type { NounType, VerbType } from './graphTypes.js'
|
|
||||||
|
|
||||||
// ============= NOUN OPERATIONS =============
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Parameters for adding a noun
|
|
||||||
*/
|
|
||||||
export interface AddNounParams {
|
|
||||||
data: any | Vector // Content or pre-computed vector
|
|
||||||
type: NounType | string // Noun type (required)
|
|
||||||
metadata?: any // Optional metadata
|
|
||||||
id?: string // Optional custom ID
|
|
||||||
service?: string // Optional service identifier
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Parameters for updating a noun
|
|
||||||
*/
|
|
||||||
export interface UpdateNounParams {
|
|
||||||
id: string // Noun ID to update
|
|
||||||
data?: any // New data
|
|
||||||
metadata?: any // New metadata
|
|
||||||
type?: NounType | string // New type
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Parameters for getting nouns
|
|
||||||
*/
|
|
||||||
export interface GetNounsParams {
|
|
||||||
ids?: string[] // Specific IDs to fetch
|
|
||||||
type?: NounType | string | string[] // Filter by type(s)
|
|
||||||
limit?: number // Maximum results
|
|
||||||
offset?: number // Pagination offset
|
|
||||||
cursor?: string // Pagination cursor
|
|
||||||
filter?: Record<string, any> // Metadata filters
|
|
||||||
service?: string // Service filter
|
|
||||||
}
|
|
||||||
|
|
||||||
// ============= VERB OPERATIONS =============
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Parameters for adding a verb (relationship)
|
|
||||||
*/
|
|
||||||
export interface AddVerbParams {
|
|
||||||
source: string // Source noun ID
|
|
||||||
target: string // Target noun ID
|
|
||||||
type: VerbType | string // Verb type (required)
|
|
||||||
weight?: number // Relationship weight (0-1)
|
|
||||||
metadata?: any // Optional metadata
|
|
||||||
service?: string // Optional service identifier
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Parameters for getting verbs
|
|
||||||
*/
|
|
||||||
export interface GetVerbsParams {
|
|
||||||
source?: string // Filter by source
|
|
||||||
target?: string // Filter by target
|
|
||||||
type?: VerbType | string | string[] // Filter by type(s)
|
|
||||||
limit?: number // Maximum results
|
|
||||||
offset?: number // Pagination offset
|
|
||||||
cursor?: string // Pagination cursor
|
|
||||||
filter?: Record<string, any> // Metadata filters
|
|
||||||
service?: string // Service filter
|
|
||||||
}
|
|
||||||
|
|
||||||
// ============= SEARCH OPERATIONS =============
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Unified search parameters
|
|
||||||
*/
|
|
||||||
export interface SearchParams {
|
|
||||||
query: string | Vector // Text query or vector
|
|
||||||
limit?: number // Maximum results (default: 10)
|
|
||||||
threshold?: number // Similarity threshold (0-1)
|
|
||||||
filter?: {
|
|
||||||
type?: NounType | string | string[] // Filter by noun type(s)
|
|
||||||
metadata?: Record<string, any> // Metadata filters
|
|
||||||
service?: string // Service filter
|
|
||||||
}
|
|
||||||
includeMetadata?: boolean // Include metadata in results
|
|
||||||
includeVectors?: boolean // Include vectors in results
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Parameters for similarity search
|
|
||||||
*/
|
|
||||||
export interface SimilarityParams {
|
|
||||||
id?: string // Find similar to this ID
|
|
||||||
data?: any | Vector // Or find similar to this data
|
|
||||||
limit?: number // Maximum results (default: 10)
|
|
||||||
threshold?: number // Similarity threshold (0-1)
|
|
||||||
filter?: {
|
|
||||||
type?: NounType | string | string[]
|
|
||||||
metadata?: Record<string, any>
|
|
||||||
service?: string
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Parameters for related items search
|
|
||||||
*/
|
|
||||||
export interface RelatedParams {
|
|
||||||
id: string // Starting noun ID
|
|
||||||
depth?: number // Traversal depth (default: 1)
|
|
||||||
limit?: number // Max results per level
|
|
||||||
types?: VerbType[] | string[] // Relationship types to follow
|
|
||||||
direction?: 'outgoing' | 'incoming' | 'both'
|
|
||||||
}
|
|
||||||
|
|
||||||
// ============= BATCH OPERATIONS =============
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Parameters for batch noun operations
|
|
||||||
*/
|
|
||||||
export interface BatchNounsParams {
|
|
||||||
items: AddNounParams[] // Array of nouns to add
|
|
||||||
parallel?: boolean // Process in parallel
|
|
||||||
chunkSize?: number // Batch size for processing
|
|
||||||
onProgress?: (completed: number, total: number) => void
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Parameters for batch verb operations
|
|
||||||
*/
|
|
||||||
export interface BatchVerbsParams {
|
|
||||||
items: AddVerbParams[] // Array of verbs to add
|
|
||||||
parallel?: boolean // Process in parallel
|
|
||||||
chunkSize?: number // Batch size for processing
|
|
||||||
onProgress?: (completed: number, total: number) => void
|
|
||||||
}
|
|
||||||
|
|
||||||
// ============= STATISTICS & METADATA =============
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Parameters for statistics queries
|
|
||||||
*/
|
|
||||||
export interface StatisticsParams {
|
|
||||||
detailed?: boolean // Include detailed breakdown
|
|
||||||
includeAugmentations?: boolean // Include augmentation stats
|
|
||||||
includeMemory?: boolean // Include memory usage
|
|
||||||
service?: string // Filter by service
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Parameters for metadata operations
|
|
||||||
*/
|
|
||||||
export interface MetadataParams {
|
|
||||||
id: string // Entity ID
|
|
||||||
metadata: any // Metadata to set/update
|
|
||||||
merge?: boolean // Merge with existing (vs replace)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ============= CONFIGURATION =============
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Dynamic configuration update parameters
|
|
||||||
*/
|
|
||||||
export interface ConfigUpdateParams {
|
|
||||||
embeddings?: {
|
|
||||||
model?: string
|
|
||||||
precision?: 'q8'
|
|
||||||
cache?: boolean
|
|
||||||
}
|
|
||||||
augmentations?: {
|
|
||||||
[name: string]: boolean | Record<string, any>
|
|
||||||
}
|
|
||||||
storage?: {
|
|
||||||
type?: string
|
|
||||||
config?: any
|
|
||||||
}
|
|
||||||
performance?: {
|
|
||||||
batchSize?: number
|
|
||||||
maxConcurrency?: number
|
|
||||||
cacheSize?: number
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ============= TRIPLE INTELLIGENCE API =============
|
|
||||||
|
|
||||||
/**
|
|
||||||
* API for Triple Intelligence Engine to access Brainy internals
|
|
||||||
* This provides type-safe access without unchecked casts
|
|
||||||
*/
|
|
||||||
export interface TripleIntelligenceAPI {
|
|
||||||
// Vector operations
|
|
||||||
vectorSearch(vector: Vector | string, limit: number): Promise<Array<{id: string, score: number, entity?: any}>>
|
|
||||||
|
|
||||||
// Graph operations
|
|
||||||
graphTraversal(options: {
|
|
||||||
start: string | string[]
|
|
||||||
type?: string | string[]
|
|
||||||
direction?: 'in' | 'out' | 'both'
|
|
||||||
maxDepth?: number
|
|
||||||
}): Promise<Array<{id: string, score: number, depth: number}>>
|
|
||||||
|
|
||||||
// Metadata operations
|
|
||||||
metadataQuery(where: Record<string, any>): Promise<Set<string>>
|
|
||||||
getEntity(id: string): Promise<any>
|
|
||||||
|
|
||||||
// Storage operations
|
|
||||||
getVerbsBySource(sourceId: string): Promise<any[]>
|
|
||||||
getVerbsByTarget(targetId: string): Promise<any[]>
|
|
||||||
|
|
||||||
// Statistics
|
|
||||||
getStatistics(): Promise<{
|
|
||||||
totalCount: number
|
|
||||||
fieldStats: Record<string, {
|
|
||||||
min: number
|
|
||||||
max: number
|
|
||||||
cardinality: number
|
|
||||||
type: string
|
|
||||||
}>
|
|
||||||
}>
|
|
||||||
|
|
||||||
// Index access
|
|
||||||
getAllNouns(): Map<string, any>
|
|
||||||
hasMetadataIndex(): boolean
|
|
||||||
}
|
|
||||||
|
|
||||||
// ============= RESULTS =============
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Unified search result
|
|
||||||
*/
|
|
||||||
export interface SearchResult<T = any> {
|
|
||||||
id: string
|
|
||||||
score: number // Similarity score (0-1)
|
|
||||||
data?: T // Original data
|
|
||||||
metadata?: any // Metadata if requested
|
|
||||||
vector?: Vector // Vector if requested
|
|
||||||
type?: string // Noun type
|
|
||||||
distance?: number // Raw distance metric
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Paginated result wrapper
|
|
||||||
*/
|
|
||||||
export interface PaginatedResult<T> {
|
|
||||||
items: T[]
|
|
||||||
total?: number
|
|
||||||
hasMore: boolean
|
|
||||||
nextCursor?: string
|
|
||||||
previousCursor?: string
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Batch operation result
|
|
||||||
*/
|
|
||||||
export interface BatchResult {
|
|
||||||
successful: string[] // Successfully processed IDs
|
|
||||||
failed: Array<{
|
|
||||||
index: number
|
|
||||||
error: string
|
|
||||||
item?: any
|
|
||||||
}>
|
|
||||||
total: number
|
|
||||||
duration: number // Total time in ms
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Statistics result
|
|
||||||
*/
|
|
||||||
export interface StatisticsResult {
|
|
||||||
nouns: {
|
|
||||||
total: number
|
|
||||||
byType: Record<string, number>
|
|
||||||
}
|
|
||||||
verbs: {
|
|
||||||
total: number
|
|
||||||
byType: Record<string, number>
|
|
||||||
}
|
|
||||||
storage: {
|
|
||||||
used: number
|
|
||||||
type: string
|
|
||||||
}
|
|
||||||
performance?: {
|
|
||||||
avgLatency: number
|
|
||||||
throughput: number
|
|
||||||
cacheHitRate?: number
|
|
||||||
}
|
|
||||||
augmentations?: Record<string, any>
|
|
||||||
memory?: {
|
|
||||||
used: number
|
|
||||||
limit: number
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ============= ERRORS =============
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Structured error for API operations
|
|
||||||
*/
|
|
||||||
export class BrainyAPIError extends Error {
|
|
||||||
constructor(
|
|
||||||
message: string,
|
|
||||||
public code: string,
|
|
||||||
public statusCode: number = 400,
|
|
||||||
public details?: any
|
|
||||||
) {
|
|
||||||
super(message)
|
|
||||||
this.name = 'BrainyAPIError'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Error codes
|
|
||||||
export const ErrorCodes = {
|
|
||||||
INVALID_TYPE: 'INVALID_TYPE',
|
|
||||||
NOT_FOUND: 'NOT_FOUND',
|
|
||||||
DUPLICATE_ID: 'DUPLICATE_ID',
|
|
||||||
INVALID_VECTOR: 'INVALID_VECTOR',
|
|
||||||
STORAGE_ERROR: 'STORAGE_ERROR',
|
|
||||||
EMBEDDING_ERROR: 'EMBEDDING_ERROR',
|
|
||||||
AUGMENTATION_ERROR: 'AUGMENTATION_ERROR',
|
|
||||||
VALIDATION_ERROR: 'VALIDATION_ERROR',
|
|
||||||
QUOTA_EXCEEDED: 'QUOTA_EXCEEDED',
|
|
||||||
UNAUTHORIZED: 'UNAUTHORIZED'
|
|
||||||
} as const
|
|
||||||
|
|
@ -1,24 +0,0 @@
|
||||||
/**
|
|
||||||
* Type declarations for the File System Access API
|
|
||||||
* Extends the FileSystemDirectoryHandle interface to include the [Symbol.asyncIterator] method
|
|
||||||
* and FileSystemHandle to include getFile() method for TypeScript compatibility
|
|
||||||
*/
|
|
||||||
|
|
||||||
// Extend the FileSystemDirectoryHandle interface
|
|
||||||
interface FileSystemDirectoryHandle {
|
|
||||||
[Symbol.asyncIterator](): AsyncIterableIterator<[string, FileSystemHandle]>;
|
|
||||||
|
|
||||||
keys(): AsyncIterableIterator<string>;
|
|
||||||
|
|
||||||
entries(): AsyncIterableIterator<[string, FileSystemHandle]>;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Extend the FileSystemHandle interface to include getFile method
|
|
||||||
// This is needed because TypeScript doesn't recognize that a FileSystemHandle
|
|
||||||
// can be a FileSystemFileHandle which has the getFile method
|
|
||||||
interface FileSystemHandle {
|
|
||||||
getFile?(): Promise<File>;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Export something to make this a module
|
|
||||||
export const fileSystemTypesLoaded = true
|
|
||||||
|
|
@ -1,288 +0,0 @@
|
||||||
/**
|
|
||||||
* Standardized Progress Reporting
|
|
||||||
*
|
|
||||||
* Provides unified progress tracking across all long-running operations
|
|
||||||
* in Brainy (imports, clustering, large searches, etc.)
|
|
||||||
*/
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Progress status states
|
|
||||||
*/
|
|
||||||
export type ProgressStatus = 'pending' | 'running' | 'completed' | 'failed' | 'cancelled'
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Standardized progress report
|
|
||||||
*/
|
|
||||||
export interface BrainyProgress<T = any> {
|
|
||||||
// Core status
|
|
||||||
status: ProgressStatus
|
|
||||||
|
|
||||||
// Progress percentage (0-100)
|
|
||||||
progress: number
|
|
||||||
|
|
||||||
// Human-readable message
|
|
||||||
message: string
|
|
||||||
|
|
||||||
// Detailed metadata
|
|
||||||
metadata: {
|
|
||||||
itemsProcessed: number
|
|
||||||
itemsTotal: number
|
|
||||||
currentItem?: string
|
|
||||||
estimatedTimeRemaining?: number // milliseconds
|
|
||||||
startedAt: number
|
|
||||||
completedAt?: number
|
|
||||||
throughput?: number // items per second
|
|
||||||
}
|
|
||||||
|
|
||||||
// Result when completed
|
|
||||||
result?: T
|
|
||||||
|
|
||||||
// Error when failed
|
|
||||||
error?: Error
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Progress tracker with automatic time estimation
|
|
||||||
*/
|
|
||||||
export class ProgressTracker<T = any> {
|
|
||||||
private status: ProgressStatus = 'pending'
|
|
||||||
private processed = 0
|
|
||||||
private total: number
|
|
||||||
private startedAt?: number
|
|
||||||
private completedAt?: number
|
|
||||||
private currentItem?: string
|
|
||||||
private result?: T
|
|
||||||
private error?: Error
|
|
||||||
private processingTimes: number[] = [] // Track last N processing times for estimation
|
|
||||||
|
|
||||||
constructor(total: number) {
|
|
||||||
if (total < 0) {
|
|
||||||
throw new Error('Total must be non-negative')
|
|
||||||
}
|
|
||||||
this.total = total
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Factory method for creating progress trackers
|
|
||||||
*/
|
|
||||||
static create<T>(total: number): ProgressTracker<T> {
|
|
||||||
return new ProgressTracker<T>(total)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Start tracking progress
|
|
||||||
*/
|
|
||||||
start(): BrainyProgress<T> {
|
|
||||||
this.status = 'running'
|
|
||||||
this.startedAt = Date.now()
|
|
||||||
return this.current()
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Update progress
|
|
||||||
*/
|
|
||||||
update(processed: number, currentItem?: string): BrainyProgress<T> {
|
|
||||||
if (processed < 0) {
|
|
||||||
throw new Error('Processed count must be non-negative')
|
|
||||||
}
|
|
||||||
if (processed > this.total) {
|
|
||||||
throw new Error(`Processed count (${processed}) exceeds total (${this.total})`)
|
|
||||||
}
|
|
||||||
|
|
||||||
const previousProcessed = this.processed
|
|
||||||
this.processed = processed
|
|
||||||
this.currentItem = currentItem
|
|
||||||
|
|
||||||
// Track processing time for estimation
|
|
||||||
if (this.startedAt && previousProcessed < processed) {
|
|
||||||
const itemsProcessed = processed - previousProcessed
|
|
||||||
const timeTaken = Date.now() - this.startedAt
|
|
||||||
const avgTimePerItem = timeTaken / processed
|
|
||||||
this.processingTimes.push(avgTimePerItem)
|
|
||||||
|
|
||||||
// Keep only last 100 measurements for rolling average
|
|
||||||
if (this.processingTimes.length > 100) {
|
|
||||||
this.processingTimes.shift()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return this.current()
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Increment progress by 1
|
|
||||||
*/
|
|
||||||
increment(currentItem?: string): BrainyProgress<T> {
|
|
||||||
return this.update(this.processed + 1, currentItem)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Mark as completed
|
|
||||||
*/
|
|
||||||
complete(result: T): BrainyProgress<T> {
|
|
||||||
this.status = 'completed'
|
|
||||||
this.completedAt = Date.now()
|
|
||||||
this.processed = this.total
|
|
||||||
this.result = result
|
|
||||||
return this.current()
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Mark as failed
|
|
||||||
*/
|
|
||||||
fail(error: Error): BrainyProgress<T> {
|
|
||||||
this.status = 'failed'
|
|
||||||
this.completedAt = Date.now()
|
|
||||||
this.error = error
|
|
||||||
return this.current()
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Mark as cancelled
|
|
||||||
*/
|
|
||||||
cancel(): BrainyProgress<T> {
|
|
||||||
this.status = 'cancelled'
|
|
||||||
this.completedAt = Date.now()
|
|
||||||
return this.current()
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get current progress state
|
|
||||||
*/
|
|
||||||
current(): BrainyProgress<T> {
|
|
||||||
const progress = this.total > 0 ? Math.round((this.processed / this.total) * 100) : 0
|
|
||||||
|
|
||||||
// Generate message based on status
|
|
||||||
let message: string
|
|
||||||
switch (this.status) {
|
|
||||||
case 'pending':
|
|
||||||
message = `Ready to process ${this.total} items`
|
|
||||||
break
|
|
||||||
case 'running':
|
|
||||||
message = this.currentItem
|
|
||||||
? `Processing: ${this.currentItem} (${this.processed}/${this.total})`
|
|
||||||
: `Processing ${this.processed}/${this.total} items`
|
|
||||||
break
|
|
||||||
case 'completed':
|
|
||||||
message = `Completed ${this.total} items`
|
|
||||||
break
|
|
||||||
case 'failed':
|
|
||||||
message = `Failed after ${this.processed} items: ${this.error?.message || 'Unknown error'}`
|
|
||||||
break
|
|
||||||
case 'cancelled':
|
|
||||||
message = `Cancelled after ${this.processed} items`
|
|
||||||
break
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
status: this.status,
|
|
||||||
progress,
|
|
||||||
message,
|
|
||||||
metadata: {
|
|
||||||
itemsProcessed: this.processed,
|
|
||||||
itemsTotal: this.total,
|
|
||||||
currentItem: this.currentItem,
|
|
||||||
estimatedTimeRemaining: this.estimateTimeRemaining(),
|
|
||||||
startedAt: this.startedAt || Date.now(),
|
|
||||||
completedAt: this.completedAt,
|
|
||||||
throughput: this.calculateThroughput()
|
|
||||||
},
|
|
||||||
result: this.result,
|
|
||||||
error: this.error
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Estimate time remaining based on processing history
|
|
||||||
*/
|
|
||||||
private estimateTimeRemaining(): number | undefined {
|
|
||||||
if (this.status !== 'running' || !this.startedAt || this.processed === 0) {
|
|
||||||
return undefined
|
|
||||||
}
|
|
||||||
|
|
||||||
const remaining = this.total - this.processed
|
|
||||||
if (remaining === 0) {
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
// Use rolling average if we have enough samples
|
|
||||||
if (this.processingTimes.length > 0) {
|
|
||||||
const avgTimePerItem = this.processingTimes.reduce((a, b) => a + b, 0) / this.processingTimes.length
|
|
||||||
return Math.round(avgTimePerItem * remaining)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Fallback to simple calculation
|
|
||||||
const elapsed = Date.now() - this.startedAt
|
|
||||||
const avgTimePerItem = elapsed / this.processed
|
|
||||||
return Math.round(avgTimePerItem * remaining)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Calculate current throughput (items/second)
|
|
||||||
*/
|
|
||||||
private calculateThroughput(): number | undefined {
|
|
||||||
if (!this.startedAt || this.processed === 0) {
|
|
||||||
return undefined
|
|
||||||
}
|
|
||||||
|
|
||||||
const elapsed = Date.now() - this.startedAt
|
|
||||||
const seconds = elapsed / 1000
|
|
||||||
return seconds > 0 ? Math.round((this.processed / seconds) * 100) / 100 : undefined
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get progress statistics
|
|
||||||
*/
|
|
||||||
getStats() {
|
|
||||||
const elapsed = this.startedAt ? Date.now() - this.startedAt : 0
|
|
||||||
return {
|
|
||||||
status: this.status,
|
|
||||||
processed: this.processed,
|
|
||||||
total: this.total,
|
|
||||||
remaining: this.total - this.processed,
|
|
||||||
progress: this.total > 0 ? this.processed / this.total : 0,
|
|
||||||
elapsed,
|
|
||||||
estimatedTotal: elapsed > 0 && this.processed > 0
|
|
||||||
? Math.round((elapsed / this.processed) * this.total)
|
|
||||||
: undefined,
|
|
||||||
throughput: this.calculateThroughput()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Helper to format time duration
|
|
||||||
*/
|
|
||||||
export function formatDuration(ms: number): string {
|
|
||||||
const seconds = Math.floor(ms / 1000)
|
|
||||||
const minutes = Math.floor(seconds / 60)
|
|
||||||
const hours = Math.floor(minutes / 60)
|
|
||||||
|
|
||||||
if (hours > 0) {
|
|
||||||
return `${hours}h ${minutes % 60}m`
|
|
||||||
} else if (minutes > 0) {
|
|
||||||
return `${minutes}m ${seconds % 60}s`
|
|
||||||
} else {
|
|
||||||
return `${seconds}s`
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Helper to format progress percentage
|
|
||||||
*/
|
|
||||||
export function formatProgress(progress: BrainyProgress): string {
|
|
||||||
const { status, progress: pct, metadata } = progress
|
|
||||||
const remaining = metadata.estimatedTimeRemaining
|
|
||||||
|
|
||||||
let str = `[${status.toUpperCase()}] ${pct}% (${metadata.itemsProcessed}/${metadata.itemsTotal})`
|
|
||||||
|
|
||||||
if (metadata.throughput) {
|
|
||||||
str += ` - ${metadata.throughput} items/s`
|
|
||||||
}
|
|
||||||
|
|
||||||
if (remaining && remaining > 0) {
|
|
||||||
str += ` - ${formatDuration(remaining)} remaining`
|
|
||||||
}
|
|
||||||
|
|
||||||
return str
|
|
||||||
}
|
|
||||||
|
|
@ -1,174 +0,0 @@
|
||||||
/**
|
|
||||||
* Type Migration Utilities for Stage 3 Taxonomy
|
|
||||||
*
|
|
||||||
* Provides migration helpers for code using removed types from v5.x
|
|
||||||
*
|
|
||||||
* ## Removed Types
|
|
||||||
*
|
|
||||||
* ### Nouns (2 removed):
|
|
||||||
* - `user` → merged into `person`
|
|
||||||
* - `topic` → merged into `concept`
|
|
||||||
*
|
|
||||||
* ### Verbs (4 removed):
|
|
||||||
* - `succeeds` → use inverse of `precedes`
|
|
||||||
* - `belongsTo` → use inverse of `owns`
|
|
||||||
* - `createdBy` → use inverse of `creates`
|
|
||||||
* - `supervises` → use inverse of `reportsTo`
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { NounType, VerbType } from './graphTypes.js'
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Migration mapping for removed noun types
|
|
||||||
*/
|
|
||||||
export const REMOVED_NOUN_TYPES = {
|
|
||||||
user: NounType.Person,
|
|
||||||
topic: NounType.Concept
|
|
||||||
} as const
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Migration mapping for removed verb types
|
|
||||||
* Note: Some verbs should use inverse relationships in Stage 3
|
|
||||||
*/
|
|
||||||
export const REMOVED_VERB_TYPES = {
|
|
||||||
succeeds: VerbType.Precedes, // Use with inverted source/target
|
|
||||||
belongsTo: VerbType.Owns, // Use with inverted source/target
|
|
||||||
createdBy: VerbType.Creates, // Use with inverted source/target
|
|
||||||
supervises: VerbType.ReportsTo // Use with inverted source/target
|
|
||||||
} as const
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Check if a type was removed in Stage 3
|
|
||||||
*/
|
|
||||||
export function isRemovedNounType(type: string): type is keyof typeof REMOVED_NOUN_TYPES {
|
|
||||||
return type in REMOVED_NOUN_TYPES
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Check if a verb type was removed in Stage 3
|
|
||||||
*/
|
|
||||||
export function isRemovedVerbType(type: string): type is keyof typeof REMOVED_VERB_TYPES {
|
|
||||||
return type in REMOVED_VERB_TYPES
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Migrate a noun type (Stage 3)
|
|
||||||
* Returns the migrated type or the original if no migration needed
|
|
||||||
*/
|
|
||||||
export function migrateNounType(type: string): NounType {
|
|
||||||
if (isRemovedNounType(type)) {
|
|
||||||
console.warn(`⚠️ NounType "${type}" was removed in v6.0. Migrating to "${REMOVED_NOUN_TYPES[type]}"`)
|
|
||||||
return REMOVED_NOUN_TYPES[type]
|
|
||||||
}
|
|
||||||
return type as NounType
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Migrate a verb type (Stage 3)
|
|
||||||
* Returns the migrated type or the original if no migration needed
|
|
||||||
*
|
|
||||||
* WARNING: Some verbs require inverting source/target relationships!
|
|
||||||
* See VERB_REQUIRES_INVERSION for details.
|
|
||||||
*/
|
|
||||||
export function migrateVerbType(type: string): VerbType {
|
|
||||||
if (isRemovedVerbType(type)) {
|
|
||||||
console.warn(`⚠️ VerbType "${type}" was removed in v6.0. Migrating to "${REMOVED_VERB_TYPES[type]}" (may require source/target inversion)`)
|
|
||||||
return REMOVED_VERB_TYPES[type]
|
|
||||||
}
|
|
||||||
return type as VerbType
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Verbs that require inverting source/target when migrating
|
|
||||||
*
|
|
||||||
* Example:
|
|
||||||
* - Old: `A createdBy B` → New: `B creates A`
|
|
||||||
* - Old: `A belongsTo B` → New: `B owns A`
|
|
||||||
* - Old: `A supervises B` → New: `B reportsTo A`
|
|
||||||
* - Old: `A succeeds B` → New: `B precedes A`
|
|
||||||
*/
|
|
||||||
export const VERB_REQUIRES_INVERSION = new Set([
|
|
||||||
'succeeds',
|
|
||||||
'belongsTo',
|
|
||||||
'createdBy',
|
|
||||||
'supervises'
|
|
||||||
])
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Check if a verb type requires inverting source/target during migration
|
|
||||||
*/
|
|
||||||
export function requiresInversion(oldVerbType: string): boolean {
|
|
||||||
return VERB_REQUIRES_INVERSION.has(oldVerbType)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Migrate a relationship, handling source/target inversion if needed
|
|
||||||
*
|
|
||||||
* @returns Object with migrated verb and potentially inverted source/target
|
|
||||||
*/
|
|
||||||
export function migrateRelationship(params: {
|
|
||||||
verb: string
|
|
||||||
source: string
|
|
||||||
target: string
|
|
||||||
}): {
|
|
||||||
verb: VerbType
|
|
||||||
source: string
|
|
||||||
target: string
|
|
||||||
inverted: boolean
|
|
||||||
} {
|
|
||||||
const verb = migrateVerbType(params.verb)
|
|
||||||
const inverted = requiresInversion(params.verb)
|
|
||||||
|
|
||||||
if (inverted) {
|
|
||||||
return {
|
|
||||||
verb,
|
|
||||||
source: params.target, // Swap source and target
|
|
||||||
target: params.source,
|
|
||||||
inverted: true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
verb,
|
|
||||||
source: params.source,
|
|
||||||
target: params.target,
|
|
||||||
inverted: false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Stage 3 Type Compatibility Check
|
|
||||||
* Helps developers identify code that needs updating
|
|
||||||
*/
|
|
||||||
export function checkTypeCompatibility(nounTypes: string[], verbTypes: string[]): {
|
|
||||||
valid: boolean
|
|
||||||
removedNouns: string[]
|
|
||||||
removedVerbs: string[]
|
|
||||||
warnings: string[]
|
|
||||||
} {
|
|
||||||
const removedNouns = nounTypes.filter(isRemovedNounType)
|
|
||||||
const removedVerbs = verbTypes.filter(isRemovedVerbType)
|
|
||||||
const warnings: string[] = []
|
|
||||||
|
|
||||||
if (removedNouns.length > 0) {
|
|
||||||
warnings.push(
|
|
||||||
`Found ${removedNouns.length} removed noun type(s): ${removedNouns.join(', ')}. ` +
|
|
||||||
`These were merged in Stage 3. Use Person instead of User, Concept instead of Topic.`
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
if (removedVerbs.length > 0) {
|
|
||||||
warnings.push(
|
|
||||||
`Found ${removedVerbs.length} removed verb type(s): ${removedVerbs.join(', ')}. ` +
|
|
||||||
`These require using inverse relationships in Stage 3. ` +
|
|
||||||
`Example: "A createdBy B" becomes "B creates A".`
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
valid: removedNouns.length === 0 && removedVerbs.length === 0,
|
|
||||||
removedNouns,
|
|
||||||
removedVerbs,
|
|
||||||
warnings
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,39 +0,0 @@
|
||||||
/**
|
|
||||||
* @module unified
|
|
||||||
* @description Unified entry point for Brainy. Re-exports the public surface
|
|
||||||
* from `index.js` and surfaces a tiny runtime-environment object the rest of
|
|
||||||
* the library can read.
|
|
||||||
*
|
|
||||||
* 8.0 supports Node-like runtimes only (Node.js, Bun, Deno); the browser
|
|
||||||
* detection helpers and Worker Thread plumbing are gone.
|
|
||||||
*/
|
|
||||||
|
|
||||||
import './setup.js'
|
|
||||||
|
|
||||||
import { isNode } from './utils/environment.js'
|
|
||||||
|
|
||||||
export const environment = {
|
|
||||||
get isNode() {
|
|
||||||
return isNode()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
declare global {
|
|
||||||
/**
|
|
||||||
* Runtime-environment marker published by the unified entry point so
|
|
||||||
* embedders and diagnostics can detect the active Brainy environment.
|
|
||||||
* Ambient `var` is required — `let`/`const` in `declare global` do not
|
|
||||||
* attach to `globalThis`.
|
|
||||||
*/
|
|
||||||
var __ENV__: typeof environment | undefined
|
|
||||||
}
|
|
||||||
|
|
||||||
if (typeof globalThis !== 'undefined') {
|
|
||||||
globalThis.__ENV__ = environment
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log(
|
|
||||||
`Brainy running on ${environment.isNode ? 'Node-like runtime' : 'unknown runtime'}`
|
|
||||||
)
|
|
||||||
|
|
||||||
export * from './index.js'
|
|
||||||
|
|
@ -1,159 +0,0 @@
|
||||||
/**
|
|
||||||
* Universal Crypto implementation
|
|
||||||
* Framework-friendly: Trusts that frameworks provide crypto polyfills
|
|
||||||
* Works in all environments: Browser (via framework), Node.js, Serverless
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { isNode } from '../utils/environment.js'
|
|
||||||
|
|
||||||
let nodeCrypto: any = null
|
|
||||||
|
|
||||||
// Dynamic import for Node.js crypto (only in Node.js environment)
|
|
||||||
if (isNode()) {
|
|
||||||
try {
|
|
||||||
// Use node: protocol to prevent bundler polyfilling (requires Node 22+)
|
|
||||||
nodeCrypto = await import('node:crypto')
|
|
||||||
} catch {
|
|
||||||
// Ignore import errors in non-Node environments
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Generate random bytes
|
|
||||||
* Framework-friendly: Assumes crypto API is available via framework polyfills
|
|
||||||
*/
|
|
||||||
export function randomBytes(size: number): Uint8Array {
|
|
||||||
if (typeof crypto !== 'undefined') {
|
|
||||||
// Use Web Crypto API (available in browsers via framework polyfills and modern Node.js)
|
|
||||||
const array = new Uint8Array(size)
|
|
||||||
crypto.getRandomValues(array)
|
|
||||||
return array
|
|
||||||
} else if (nodeCrypto) {
|
|
||||||
// Use Node.js crypto
|
|
||||||
return new Uint8Array(nodeCrypto.randomBytes(size))
|
|
||||||
} else {
|
|
||||||
throw new Error('Crypto API not available. Framework bundlers should provide crypto polyfills.')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Generate random UUID
|
|
||||||
* Framework-friendly: Assumes crypto.randomUUID is available via framework polyfills
|
|
||||||
*/
|
|
||||||
export function randomUUID(): string {
|
|
||||||
if (typeof crypto !== 'undefined' && crypto.randomUUID) {
|
|
||||||
return crypto.randomUUID()
|
|
||||||
} else if (nodeCrypto && nodeCrypto.randomUUID) {
|
|
||||||
return nodeCrypto.randomUUID()
|
|
||||||
} else {
|
|
||||||
throw new Error('crypto.randomUUID not available. Framework bundlers should provide crypto polyfills.')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create hash (simplified interface)
|
|
||||||
* Framework-friendly: Relies on Node.js crypto or framework-provided implementations
|
|
||||||
*/
|
|
||||||
export function createHash(algorithm: string): {
|
|
||||||
update: (data: string | Uint8Array) => any
|
|
||||||
digest: (encoding: string) => string
|
|
||||||
} {
|
|
||||||
if (nodeCrypto && nodeCrypto.createHash) {
|
|
||||||
return nodeCrypto.createHash(algorithm)
|
|
||||||
} else {
|
|
||||||
throw new Error(`createHash not available. For browser environments, frameworks should provide crypto polyfills or use Web Crypto API directly.`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create HMAC
|
|
||||||
* Framework-friendly: Relies on Node.js crypto or framework-provided implementations
|
|
||||||
*/
|
|
||||||
export function createHmac(algorithm: string, key: string | Uint8Array): {
|
|
||||||
update: (data: string | Uint8Array) => any
|
|
||||||
digest: (encoding: string) => string
|
|
||||||
} {
|
|
||||||
if (nodeCrypto && nodeCrypto.createHmac) {
|
|
||||||
return nodeCrypto.createHmac(algorithm, key)
|
|
||||||
} else {
|
|
||||||
throw new Error(`createHmac not available. For browser environments, frameworks should provide crypto polyfills or use Web Crypto API directly.`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* PBKDF2 synchronous
|
|
||||||
* Framework-friendly: Relies on Node.js crypto or framework-provided implementations
|
|
||||||
*/
|
|
||||||
export function pbkdf2Sync(password: string | Uint8Array, salt: string | Uint8Array, iterations: number, keylen: number, digest: string): Uint8Array {
|
|
||||||
if (nodeCrypto && nodeCrypto.pbkdf2Sync) {
|
|
||||||
return new Uint8Array(nodeCrypto.pbkdf2Sync(password, salt, iterations, keylen, digest))
|
|
||||||
} else {
|
|
||||||
throw new Error(`pbkdf2Sync not available. For browser environments, frameworks should provide crypto polyfills or use Web Crypto API directly.`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Scrypt synchronous
|
|
||||||
* Framework-friendly: Relies on Node.js crypto or framework-provided implementations
|
|
||||||
*/
|
|
||||||
export function scryptSync(password: string | Uint8Array, salt: string | Uint8Array, keylen: number, options?: any): Uint8Array {
|
|
||||||
if (nodeCrypto && nodeCrypto.scryptSync) {
|
|
||||||
return new Uint8Array(nodeCrypto.scryptSync(password, salt, keylen, options))
|
|
||||||
} else {
|
|
||||||
throw new Error(`scryptSync not available. For browser environments, frameworks should provide crypto polyfills or use Web Crypto API directly.`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create cipher
|
|
||||||
* Framework-friendly: Relies on Node.js crypto or framework-provided implementations
|
|
||||||
*/
|
|
||||||
export function createCipheriv(algorithm: string, key: Uint8Array, iv: Uint8Array): {
|
|
||||||
update: (data: string, inputEncoding?: string, outputEncoding?: string) => string
|
|
||||||
final: (outputEncoding?: string) => string
|
|
||||||
} {
|
|
||||||
if (nodeCrypto && nodeCrypto.createCipheriv) {
|
|
||||||
return nodeCrypto.createCipheriv(algorithm, key, iv)
|
|
||||||
} else {
|
|
||||||
throw new Error(`createCipheriv not available. For browser environments, frameworks should provide crypto polyfills or use Web Crypto API directly.`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create decipher
|
|
||||||
* Framework-friendly: Relies on Node.js crypto or framework-provided implementations
|
|
||||||
*/
|
|
||||||
export function createDecipheriv(algorithm: string, key: Uint8Array, iv: Uint8Array): {
|
|
||||||
update: (data: string, inputEncoding?: string, outputEncoding?: string) => string
|
|
||||||
final: (outputEncoding?: string) => string
|
|
||||||
} {
|
|
||||||
if (nodeCrypto && nodeCrypto.createDecipheriv) {
|
|
||||||
return nodeCrypto.createDecipheriv(algorithm, key, iv)
|
|
||||||
} else {
|
|
||||||
throw new Error(`createDecipheriv not available. For browser environments, frameworks should provide crypto polyfills or use Web Crypto API directly.`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Timing safe equal
|
|
||||||
* Framework-friendly: Relies on Node.js crypto or framework-provided implementations
|
|
||||||
*/
|
|
||||||
export function timingSafeEqual(a: Uint8Array, b: Uint8Array): boolean {
|
|
||||||
if (nodeCrypto && nodeCrypto.timingSafeEqual) {
|
|
||||||
return nodeCrypto.timingSafeEqual(a, b)
|
|
||||||
} else {
|
|
||||||
throw new Error(`timingSafeEqual not available. For browser environments, frameworks should provide crypto polyfills or use Web Crypto API directly.`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export default {
|
|
||||||
randomBytes,
|
|
||||||
randomUUID,
|
|
||||||
createHash,
|
|
||||||
createHmac,
|
|
||||||
pbkdf2Sync,
|
|
||||||
scryptSync,
|
|
||||||
createCipheriv,
|
|
||||||
createDecipheriv,
|
|
||||||
timingSafeEqual
|
|
||||||
}
|
|
||||||
|
|
@ -1,62 +0,0 @@
|
||||||
/**
|
|
||||||
* Bounded Registry with LRU eviction
|
|
||||||
* Prevents unbounded memory growth in production
|
|
||||||
*/
|
|
||||||
export class BoundedRegistry<T> {
|
|
||||||
private items = new Map<T, number>() // item -> last accessed timestamp
|
|
||||||
private readonly maxSize: number
|
|
||||||
|
|
||||||
constructor(maxSize: number = 10000) {
|
|
||||||
this.maxSize = maxSize
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Add item to registry, evicting oldest if at capacity
|
|
||||||
*/
|
|
||||||
add(item: T): void {
|
|
||||||
// Update timestamp if already exists
|
|
||||||
if (this.items.has(item)) {
|
|
||||||
this.items.delete(item) // Remove to re-add at end
|
|
||||||
this.items.set(item, Date.now())
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Evict oldest if at capacity
|
|
||||||
if (this.items.size >= this.maxSize) {
|
|
||||||
const oldest = this.items.entries().next().value
|
|
||||||
if (oldest) {
|
|
||||||
this.items.delete(oldest[0])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
this.items.set(item, Date.now())
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Check if item exists
|
|
||||||
*/
|
|
||||||
has(item: T): boolean {
|
|
||||||
return this.items.has(item)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get all items
|
|
||||||
*/
|
|
||||||
getAll(): T[] {
|
|
||||||
return Array.from(this.items.keys())
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get size
|
|
||||||
*/
|
|
||||||
get size(): number {
|
|
||||||
return this.items.size
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Clear all items
|
|
||||||
*/
|
|
||||||
clear(): void {
|
|
||||||
this.items.clear()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,47 +0,0 @@
|
||||||
/**
|
|
||||||
* Cross-platform crypto utilities
|
|
||||||
* Provides hashing functions that work in both Node.js and browser environments
|
|
||||||
*/
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Simple string hash function that works in all environments
|
|
||||||
* Uses djb2 algorithm - fast and good distribution
|
|
||||||
* @param str - String to hash
|
|
||||||
* @returns Positive integer hash
|
|
||||||
*/
|
|
||||||
export function hashString(str: string): number {
|
|
||||||
let hash = 5381
|
|
||||||
for (let i = 0; i < str.length; i++) {
|
|
||||||
const char = str.charCodeAt(i)
|
|
||||||
hash = ((hash << 5) + hash) + char // hash * 33 + char
|
|
||||||
}
|
|
||||||
// Ensure positive number
|
|
||||||
return Math.abs(hash)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Alternative: FNV-1a hash algorithm
|
|
||||||
* Good distribution and fast
|
|
||||||
* @param str - String to hash
|
|
||||||
* @returns Positive integer hash
|
|
||||||
*/
|
|
||||||
export function fnv1aHash(str: string): number {
|
|
||||||
let hash = 2166136261
|
|
||||||
for (let i = 0; i < str.length; i++) {
|
|
||||||
hash ^= str.charCodeAt(i)
|
|
||||||
hash = (hash * 16777619) >>> 0
|
|
||||||
}
|
|
||||||
return hash
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Generate a deterministic hash for partitioning
|
|
||||||
* Uses the most appropriate algorithm for the environment
|
|
||||||
* @param input - Input string to hash
|
|
||||||
* @returns Positive integer hash suitable for modulo operations
|
|
||||||
*/
|
|
||||||
export function getPartitionHash(input: string): number {
|
|
||||||
// Use djb2 by default as it's fast and has good distribution
|
|
||||||
// This ensures consistent partitioning across all environments
|
|
||||||
return hashString(input)
|
|
||||||
}
|
|
||||||
|
|
@ -1,106 +0,0 @@
|
||||||
/**
|
|
||||||
* Dedicated index for tracking soft-deleted items
|
|
||||||
* This is MUCH more efficient than checking every item in the database
|
|
||||||
*
|
|
||||||
* Performance characteristics:
|
|
||||||
* - Add deleted item: O(1)
|
|
||||||
* - Remove deleted item: O(1)
|
|
||||||
* - Check if deleted: O(1)
|
|
||||||
* - Get all deleted: O(d) where d = number of deleted items << total items
|
|
||||||
*/
|
|
||||||
|
|
||||||
export class DeletedItemsIndex {
|
|
||||||
private deletedIds: Set<string> = new Set()
|
|
||||||
private deletedCount: number = 0
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Mark an item as deleted
|
|
||||||
*/
|
|
||||||
markDeleted(id: string): void {
|
|
||||||
if (!this.deletedIds.has(id)) {
|
|
||||||
this.deletedIds.add(id)
|
|
||||||
this.deletedCount++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Mark an item as not deleted (restored)
|
|
||||||
*/
|
|
||||||
markRestored(id: string): void {
|
|
||||||
if (this.deletedIds.delete(id)) {
|
|
||||||
this.deletedCount--
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Check if an item is deleted - O(1)
|
|
||||||
*/
|
|
||||||
isDeleted(id: string): boolean {
|
|
||||||
return this.deletedIds.has(id)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get all deleted item IDs - O(d)
|
|
||||||
*/
|
|
||||||
getAllDeleted(): string[] {
|
|
||||||
return Array.from(this.deletedIds)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Filter out deleted items from results - O(k) where k = result count
|
|
||||||
*/
|
|
||||||
filterDeleted<T extends { id?: string }>(items: T[]): T[] {
|
|
||||||
if (this.deletedCount === 0) {
|
|
||||||
// Fast path - no deleted items
|
|
||||||
return items
|
|
||||||
}
|
|
||||||
|
|
||||||
return items.filter(item => {
|
|
||||||
const id = item.id
|
|
||||||
return id ? !this.deletedIds.has(id) : true
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get statistics
|
|
||||||
*/
|
|
||||||
getStats() {
|
|
||||||
return {
|
|
||||||
deletedCount: this.deletedCount,
|
|
||||||
memoryUsage: this.deletedCount * 100 // Rough estimate: 100 bytes per ID
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Clear all deleted items (for testing)
|
|
||||||
*/
|
|
||||||
clear(): void {
|
|
||||||
this.deletedIds.clear()
|
|
||||||
this.deletedCount = 0
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Serialize for persistence
|
|
||||||
*/
|
|
||||||
serialize(): string {
|
|
||||||
return JSON.stringify(Array.from(this.deletedIds))
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Deserialize from persistence
|
|
||||||
*/
|
|
||||||
deserialize(data: string): void {
|
|
||||||
try {
|
|
||||||
const ids = JSON.parse(data)
|
|
||||||
this.deletedIds = new Set(ids)
|
|
||||||
this.deletedCount = this.deletedIds.size
|
|
||||||
} catch (e) {
|
|
||||||
console.warn('Failed to deserialize deleted items index')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Global singleton for deleted items tracking
|
|
||||||
*/
|
|
||||||
export const deletedItemsIndex = new DeletedItemsIndex()
|
|
||||||
|
|
@ -1,88 +0,0 @@
|
||||||
/**
|
|
||||||
* Utility to ensure all metadata has the deleted field set properly
|
|
||||||
* This is CRITICAL for O(1) soft delete filtering performance
|
|
||||||
*
|
|
||||||
* Uses _brainy namespace to avoid conflicts with user metadata
|
|
||||||
*/
|
|
||||||
|
|
||||||
const BRAINY_NAMESPACE = '_brainy'
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Ensure metadata has internal Brainy fields set
|
|
||||||
* @param metadata The metadata object (could be null/undefined)
|
|
||||||
* @param preserveExisting If true, preserve existing deleted value
|
|
||||||
* @returns Metadata with internal fields guaranteed
|
|
||||||
*/
|
|
||||||
export function ensureDeletedField(metadata: any, preserveExisting: boolean = true): any {
|
|
||||||
// Handle null/undefined metadata
|
|
||||||
if (!metadata) {
|
|
||||||
return {
|
|
||||||
[BRAINY_NAMESPACE]: {
|
|
||||||
deleted: false,
|
|
||||||
version: 1
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Clone to avoid mutation
|
|
||||||
const result = { ...metadata }
|
|
||||||
|
|
||||||
// Ensure _brainy namespace exists
|
|
||||||
if (!result[BRAINY_NAMESPACE]) {
|
|
||||||
result[BRAINY_NAMESPACE] = {}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Set deleted field if not present
|
|
||||||
if (!('deleted' in result[BRAINY_NAMESPACE])) {
|
|
||||||
result[BRAINY_NAMESPACE].deleted = false
|
|
||||||
} else if (!preserveExisting) {
|
|
||||||
// Force to false if not preserving
|
|
||||||
result[BRAINY_NAMESPACE].deleted = false
|
|
||||||
}
|
|
||||||
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Mark an item as soft deleted
|
|
||||||
* @param metadata The metadata object
|
|
||||||
* @returns Metadata with _brainy.deleted=true
|
|
||||||
*/
|
|
||||||
export function markAsDeleted(metadata: any): any {
|
|
||||||
const result = ensureDeletedField(metadata)
|
|
||||||
result[BRAINY_NAMESPACE].deleted = true
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Mark an item as restored (not deleted)
|
|
||||||
* @param metadata The metadata object
|
|
||||||
* @returns Metadata with _brainy.deleted=false
|
|
||||||
*/
|
|
||||||
export function markAsRestored(metadata: any): any {
|
|
||||||
const result = ensureDeletedField(metadata)
|
|
||||||
result[BRAINY_NAMESPACE].deleted = false
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Check if an item is deleted
|
|
||||||
* @param metadata The metadata object
|
|
||||||
* @returns true if deleted, false otherwise (including if field missing)
|
|
||||||
*/
|
|
||||||
export function isDeleted(metadata: any): boolean {
|
|
||||||
return metadata?.[BRAINY_NAMESPACE]?.deleted === true
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Check if an item is active (not deleted)
|
|
||||||
* @param metadata The metadata object
|
|
||||||
* @returns true if not deleted (default), false if deleted
|
|
||||||
*/
|
|
||||||
export function isActive(metadata: any): boolean {
|
|
||||||
// If no deleted field or deleted=false, item is active
|
|
||||||
return !isDeleted(metadata)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Export the namespace constant for use in queries
|
|
||||||
export const BRAINY_DELETED_FIELD = `${BRAINY_NAMESPACE}.deleted`
|
|
||||||
|
|
@ -1,255 +0,0 @@
|
||||||
/**
|
|
||||||
* Clean Metadata Architecture for Brainy 2.2
|
|
||||||
* No backward compatibility - doing it RIGHT from the start!
|
|
||||||
*/
|
|
||||||
|
|
||||||
// Namespace constants
|
|
||||||
export const BRAINY_NS = '_brainy' as const
|
|
||||||
export const AUG_NS = '_augmentations' as const
|
|
||||||
export const AUDIT_NS = '_audit' as const
|
|
||||||
|
|
||||||
// Field paths for O(1) indexing
|
|
||||||
export const DELETED_FIELD = `${BRAINY_NS}.deleted` as const
|
|
||||||
export const INDEXED_FIELD = `${BRAINY_NS}.indexed` as const
|
|
||||||
export const VERSION_FIELD = `${BRAINY_NS}.version` as const
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Internal Brainy metadata structure
|
|
||||||
* These fields are ALWAYS present and indexed for O(1) access
|
|
||||||
*/
|
|
||||||
export interface BrainyInternalMetadata {
|
|
||||||
deleted: boolean // ALWAYS boolean, enables O(1) soft delete
|
|
||||||
indexed: boolean // Whether in search index
|
|
||||||
version: number // Schema version
|
|
||||||
created: number // Unix timestamp
|
|
||||||
updated: number // Unix timestamp
|
|
||||||
|
|
||||||
// Optional internal fields
|
|
||||||
domain?: string // Domain classification
|
|
||||||
priority?: number // Query priority hint
|
|
||||||
ttl?: number // Time to live
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Complete metadata structure with namespaces
|
|
||||||
*/
|
|
||||||
export interface NamespacedMetadata<T = any> {
|
|
||||||
// User metadata - any fields they want
|
|
||||||
[key: string]: any
|
|
||||||
|
|
||||||
// Internal metadata - our fields
|
|
||||||
[BRAINY_NS]: BrainyInternalMetadata
|
|
||||||
|
|
||||||
// Augmentation metadata - isolated per augmentation
|
|
||||||
[AUG_NS]?: {
|
|
||||||
[augmentationName: string]: any
|
|
||||||
}
|
|
||||||
|
|
||||||
// Audit trail - optional
|
|
||||||
[AUDIT_NS]?: Array<{
|
|
||||||
timestamp: number
|
|
||||||
augmentation: string
|
|
||||||
field: string
|
|
||||||
oldValue: any
|
|
||||||
newValue: any
|
|
||||||
}>
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create properly namespaced metadata
|
|
||||||
* This is called for EVERY noun/verb creation
|
|
||||||
*/
|
|
||||||
export function createNamespacedMetadata<T = any>(
|
|
||||||
userMetadata?: T
|
|
||||||
): NamespacedMetadata<T> {
|
|
||||||
const now = Date.now()
|
|
||||||
|
|
||||||
// Start with user metadata or empty object
|
|
||||||
const result: any = userMetadata ? { ...userMetadata } : {}
|
|
||||||
|
|
||||||
// ALWAYS add internal namespace with required fields
|
|
||||||
result[BRAINY_NS] = {
|
|
||||||
deleted: false, // CRITICAL: Always false for new items
|
|
||||||
indexed: true, // New items are indexed
|
|
||||||
version: 1, // Current schema version
|
|
||||||
created: now,
|
|
||||||
updated: now
|
|
||||||
}
|
|
||||||
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Update metadata while preserving namespaces
|
|
||||||
*/
|
|
||||||
export function updateNamespacedMetadata<T = any>(
|
|
||||||
existing: NamespacedMetadata<T>,
|
|
||||||
updates: Partial<T>
|
|
||||||
): NamespacedMetadata<T> {
|
|
||||||
const now = Date.now()
|
|
||||||
|
|
||||||
// Merge user fields
|
|
||||||
const result: any = {
|
|
||||||
...existing,
|
|
||||||
...updates
|
|
||||||
}
|
|
||||||
|
|
||||||
// Preserve internal namespace but update timestamp
|
|
||||||
result[BRAINY_NS] = {
|
|
||||||
...existing[BRAINY_NS],
|
|
||||||
updated: now
|
|
||||||
}
|
|
||||||
|
|
||||||
// Preserve augmentation namespace
|
|
||||||
if (existing[AUG_NS]) {
|
|
||||||
result[AUG_NS] = existing[AUG_NS]
|
|
||||||
}
|
|
||||||
|
|
||||||
// Preserve audit trail
|
|
||||||
if (existing[AUDIT_NS]) {
|
|
||||||
result[AUDIT_NS] = existing[AUDIT_NS]
|
|
||||||
}
|
|
||||||
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Soft delete a noun (O(1) operation)
|
|
||||||
*/
|
|
||||||
export function markDeleted<T = any>(
|
|
||||||
metadata: NamespacedMetadata<T>
|
|
||||||
): NamespacedMetadata<T> {
|
|
||||||
return {
|
|
||||||
...metadata,
|
|
||||||
[BRAINY_NS]: {
|
|
||||||
...metadata[BRAINY_NS],
|
|
||||||
deleted: true,
|
|
||||||
updated: Date.now()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Restore a soft-deleted noun (O(1) operation)
|
|
||||||
*/
|
|
||||||
export function markRestored<T = any>(
|
|
||||||
metadata: NamespacedMetadata<T>
|
|
||||||
): NamespacedMetadata<T> {
|
|
||||||
return {
|
|
||||||
...metadata,
|
|
||||||
[BRAINY_NS]: {
|
|
||||||
...metadata[BRAINY_NS],
|
|
||||||
deleted: false,
|
|
||||||
updated: Date.now()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Check if a noun is deleted (O(1) check)
|
|
||||||
*/
|
|
||||||
export function isDeleted<T = any>(
|
|
||||||
metadata: NamespacedMetadata<T>
|
|
||||||
): boolean {
|
|
||||||
return metadata[BRAINY_NS]?.deleted === true
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get user metadata without internal fields
|
|
||||||
* Used by augmentations to get clean user data
|
|
||||||
*/
|
|
||||||
export function getUserMetadata<T = any>(
|
|
||||||
metadata: NamespacedMetadata<T>
|
|
||||||
): T {
|
|
||||||
const { [BRAINY_NS]: _, [AUG_NS]: __, [AUDIT_NS]: ___, ...userMeta } = metadata
|
|
||||||
return userMeta as T
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Set augmentation data in isolated namespace
|
|
||||||
*/
|
|
||||||
export function setAugmentationData<T = any>(
|
|
||||||
metadata: NamespacedMetadata<T>,
|
|
||||||
augmentationName: string,
|
|
||||||
data: any
|
|
||||||
): NamespacedMetadata<T> {
|
|
||||||
const result = { ...metadata }
|
|
||||||
|
|
||||||
if (!result[AUG_NS]) {
|
|
||||||
result[AUG_NS] = {}
|
|
||||||
}
|
|
||||||
|
|
||||||
result[AUG_NS][augmentationName] = data
|
|
||||||
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Add audit entry for tracking
|
|
||||||
*/
|
|
||||||
export function addAuditEntry<T = any>(
|
|
||||||
metadata: NamespacedMetadata<T>,
|
|
||||||
entry: {
|
|
||||||
augmentation: string
|
|
||||||
field: string
|
|
||||||
oldValue: any
|
|
||||||
newValue: any
|
|
||||||
}
|
|
||||||
): NamespacedMetadata<T> {
|
|
||||||
const result = { ...metadata }
|
|
||||||
|
|
||||||
if (!result[AUDIT_NS]) {
|
|
||||||
result[AUDIT_NS] = []
|
|
||||||
}
|
|
||||||
|
|
||||||
result[AUDIT_NS].push({
|
|
||||||
...entry,
|
|
||||||
timestamp: Date.now()
|
|
||||||
})
|
|
||||||
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* INDEXING EXPLANATION:
|
|
||||||
*
|
|
||||||
* The MetadataIndex flattens nested objects into dot-notation keys:
|
|
||||||
*
|
|
||||||
* Input metadata:
|
|
||||||
* {
|
|
||||||
* name: "Django",
|
|
||||||
* _brainy: {
|
|
||||||
* deleted: false,
|
|
||||||
* indexed: true
|
|
||||||
* }
|
|
||||||
* }
|
|
||||||
*
|
|
||||||
* Creates index entries:
|
|
||||||
* - "name" -> "django" -> Set([id1, id2...])
|
|
||||||
* - "_brainy.deleted" -> "false" -> Set([id1, id2...]) // O(1) lookup!
|
|
||||||
* - "_brainy.indexed" -> "true" -> Set([id1, id2...])
|
|
||||||
*
|
|
||||||
* Query: { "_brainy.deleted": false }
|
|
||||||
* Lookup: index["_brainy.deleted"]["false"] -> Set of IDs in O(1)
|
|
||||||
*
|
|
||||||
* This is why namespacing doesn't hurt performance - it's all flattened!
|
|
||||||
*/
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Fields that should ALWAYS be indexed for O(1) access
|
|
||||||
*/
|
|
||||||
export const ALWAYS_INDEXED_FIELDS = [
|
|
||||||
DELETED_FIELD, // For soft delete filtering
|
|
||||||
INDEXED_FIELD, // For index management
|
|
||||||
VERSION_FIELD // For schema versioning
|
|
||||||
]
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Fields that should use sorted index for O(log n) range queries
|
|
||||||
*/
|
|
||||||
export const SORTED_INDEX_FIELDS = [
|
|
||||||
`${BRAINY_NS}.created`,
|
|
||||||
`${BRAINY_NS}.updated`,
|
|
||||||
`${BRAINY_NS}.priority`,
|
|
||||||
`${BRAINY_NS}.ttl`
|
|
||||||
]
|
|
||||||
|
|
@ -1,545 +0,0 @@
|
||||||
/**
|
|
||||||
* Enhanced Structured Logging System for Brainy
|
|
||||||
* Provides production-ready logging with structured output, context preservation,
|
|
||||||
* performance tracking, and multiple transport support
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { performance } from 'perf_hooks'
|
|
||||||
import { randomUUID } from '../universal/crypto.js'
|
|
||||||
|
|
||||||
export enum LogLevel {
|
|
||||||
SILENT = -1,
|
|
||||||
FATAL = 0,
|
|
||||||
ERROR = 1,
|
|
||||||
WARN = 2,
|
|
||||||
INFO = 3,
|
|
||||||
DEBUG = 4,
|
|
||||||
TRACE = 5
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface LogContext {
|
|
||||||
requestId?: string
|
|
||||||
userId?: string
|
|
||||||
operation?: string
|
|
||||||
entityId?: string
|
|
||||||
entityType?: string
|
|
||||||
[key: string]: any
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface LogEntry {
|
|
||||||
timestamp: string
|
|
||||||
level: string
|
|
||||||
levelNumeric: number
|
|
||||||
module: string
|
|
||||||
message: string
|
|
||||||
context?: LogContext
|
|
||||||
data?: any
|
|
||||||
error?: {
|
|
||||||
name: string
|
|
||||||
message: string
|
|
||||||
stack?: string
|
|
||||||
code?: string
|
|
||||||
}
|
|
||||||
performance?: {
|
|
||||||
duration?: number
|
|
||||||
memory?: {
|
|
||||||
used: number
|
|
||||||
total: number
|
|
||||||
}
|
|
||||||
}
|
|
||||||
host?: string
|
|
||||||
pid: number
|
|
||||||
version?: string
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface LogTransport {
|
|
||||||
name: string
|
|
||||||
log(entry: LogEntry): void | Promise<void>
|
|
||||||
flush?(): Promise<void>
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface StructuredLoggerConfig {
|
|
||||||
level: LogLevel
|
|
||||||
modules?: Record<string, LogLevel>
|
|
||||||
format: 'json' | 'pretty' | 'simple'
|
|
||||||
transports: LogTransport[]
|
|
||||||
context?: LogContext
|
|
||||||
includeHost?: boolean
|
|
||||||
includeMemory?: boolean
|
|
||||||
bufferSize?: number
|
|
||||||
flushInterval?: number
|
|
||||||
version?: string
|
|
||||||
}
|
|
||||||
|
|
||||||
class ConsoleTransport implements LogTransport {
|
|
||||||
name = 'console'
|
|
||||||
private format: 'json' | 'pretty' | 'simple'
|
|
||||||
|
|
||||||
constructor(format: 'json' | 'pretty' | 'simple' = 'json') {
|
|
||||||
this.format = format
|
|
||||||
}
|
|
||||||
|
|
||||||
log(entry: LogEntry): void {
|
|
||||||
const method = this.getConsoleMethod(entry.levelNumeric)
|
|
||||||
|
|
||||||
if (this.format === 'json') {
|
|
||||||
method(JSON.stringify(entry))
|
|
||||||
} else if (this.format === 'pretty') {
|
|
||||||
const color = this.getColor(entry.levelNumeric)
|
|
||||||
const prefix = `${entry.timestamp} ${color}[${entry.level}]\\x1b[0m [${entry.module}]`
|
|
||||||
const message = entry.message
|
|
||||||
|
|
||||||
if (entry.error) {
|
|
||||||
method(`${prefix} ${message}`, entry.error)
|
|
||||||
} else if (entry.data) {
|
|
||||||
method(`${prefix} ${message}`, entry.data)
|
|
||||||
} else {
|
|
||||||
method(`${prefix} ${message}`)
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
// Simple format
|
|
||||||
method(`[${entry.level}] ${entry.message}`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private getConsoleMethod(level: number): (...args: any[]) => void {
|
|
||||||
switch (level) {
|
|
||||||
case LogLevel.FATAL:
|
|
||||||
case LogLevel.ERROR:
|
|
||||||
return console.error
|
|
||||||
case LogLevel.WARN:
|
|
||||||
return console.warn
|
|
||||||
case LogLevel.INFO:
|
|
||||||
return console.info
|
|
||||||
default:
|
|
||||||
return console.log
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private getColor(level: number): string {
|
|
||||||
switch (level) {
|
|
||||||
case LogLevel.FATAL:
|
|
||||||
return '\\x1b[35m' // Magenta
|
|
||||||
case LogLevel.ERROR:
|
|
||||||
return '\\x1b[31m' // Red
|
|
||||||
case LogLevel.WARN:
|
|
||||||
return '\\x1b[33m' // Yellow
|
|
||||||
case LogLevel.INFO:
|
|
||||||
return '\\x1b[36m' // Cyan
|
|
||||||
case LogLevel.DEBUG:
|
|
||||||
return '\\x1b[32m' // Green
|
|
||||||
case LogLevel.TRACE:
|
|
||||||
return '\\x1b[90m' // Gray
|
|
||||||
default:
|
|
||||||
return '\\x1b[0m' // Reset
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
class BufferedTransport implements LogTransport {
|
|
||||||
name = 'buffered'
|
|
||||||
private buffer: LogEntry[] = []
|
|
||||||
private innerTransport: LogTransport
|
|
||||||
private bufferSize: number
|
|
||||||
private flushTimer?: NodeJS.Timeout
|
|
||||||
|
|
||||||
constructor(innerTransport: LogTransport, bufferSize: number = 100, flushInterval: number = 5000) {
|
|
||||||
this.innerTransport = innerTransport
|
|
||||||
this.bufferSize = bufferSize
|
|
||||||
|
|
||||||
if (flushInterval > 0) {
|
|
||||||
this.flushTimer = setInterval(() => this.flush(), flushInterval)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
log(entry: LogEntry): void {
|
|
||||||
this.buffer.push(entry)
|
|
||||||
|
|
||||||
if (this.buffer.length >= this.bufferSize) {
|
|
||||||
this.flush()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async flush(): Promise<void> {
|
|
||||||
const entries = this.buffer.splice(0)
|
|
||||||
for (const entry of entries) {
|
|
||||||
await this.innerTransport.log(entry)
|
|
||||||
}
|
|
||||||
|
|
||||||
if (this.innerTransport.flush) {
|
|
||||||
await this.innerTransport.flush()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
destroy(): void {
|
|
||||||
if (this.flushTimer) {
|
|
||||||
clearInterval(this.flushTimer)
|
|
||||||
}
|
|
||||||
this.flush()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export class StructuredLogger {
|
|
||||||
private static instance: StructuredLogger
|
|
||||||
private config: StructuredLoggerConfig
|
|
||||||
private defaultContext: LogContext = {}
|
|
||||||
private performanceMarks = new Map<string, number>()
|
|
||||||
|
|
||||||
private constructor() {
|
|
||||||
const isDevelopment = process.env.NODE_ENV !== 'production'
|
|
||||||
const format = isDevelopment ? 'pretty' : 'json'
|
|
||||||
|
|
||||||
this.config = {
|
|
||||||
level: isDevelopment ? LogLevel.DEBUG : LogLevel.INFO,
|
|
||||||
format,
|
|
||||||
transports: [new ConsoleTransport(format)],
|
|
||||||
includeHost: !isDevelopment,
|
|
||||||
includeMemory: false,
|
|
||||||
bufferSize: 100,
|
|
||||||
flushInterval: 5000,
|
|
||||||
version: process.env.npm_package_version
|
|
||||||
}
|
|
||||||
|
|
||||||
// Load from environment
|
|
||||||
this.loadEnvironmentConfig()
|
|
||||||
}
|
|
||||||
|
|
||||||
private loadEnvironmentConfig(): void {
|
|
||||||
const envLevel = process.env.BRAINY_LOG_LEVEL
|
|
||||||
if (envLevel) {
|
|
||||||
const level = LogLevel[envLevel.toUpperCase() as keyof typeof LogLevel]
|
|
||||||
if (level !== undefined) {
|
|
||||||
this.config.level = level
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const envFormat = process.env.BRAINY_LOG_FORMAT
|
|
||||||
if (envFormat && ['json', 'pretty', 'simple'].includes(envFormat)) {
|
|
||||||
this.config.format = envFormat as 'json' | 'pretty' | 'simple'
|
|
||||||
}
|
|
||||||
|
|
||||||
const moduleConfig = process.env.BRAINY_MODULE_LOG_LEVELS
|
|
||||||
if (moduleConfig) {
|
|
||||||
try {
|
|
||||||
this.config.modules = JSON.parse(moduleConfig)
|
|
||||||
} catch {
|
|
||||||
// Ignore parse errors
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
static getInstance(): StructuredLogger {
|
|
||||||
if (!StructuredLogger.instance) {
|
|
||||||
StructuredLogger.instance = new StructuredLogger()
|
|
||||||
}
|
|
||||||
return StructuredLogger.instance
|
|
||||||
}
|
|
||||||
|
|
||||||
configure(config: Partial<StructuredLoggerConfig>): void {
|
|
||||||
this.config = { ...this.config, ...config }
|
|
||||||
}
|
|
||||||
|
|
||||||
setContext(context: LogContext): void {
|
|
||||||
this.defaultContext = { ...this.defaultContext, ...context }
|
|
||||||
}
|
|
||||||
|
|
||||||
clearContext(): void {
|
|
||||||
this.defaultContext = {}
|
|
||||||
}
|
|
||||||
|
|
||||||
withContext(context: LogContext): StructuredLogger {
|
|
||||||
const contextualLogger = Object.create(this)
|
|
||||||
contextualLogger.defaultContext = { ...this.defaultContext, ...context }
|
|
||||||
return contextualLogger
|
|
||||||
}
|
|
||||||
|
|
||||||
startTimer(label: string): void {
|
|
||||||
this.performanceMarks.set(label, performance.now())
|
|
||||||
}
|
|
||||||
|
|
||||||
endTimer(label: string): number | undefined {
|
|
||||||
const start = this.performanceMarks.get(label)
|
|
||||||
if (start === undefined) return undefined
|
|
||||||
|
|
||||||
const duration = performance.now() - start
|
|
||||||
this.performanceMarks.delete(label)
|
|
||||||
return duration
|
|
||||||
}
|
|
||||||
|
|
||||||
private shouldLog(level: LogLevel, module: string): boolean {
|
|
||||||
if (this.config.modules?.[module] !== undefined) {
|
|
||||||
return level <= this.config.modules[module]
|
|
||||||
}
|
|
||||||
return level <= this.config.level
|
|
||||||
}
|
|
||||||
|
|
||||||
private createLogEntry(
|
|
||||||
level: LogLevel,
|
|
||||||
module: string,
|
|
||||||
message: string,
|
|
||||||
context?: LogContext,
|
|
||||||
data?: any,
|
|
||||||
error?: Error
|
|
||||||
): LogEntry {
|
|
||||||
const entry: LogEntry = {
|
|
||||||
timestamp: new Date().toISOString(),
|
|
||||||
level: LogLevel[level],
|
|
||||||
levelNumeric: level,
|
|
||||||
module,
|
|
||||||
message,
|
|
||||||
pid: process.pid,
|
|
||||||
version: this.config.version
|
|
||||||
}
|
|
||||||
|
|
||||||
// Merge contexts
|
|
||||||
const mergedContext = { ...this.defaultContext, ...context }
|
|
||||||
if (Object.keys(mergedContext).length > 0) {
|
|
||||||
entry.context = mergedContext
|
|
||||||
}
|
|
||||||
|
|
||||||
if (data !== undefined) {
|
|
||||||
entry.data = data
|
|
||||||
}
|
|
||||||
|
|
||||||
if (error) {
|
|
||||||
entry.error = {
|
|
||||||
name: error.name,
|
|
||||||
message: error.message,
|
|
||||||
stack: error.stack,
|
|
||||||
code: (error as Error & { code?: string }).code
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (this.config.includeHost) {
|
|
||||||
try {
|
|
||||||
const os = require('node:os')
|
|
||||||
entry.host = os.hostname()
|
|
||||||
} catch {
|
|
||||||
entry.host = 'unknown'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (this.config.includeMemory && typeof process !== 'undefined' && process.memoryUsage) {
|
|
||||||
const mem = process.memoryUsage()
|
|
||||||
entry.performance = {
|
|
||||||
memory: {
|
|
||||||
used: Math.round(mem.heapUsed / 1024 / 1024),
|
|
||||||
total: Math.round(mem.heapTotal / 1024 / 1024)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return entry
|
|
||||||
}
|
|
||||||
|
|
||||||
private log(
|
|
||||||
level: LogLevel,
|
|
||||||
module: string,
|
|
||||||
message: string,
|
|
||||||
contextOrData?: LogContext | any,
|
|
||||||
data?: any
|
|
||||||
): void {
|
|
||||||
if (!this.shouldLog(level, module)) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Handle overloaded parameters
|
|
||||||
let context: LogContext | undefined
|
|
||||||
let logData: any
|
|
||||||
|
|
||||||
if (contextOrData && typeof contextOrData === 'object' && !Array.isArray(contextOrData)) {
|
|
||||||
// Check if it looks like a context object
|
|
||||||
const hasContextKeys = ['requestId', 'userId', 'operation', 'entityId', 'entityType']
|
|
||||||
.some(key => key in contextOrData)
|
|
||||||
|
|
||||||
if (hasContextKeys) {
|
|
||||||
context = contextOrData
|
|
||||||
logData = data
|
|
||||||
} else {
|
|
||||||
logData = contextOrData
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
logData = contextOrData
|
|
||||||
}
|
|
||||||
|
|
||||||
// Extract error if present
|
|
||||||
let error: Error | undefined
|
|
||||||
if (logData instanceof Error) {
|
|
||||||
error = logData
|
|
||||||
logData = undefined
|
|
||||||
} else if (logData?.error instanceof Error) {
|
|
||||||
error = logData.error
|
|
||||||
delete logData.error
|
|
||||||
}
|
|
||||||
|
|
||||||
const entry = this.createLogEntry(level, module, message, context, logData, error)
|
|
||||||
|
|
||||||
// Send to all transports
|
|
||||||
for (const transport of this.config.transports) {
|
|
||||||
try {
|
|
||||||
transport.log(entry)
|
|
||||||
} catch (err) {
|
|
||||||
// Fallback to console.error if transport fails
|
|
||||||
console.error('Logger transport error:', err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fatal(module: string, message: string, contextOrData?: LogContext | any, data?: any): void {
|
|
||||||
this.log(LogLevel.FATAL, module, message, contextOrData, data)
|
|
||||||
}
|
|
||||||
|
|
||||||
error(module: string, message: string, contextOrData?: LogContext | any, data?: any): void {
|
|
||||||
this.log(LogLevel.ERROR, module, message, contextOrData, data)
|
|
||||||
}
|
|
||||||
|
|
||||||
warn(module: string, message: string, contextOrData?: LogContext | any, data?: any): void {
|
|
||||||
this.log(LogLevel.WARN, module, message, contextOrData, data)
|
|
||||||
}
|
|
||||||
|
|
||||||
info(module: string, message: string, contextOrData?: LogContext | any, data?: any): void {
|
|
||||||
this.log(LogLevel.INFO, module, message, contextOrData, data)
|
|
||||||
}
|
|
||||||
|
|
||||||
debug(module: string, message: string, contextOrData?: LogContext | any, data?: any): void {
|
|
||||||
this.log(LogLevel.DEBUG, module, message, contextOrData, data)
|
|
||||||
}
|
|
||||||
|
|
||||||
trace(module: string, message: string, contextOrData?: LogContext | any, data?: any): void {
|
|
||||||
this.log(LogLevel.TRACE, module, message, contextOrData, data)
|
|
||||||
}
|
|
||||||
|
|
||||||
createModuleLogger(module: string) {
|
|
||||||
const self = this
|
|
||||||
return {
|
|
||||||
fatal: (message: string, contextOrData?: LogContext | any, data?: any) =>
|
|
||||||
self.fatal(module, message, contextOrData, data),
|
|
||||||
error: (message: string, contextOrData?: LogContext | any, data?: any) =>
|
|
||||||
self.error(module, message, contextOrData, data),
|
|
||||||
warn: (message: string, contextOrData?: LogContext | any, data?: any) =>
|
|
||||||
self.warn(module, message, contextOrData, data),
|
|
||||||
info: (message: string, contextOrData?: LogContext | any, data?: any) =>
|
|
||||||
self.info(module, message, contextOrData, data),
|
|
||||||
debug: (message: string, contextOrData?: LogContext | any, data?: any) =>
|
|
||||||
self.debug(module, message, contextOrData, data),
|
|
||||||
trace: (message: string, contextOrData?: LogContext | any, data?: any) =>
|
|
||||||
self.trace(module, message, contextOrData, data),
|
|
||||||
withContext: (context: LogContext) => {
|
|
||||||
const contextual = self.withContext(context)
|
|
||||||
return contextual.createModuleLogger(module)
|
|
||||||
},
|
|
||||||
startTimer: (label: string) => self.startTimer(`${module}:${label}`),
|
|
||||||
endTimer: (label: string) => self.endTimer(`${module}:${label}`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async flush(): Promise<void> {
|
|
||||||
const flushPromises = this.config.transports
|
|
||||||
.filter(t => t.flush)
|
|
||||||
.map(t => t.flush!())
|
|
||||||
|
|
||||||
await Promise.all(flushPromises)
|
|
||||||
}
|
|
||||||
|
|
||||||
addTransport(transport: LogTransport): void {
|
|
||||||
this.config.transports.push(transport)
|
|
||||||
}
|
|
||||||
|
|
||||||
removeTransport(name: string): void {
|
|
||||||
this.config.transports = this.config.transports.filter(t => t.name !== name)
|
|
||||||
}
|
|
||||||
|
|
||||||
child(context: LogContext): StructuredLogger {
|
|
||||||
return this.withContext(context)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Singleton instance
|
|
||||||
export const structuredLogger = StructuredLogger.getInstance()
|
|
||||||
|
|
||||||
// Convenience functions
|
|
||||||
export function createModuleLogger(module: string) {
|
|
||||||
return structuredLogger.createModuleLogger(module)
|
|
||||||
}
|
|
||||||
|
|
||||||
export function setLogContext(context: LogContext) {
|
|
||||||
structuredLogger.setContext(context)
|
|
||||||
}
|
|
||||||
|
|
||||||
export function withLogContext(context: LogContext) {
|
|
||||||
return structuredLogger.withContext(context)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Correlation ID middleware helper
|
|
||||||
export function createCorrelationId(): string {
|
|
||||||
return randomUUID()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Performance logging helper
|
|
||||||
export function logPerformance(
|
|
||||||
logger: ReturnType<typeof createModuleLogger>,
|
|
||||||
operation: string,
|
|
||||||
fn: () => any
|
|
||||||
): any {
|
|
||||||
logger.startTimer(operation)
|
|
||||||
try {
|
|
||||||
const result = fn()
|
|
||||||
if (result && typeof result.then === 'function') {
|
|
||||||
return result.finally(() => {
|
|
||||||
const duration = logger.endTimer(operation)
|
|
||||||
logger.debug(`${operation} completed`, { duration })
|
|
||||||
})
|
|
||||||
}
|
|
||||||
const duration = logger.endTimer(operation)
|
|
||||||
logger.debug(`${operation} completed`, { duration })
|
|
||||||
return result
|
|
||||||
} catch (error) {
|
|
||||||
const duration = logger.endTimer(operation)
|
|
||||||
logger.error(`${operation} failed`, { duration, error })
|
|
||||||
throw error
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Backward compatibility wrapper for existing logger
|
|
||||||
export class LoggerCompatibilityWrapper {
|
|
||||||
private moduleLogger: ReturnType<typeof createModuleLogger>
|
|
||||||
|
|
||||||
constructor(module: string = 'legacy') {
|
|
||||||
this.moduleLogger = createModuleLogger(module)
|
|
||||||
}
|
|
||||||
|
|
||||||
error(module: string, message: string, ...args: any[]): void {
|
|
||||||
this.moduleLogger.error(message, { module, data: args })
|
|
||||||
}
|
|
||||||
|
|
||||||
warn(module: string, message: string, ...args: any[]): void {
|
|
||||||
this.moduleLogger.warn(message, { module, data: args })
|
|
||||||
}
|
|
||||||
|
|
||||||
info(module: string, message: string, ...args: any[]): void {
|
|
||||||
this.moduleLogger.info(message, { module, data: args })
|
|
||||||
}
|
|
||||||
|
|
||||||
debug(module: string, message: string, ...args: any[]): void {
|
|
||||||
this.moduleLogger.debug(message, { module, data: args })
|
|
||||||
}
|
|
||||||
|
|
||||||
trace(module: string, message: string, ...args: any[]): void {
|
|
||||||
this.moduleLogger.trace(message, { module, data: args })
|
|
||||||
}
|
|
||||||
|
|
||||||
createModuleLogger(module: string) {
|
|
||||||
const logger = createModuleLogger(module)
|
|
||||||
return {
|
|
||||||
error: (message: string, ...args: any[]) => logger.error(message, { data: args }),
|
|
||||||
warn: (message: string, ...args: any[]) => logger.warn(message, { data: args }),
|
|
||||||
info: (message: string, ...args: any[]) => logger.info(message, { data: args }),
|
|
||||||
debug: (message: string, ...args: any[]) => logger.debug(message, { data: args }),
|
|
||||||
trace: (message: string, ...args: any[]) => logger.trace(message, { data: args })
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Export types for external use
|
|
||||||
export type ModuleLogger = ReturnType<typeof createModuleLogger>
|
|
||||||
// Types are already exported above, no need to re-export
|
|
||||||
|
|
@ -1,411 +0,0 @@
|
||||||
/**
|
|
||||||
* Write Buffer
|
|
||||||
* Accumulates writes and flushes them in bulk to reduce S3 operations
|
|
||||||
* Implements intelligent deduplication and compression
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { HNSWNoun, HNSWVerb } from '../coreTypes.js'
|
|
||||||
import { createModuleLogger } from './logger.js'
|
|
||||||
import { getGlobalBackpressure } from './adaptiveBackpressure.js'
|
|
||||||
|
|
||||||
interface BufferedWrite<T> {
|
|
||||||
id: string
|
|
||||||
data: T
|
|
||||||
timestamp: number
|
|
||||||
type: 'noun' | 'verb' | 'metadata'
|
|
||||||
retryCount: number
|
|
||||||
}
|
|
||||||
|
|
||||||
interface FlushResult {
|
|
||||||
successful: number
|
|
||||||
failed: number
|
|
||||||
duration: number
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* High-performance write buffer for bulk operations
|
|
||||||
*/
|
|
||||||
export class WriteBuffer<T> {
|
|
||||||
private logger = createModuleLogger('WriteBuffer')
|
|
||||||
|
|
||||||
// Buffer storage
|
|
||||||
private buffer = new Map<string, BufferedWrite<T>>()
|
|
||||||
|
|
||||||
// Configuration - More aggressive for high volume
|
|
||||||
private maxBufferSize = 2000 // Allow larger buffers
|
|
||||||
private flushInterval = 500 // Flush more frequently (0.5 seconds)
|
|
||||||
private minFlushSize = 50 // Lower minimum to flush sooner
|
|
||||||
private maxRetries = 3 // Maximum retry attempts
|
|
||||||
|
|
||||||
// State
|
|
||||||
private flushTimer: NodeJS.Timeout | null = null
|
|
||||||
private isFlushing = false
|
|
||||||
private lastFlush = Date.now()
|
|
||||||
private pendingFlush: Promise<FlushResult> | null = null
|
|
||||||
|
|
||||||
// Statistics
|
|
||||||
private totalWrites = 0
|
|
||||||
private totalFlushes = 0
|
|
||||||
private failedWrites = 0
|
|
||||||
private duplicatesRemoved = 0
|
|
||||||
|
|
||||||
// Write function
|
|
||||||
private writeFunction: (items: Map<string, T>) => Promise<void>
|
|
||||||
private type: 'noun' | 'verb' | 'metadata'
|
|
||||||
|
|
||||||
// Backpressure integration
|
|
||||||
private backpressure = getGlobalBackpressure()
|
|
||||||
|
|
||||||
constructor(
|
|
||||||
type: 'noun' | 'verb' | 'metadata',
|
|
||||||
writeFunction: (items: Map<string, T>) => Promise<void>,
|
|
||||||
options?: {
|
|
||||||
maxBufferSize?: number
|
|
||||||
flushInterval?: number
|
|
||||||
minFlushSize?: number
|
|
||||||
}
|
|
||||||
) {
|
|
||||||
this.type = type
|
|
||||||
this.writeFunction = writeFunction
|
|
||||||
|
|
||||||
if (options) {
|
|
||||||
this.maxBufferSize = options.maxBufferSize || this.maxBufferSize
|
|
||||||
this.flushInterval = options.flushInterval || this.flushInterval
|
|
||||||
this.minFlushSize = options.minFlushSize || this.minFlushSize
|
|
||||||
}
|
|
||||||
|
|
||||||
// Start periodic flush
|
|
||||||
this.startPeriodicFlush()
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Add item to buffer
|
|
||||||
*/
|
|
||||||
public async add(id: string, data: T): Promise<void> {
|
|
||||||
// Check if we're already at capacity
|
|
||||||
if (this.buffer.size >= this.maxBufferSize) {
|
|
||||||
// Wait for current flush to complete
|
|
||||||
if (this.pendingFlush) {
|
|
||||||
await this.pendingFlush
|
|
||||||
}
|
|
||||||
|
|
||||||
// Force flush if still at capacity
|
|
||||||
if (this.buffer.size >= this.maxBufferSize) {
|
|
||||||
await this.flush('capacity')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check for duplicate and update if newer
|
|
||||||
const existing = this.buffer.get(id)
|
|
||||||
if (existing) {
|
|
||||||
// Update with newer data
|
|
||||||
existing.data = data
|
|
||||||
existing.timestamp = Date.now()
|
|
||||||
this.duplicatesRemoved++
|
|
||||||
} else {
|
|
||||||
// Add new item
|
|
||||||
this.buffer.set(id, {
|
|
||||||
id,
|
|
||||||
data,
|
|
||||||
timestamp: Date.now(),
|
|
||||||
type: this.type,
|
|
||||||
retryCount: 0
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
this.totalWrites++
|
|
||||||
|
|
||||||
// Log buffer growth periodically
|
|
||||||
if (this.totalWrites % 100 === 0) {
|
|
||||||
this.logger.info(`📈 BUFFER GROWTH: ${this.buffer.size} ${this.type} items buffered (${this.totalWrites} total writes, ${this.duplicatesRemoved} deduplicated)`)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check if we should flush
|
|
||||||
this.checkFlush()
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Check if we should flush
|
|
||||||
*/
|
|
||||||
private checkFlush(): void {
|
|
||||||
const bufferSize = this.buffer.size
|
|
||||||
const timeSinceFlush = Date.now() - this.lastFlush
|
|
||||||
|
|
||||||
// Immediate flush conditions
|
|
||||||
if (bufferSize >= this.maxBufferSize) {
|
|
||||||
this.flush('size')
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Time-based flush with minimum size
|
|
||||||
if (timeSinceFlush >= this.flushInterval && bufferSize >= this.minFlushSize) {
|
|
||||||
this.flush('time')
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Adaptive flush based on system load
|
|
||||||
const backpressureStatus = this.backpressure.getStatus()
|
|
||||||
if (backpressureStatus.queueLength > 1000 && bufferSize > 10) {
|
|
||||||
// System under pressure - flush smaller batches more frequently
|
|
||||||
this.flush('pressure')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Flush buffer to storage
|
|
||||||
*/
|
|
||||||
public async flush(reason: string = 'manual'): Promise<FlushResult> {
|
|
||||||
// Prevent concurrent flushes
|
|
||||||
if (this.isFlushing) {
|
|
||||||
if (this.pendingFlush) {
|
|
||||||
return this.pendingFlush
|
|
||||||
}
|
|
||||||
return { successful: 0, failed: 0, duration: 0 }
|
|
||||||
}
|
|
||||||
|
|
||||||
// Nothing to flush
|
|
||||||
if (this.buffer.size === 0) {
|
|
||||||
return { successful: 0, failed: 0, duration: 0 }
|
|
||||||
}
|
|
||||||
|
|
||||||
this.isFlushing = true
|
|
||||||
const startTime = Date.now()
|
|
||||||
|
|
||||||
// Create flush promise
|
|
||||||
this.pendingFlush = this.doFlush(reason, startTime)
|
|
||||||
|
|
||||||
try {
|
|
||||||
const result = await this.pendingFlush
|
|
||||||
return result
|
|
||||||
} finally {
|
|
||||||
this.isFlushing = false
|
|
||||||
this.pendingFlush = null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Perform the actual flush
|
|
||||||
*/
|
|
||||||
private async doFlush(reason: string, startTime: number): Promise<FlushResult> {
|
|
||||||
const itemsToFlush = new Map<string, T>()
|
|
||||||
const flushingItems = new Map<string, BufferedWrite<T>>()
|
|
||||||
|
|
||||||
// Take items from buffer
|
|
||||||
let count = 0
|
|
||||||
for (const [id, item] of this.buffer.entries()) {
|
|
||||||
itemsToFlush.set(id, item.data)
|
|
||||||
flushingItems.set(id, item)
|
|
||||||
count++
|
|
||||||
|
|
||||||
// Limit batch size for better performance
|
|
||||||
if (count >= 500) {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Remove from buffer
|
|
||||||
for (const id of itemsToFlush.keys()) {
|
|
||||||
this.buffer.delete(id)
|
|
||||||
}
|
|
||||||
|
|
||||||
this.logger.warn(`🔄 BUFFERING: Flushing ${itemsToFlush.size} ${this.type} items (buffer had ${this.buffer.size + itemsToFlush.size}) - reason: ${reason}`)
|
|
||||||
|
|
||||||
try {
|
|
||||||
// Request permission from backpressure system
|
|
||||||
const opId = `flush-${Date.now()}`
|
|
||||||
await this.backpressure.requestPermission(opId, 2) // Higher priority
|
|
||||||
|
|
||||||
try {
|
|
||||||
// Perform bulk write
|
|
||||||
await this.writeFunction(itemsToFlush)
|
|
||||||
|
|
||||||
// Success
|
|
||||||
this.backpressure.releasePermission(opId, true)
|
|
||||||
this.totalFlushes++
|
|
||||||
this.lastFlush = Date.now()
|
|
||||||
|
|
||||||
const duration = Date.now() - startTime
|
|
||||||
this.logger.warn(`🚀 BATCH FLUSH: ${itemsToFlush.size} ${this.type} items → 1 bulk S3 operation (${duration}ms, reason: ${reason})`)
|
|
||||||
|
|
||||||
return {
|
|
||||||
successful: itemsToFlush.size,
|
|
||||||
failed: 0,
|
|
||||||
duration
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
// Release with error
|
|
||||||
this.backpressure.releasePermission(opId, false)
|
|
||||||
throw error
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
this.logger.error(`Flush failed: ${error}`)
|
|
||||||
|
|
||||||
// Put items back with retry count
|
|
||||||
for (const [id, item] of flushingItems.entries()) {
|
|
||||||
item.retryCount++
|
|
||||||
|
|
||||||
if (item.retryCount < this.maxRetries) {
|
|
||||||
// Put back for retry
|
|
||||||
this.buffer.set(id, item)
|
|
||||||
} else {
|
|
||||||
// Max retries exceeded
|
|
||||||
this.failedWrites++
|
|
||||||
this.logger.error(`Max retries exceeded for ${this.type} ${id}`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const duration = Date.now() - startTime
|
|
||||||
|
|
||||||
return {
|
|
||||||
successful: 0,
|
|
||||||
failed: itemsToFlush.size,
|
|
||||||
duration
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Start periodic flush timer
|
|
||||||
*/
|
|
||||||
private startPeriodicFlush(): void {
|
|
||||||
if (this.flushTimer) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
this.flushTimer = setInterval(() => {
|
|
||||||
if (this.buffer.size > 0) {
|
|
||||||
const timeSinceFlush = Date.now() - this.lastFlush
|
|
||||||
|
|
||||||
// Flush if we have items and enough time has passed
|
|
||||||
if (timeSinceFlush >= this.flushInterval) {
|
|
||||||
this.flush('periodic').catch(error => {
|
|
||||||
this.logger.error('Periodic flush failed:', error)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}, Math.min(100, this.flushInterval / 2))
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Stop periodic flush timer
|
|
||||||
*/
|
|
||||||
public stop(): void {
|
|
||||||
if (this.flushTimer) {
|
|
||||||
clearInterval(this.flushTimer)
|
|
||||||
this.flushTimer = null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Force flush all pending writes
|
|
||||||
*/
|
|
||||||
public async forceFlush(): Promise<FlushResult> {
|
|
||||||
// Flush everything regardless of size
|
|
||||||
const oldMinSize = this.minFlushSize
|
|
||||||
this.minFlushSize = 0
|
|
||||||
|
|
||||||
try {
|
|
||||||
const result = await this.flush('force')
|
|
||||||
|
|
||||||
// Flush any remaining items
|
|
||||||
while (this.buffer.size > 0) {
|
|
||||||
const additionalResult = await this.flush('force-remaining')
|
|
||||||
result.successful += additionalResult.successful
|
|
||||||
result.failed += additionalResult.failed
|
|
||||||
result.duration += additionalResult.duration
|
|
||||||
}
|
|
||||||
|
|
||||||
return result
|
|
||||||
} finally {
|
|
||||||
this.minFlushSize = oldMinSize
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get buffer statistics
|
|
||||||
*/
|
|
||||||
public getStats(): {
|
|
||||||
bufferSize: number
|
|
||||||
totalWrites: number
|
|
||||||
totalFlushes: number
|
|
||||||
failedWrites: number
|
|
||||||
duplicatesRemoved: number
|
|
||||||
avgFlushSize: number
|
|
||||||
} {
|
|
||||||
return {
|
|
||||||
bufferSize: this.buffer.size,
|
|
||||||
totalWrites: this.totalWrites,
|
|
||||||
totalFlushes: this.totalFlushes,
|
|
||||||
failedWrites: this.failedWrites,
|
|
||||||
duplicatesRemoved: this.duplicatesRemoved,
|
|
||||||
avgFlushSize: this.totalFlushes > 0 ? this.totalWrites / this.totalFlushes : 0
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Adjust parameters based on load
|
|
||||||
*/
|
|
||||||
public adjustForLoad(pendingRequests: number): void {
|
|
||||||
if (pendingRequests > 10000) {
|
|
||||||
// Extreme load - buffer more aggressively
|
|
||||||
this.maxBufferSize = 5000
|
|
||||||
this.flushInterval = 500
|
|
||||||
this.minFlushSize = 500
|
|
||||||
} else if (pendingRequests > 1000) {
|
|
||||||
// High load
|
|
||||||
this.maxBufferSize = 2000
|
|
||||||
this.flushInterval = 1000
|
|
||||||
this.minFlushSize = 200
|
|
||||||
} else if (pendingRequests > 100) {
|
|
||||||
// Moderate load
|
|
||||||
this.maxBufferSize = 1000
|
|
||||||
this.flushInterval = 2000
|
|
||||||
this.minFlushSize = 100
|
|
||||||
} else {
|
|
||||||
// Low load - optimize for latency
|
|
||||||
this.maxBufferSize = 500
|
|
||||||
this.flushInterval = 5000
|
|
||||||
this.minFlushSize = 50
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Global write buffers
|
|
||||||
const writeBuffers = new Map<string, WriteBuffer<any>>()
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get or create a write buffer
|
|
||||||
*/
|
|
||||||
export function getWriteBuffer<T>(
|
|
||||||
id: string,
|
|
||||||
type: 'noun' | 'verb' | 'metadata',
|
|
||||||
writeFunction: (items: Map<string, T>) => Promise<void>
|
|
||||||
): WriteBuffer<T> {
|
|
||||||
if (!writeBuffers.has(id)) {
|
|
||||||
writeBuffers.set(id, new WriteBuffer<T>(type, writeFunction))
|
|
||||||
}
|
|
||||||
return writeBuffers.get(id)!
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Flush all write buffers
|
|
||||||
*/
|
|
||||||
export async function flushAllBuffers(): Promise<void> {
|
|
||||||
const promises: Promise<FlushResult>[] = []
|
|
||||||
|
|
||||||
for (const buffer of writeBuffers.values()) {
|
|
||||||
promises.push(buffer.forceFlush())
|
|
||||||
}
|
|
||||||
|
|
||||||
await Promise.all(promises)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Clear all write buffers
|
|
||||||
*/
|
|
||||||
export function clearWriteBuffers(): void {
|
|
||||||
for (const buffer of writeBuffers.values()) {
|
|
||||||
buffer.stop()
|
|
||||||
}
|
|
||||||
writeBuffers.clear()
|
|
||||||
}
|
|
||||||
|
|
@ -1,333 +0,0 @@
|
||||||
/**
|
|
||||||
* fs-Compatible Interface for VFS
|
|
||||||
*
|
|
||||||
* Provides a drop-in replacement for Node's fs module
|
|
||||||
* that uses VFS for storage instead of the real filesystem.
|
|
||||||
*
|
|
||||||
* Usage:
|
|
||||||
* import { FSCompat } from '@soulcraft/brainy/vfs'
|
|
||||||
* const fs = new FSCompat(brain.vfs)
|
|
||||||
*
|
|
||||||
* // Now use like Node's fs
|
|
||||||
* await fs.promises.readFile('/path')
|
|
||||||
* fs.createReadStream('/path').pipe(output)
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { VirtualFileSystem } from './VirtualFileSystem.js'
|
|
||||||
import type {
|
|
||||||
ReadOptions,
|
|
||||||
WriteOptions,
|
|
||||||
MkdirOptions,
|
|
||||||
VFSStats,
|
|
||||||
VFSDirent
|
|
||||||
} from './types.js'
|
|
||||||
|
|
||||||
export class FSCompat {
|
|
||||||
/**
|
|
||||||
* Promise-based API (like fs.promises)
|
|
||||||
*/
|
|
||||||
public readonly promises: FSPromises
|
|
||||||
|
|
||||||
constructor(private vfs: VirtualFileSystem) {
|
|
||||||
this.promises = new FSPromises(vfs)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ============= Callback-style methods (for compatibility) =============
|
|
||||||
|
|
||||||
readFile(path: string, callback: (err: Error | null, data?: Buffer) => void): void
|
|
||||||
readFile(path: string, encoding: BufferEncoding, callback: (err: Error | null, data?: string) => void): void
|
|
||||||
readFile(path: string, options: any, callback?: any): void {
|
|
||||||
if (typeof options === 'function') {
|
|
||||||
callback = options
|
|
||||||
options = {}
|
|
||||||
}
|
|
||||||
|
|
||||||
this.vfs.readFile(path, options)
|
|
||||||
.then(data => {
|
|
||||||
if (options?.encoding) {
|
|
||||||
callback(null, data.toString(options.encoding))
|
|
||||||
} else {
|
|
||||||
callback(null, data)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.catch(err => callback(err))
|
|
||||||
}
|
|
||||||
|
|
||||||
writeFile(path: string, data: Buffer | string, callback: (err: Error | null) => void): void
|
|
||||||
writeFile(path: string, data: Buffer | string, options: any, callback?: any): void {
|
|
||||||
if (typeof options === 'function') {
|
|
||||||
callback = options
|
|
||||||
options = {}
|
|
||||||
}
|
|
||||||
|
|
||||||
const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, options?.encoding || 'utf8')
|
|
||||||
|
|
||||||
this.vfs.writeFile(path, buffer, options)
|
|
||||||
.then(() => callback(null))
|
|
||||||
.catch(err => callback(err))
|
|
||||||
}
|
|
||||||
|
|
||||||
mkdir(path: string, callback: (err: Error | null) => void): void
|
|
||||||
mkdir(path: string, options: any, callback?: any): void {
|
|
||||||
if (typeof options === 'function') {
|
|
||||||
callback = options
|
|
||||||
options = {}
|
|
||||||
}
|
|
||||||
|
|
||||||
this.vfs.mkdir(path, options)
|
|
||||||
.then(() => callback(null))
|
|
||||||
.catch(err => callback(err))
|
|
||||||
}
|
|
||||||
|
|
||||||
rmdir(path: string, callback: (err: Error | null) => void): void
|
|
||||||
rmdir(path: string, options: any, callback?: any): void {
|
|
||||||
if (typeof options === 'function') {
|
|
||||||
callback = options
|
|
||||||
options = {}
|
|
||||||
}
|
|
||||||
|
|
||||||
this.vfs.rmdir(path, options)
|
|
||||||
.then(() => callback(null))
|
|
||||||
.catch(err => callback(err))
|
|
||||||
}
|
|
||||||
|
|
||||||
readdir(path: string, callback: (err: Error | null, files?: string[]) => void): void
|
|
||||||
readdir(path: string, options: any, callback?: any): void {
|
|
||||||
if (typeof options === 'function') {
|
|
||||||
callback = options
|
|
||||||
options = {}
|
|
||||||
}
|
|
||||||
|
|
||||||
this.vfs.readdir(path, options)
|
|
||||||
.then(files => callback(null, files))
|
|
||||||
.catch(err => callback(err))
|
|
||||||
}
|
|
||||||
|
|
||||||
stat(path: string, callback: (err: Error | null, stats?: VFSStats) => void): void {
|
|
||||||
this.vfs.stat(path)
|
|
||||||
.then(stats => callback(null, stats))
|
|
||||||
.catch(err => callback(err))
|
|
||||||
}
|
|
||||||
|
|
||||||
lstat(path: string, callback: (err: Error | null, stats?: VFSStats) => void): void {
|
|
||||||
this.vfs.lstat(path)
|
|
||||||
.then(stats => callback(null, stats))
|
|
||||||
.catch(err => callback(err))
|
|
||||||
}
|
|
||||||
|
|
||||||
unlink(path: string, callback: (err: Error | null) => void): void {
|
|
||||||
this.vfs.unlink(path)
|
|
||||||
.then(() => callback(null))
|
|
||||||
.catch(err => callback(err))
|
|
||||||
}
|
|
||||||
|
|
||||||
rename(oldPath: string, newPath: string, callback: (err: Error | null) => void): void {
|
|
||||||
this.vfs.rename(oldPath, newPath)
|
|
||||||
.then(() => callback(null))
|
|
||||||
.catch(err => callback(err))
|
|
||||||
}
|
|
||||||
|
|
||||||
copyFile(src: string, dest: string, callback: (err: Error | null) => void): void
|
|
||||||
copyFile(src: string, dest: string, flags: number, callback: (err: Error | null) => void): void
|
|
||||||
copyFile(src: string, dest: string, flagsOrCallback: any, callback?: any): void {
|
|
||||||
if (typeof flagsOrCallback === 'function') {
|
|
||||||
callback = flagsOrCallback
|
|
||||||
} else {
|
|
||||||
// flags provided but not used
|
|
||||||
}
|
|
||||||
|
|
||||||
this.vfs.copy(src, dest)
|
|
||||||
.then(() => callback(null))
|
|
||||||
.catch(err => callback(err))
|
|
||||||
}
|
|
||||||
|
|
||||||
exists(path: string, callback: (exists: boolean) => void): void {
|
|
||||||
this.vfs.exists(path)
|
|
||||||
.then(exists => callback(exists))
|
|
||||||
.catch(() => callback(false))
|
|
||||||
}
|
|
||||||
|
|
||||||
access(path: string, callback: (err: Error | null) => void): void
|
|
||||||
access(path: string, mode: number, callback: (err: Error | null) => void): void
|
|
||||||
access(path: string, modeOrCallback: any, callback?: any): void {
|
|
||||||
if (typeof modeOrCallback === 'function') {
|
|
||||||
callback = modeOrCallback
|
|
||||||
} else {
|
|
||||||
// mode provided but not used
|
|
||||||
}
|
|
||||||
|
|
||||||
this.vfs.exists(path)
|
|
||||||
.then(exists => {
|
|
||||||
if (exists) {
|
|
||||||
callback(null)
|
|
||||||
} else {
|
|
||||||
const err: any = new Error('ENOENT: no such file or directory')
|
|
||||||
err.code = 'ENOENT'
|
|
||||||
callback(err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.catch(err => callback(err))
|
|
||||||
}
|
|
||||||
|
|
||||||
appendFile(path: string, data: Buffer | string, callback: (err: Error | null) => void): void
|
|
||||||
appendFile(path: string, data: Buffer | string, options: any, callback?: any): void {
|
|
||||||
if (typeof options === 'function') {
|
|
||||||
callback = options
|
|
||||||
options = {}
|
|
||||||
}
|
|
||||||
|
|
||||||
const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, options?.encoding || 'utf8')
|
|
||||||
|
|
||||||
this.vfs.appendFile(path, buffer, options)
|
|
||||||
.then(() => callback(null))
|
|
||||||
.catch(err => callback(err))
|
|
||||||
}
|
|
||||||
|
|
||||||
// ============= Stream methods =============
|
|
||||||
|
|
||||||
createReadStream(path: string, options?: any) {
|
|
||||||
return this.vfs.createReadStream(path, options)
|
|
||||||
}
|
|
||||||
|
|
||||||
createWriteStream(path: string, options?: any) {
|
|
||||||
return this.vfs.createWriteStream(path, options)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ============= Watch methods =============
|
|
||||||
|
|
||||||
watch(path: string, listener?: any) {
|
|
||||||
return this.vfs.watch(path, listener)
|
|
||||||
}
|
|
||||||
|
|
||||||
watchFile(path: string, listener: any) {
|
|
||||||
return this.vfs.watchFile(path, listener)
|
|
||||||
}
|
|
||||||
|
|
||||||
unwatchFile(path: string) {
|
|
||||||
return this.vfs.unwatchFile(path)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ============= Additional methods =============
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Import a directory from real filesystem (VFS extension)
|
|
||||||
*/
|
|
||||||
async importDirectory(sourcePath: string, options?: any) {
|
|
||||||
return this.vfs.importDirectory(sourcePath, options)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Search files semantically (VFS extension)
|
|
||||||
*/
|
|
||||||
async search(query: string, options?: any) {
|
|
||||||
return this.vfs.search(query, options)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Promise-based fs API (like fs.promises)
|
|
||||||
*/
|
|
||||||
class FSPromises {
|
|
||||||
constructor(private vfs: VirtualFileSystem) {}
|
|
||||||
|
|
||||||
async readFile(path: string, options?: any): Promise<Buffer | string> {
|
|
||||||
const buffer = await this.vfs.readFile(path, options)
|
|
||||||
if (options?.encoding) {
|
|
||||||
return buffer.toString(options.encoding)
|
|
||||||
}
|
|
||||||
return buffer
|
|
||||||
}
|
|
||||||
|
|
||||||
async writeFile(path: string, data: Buffer | string, options?: any): Promise<void> {
|
|
||||||
const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, options?.encoding || 'utf8')
|
|
||||||
return this.vfs.writeFile(path, buffer, options)
|
|
||||||
}
|
|
||||||
|
|
||||||
async mkdir(path: string, options?: any): Promise<void> {
|
|
||||||
return this.vfs.mkdir(path, options)
|
|
||||||
}
|
|
||||||
|
|
||||||
async rmdir(path: string, options?: any): Promise<void> {
|
|
||||||
return this.vfs.rmdir(path, options)
|
|
||||||
}
|
|
||||||
|
|
||||||
async readdir(path: string, options?: any): Promise<string[] | VFSDirent[]> {
|
|
||||||
return this.vfs.readdir(path, options)
|
|
||||||
}
|
|
||||||
|
|
||||||
async stat(path: string): Promise<VFSStats> {
|
|
||||||
return this.vfs.stat(path)
|
|
||||||
}
|
|
||||||
|
|
||||||
async lstat(path: string): Promise<VFSStats> {
|
|
||||||
return this.vfs.lstat(path)
|
|
||||||
}
|
|
||||||
|
|
||||||
async unlink(path: string): Promise<void> {
|
|
||||||
return this.vfs.unlink(path)
|
|
||||||
}
|
|
||||||
|
|
||||||
async rename(oldPath: string, newPath: string): Promise<void> {
|
|
||||||
return this.vfs.rename(oldPath, newPath)
|
|
||||||
}
|
|
||||||
|
|
||||||
async copyFile(src: string, dest: string, flags?: number): Promise<void> {
|
|
||||||
return this.vfs.copy(src, dest)
|
|
||||||
}
|
|
||||||
|
|
||||||
async access(path: string, mode?: number): Promise<void> {
|
|
||||||
const exists = await this.vfs.exists(path)
|
|
||||||
if (!exists) {
|
|
||||||
const err: any = new Error('ENOENT: no such file or directory')
|
|
||||||
err.code = 'ENOENT'
|
|
||||||
throw err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async appendFile(path: string, data: Buffer | string, options?: any): Promise<void> {
|
|
||||||
const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, options?.encoding || 'utf8')
|
|
||||||
return this.vfs.appendFile(path, buffer, options)
|
|
||||||
}
|
|
||||||
|
|
||||||
async realpath(path: string): Promise<string> {
|
|
||||||
return this.vfs.realpath(path)
|
|
||||||
}
|
|
||||||
|
|
||||||
async chmod(path: string, mode: number): Promise<void> {
|
|
||||||
return this.vfs.chmod(path, mode)
|
|
||||||
}
|
|
||||||
|
|
||||||
async chown(path: string, uid: number, gid: number): Promise<void> {
|
|
||||||
return this.vfs.chown(path, uid, gid)
|
|
||||||
}
|
|
||||||
|
|
||||||
async utimes(path: string, atime: Date, mtime: Date): Promise<void> {
|
|
||||||
return this.vfs.utimes(path, atime, mtime)
|
|
||||||
}
|
|
||||||
|
|
||||||
async symlink(target: string, path: string): Promise<void> {
|
|
||||||
return this.vfs.symlink(target, path)
|
|
||||||
}
|
|
||||||
|
|
||||||
async readlink(path: string): Promise<string> {
|
|
||||||
return this.vfs.readlink(path)
|
|
||||||
}
|
|
||||||
|
|
||||||
// VFS Extensions
|
|
||||||
async search(query: string, options?: any) {
|
|
||||||
return this.vfs.search(query, options)
|
|
||||||
}
|
|
||||||
|
|
||||||
async findSimilar(path: string, options?: any) {
|
|
||||||
return this.vfs.findSimilar(path, options)
|
|
||||||
}
|
|
||||||
|
|
||||||
async importDirectory(sourcePath: string, options?: any) {
|
|
||||||
return this.vfs.importDirectory(sourcePath, options)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Export a convenience function to create fs replacement
|
|
||||||
export function createFS(vfs: VirtualFileSystem): FSCompat {
|
|
||||||
return new FSCompat(vfs)
|
|
||||||
}
|
|
||||||
|
|
@ -1,27 +0,0 @@
|
||||||
/**
|
|
||||||
* Brainy Virtual Filesystem
|
|
||||||
*
|
|
||||||
* A simplified fs-compatible filesystem that stores data in Brainy
|
|
||||||
* Works across all storage adapters and scales to millions of files
|
|
||||||
*/
|
|
||||||
|
|
||||||
// Core VFS
|
|
||||||
export { VirtualFileSystem } from './VirtualFileSystem.js'
|
|
||||||
export { PathResolver } from './PathResolver.js'
|
|
||||||
export * from './types.js'
|
|
||||||
|
|
||||||
// MIME Type Detection
|
|
||||||
export { MimeTypeDetector, mimeDetector } from './MimeTypeDetector.js'
|
|
||||||
|
|
||||||
// fs compatibility layer
|
|
||||||
export { FSCompat, createFS } from './FSCompat.js'
|
|
||||||
|
|
||||||
// Directory import
|
|
||||||
export { DirectoryImporter } from './importers/DirectoryImporter.js'
|
|
||||||
|
|
||||||
// Streaming
|
|
||||||
export { VFSReadStream } from './streams/VFSReadStream.js'
|
|
||||||
export { VFSWriteStream } from './streams/VFSWriteStream.js'
|
|
||||||
|
|
||||||
// Convenience alias
|
|
||||||
export { VirtualFileSystem as VFS } from './VirtualFileSystem.js'
|
|
||||||
|
|
@ -1,317 +0,0 @@
|
||||||
import { describe, it, expect, beforeEach } from 'vitest'
|
|
||||||
import { Brainy } from '../../../src/brainy.js'
|
|
||||||
import { InstancePool, createInstancePool } from '../../../src/import/InstancePool.js'
|
|
||||||
|
|
||||||
describe('InstancePool', () => {
|
|
||||||
let brain: Brainy
|
|
||||||
let pool: InstancePool
|
|
||||||
|
|
||||||
beforeEach(async () => {
|
|
||||||
brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
|
|
||||||
await brain.init()
|
|
||||||
pool = new InstancePool(brain)
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('lazy initialization', () => {
|
|
||||||
it('should not create instances until requested', () => {
|
|
||||||
const stats = pool.getStats()
|
|
||||||
expect(stats.nlpCreated).toBe(false)
|
|
||||||
expect(stats.extractorCreated).toBe(false)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should create NLP instance on first access', async () => {
|
|
||||||
const nlp = await pool.getNLP()
|
|
||||||
expect(nlp).toBeDefined()
|
|
||||||
|
|
||||||
const stats = pool.getStats()
|
|
||||||
expect(stats.nlpCreated).toBe(true)
|
|
||||||
expect(stats.nlpReuses).toBe(1)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should create extractor instance on first access', () => {
|
|
||||||
const extractor = pool.getExtractor()
|
|
||||||
expect(extractor).toBeDefined()
|
|
||||||
|
|
||||||
const stats = pool.getStats()
|
|
||||||
expect(stats.extractorCreated).toBe(true)
|
|
||||||
expect(stats.extractorReuses).toBe(1)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('instance reuse', () => {
|
|
||||||
it('should return same NLP instance on multiple calls', async () => {
|
|
||||||
const nlp1 = await pool.getNLP()
|
|
||||||
const nlp2 = await pool.getNLP()
|
|
||||||
const nlp3 = await pool.getNLP()
|
|
||||||
|
|
||||||
expect(nlp1).toBe(nlp2)
|
|
||||||
expect(nlp2).toBe(nlp3)
|
|
||||||
|
|
||||||
const stats = pool.getStats()
|
|
||||||
expect(stats.nlpReuses).toBe(3)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should return same extractor instance on multiple calls', () => {
|
|
||||||
const extractor1 = pool.getExtractor()
|
|
||||||
const extractor2 = pool.getExtractor()
|
|
||||||
const extractor3 = pool.getExtractor()
|
|
||||||
|
|
||||||
expect(extractor1).toBe(extractor2)
|
|
||||||
expect(extractor2).toBe(extractor3)
|
|
||||||
|
|
||||||
const stats = pool.getStats()
|
|
||||||
expect(stats.extractorReuses).toBe(3)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should track reuse counts correctly', async () => {
|
|
||||||
await pool.getNLP()
|
|
||||||
await pool.getNLP()
|
|
||||||
pool.getExtractor()
|
|
||||||
pool.getExtractor()
|
|
||||||
pool.getExtractor()
|
|
||||||
|
|
||||||
const stats = pool.getStats()
|
|
||||||
expect(stats.nlpReuses).toBe(2)
|
|
||||||
expect(stats.extractorReuses).toBe(3)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('initialization', () => {
|
|
||||||
it('should initialize all instances with init()', async () => {
|
|
||||||
await pool.init()
|
|
||||||
|
|
||||||
expect(pool.isInitialized()).toBe(true)
|
|
||||||
|
|
||||||
const stats = pool.getStats()
|
|
||||||
expect(stats.nlpCreated).toBe(true)
|
|
||||||
expect(stats.extractorCreated).toBe(true)
|
|
||||||
expect(stats.initialized).toBe(true)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should handle concurrent init calls safely', async () => {
|
|
||||||
// Call init multiple times concurrently
|
|
||||||
const promises = [
|
|
||||||
pool.init(),
|
|
||||||
pool.init(),
|
|
||||||
pool.init()
|
|
||||||
]
|
|
||||||
|
|
||||||
await Promise.all(promises)
|
|
||||||
|
|
||||||
// Should only initialize once
|
|
||||||
expect(pool.isInitialized()).toBe(true)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should auto-initialize NLP when accessed', async () => {
|
|
||||||
const nlp = await pool.getNLP()
|
|
||||||
|
|
||||||
// NLP is lazy-initialized but extractor might not be
|
|
||||||
const stats = pool.getStats()
|
|
||||||
expect(stats.nlpCreated).toBe(true)
|
|
||||||
expect(stats.initialized).toBe(true)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should provide sync access to NLP', () => {
|
|
||||||
const nlp = pool.getNLPSync()
|
|
||||||
expect(nlp).toBeDefined()
|
|
||||||
|
|
||||||
const stats = pool.getStats()
|
|
||||||
expect(stats.nlpCreated).toBe(true)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('statistics', () => {
|
|
||||||
it('should track creation time', async () => {
|
|
||||||
await pool.init()
|
|
||||||
|
|
||||||
const stats = pool.getStats()
|
|
||||||
expect(stats.creationTime).toBeGreaterThanOrEqual(0)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should calculate memory saved', async () => {
|
|
||||||
// Use instances multiple times
|
|
||||||
await pool.getNLP()
|
|
||||||
await pool.getNLP()
|
|
||||||
await pool.getNLP()
|
|
||||||
pool.getExtractor()
|
|
||||||
pool.getExtractor()
|
|
||||||
|
|
||||||
const stats = pool.getStats()
|
|
||||||
expect(stats.memorySaved).toBeGreaterThan(0)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should reset statistics', async () => {
|
|
||||||
await pool.getNLP()
|
|
||||||
pool.getExtractor()
|
|
||||||
|
|
||||||
pool.resetStats()
|
|
||||||
|
|
||||||
const stats = pool.getStats()
|
|
||||||
expect(stats.nlpReuses).toBe(0)
|
|
||||||
expect(stats.extractorReuses).toBe(0)
|
|
||||||
expect(stats.creationTime).toBe(0)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should provide string representation', async () => {
|
|
||||||
await pool.init()
|
|
||||||
|
|
||||||
const str = pool.toString()
|
|
||||||
expect(str).toContain('InstancePool')
|
|
||||||
expect(str).toContain('nlp=true')
|
|
||||||
expect(str).toContain('extractor=true')
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('memory efficiency', () => {
|
|
||||||
it('should reuse instances in loop (no memory leak)', async () => {
|
|
||||||
const initialStats = pool.getStats()
|
|
||||||
|
|
||||||
// Simulate import loop
|
|
||||||
for (let i = 0; i < 1000; i++) {
|
|
||||||
const nlp = await pool.getNLP()
|
|
||||||
const extractor = pool.getExtractor()
|
|
||||||
|
|
||||||
// All iterations should get same instances
|
|
||||||
expect(nlp).toBeDefined()
|
|
||||||
expect(extractor).toBeDefined()
|
|
||||||
}
|
|
||||||
|
|
||||||
const finalStats = pool.getStats()
|
|
||||||
expect(finalStats.nlpReuses).toBe(1000)
|
|
||||||
expect(finalStats.extractorReuses).toBe(1000)
|
|
||||||
|
|
||||||
// Should have saved ~60GB of memory (1000 iterations × ~60MB)
|
|
||||||
expect(finalStats.memorySaved).toBeGreaterThan(50 * 1024 * 1024 * 1000) // > 50GB
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should handle rapid concurrent access', async () => {
|
|
||||||
// Simulate concurrent row processing
|
|
||||||
const promises = []
|
|
||||||
for (let i = 0; i < 100; i++) {
|
|
||||||
promises.push(pool.getNLP())
|
|
||||||
promises.push(Promise.resolve(pool.getExtractor()))
|
|
||||||
}
|
|
||||||
|
|
||||||
await Promise.all(promises)
|
|
||||||
|
|
||||||
const stats = pool.getStats()
|
|
||||||
expect(stats.nlpReuses).toBe(100)
|
|
||||||
expect(stats.extractorReuses).toBe(100)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('cleanup', () => {
|
|
||||||
it('should cleanup instances', async () => {
|
|
||||||
await pool.init()
|
|
||||||
expect(pool.isInitialized()).toBe(true)
|
|
||||||
|
|
||||||
pool.cleanup()
|
|
||||||
|
|
||||||
expect(pool.isInitialized()).toBe(false)
|
|
||||||
const stats = pool.getStats()
|
|
||||||
expect(stats.nlpCreated).toBe(false)
|
|
||||||
expect(stats.extractorCreated).toBe(false)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should allow reinitialization after cleanup', async () => {
|
|
||||||
await pool.init()
|
|
||||||
pool.cleanup()
|
|
||||||
|
|
||||||
await pool.init()
|
|
||||||
expect(pool.isInitialized()).toBe(true)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('factory function', () => {
|
|
||||||
it('should create pool with auto-init', async () => {
|
|
||||||
const newPool = await createInstancePool(brain, true)
|
|
||||||
|
|
||||||
expect(newPool.isInitialized()).toBe(true)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should create pool without auto-init', async () => {
|
|
||||||
const newPool = await createInstancePool(brain, false)
|
|
||||||
|
|
||||||
expect(newPool.isInitialized()).toBe(false)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should default to auto-init', async () => {
|
|
||||||
const newPool = await createInstancePool(brain)
|
|
||||||
|
|
||||||
expect(newPool.isInitialized()).toBe(true)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('error handling', () => {
|
|
||||||
it('should handle missing NLP instance gracefully', async () => {
|
|
||||||
const emptyPool = new InstancePool(brain)
|
|
||||||
|
|
||||||
// Should create NLP on first access
|
|
||||||
const nlp = await emptyPool.getNLP()
|
|
||||||
expect(nlp).toBeDefined()
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should handle missing extractor instance gracefully', () => {
|
|
||||||
const emptyPool = new InstancePool(brain)
|
|
||||||
|
|
||||||
// Should create extractor on first access
|
|
||||||
const extractor = emptyPool.getExtractor()
|
|
||||||
expect(extractor).toBeDefined()
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('real-world usage', () => {
|
|
||||||
it('should work with actual NLP operations', async () => {
|
|
||||||
const nlp = await pool.getNLP()
|
|
||||||
|
|
||||||
// Should be initialized and ready to use
|
|
||||||
expect(nlp).toBeDefined()
|
|
||||||
|
|
||||||
// NLP should have init method
|
|
||||||
expect(typeof nlp.init).toBe('function')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should work with actual entity extraction', async () => {
|
|
||||||
const extractor = pool.getExtractor()
|
|
||||||
|
|
||||||
// Should be ready to use
|
|
||||||
expect(extractor).toBeDefined()
|
|
||||||
|
|
||||||
// Can call extractor methods
|
|
||||||
const entities = await extractor.extract('Paris is a beautiful city', {
|
|
||||||
confidence: 0.5
|
|
||||||
})
|
|
||||||
|
|
||||||
expect(Array.isArray(entities)).toBe(true)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should handle full import workflow', async () => {
|
|
||||||
// Initialize pool
|
|
||||||
await pool.init()
|
|
||||||
|
|
||||||
// Simulate processing multiple rows
|
|
||||||
const rows = [
|
|
||||||
{ text: 'Paris is beautiful' },
|
|
||||||
{ text: 'London is historic' },
|
|
||||||
{ text: 'Tokyo is modern' }
|
|
||||||
]
|
|
||||||
|
|
||||||
for (const row of rows) {
|
|
||||||
const nlp = await pool.getNLP()
|
|
||||||
const extractor = pool.getExtractor()
|
|
||||||
|
|
||||||
// Process row - extract entities
|
|
||||||
const entities = await extractor.extract(row.text, { confidence: 0.5 })
|
|
||||||
|
|
||||||
expect(nlp).toBeDefined()
|
|
||||||
expect(extractor).toBeDefined()
|
|
||||||
expect(entities).toBeDefined()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Verify instances were reused
|
|
||||||
const stats = pool.getStats()
|
|
||||||
expect(stats.nlpReuses).toBe(3)
|
|
||||||
expect(stats.extractorReuses).toBe(3)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
@ -1,561 +0,0 @@
|
||||||
import { describe, it, expect } from 'vitest'
|
|
||||||
import {
|
|
||||||
autoDetectPreset,
|
|
||||||
getPreset,
|
|
||||||
getPresetNames,
|
|
||||||
explainPresetChoice,
|
|
||||||
createCustomPreset,
|
|
||||||
validatePreset,
|
|
||||||
formatPreset,
|
|
||||||
FAST_PRESET,
|
|
||||||
BALANCED_PRESET,
|
|
||||||
ACCURATE_PRESET,
|
|
||||||
EXPLICIT_PRESET,
|
|
||||||
PATTERN_PRESET,
|
|
||||||
PRESETS,
|
|
||||||
type ImportContext,
|
|
||||||
type PresetConfig
|
|
||||||
} from '../../../src/neural/presets.js'
|
|
||||||
|
|
||||||
describe('Presets', () => {
|
|
||||||
describe('preset definitions', () => {
|
|
||||||
it('should have all 5 presets defined', () => {
|
|
||||||
expect(PRESETS).toBeDefined()
|
|
||||||
expect(Object.keys(PRESETS)).toHaveLength(5)
|
|
||||||
expect(PRESETS.fast).toBe(FAST_PRESET)
|
|
||||||
expect(PRESETS.balanced).toBe(BALANCED_PRESET)
|
|
||||||
expect(PRESETS.accurate).toBe(ACCURATE_PRESET)
|
|
||||||
expect(PRESETS.explicit).toBe(EXPLICIT_PRESET)
|
|
||||||
expect(PRESETS.pattern).toBe(PATTERN_PRESET)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should have valid fast preset', () => {
|
|
||||||
expect(FAST_PRESET.name).toBe('fast')
|
|
||||||
expect(FAST_PRESET.signals.enabled).toEqual(['exact', 'pattern'])
|
|
||||||
expect(FAST_PRESET.strategies.enabled).toEqual(['explicit'])
|
|
||||||
expect(FAST_PRESET.streaming).toBe(true)
|
|
||||||
expect(FAST_PRESET.strategies.earlyTermination).toBe(true)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should have valid balanced preset', () => {
|
|
||||||
expect(BALANCED_PRESET.name).toBe('balanced')
|
|
||||||
expect(BALANCED_PRESET.signals.enabled).toEqual(['exact', 'embedding', 'pattern'])
|
|
||||||
expect(BALANCED_PRESET.strategies.enabled).toEqual(['explicit', 'pattern', 'embedding'])
|
|
||||||
expect(BALANCED_PRESET.streaming).toBe(false)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should have valid accurate preset', () => {
|
|
||||||
expect(ACCURATE_PRESET.name).toBe('accurate')
|
|
||||||
expect(ACCURATE_PRESET.signals.enabled).toEqual(['exact', 'embedding', 'pattern', 'context'])
|
|
||||||
expect(ACCURATE_PRESET.strategies.enabled).toEqual(['explicit', 'pattern', 'embedding'])
|
|
||||||
expect(ACCURATE_PRESET.strategies.earlyTermination).toBe(false)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should have valid explicit preset', () => {
|
|
||||||
expect(EXPLICIT_PRESET.name).toBe('explicit')
|
|
||||||
expect(EXPLICIT_PRESET.signals.enabled).toEqual(['exact', 'pattern'])
|
|
||||||
expect(EXPLICIT_PRESET.strategies.enabled).toEqual(['explicit', 'pattern'])
|
|
||||||
expect(EXPLICIT_PRESET.strategies.minConfidence).toBeGreaterThanOrEqual(0.80)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should have valid pattern preset', () => {
|
|
||||||
expect(PATTERN_PRESET.name).toBe('pattern')
|
|
||||||
expect(PATTERN_PRESET.signals.enabled).toEqual(['embedding', 'pattern', 'context'])
|
|
||||||
expect(PATTERN_PRESET.strategies.enabled).toEqual(['pattern', 'embedding'])
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('autoDetectPreset', () => {
|
|
||||||
it('should return fast preset for large datasets', () => {
|
|
||||||
const context: ImportContext = {
|
|
||||||
rowCount: 15000,
|
|
||||||
fileSize: 5_000_000
|
|
||||||
}
|
|
||||||
|
|
||||||
const preset = autoDetectPreset(context)
|
|
||||||
expect(preset.name).toBe('fast')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should return fast preset for large files', () => {
|
|
||||||
const context: ImportContext = {
|
|
||||||
fileSize: 15_000_000
|
|
||||||
}
|
|
||||||
|
|
||||||
const preset = autoDetectPreset(context)
|
|
||||||
expect(preset.name).toBe('fast')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should return accurate preset for small datasets', () => {
|
|
||||||
const context: ImportContext = {
|
|
||||||
rowCount: 50
|
|
||||||
}
|
|
||||||
|
|
||||||
const preset = autoDetectPreset(context)
|
|
||||||
expect(preset.name).toBe('accurate')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should return explicit preset for Excel with explicit columns', () => {
|
|
||||||
const context: ImportContext = {
|
|
||||||
fileType: 'excel',
|
|
||||||
hasExplicitColumns: true,
|
|
||||||
rowCount: 500
|
|
||||||
}
|
|
||||||
|
|
||||||
const preset = autoDetectPreset(context)
|
|
||||||
expect(preset.name).toBe('explicit')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should return explicit preset for CSV with explicit columns', () => {
|
|
||||||
const context: ImportContext = {
|
|
||||||
fileType: 'csv',
|
|
||||||
hasExplicitColumns: true
|
|
||||||
}
|
|
||||||
|
|
||||||
const preset = autoDetectPreset(context)
|
|
||||||
expect(preset.name).toBe('explicit')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should return pattern preset for PDF files', () => {
|
|
||||||
const context: ImportContext = {
|
|
||||||
fileType: 'pdf',
|
|
||||||
rowCount: 200
|
|
||||||
}
|
|
||||||
|
|
||||||
const preset = autoDetectPreset(context)
|
|
||||||
expect(preset.name).toBe('pattern')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should return pattern preset for Markdown files', () => {
|
|
||||||
const context: ImportContext = {
|
|
||||||
fileType: 'markdown'
|
|
||||||
}
|
|
||||||
|
|
||||||
const preset = autoDetectPreset(context)
|
|
||||||
expect(preset.name).toBe('pattern')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should return pattern preset for narrative content', () => {
|
|
||||||
const context: ImportContext = {
|
|
||||||
hasNarrativeContent: true,
|
|
||||||
rowCount: 300
|
|
||||||
}
|
|
||||||
|
|
||||||
const preset = autoDetectPreset(context)
|
|
||||||
expect(preset.name).toBe('pattern')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should return pattern preset for long definitions', () => {
|
|
||||||
const context: ImportContext = {
|
|
||||||
avgDefinitionLength: 800,
|
|
||||||
fileType: 'csv'
|
|
||||||
}
|
|
||||||
|
|
||||||
const preset = autoDetectPreset(context)
|
|
||||||
expect(preset.name).toBe('pattern')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should return balanced preset for JSON', () => {
|
|
||||||
const context: ImportContext = {
|
|
||||||
fileType: 'json',
|
|
||||||
rowCount: 500
|
|
||||||
}
|
|
||||||
|
|
||||||
const preset = autoDetectPreset(context)
|
|
||||||
expect(preset.name).toBe('balanced')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should return balanced preset for medium datasets', () => {
|
|
||||||
const context: ImportContext = {
|
|
||||||
fileType: 'excel',
|
|
||||||
rowCount: 2000
|
|
||||||
}
|
|
||||||
|
|
||||||
const preset = autoDetectPreset(context)
|
|
||||||
expect(preset.name).toBe('balanced')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should return balanced preset for empty context', () => {
|
|
||||||
const preset = autoDetectPreset()
|
|
||||||
expect(preset.name).toBe('balanced')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should return balanced preset for unknown file type', () => {
|
|
||||||
const context: ImportContext = {
|
|
||||||
fileType: 'unknown',
|
|
||||||
rowCount: 500
|
|
||||||
}
|
|
||||||
|
|
||||||
const preset = autoDetectPreset(context)
|
|
||||||
expect(preset.name).toBe('balanced')
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('getPreset', () => {
|
|
||||||
it('should get preset by name', () => {
|
|
||||||
expect(getPreset('fast')).toBe(FAST_PRESET)
|
|
||||||
expect(getPreset('balanced')).toBe(BALANCED_PRESET)
|
|
||||||
expect(getPreset('accurate')).toBe(ACCURATE_PRESET)
|
|
||||||
expect(getPreset('explicit')).toBe(EXPLICIT_PRESET)
|
|
||||||
expect(getPreset('pattern')).toBe(PATTERN_PRESET)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should be case-insensitive', () => {
|
|
||||||
expect(getPreset('FAST')).toBe(FAST_PRESET)
|
|
||||||
expect(getPreset('Balanced')).toBe(BALANCED_PRESET)
|
|
||||||
expect(getPreset('EXPLICIT')).toBe(EXPLICIT_PRESET)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should throw error for unknown preset', () => {
|
|
||||||
expect(() => getPreset('unknown')).toThrow('Unknown preset: unknown')
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('getPresetNames', () => {
|
|
||||||
it('should return all preset names', () => {
|
|
||||||
const names = getPresetNames()
|
|
||||||
expect(names).toHaveLength(5)
|
|
||||||
expect(names).toContain('fast')
|
|
||||||
expect(names).toContain('balanced')
|
|
||||||
expect(names).toContain('accurate')
|
|
||||||
expect(names).toContain('explicit')
|
|
||||||
expect(names).toContain('pattern')
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('explainPresetChoice', () => {
|
|
||||||
it('should explain large dataset choice', () => {
|
|
||||||
const context: ImportContext = {
|
|
||||||
rowCount: 15000,
|
|
||||||
fileSize: 12_000_000
|
|
||||||
}
|
|
||||||
|
|
||||||
const explanation = explainPresetChoice(context)
|
|
||||||
expect(explanation).toContain('Large dataset')
|
|
||||||
expect(explanation).toContain('15000 rows')
|
|
||||||
expect(explanation).toContain('fast preset')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should explain small dataset choice', () => {
|
|
||||||
const context: ImportContext = {
|
|
||||||
rowCount: 50
|
|
||||||
}
|
|
||||||
|
|
||||||
const explanation = explainPresetChoice(context)
|
|
||||||
expect(explanation).toContain('Small critical dataset')
|
|
||||||
expect(explanation).toContain('50 rows')
|
|
||||||
expect(explanation).toContain('accurate preset')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should explain explicit columns choice', () => {
|
|
||||||
const context: ImportContext = {
|
|
||||||
fileType: 'excel',
|
|
||||||
hasExplicitColumns: true
|
|
||||||
}
|
|
||||||
|
|
||||||
const explanation = explainPresetChoice(context)
|
|
||||||
expect(explanation).toContain('EXCEL')
|
|
||||||
expect(explanation).toContain('explicit relationship columns')
|
|
||||||
expect(explanation).toContain('explicit preset')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should explain narrative content choice', () => {
|
|
||||||
const context: ImportContext = {
|
|
||||||
fileType: 'pdf',
|
|
||||||
hasNarrativeContent: true
|
|
||||||
}
|
|
||||||
|
|
||||||
const explanation = explainPresetChoice(context)
|
|
||||||
expect(explanation).toContain('Narrative content')
|
|
||||||
expect(explanation).toContain('pattern preset')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should explain default choice', () => {
|
|
||||||
const context: ImportContext = {
|
|
||||||
rowCount: 500
|
|
||||||
}
|
|
||||||
|
|
||||||
const explanation = explainPresetChoice(context)
|
|
||||||
expect(explanation).toContain('balanced preset')
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('createCustomPreset', () => {
|
|
||||||
it('should create custom preset from base', () => {
|
|
||||||
const custom = createCustomPreset('balanced', {
|
|
||||||
name: 'my-custom',
|
|
||||||
batchSize: 2000
|
|
||||||
})
|
|
||||||
|
|
||||||
expect(custom.name).toBe('my-custom')
|
|
||||||
expect(custom.batchSize).toBe(2000)
|
|
||||||
expect(custom.signals).toEqual(BALANCED_PRESET.signals)
|
|
||||||
expect(custom.strategies).toEqual(BALANCED_PRESET.strategies)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should override signals', () => {
|
|
||||||
const custom = createCustomPreset('fast', {
|
|
||||||
signals: {
|
|
||||||
enabled: ['embedding'],
|
|
||||||
weights: { embedding: 1.0, exact: 0, pattern: 0, context: 0 },
|
|
||||||
timeout: 200
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
expect(custom.signals.enabled).toEqual(['embedding'])
|
|
||||||
expect(custom.signals.timeout).toBe(200)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should override strategies', () => {
|
|
||||||
const custom = createCustomPreset('balanced', {
|
|
||||||
strategies: {
|
|
||||||
enabled: ['pattern'],
|
|
||||||
timeout: 500,
|
|
||||||
earlyTermination: false,
|
|
||||||
minConfidence: 0.75
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
expect(custom.strategies.enabled).toEqual(['pattern'])
|
|
||||||
expect(custom.strategies.timeout).toBe(500)
|
|
||||||
expect(custom.strategies.earlyTermination).toBe(false)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should merge partial signal overrides', () => {
|
|
||||||
const custom = createCustomPreset('balanced', {
|
|
||||||
signals: {
|
|
||||||
timeout: 300
|
|
||||||
} as any
|
|
||||||
})
|
|
||||||
|
|
||||||
expect(custom.signals.enabled).toEqual(BALANCED_PRESET.signals.enabled)
|
|
||||||
expect(custom.signals.timeout).toBe(300)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('validatePreset', () => {
|
|
||||||
it('should validate all built-in presets', () => {
|
|
||||||
expect(() => validatePreset(FAST_PRESET)).not.toThrow()
|
|
||||||
expect(() => validatePreset(BALANCED_PRESET)).not.toThrow()
|
|
||||||
expect(() => validatePreset(ACCURATE_PRESET)).not.toThrow()
|
|
||||||
expect(() => validatePreset(EXPLICIT_PRESET)).not.toThrow()
|
|
||||||
expect(() => validatePreset(PATTERN_PRESET)).not.toThrow()
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should reject preset with no signals', () => {
|
|
||||||
const invalid: PresetConfig = {
|
|
||||||
...BALANCED_PRESET,
|
|
||||||
signals: {
|
|
||||||
...BALANCED_PRESET.signals,
|
|
||||||
enabled: []
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
expect(() => validatePreset(invalid)).toThrow('at least one enabled signal')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should reject preset with no strategies', () => {
|
|
||||||
const invalid: PresetConfig = {
|
|
||||||
...BALANCED_PRESET,
|
|
||||||
strategies: {
|
|
||||||
...BALANCED_PRESET.strategies,
|
|
||||||
enabled: []
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
expect(() => validatePreset(invalid)).toThrow('at least one enabled strategy')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should reject preset with invalid weight sum', () => {
|
|
||||||
const invalid: PresetConfig = {
|
|
||||||
...BALANCED_PRESET,
|
|
||||||
signals: {
|
|
||||||
enabled: ['exact', 'embedding'],
|
|
||||||
weights: {
|
|
||||||
exact: 0.3,
|
|
||||||
embedding: 0.5,
|
|
||||||
pattern: 0,
|
|
||||||
context: 0
|
|
||||||
},
|
|
||||||
timeout: 100
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
expect(() => validatePreset(invalid)).toThrow('weights must sum to 1.0')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should reject preset with negative timeout', () => {
|
|
||||||
const invalid: PresetConfig = {
|
|
||||||
...BALANCED_PRESET,
|
|
||||||
signals: {
|
|
||||||
...BALANCED_PRESET.signals,
|
|
||||||
timeout: -100
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
expect(() => validatePreset(invalid)).toThrow('Timeouts must be positive')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should reject preset with invalid batch size', () => {
|
|
||||||
const invalid: PresetConfig = {
|
|
||||||
...BALANCED_PRESET,
|
|
||||||
batchSize: 0
|
|
||||||
}
|
|
||||||
|
|
||||||
expect(() => validatePreset(invalid)).toThrow('Batch size must be positive')
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('formatPreset', () => {
|
|
||||||
it('should format preset for display', () => {
|
|
||||||
const formatted = formatPreset(BALANCED_PRESET)
|
|
||||||
|
|
||||||
expect(formatted).toContain('Preset: balanced')
|
|
||||||
expect(formatted).toContain('Description:')
|
|
||||||
expect(formatted).toContain('Signals:')
|
|
||||||
expect(formatted).toContain('exact: 40%')
|
|
||||||
expect(formatted).toContain('embedding: 35%')
|
|
||||||
expect(formatted).toContain('Strategies:')
|
|
||||||
expect(formatted).toContain('explicit')
|
|
||||||
expect(formatted).toContain('pattern')
|
|
||||||
expect(formatted).toContain('embedding')
|
|
||||||
expect(formatted).toContain('Streaming: false')
|
|
||||||
expect(formatted).toContain('Batch size: 500')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should format fast preset correctly', () => {
|
|
||||||
const formatted = formatPreset(FAST_PRESET)
|
|
||||||
|
|
||||||
expect(formatted).toContain('fast')
|
|
||||||
expect(formatted).toContain('Streaming: true')
|
|
||||||
expect(formatted).toContain('Early termination: true')
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('preset priorities', () => {
|
|
||||||
it('should prioritize size over explicit columns for large datasets', () => {
|
|
||||||
const context: ImportContext = {
|
|
||||||
rowCount: 20000,
|
|
||||||
fileType: 'excel',
|
|
||||||
hasExplicitColumns: true
|
|
||||||
}
|
|
||||||
|
|
||||||
const preset = autoDetectPreset(context)
|
|
||||||
expect(preset.name).toBe('fast') // Size trumps explicit
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should prioritize small size over other factors', () => {
|
|
||||||
const context: ImportContext = {
|
|
||||||
rowCount: 50,
|
|
||||||
fileType: 'pdf',
|
|
||||||
hasNarrativeContent: true
|
|
||||||
}
|
|
||||||
|
|
||||||
const preset = autoDetectPreset(context)
|
|
||||||
expect(preset.name).toBe('accurate') // Small size trumps pattern
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should prioritize explicit columns over narrative for Excel', () => {
|
|
||||||
const context: ImportContext = {
|
|
||||||
rowCount: 500,
|
|
||||||
fileType: 'excel',
|
|
||||||
hasExplicitColumns: true,
|
|
||||||
hasNarrativeContent: true
|
|
||||||
}
|
|
||||||
|
|
||||||
const preset = autoDetectPreset(context)
|
|
||||||
expect(preset.name).toBe('explicit') // Explicit trumps narrative
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('edge cases', () => {
|
|
||||||
it('should handle zero row count', () => {
|
|
||||||
const context: ImportContext = {
|
|
||||||
rowCount: 0,
|
|
||||||
fileType: 'csv'
|
|
||||||
}
|
|
||||||
|
|
||||||
const preset = autoDetectPreset(context)
|
|
||||||
expect(preset.name).toBe('balanced')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should handle boundary row count (exactly 100)', () => {
|
|
||||||
const context: ImportContext = {
|
|
||||||
rowCount: 100
|
|
||||||
}
|
|
||||||
|
|
||||||
const preset = autoDetectPreset(context)
|
|
||||||
expect(preset.name).toBe('balanced') // Not accurate (< 100)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should handle boundary row count (exactly 10000)', () => {
|
|
||||||
const context: ImportContext = {
|
|
||||||
rowCount: 10000
|
|
||||||
}
|
|
||||||
|
|
||||||
const preset = autoDetectPreset(context)
|
|
||||||
expect(preset.name).toBe('balanced') // Not fast (> 10000)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should handle missing hasExplicitColumns flag', () => {
|
|
||||||
const context: ImportContext = {
|
|
||||||
fileType: 'excel',
|
|
||||||
rowCount: 500
|
|
||||||
}
|
|
||||||
|
|
||||||
const preset = autoDetectPreset(context)
|
|
||||||
expect(preset.name).toBe('balanced')
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('real-world scenarios', () => {
|
|
||||||
it('should handle glossary correctly', () => {
|
|
||||||
const context: ImportContext = {
|
|
||||||
fileType: 'excel',
|
|
||||||
rowCount: 567,
|
|
||||||
hasExplicitColumns: true, // Has "Related Terms" column
|
|
||||||
fileSize: 50_000
|
|
||||||
}
|
|
||||||
|
|
||||||
const preset = autoDetectPreset(context)
|
|
||||||
expect(preset.name).toBe('explicit')
|
|
||||||
|
|
||||||
const explanation = explainPresetChoice(context)
|
|
||||||
expect(explanation).toContain('explicit')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should handle large CSV import', () => {
|
|
||||||
const context: ImportContext = {
|
|
||||||
fileType: 'csv',
|
|
||||||
rowCount: 50000,
|
|
||||||
fileSize: 25_000_000
|
|
||||||
}
|
|
||||||
|
|
||||||
const preset = autoDetectPreset(context)
|
|
||||||
expect(preset.name).toBe('fast')
|
|
||||||
expect(preset.streaming).toBe(true)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should handle PDF documentation', () => {
|
|
||||||
const context: ImportContext = {
|
|
||||||
fileType: 'pdf',
|
|
||||||
rowCount: 150,
|
|
||||||
hasNarrativeContent: true,
|
|
||||||
avgDefinitionLength: 600
|
|
||||||
}
|
|
||||||
|
|
||||||
const preset = autoDetectPreset(context)
|
|
||||||
expect(preset.name).toBe('pattern')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should handle JSON API import', () => {
|
|
||||||
const context: ImportContext = {
|
|
||||||
fileType: 'json',
|
|
||||||
rowCount: 1000,
|
|
||||||
fileSize: 500_000
|
|
||||||
}
|
|
||||||
|
|
||||||
const preset = autoDetectPreset(context)
|
|
||||||
expect(preset.name).toBe('balanced')
|
|
||||||
})
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
@ -5,7 +5,7 @@
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { describe, it, expect } from 'vitest'
|
import { describe, it, expect } from 'vitest'
|
||||||
import { VirtualFileSystem } from '../../src/vfs/index.js'
|
import { VirtualFileSystem } from '../../src/vfs/VirtualFileSystem.js'
|
||||||
import { Brainy } from '../../src/brainy.js'
|
import { Brainy } from '../../src/brainy.js'
|
||||||
|
|
||||||
describe('VFS Initialization', () => {
|
describe('VFS Initialization', () => {
|
||||||
|
|
|
||||||
|
|
@ -6,7 +6,7 @@
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||||
import { VirtualFileSystem } from '../../src/vfs/index.js'
|
import { VirtualFileSystem } from '../../src/vfs/VirtualFileSystem.js'
|
||||||
import { Brainy } from '../../src/brainy.js'
|
import { Brainy } from '../../src/brainy.js'
|
||||||
import { VFSErrorCode } from '../../src/vfs/types.js'
|
import { VFSErrorCode } from '../../src/vfs/types.js'
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue