feat: remove legacy ImportManager, standardize getStats() API
- Removed ImportManager class and exports (use brain.import() instead) - Fixed all documentation: getStatistics() → getStats() - Updated 41 files across codebase for consistency - Removed ImportManager section from API docs - Added v3.30.0 migration guide to CHANGELOG Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
parent
68c989e4f7
commit
58daf09403
31 changed files with 87 additions and 481 deletions
|
|
@ -270,7 +270,7 @@ export class APIServerAugmentation extends BaseAugmentation {
|
|||
// Statistics endpoint
|
||||
app.get('/api/stats', async (_req: any, res: any) => {
|
||||
try {
|
||||
const stats = await this.context!.brain.getStatistics()
|
||||
const stats = this.context!.brain.getStats()
|
||||
res.json({ success: true, stats })
|
||||
} catch (error: any) {
|
||||
res.status(500).json({ success: false, error: error.message })
|
||||
|
|
|
|||
|
|
@ -1,342 +0,0 @@
|
|||
/**
|
||||
* Import Manager - Comprehensive data import with intelligent type detection
|
||||
*
|
||||
* Handles multiple data sources:
|
||||
* - Direct data (objects, arrays)
|
||||
* - Files (JSON, CSV, text)
|
||||
* - URLs (fetch and parse)
|
||||
* - Streams (for large files)
|
||||
*
|
||||
* Uses NeuralImportAugmentation for intelligent processing
|
||||
*/
|
||||
|
||||
import { NounType, VerbType } from './types/graphTypes.js'
|
||||
import { NeuralImportAugmentation } from './augmentations/neuralImport.js'
|
||||
import { BrainyTypes } from './augmentations/typeMatching/brainyTypes.js'
|
||||
import * as fs from './universal/fs.js'
|
||||
import * as path from './universal/path.js'
|
||||
import { prodLog } from './utils/logger.js'
|
||||
|
||||
export interface ImportOptions {
|
||||
// Source type
|
||||
source?: 'data' | 'file' | 'url' | 'auto'
|
||||
|
||||
// Data format
|
||||
format?: 'json' | 'csv' | 'text' | 'yaml' | 'auto'
|
||||
|
||||
// Processing
|
||||
batchSize?: number
|
||||
autoDetect?: boolean
|
||||
typeHint?: NounType
|
||||
extractRelationships?: boolean
|
||||
|
||||
// CSV specific
|
||||
csvDelimiter?: string
|
||||
csvHeaders?: boolean
|
||||
|
||||
// Performance
|
||||
parallel?: boolean
|
||||
maxConcurrency?: number
|
||||
}
|
||||
|
||||
export interface ImportResult {
|
||||
success: boolean
|
||||
nouns: string[]
|
||||
verbs: string[]
|
||||
errors: string[]
|
||||
stats: {
|
||||
total: number
|
||||
imported: number
|
||||
failed: number
|
||||
relationships: number
|
||||
}
|
||||
}
|
||||
|
||||
export class ImportManager {
|
||||
private neuralImport: NeuralImportAugmentation
|
||||
private typeMatcher: BrainyTypes | null = null
|
||||
private brain: any // Brainy instance
|
||||
|
||||
constructor(brain: any) {
|
||||
this.brain = brain
|
||||
this.neuralImport = new NeuralImportAugmentation()
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the import manager
|
||||
*/
|
||||
async init(): Promise<void> {
|
||||
// Initialize neural import with proper context
|
||||
const context = {
|
||||
brain: this.brain,
|
||||
storage: this.brain.storage,
|
||||
config: {},
|
||||
log: (message: string, level?: string) => {
|
||||
if (level === 'error') {
|
||||
prodLog.error(message)
|
||||
} else if (level === 'warn') {
|
||||
prodLog.warn(message)
|
||||
} else {
|
||||
prodLog.info(message)
|
||||
}
|
||||
}
|
||||
}
|
||||
await this.neuralImport.initialize(context as any)
|
||||
|
||||
// Get type matcher
|
||||
const { getBrainyTypes } = await import('./augmentations/typeMatching/brainyTypes.js')
|
||||
this.typeMatcher = await getBrainyTypes()
|
||||
}
|
||||
|
||||
/**
|
||||
* Main import method - handles all sources
|
||||
*/
|
||||
async import(
|
||||
source: string | Buffer | any[] | any,
|
||||
options: ImportOptions = {}
|
||||
): Promise<ImportResult> {
|
||||
const result: ImportResult = {
|
||||
success: false,
|
||||
nouns: [],
|
||||
verbs: [],
|
||||
errors: [],
|
||||
stats: {
|
||||
total: 0,
|
||||
imported: 0,
|
||||
failed: 0,
|
||||
relationships: 0
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
// Detect source type
|
||||
const sourceType = await this.detectSourceType(source, options.source)
|
||||
|
||||
// Get data based on source type
|
||||
let data: any
|
||||
let format = options.format || 'auto'
|
||||
|
||||
switch (sourceType) {
|
||||
case 'url':
|
||||
data = await this.fetchFromUrl(source as string)
|
||||
break
|
||||
|
||||
case 'file':
|
||||
const filePath = source as string
|
||||
data = await this.readFile(filePath)
|
||||
if (format === 'auto') {
|
||||
format = this.detectFormatFromPath(filePath)
|
||||
}
|
||||
break
|
||||
|
||||
case 'data':
|
||||
default:
|
||||
data = source
|
||||
break
|
||||
}
|
||||
|
||||
// Process data through neural import
|
||||
let items: any[]
|
||||
let relationships: any[] = []
|
||||
|
||||
if (Buffer.isBuffer(data) || typeof data === 'string') {
|
||||
// Use neural import for parsing and analysis
|
||||
const analysis = await this.neuralImport.getNeuralAnalysis(data, format as string)
|
||||
|
||||
// Extract items and relationships
|
||||
items = analysis.detectedEntities.map(entity => ({
|
||||
data: entity.originalData,
|
||||
type: entity.nounType,
|
||||
confidence: entity.confidence,
|
||||
id: entity.suggestedId
|
||||
}))
|
||||
|
||||
if (options.extractRelationships !== false) {
|
||||
relationships = analysis.detectedRelationships
|
||||
}
|
||||
|
||||
// Log insights
|
||||
for (const insight of analysis.insights) {
|
||||
prodLog.info(`🧠 ${insight.description} (confidence: ${insight.confidence})`)
|
||||
}
|
||||
} else if (Array.isArray(data)) {
|
||||
items = data
|
||||
} else {
|
||||
items = [data]
|
||||
}
|
||||
|
||||
result.stats.total = items.length
|
||||
|
||||
// Import items in batches
|
||||
const batchSize = options.batchSize || 50
|
||||
for (let i = 0; i < items.length; i += batchSize) {
|
||||
const batch = items.slice(i, i + batchSize)
|
||||
|
||||
// Process batch in parallel if enabled
|
||||
const promises = batch.map(async (item) => {
|
||||
try {
|
||||
// Detect type if needed
|
||||
let nounType = item.type || options.typeHint
|
||||
if (!nounType && options.autoDetect !== false && this.typeMatcher) {
|
||||
const match = await this.typeMatcher.matchNounType(item.data || item)
|
||||
nounType = match.type
|
||||
}
|
||||
|
||||
// Prepare the data to import
|
||||
const dataToImport = item.data || item
|
||||
|
||||
// Create metadata combining original data with import metadata
|
||||
const metadata: any = {
|
||||
...(typeof dataToImport === 'object' ? dataToImport : {}),
|
||||
...(item.data?.metadata || {}),
|
||||
nounType,
|
||||
_importedAt: new Date().toISOString(),
|
||||
_confidence: item.confidence
|
||||
}
|
||||
|
||||
// Add to brain using modern API signature
|
||||
const id = await this.brain.add({ data: dataToImport, type: nounType || 'content', metadata })
|
||||
result.nouns.push(id)
|
||||
result.stats.imported++
|
||||
return id
|
||||
} catch (error: any) {
|
||||
result.errors.push(`Failed to import item: ${error.message}`)
|
||||
result.stats.failed++
|
||||
return null
|
||||
}
|
||||
})
|
||||
|
||||
if (options.parallel !== false) {
|
||||
await Promise.all(promises)
|
||||
} else {
|
||||
for (const promise of promises) {
|
||||
await promise
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Import relationships
|
||||
for (const rel of relationships) {
|
||||
try {
|
||||
// Match verb type if needed
|
||||
let verbType = rel.verbType
|
||||
if (!Object.values(VerbType).includes(verbType) && this.typeMatcher) {
|
||||
const match = await this.typeMatcher.matchVerbType(
|
||||
{ id: rel.sourceId },
|
||||
{ id: rel.targetId },
|
||||
rel.verbType
|
||||
)
|
||||
verbType = match.type
|
||||
}
|
||||
|
||||
const verbId = await this.brain.relate({
|
||||
from: rel.sourceId,
|
||||
to: rel.targetId,
|
||||
type: verbType as VerbType,
|
||||
metadata: rel.metadata,
|
||||
weight: rel.weight
|
||||
})
|
||||
|
||||
result.verbs.push(verbId)
|
||||
result.stats.relationships++
|
||||
} catch (error: any) {
|
||||
result.errors.push(`Failed to create relationship: ${error.message}`)
|
||||
}
|
||||
}
|
||||
|
||||
result.success = result.stats.imported > 0
|
||||
|
||||
prodLog.info(`✨ Import complete: ${result.stats.imported}/${result.stats.total} items, ${result.stats.relationships} relationships`)
|
||||
|
||||
} catch (error: any) {
|
||||
result.errors.push(`Import failed: ${error.message}`)
|
||||
prodLog.error('Import failed:', error)
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* Import from file
|
||||
*/
|
||||
async importFile(filePath: string, options: ImportOptions = {}): Promise<ImportResult> {
|
||||
return this.import(filePath, { ...options, source: 'file' })
|
||||
}
|
||||
|
||||
/**
|
||||
* Import from URL
|
||||
*/
|
||||
async importUrl(url: string, options: ImportOptions = {}): Promise<ImportResult> {
|
||||
return this.import(url, { ...options, source: 'url' })
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect source type
|
||||
*/
|
||||
private async detectSourceType(source: any, hint?: string): Promise<'url' | 'file' | 'data'> {
|
||||
if (hint && hint !== 'auto') {
|
||||
return hint as any
|
||||
}
|
||||
|
||||
if (typeof source === 'string') {
|
||||
// Check if URL
|
||||
if (source.startsWith('http://') || source.startsWith('https://')) {
|
||||
return 'url'
|
||||
}
|
||||
|
||||
// Check if file path exists
|
||||
try {
|
||||
if (await fs.exists(source)) {
|
||||
return 'file'
|
||||
}
|
||||
} catch (error) {
|
||||
// File system check failed, not a file path
|
||||
console.debug('File path check failed:', error)
|
||||
}
|
||||
}
|
||||
|
||||
return 'data'
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect format from file path
|
||||
*/
|
||||
private detectFormatFromPath(filePath: string): 'json' | 'csv' | 'text' | 'yaml' | 'auto' {
|
||||
const ext = path.extname(filePath).toLowerCase()
|
||||
switch (ext) {
|
||||
case '.json': return 'json'
|
||||
case '.csv': return 'csv'
|
||||
case '.txt': return 'text'
|
||||
case '.md': return 'text'
|
||||
case '.yaml':
|
||||
case '.yml': return 'yaml'
|
||||
default: return 'auto'
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Read file
|
||||
*/
|
||||
private async readFile(filePath: string): Promise<Buffer> {
|
||||
const content = await fs.readFile(filePath, 'utf8')
|
||||
return Buffer.from(content, 'utf8')
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch from URL
|
||||
*/
|
||||
private async fetchFromUrl(url: string): Promise<string> {
|
||||
const response = await fetch(url)
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch ${url}: ${response.statusText}`)
|
||||
}
|
||||
return response.text()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create an import manager instance
|
||||
*/
|
||||
export function createImportManager(brain: any): ImportManager {
|
||||
return new ImportManager(brain)
|
||||
}
|
||||
|
|
@ -70,9 +70,7 @@ export type {
|
|||
NeuralImportOptions
|
||||
} from './cortex/neuralImport.js'
|
||||
|
||||
// Export Import Manager (comprehensive data import)
|
||||
export { ImportManager, createImportManager } from './importManager.js'
|
||||
export type { ImportOptions, ImportResult } from './importManager.js'
|
||||
// Import Manager removed - use brain.import() instead (available on all Brainy instances)
|
||||
|
||||
// Augmentation types are already exported later in the file
|
||||
|
||||
|
|
|
|||
|
|
@ -279,8 +279,8 @@ export class NeuralAPI {
|
|||
*/
|
||||
async outliers(threshold: number = 0.3): Promise<string[]> {
|
||||
// Get all items
|
||||
const stats = await this.brain.getStatistics()
|
||||
const totalItems = stats.nounCount
|
||||
const stats = this.brain.getStats()
|
||||
const totalItems = stats.entities.total
|
||||
|
||||
if (totalItems === 0) return []
|
||||
|
||||
|
|
@ -539,8 +539,8 @@ export class NeuralAPI {
|
|||
// Enterprise clustering implementations
|
||||
private async getOptimalClusteringLevel(): Promise<number> {
|
||||
// Analyze dataset size and return optimal HNSW level
|
||||
const stats = await this.brain.getStatistics()
|
||||
const itemCount = stats.nounCount
|
||||
const stats = this.brain.getStats()
|
||||
const itemCount = stats.entities.total
|
||||
|
||||
if (itemCount < 1000) return 0
|
||||
if (itemCount < 10000) return 1
|
||||
|
|
@ -551,8 +551,8 @@ export class NeuralAPI {
|
|||
private async getHNSWLevelNodes(level: number): Promise<any[]> {
|
||||
// Get nodes from specific HNSW level
|
||||
// For now, use search to get a representative sample
|
||||
const stats = await this.brain.getStatistics()
|
||||
const sampleSize = Math.min(100, Math.floor(stats.nounCount / (level + 1)))
|
||||
const stats = this.brain.getStats()
|
||||
const sampleSize = Math.min(100, Math.floor(stats.entities.total / (level + 1)))
|
||||
|
||||
// Use search with a general query to get representative items
|
||||
const queryVector = await this.brain.embed('data information content')
|
||||
|
|
@ -568,8 +568,8 @@ export class NeuralAPI {
|
|||
|
||||
private async getSample(size: number, strategy: string): Promise<any[]> {
|
||||
// Use search to get a sample of items
|
||||
const stats = await this.brain.getStatistics()
|
||||
const maxSize = Math.min(size * 3, stats.nounCount) // Get more than needed for sampling
|
||||
const stats = this.brain.getStats()
|
||||
const maxSize = Math.min(size * 3, stats.entities.total) // Get more than needed for sampling
|
||||
const queryVector = await this.brain.embed('sample data content')
|
||||
const allItems = await this.brain.search(queryVector, maxSize)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue