chore: recovery checkpoint - v3.0 API successfully recovered
CRITICAL CHECKPOINT - DO NOT PUSH TO GITHUB Recovery Status: - Successfully recovered brainy.ts from compiled JavaScript - All core v3.0 API methods functional (add, get, update, delete, relate, find, etc.) - Neural subsystem intact (562KB embedded patterns, NLP working) - Augmentation pipeline operational (20+ augmentations) - HNSW clustering system complete - Triple Intelligence compiled (needs constructor fix) - Test suite validates functionality Changes preserved: - 898 files with changes from last 3 days - 144,475 insertions - All augmentation improvements - All test coverage enhancements - Complete v3.0 feature set This is a LOCAL checkpoint only - contains recovered work after corruption incident. Created backup in .backups/brainy-full-20250910-151314.tar.gz Branch: recovery-checkpoint-20250910-151433 Date: Wed Sep 10 03:18:04 PM PDT 2025
This commit is contained in:
parent
f65455fb22
commit
8ff382ca3b
895 changed files with 143654 additions and 28268 deletions
258
.recovery-workspace/dist-backup-20250910-141917/importManager.js
Normal file
258
.recovery-workspace/dist-backup-20250910-141917/importManager.js
Normal file
|
|
@ -0,0 +1,258 @@
|
|||
/**
|
||||
* Import Manager - Comprehensive data import with intelligent type detection
|
||||
*
|
||||
* Handles multiple data sources:
|
||||
* - Direct data (objects, arrays)
|
||||
* - Files (JSON, CSV, text)
|
||||
* - URLs (fetch and parse)
|
||||
* - Streams (for large files)
|
||||
*
|
||||
* Uses NeuralImportAugmentation for intelligent processing
|
||||
*/
|
||||
import { VerbType } from './types/graphTypes.js';
|
||||
import { NeuralImportAugmentation } from './augmentations/neuralImport.js';
|
||||
import * as fs from './universal/fs.js';
|
||||
import * as path from './universal/path.js';
|
||||
import { prodLog } from './utils/logger.js';
|
||||
export class ImportManager {
|
||||
constructor(brain) {
|
||||
this.typeMatcher = null;
|
||||
this.brain = brain;
|
||||
this.neuralImport = new NeuralImportAugmentation();
|
||||
}
|
||||
/**
|
||||
* Initialize the import manager
|
||||
*/
|
||||
async init() {
|
||||
// Initialize neural import with proper context
|
||||
const context = {
|
||||
brain: this.brain,
|
||||
storage: this.brain.storage,
|
||||
config: {},
|
||||
log: (message, level) => {
|
||||
if (level === 'error') {
|
||||
prodLog.error(message);
|
||||
}
|
||||
else if (level === 'warn') {
|
||||
prodLog.warn(message);
|
||||
}
|
||||
else {
|
||||
prodLog.info(message);
|
||||
}
|
||||
}
|
||||
};
|
||||
await this.neuralImport.initialize(context);
|
||||
// Get type matcher
|
||||
const { getBrainyTypes } = await import('./augmentations/typeMatching/brainyTypes.js');
|
||||
this.typeMatcher = await getBrainyTypes();
|
||||
}
|
||||
/**
|
||||
* Main import method - handles all sources
|
||||
*/
|
||||
async import(source, options = {}) {
|
||||
const result = {
|
||||
success: false,
|
||||
nouns: [],
|
||||
verbs: [],
|
||||
errors: [],
|
||||
stats: {
|
||||
total: 0,
|
||||
imported: 0,
|
||||
failed: 0,
|
||||
relationships: 0
|
||||
}
|
||||
};
|
||||
try {
|
||||
// Detect source type
|
||||
const sourceType = await this.detectSourceType(source, options.source);
|
||||
// Get data based on source type
|
||||
let data;
|
||||
let format = options.format || 'auto';
|
||||
switch (sourceType) {
|
||||
case 'url':
|
||||
data = await this.fetchFromUrl(source);
|
||||
break;
|
||||
case 'file':
|
||||
const filePath = source;
|
||||
data = await this.readFile(filePath);
|
||||
if (format === 'auto') {
|
||||
format = this.detectFormatFromPath(filePath);
|
||||
}
|
||||
break;
|
||||
case 'data':
|
||||
default:
|
||||
data = source;
|
||||
break;
|
||||
}
|
||||
// Process data through neural import
|
||||
let items;
|
||||
let relationships = [];
|
||||
if (Buffer.isBuffer(data) || typeof data === 'string') {
|
||||
// Use neural import for parsing and analysis
|
||||
const analysis = await this.neuralImport.getNeuralAnalysis(data, format);
|
||||
// Extract items and relationships
|
||||
items = analysis.detectedEntities.map(entity => ({
|
||||
data: entity.originalData,
|
||||
type: entity.nounType,
|
||||
confidence: entity.confidence,
|
||||
id: entity.suggestedId
|
||||
}));
|
||||
if (options.extractRelationships !== false) {
|
||||
relationships = analysis.detectedRelationships;
|
||||
}
|
||||
// Log insights
|
||||
for (const insight of analysis.insights) {
|
||||
prodLog.info(`🧠 ${insight.description} (confidence: ${insight.confidence})`);
|
||||
}
|
||||
}
|
||||
else if (Array.isArray(data)) {
|
||||
items = data;
|
||||
}
|
||||
else {
|
||||
items = [data];
|
||||
}
|
||||
result.stats.total = items.length;
|
||||
// Import items in batches
|
||||
const batchSize = options.batchSize || 50;
|
||||
for (let i = 0; i < items.length; i += batchSize) {
|
||||
const batch = items.slice(i, i + batchSize);
|
||||
// Process batch in parallel if enabled
|
||||
const promises = batch.map(async (item) => {
|
||||
try {
|
||||
// Detect type if needed
|
||||
let nounType = item.type || options.typeHint;
|
||||
if (!nounType && options.autoDetect !== false && this.typeMatcher) {
|
||||
const match = await this.typeMatcher.matchNounType(item.data || item);
|
||||
nounType = match.type;
|
||||
}
|
||||
// Prepare the data to import
|
||||
const dataToImport = item.data || item;
|
||||
// Create metadata combining original data with import metadata
|
||||
const metadata = {
|
||||
...(typeof dataToImport === 'object' ? dataToImport : {}),
|
||||
...(item.data?.metadata || {}),
|
||||
nounType,
|
||||
_importedAt: new Date().toISOString(),
|
||||
_confidence: item.confidence
|
||||
};
|
||||
// Add to brain using proper API signature: addNoun(vectorOrData, nounType, metadata)
|
||||
const id = await this.brain.addNoun(dataToImport, nounType || 'content', metadata);
|
||||
result.nouns.push(id);
|
||||
result.stats.imported++;
|
||||
return id;
|
||||
}
|
||||
catch (error) {
|
||||
result.errors.push(`Failed to import item: ${error.message}`);
|
||||
result.stats.failed++;
|
||||
return null;
|
||||
}
|
||||
});
|
||||
if (options.parallel !== false) {
|
||||
await Promise.all(promises);
|
||||
}
|
||||
else {
|
||||
for (const promise of promises) {
|
||||
await promise;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Import relationships
|
||||
for (const rel of relationships) {
|
||||
try {
|
||||
// Match verb type if needed
|
||||
let verbType = rel.verbType;
|
||||
if (!Object.values(VerbType).includes(verbType) && this.typeMatcher) {
|
||||
const match = await this.typeMatcher.matchVerbType({ id: rel.sourceId }, { id: rel.targetId }, rel.verbType);
|
||||
verbType = match.type;
|
||||
}
|
||||
const verbId = await this.brain.addVerb(rel.sourceId, rel.targetId, verbType, rel.metadata, rel.weight);
|
||||
result.verbs.push(verbId);
|
||||
result.stats.relationships++;
|
||||
}
|
||||
catch (error) {
|
||||
result.errors.push(`Failed to create relationship: ${error.message}`);
|
||||
}
|
||||
}
|
||||
result.success = result.stats.imported > 0;
|
||||
prodLog.info(`✨ Import complete: ${result.stats.imported}/${result.stats.total} items, ${result.stats.relationships} relationships`);
|
||||
}
|
||||
catch (error) {
|
||||
result.errors.push(`Import failed: ${error.message}`);
|
||||
prodLog.error('Import failed:', error);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
/**
|
||||
* Import from file
|
||||
*/
|
||||
async importFile(filePath, options = {}) {
|
||||
return this.import(filePath, { ...options, source: 'file' });
|
||||
}
|
||||
/**
|
||||
* Import from URL
|
||||
*/
|
||||
async importUrl(url, options = {}) {
|
||||
return this.import(url, { ...options, source: 'url' });
|
||||
}
|
||||
/**
|
||||
* Detect source type
|
||||
*/
|
||||
async detectSourceType(source, hint) {
|
||||
if (hint && hint !== 'auto') {
|
||||
return hint;
|
||||
}
|
||||
if (typeof source === 'string') {
|
||||
// Check if URL
|
||||
if (source.startsWith('http://') || source.startsWith('https://')) {
|
||||
return 'url';
|
||||
}
|
||||
// Check if file path exists
|
||||
try {
|
||||
if (await fs.exists(source)) {
|
||||
return 'file';
|
||||
}
|
||||
}
|
||||
catch { }
|
||||
}
|
||||
return 'data';
|
||||
}
|
||||
/**
|
||||
* Detect format from file path
|
||||
*/
|
||||
detectFormatFromPath(filePath) {
|
||||
const ext = path.extname(filePath).toLowerCase();
|
||||
switch (ext) {
|
||||
case '.json': return 'json';
|
||||
case '.csv': return 'csv';
|
||||
case '.txt': return 'text';
|
||||
case '.md': return 'text';
|
||||
case '.yaml':
|
||||
case '.yml': return 'yaml';
|
||||
default: return 'auto';
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Read file
|
||||
*/
|
||||
async readFile(filePath) {
|
||||
const content = await fs.readFile(filePath, 'utf8');
|
||||
return Buffer.from(content, 'utf8');
|
||||
}
|
||||
/**
|
||||
* Fetch from URL
|
||||
*/
|
||||
async fetchFromUrl(url) {
|
||||
const response = await fetch(url);
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch ${url}: ${response.statusText}`);
|
||||
}
|
||||
return response.text();
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Create an import manager instance
|
||||
*/
|
||||
export function createImportManager(brain) {
|
||||
return new ImportManager(brain);
|
||||
}
|
||||
//# sourceMappingURL=importManager.js.map
|
||||
Loading…
Add table
Add a link
Reference in a new issue