soulcraft/brainy
2
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

chore: recovery checkpoint - v3.0 API successfully recovered

Browse source 
CRITICAL CHECKPOINT - DO NOT PUSH TO GITHUB

Recovery Status:
- Successfully recovered brainy.ts from compiled JavaScript
- All core v3.0 API methods functional (add, get, update, delete, relate, find, etc.)
- Neural subsystem intact (562KB embedded patterns, NLP working)
- Augmentation pipeline operational (20+ augmentations)
- HNSW clustering system complete
- Triple Intelligence compiled (needs constructor fix)
- Test suite validates functionality

Changes preserved:
- 898 files with changes from last 3 days
- 144,475 insertions
- All augmentation improvements
- All test coverage enhancements
- Complete v3.0 feature set

This is a LOCAL checkpoint only - contains recovered work after corruption incident.
Created backup in .backups/brainy-full-20250910-151314.tar.gz

Branch: recovery-checkpoint-20250910-151433
Date: Wed Sep 10 03:18:04 PM PDT 2025
This commit is contained in:
David Snelling 2025-09-10 15:18:04 -07:00
parent f65455fb22
commit 8ff382ca3b
895 changed files with 143654 additions and 28268 deletions
Download patch file Download diff file Expand all files Collapse all files

67
.recovery-workspace/dist-backup-20250910-141917/api/ConfigAPI.d.ts vendored Normal file
View file

@ -0,0 +1,67 @@
/**
* Configuration API for Brainy 3.0
* Provides configuration storage with optional encryption
*/
import { StorageAdapter } from '../coreTypes.js';
export interface ConfigOptions {
encrypt?: boolean;
decrypt?: boolean;
}
export interface ConfigEntry {
key: string;
value: any;
encrypted: boolean;
createdAt: number;
updatedAt: number;
}
export declare class ConfigAPI {
private storage;
private security;
private configCache;
private CONFIG_NOUN_PREFIX;
constructor(storage: StorageAdapter);
/**
* Set a configuration value with optional encryption
*/
set(params: {
key: string;
value: any;
encrypt?: boolean;
}): Promise<void>;
/**
* Get a configuration value with optional decryption
*/
get(params: {
key: string;
decrypt?: boolean;
defaultValue?: any;
}): Promise<any>;
/**
* Delete a configuration value
*/
delete(key: string): Promise<void>;
/**
* List all configuration keys
*/
list(): Promise<string[]>;
/**
* Check if a configuration key exists
*/
has(key: string): Promise<boolean>;
/**
* Clear all configuration
*/
clear(): Promise<void>;
/**
* Export all configuration
*/
export(): Promise<Record<string, ConfigEntry>>;
/**
* Import configuration
*/
import(config: Record<string, ConfigEntry>): Promise<void>;
/**
* Get raw config entry (without decryption)
*/
private getEntry;
}

166
.recovery-workspace/dist-backup-20250910-141917/api/ConfigAPI.js Normal file
View file

@ -0,0 +1,166 @@
/**
* Configuration API for Brainy 3.0
* Provides configuration storage with optional encryption
*/
import { SecurityAPI } from './SecurityAPI.js';
export class ConfigAPI {
constructor(storage) {
this.storage = storage;
this.configCache = new Map();
this.CONFIG_NOUN_PREFIX = '_config_';
this.security = new SecurityAPI();
}
/**
* Set a configuration value with optional encryption
*/
async set(params) {
const { key, value, encrypt = false } = params;
// Serialize and optionally encrypt the value
let storedValue = value;
if (typeof value !== 'string') {
storedValue = JSON.stringify(value);
}
if (encrypt) {
storedValue = await this.security.encrypt(storedValue);
}
// Create config entry
const entry = {
key,
value: storedValue,
encrypted: encrypt,
createdAt: this.configCache.get(key)?.createdAt || Date.now(),
updatedAt: Date.now()
};
// Store in cache
this.configCache.set(key, entry);
// Persist to storage
const configId = this.CONFIG_NOUN_PREFIX + key;
await this.storage.saveMetadata(configId, entry);
}
/**
* Get a configuration value with optional decryption
*/
async get(params) {
const { key, decrypt, defaultValue } = params;
// Check cache first
let entry = this.configCache.get(key);
// If not in cache, load from storage
if (!entry) {
const configId = this.CONFIG_NOUN_PREFIX + key;
const metadata = await this.storage.getMetadata(configId);
if (!metadata) {
return defaultValue;
}
entry = metadata;
this.configCache.set(key, entry);
}
let value = entry.value;
// Decrypt if needed
const shouldDecrypt = decrypt !== undefined ? decrypt : entry.encrypted;
if (shouldDecrypt && entry.encrypted && typeof value === 'string') {
value = await this.security.decrypt(value);
}
// Try to parse JSON if it looks like JSON
if (typeof value === 'string' && (value.startsWith('{') || value.startsWith('['))) {
try {
value = JSON.parse(value);
}
catch {
// Not JSON, return as string
}
}
return value;
}
/**
* Delete a configuration value
*/
async delete(key) {
// Remove from cache
this.configCache.delete(key);
// Remove from storage
const configId = this.CONFIG_NOUN_PREFIX + key;
await this.storage.saveMetadata(configId, null);
}
/**
* List all configuration keys
*/
async list() {
// Get all metadata keys from storage
const allMetadata = await this.storage.getMetadata('');
if (!allMetadata || typeof allMetadata !== 'object') {
return [];
}
// Filter for config keys
const configKeys = [];
for (const key of Object.keys(allMetadata)) {
if (key.startsWith(this.CONFIG_NOUN_PREFIX)) {
configKeys.push(key.substring(this.CONFIG_NOUN_PREFIX.length));
}
}
return configKeys;
}
/**
* Check if a configuration key exists
*/
async has(key) {
if (this.configCache.has(key)) {
return true;
}
const configId = this.CONFIG_NOUN_PREFIX + key;
const metadata = await this.storage.getMetadata(configId);
return metadata !== null && metadata !== undefined;
}
/**
* Clear all configuration
*/
async clear() {
// Clear cache
this.configCache.clear();
// Clear from storage
const keys = await this.list();
for (const key of keys) {
await this.delete(key);
}
}
/**
* Export all configuration
*/
async export() {
const keys = await this.list();
const config = {};
for (const key of keys) {
const entry = await this.getEntry(key);
if (entry) {
config[key] = entry;
}
}
return config;
}
/**
* Import configuration
*/
async import(config) {
for (const [key, entry] of Object.entries(config)) {
this.configCache.set(key, entry);
const configId = this.CONFIG_NOUN_PREFIX + key;
await this.storage.saveMetadata(configId, entry);
}
}
/**
* Get raw config entry (without decryption)
*/
async getEntry(key) {
if (this.configCache.has(key)) {
return this.configCache.get(key);
}
const configId = this.CONFIG_NOUN_PREFIX + key;
const metadata = await this.storage.getMetadata(configId);
if (!metadata) {
return null;
}
const entry = metadata;
this.configCache.set(key, entry);
return entry;
}
}
//# sourceMappingURL=ConfigAPI.js.map

1
.recovery-workspace/dist-backup-20250910-141917/api/ConfigAPI.js.map Normal file
View file

@ -0,0 +1 @@
{"version":3,"file":"ConfigAPI.js","sourceRoot":"","sources":["../../src/api/ConfigAPI.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAA;AAgB9C,MAAM,OAAO,SAAS;IAKpB,YAAoB,OAAuB;QAAvB,YAAO,GAAP,OAAO,CAAgB;QAHnC,gBAAW,GAA6B,IAAI,GAAG,EAAE,CAAA;QACjD,uBAAkB,GAAG,UAAU,CAAA;QAGrC,IAAI,CAAC,QAAQ,GAAG,IAAI,WAAW,EAAE,CAAA;IACnC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,GAAG,CAAC,MAIT;QACC,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,GAAG,KAAK,EAAE,GAAG,MAAM,CAAA;QAE9C,6CAA6C;QAC7C,IAAI,WAAW,GAAQ,KAAK,CAAA;QAC5B,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC9B,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;QACrC,CAAC;QAED,IAAI,OAAO,EAAE,CAAC;YACZ,WAAW,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,WAAW,CAAC,CAAA;QACxD,CAAC;QAED,sBAAsB;QACtB,MAAM,KAAK,GAAgB;YACzB,GAAG;YACH,KAAK,EAAE,WAAW;YAClB,SAAS,EAAE,OAAO;YAClB,SAAS,EAAE,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,SAAS,IAAI,IAAI,CAAC,GAAG,EAAE;YAC7D,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;SACtB,CAAA;QAED,iBAAiB;QACjB,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;QAEhC,qBAAqB;QACrB,MAAM,QAAQ,GAAG,IAAI,CAAC,kBAAkB,GAAG,GAAG,CAAA;QAC9C,MAAM,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAA;IAClD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,GAAG,CAAC,MAIT;QACC,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,YAAY,EAAE,GAAG,MAAM,CAAA;QAE7C,oBAAoB;QACpB,IAAI,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QAErC,qCAAqC;QACrC,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,MAAM,QAAQ,GAAG,IAAI,CAAC,kBAAkB,GAAG,GAAG,CAAA;YAC9C,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAA;YAEzD,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,OAAO,YAAY,CAAA;YACrB,CAAC;YAED,KAAK,GAAG,QAAuB,CAAA;YAC/B,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;QAClC,CAAC;QAED,IAAI,KAAK,GAAG,KAAK,CAAC,KAAK,CAAA;QAEvB,oBAAoB;QACpB,MAAM,aAAa,GAAG,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,CAAA;QACvE,IAAI,aAAa,IAAI,KAAK,CAAC,SAAS,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAClE,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;QAC5C,CAAC;QAED,0CAA0C;QAC1C,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;YAClF,IAAI,CAAC;gBACH,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;YAC3B,CAAC;YAAC,MAAM,CAAC;gBACP,6BAA6B;YAC/B,CAAC;QACH,CAAC;QAED,OAAO,KAAK,CAAA;IACd,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,GAAW;QACtB,oBAAoB;QACpB,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;QAE5B,sBAAsB;QACtB,MAAM,QAAQ,GAAG,IAAI,CAAC,kBAAkB,GAAG,GAAG,CAAA;QAC9C,MAAM,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,QAAQ,EAAE,IAAW,CAAC,CAAA;IACxD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI;QACR,qCAAqC;QACrC,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,CAAC,CAAA;QAEtD,IAAI,CAAC,WAAW,IAAI,OAAO,WAAW,KAAK,QAAQ,EAAE,CAAC;YACpD,OAAO,EAAE,CAAA;QACX,CAAC;QAED,yBAAyB;QACzB,MAAM,UAAU,GAAa,EAAE,CAAA;QAC/B,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;YAC3C,IAAI,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,kBAAkB,CAAC,EAAE,CAAC;gBAC5C,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC,CAAA;YAChE,CAAC;QACH,CAAC;QAED,OAAO,UAAU,CAAA;IACnB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,GAAG,CAAC,GAAW;QACnB,IAAI,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAC9B,OAAO,IAAI,CAAA;QACb,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,kBAAkB,GAAG,GAAG,CAAA;QAC9C,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAA;QACzD,OAAO,QAAQ,KAAK,IAAI,IAAI,QAAQ,KAAK,SAAS,CAAA;IACpD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,cAAc;QACd,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAA;QAExB,qBAAqB;QACrB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAA;QAC9B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;QACxB,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM;QACV,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAA;QAC9B,MAAM,MAAM,GAAgC,EAAE,CAAA;QAE9C,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAA;YACtC,IAAI,KAAK,EAAE,CAAC;gBACV,MAAM,CAAC,GAAG,CAAC,GAAG,KAAK,CAAA;YACrB,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAA;IACf,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,MAAmC;QAC9C,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YAClD,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;YAChC,MAAM,QAAQ,GAAG,IAAI,CAAC,kBAAkB,GAAG,GAAG,CAAA;YAC9C,MAAM,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAA;QAClD,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,QAAQ,CAAC,GAAW;QAChC,IAAI,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAC9B,OAAO,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAE,CAAA;QACnC,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,kBAAkB,GAAG,GAAG,CAAA;QAC9C,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAA;QAEzD,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,OAAO,IAAI,CAAA;QACb,CAAC;QAED,MAAM,KAAK,GAAG,QAAuB,CAAA;QACrC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;QAChC,OAAO,KAAK,CAAA;IACd,CAAC;CACF"}

118
.recovery-workspace/dist-backup-20250910-141917/api/DataAPI.d.ts vendored Normal file
View file

@ -0,0 +1,118 @@
/**
* Data Management API for Brainy 3.0
* Provides backup, restore, import, export, and data management
*/
import { StorageAdapter } from '../coreTypes.js';
import { Entity, Relation } from '../types/brainy.types.js';
import { NounType } from '../types/graphTypes.js';
export interface BackupOptions {
includeVectors?: boolean;
compress?: boolean;
format?: 'json' | 'binary';
}
export interface RestoreOptions {
merge?: boolean;
overwrite?: boolean;
validate?: boolean;
}
export interface ImportOptions {
format: 'json' | 'csv' | 'parquet';
mapping?: Record<string, string>;
batchSize?: number;
validate?: boolean;
}
export interface ExportOptions {
format?: 'json' | 'csv' | 'parquet';
filter?: {
type?: NounType | NounType[];
where?: Record<string, any>;
service?: string;
};
includeVectors?: boolean;
}
export interface BackupData {
version: string;
timestamp: number;
entities: Array<{
id: string;
vector?: number[];
type: string;
metadata: any;
service?: string;
}>;
relations: Array<{
id: string;
from: string;
to: string;
type: string;
weight: number;
metadata?: any;
}>;
config?: Record<string, any>;
stats: {
entityCount: number;
relationCount: number;
vectorDimensions?: number;
};
}
export interface ImportResult {
successful: number;
failed: number;
errors: Array<{
item: any;
error: string;
}>;
duration: number;
}
export declare class DataAPI {
private storage;
private getEntity;
private getRelation?;
private brain;
constructor(storage: StorageAdapter, getEntity: (id: string) => Promise<Entity | null>, getRelation?: ((id: string) => Promise<Relation | null>) | undefined, brain?: any);
/**
* Create a backup of all data
*/
backup(options?: BackupOptions): Promise<BackupData>;
/**
* Restore data from a backup
*/
restore(params: {
backup: BackupData;
merge?: boolean;
overwrite?: boolean;
validate?: boolean;
}): Promise<void>;
/**
* Clear data
*/
clear(params?: {
entities?: boolean;
relations?: boolean;
config?: boolean;
}): Promise<void>;
/**
* Import data from various formats
*/
import(params: ImportOptions & {
data: any;
}): Promise<ImportResult>;
/**
* Export data to various formats
*/
export(params?: ExportOptions): Promise<any>;
/**
* Get storage statistics
*/
getStats(): Promise<{
entities: number;
relations: number;
storageSize?: number;
vectorDimensions?: number;
}>;
private applyMapping;
private validateImportItem;
private matchesFilter;
private convertToCSV;
private generateId;
}

386
.recovery-workspace/dist-backup-20250910-141917/api/DataAPI.js Normal file
View file

@ -0,0 +1,386 @@
/**
* Data Management API for Brainy 3.0
* Provides backup, restore, import, export, and data management
*/
import { NounType } from '../types/graphTypes.js';
export class DataAPI {
constructor(storage, getEntity, getRelation, brain) {
this.storage = storage;
this.getEntity = getEntity;
this.getRelation = getRelation;
this.brain = brain;
}
/**
* Create a backup of all data
*/
async backup(options = {}) {
const { includeVectors = true, compress = false, format = 'json' } = options;
const startTime = Date.now();
// Get all entities
const nounsResult = await this.storage.getNouns({
pagination: { limit: 1000000 }
});
const entities = [];
for (const noun of nounsResult.items) {
const entity = {
id: noun.id,
vector: includeVectors ? noun.vector : undefined,
type: noun.metadata?.noun || NounType.Thing,
metadata: noun.metadata,
service: noun.metadata?.service
};
entities.push(entity);
}
// Get all relations
const verbsResult = await this.storage.getVerbs({
pagination: { limit: 1000000 }
});
const relations = [];
for (const verb of verbsResult.items) {
relations.push({
id: verb.id,
from: verb.sourceId,
to: verb.targetId,
type: (verb.verb || verb.type),
weight: verb.weight || 1.0,
metadata: verb.metadata
});
}
// Create backup data
const backupData = {
version: '3.0.0',
timestamp: Date.now(),
entities,
relations,
stats: {
entityCount: entities.length,
relationCount: relations.length,
vectorDimensions: entities[0]?.vector?.length
}
};
// Compress if requested (simplified - just stringify for now)
if (compress) {
// In production, use proper compression like gzip
// For now, just return the data
}
return backupData;
}
/**
* Restore data from a backup
*/
async restore(params) {
const { backup, merge = false, overwrite = false, validate = true } = params;
// Validate backup format
if (validate) {
if (!backup.version || !backup.entities || !backup.relations) {
throw new Error('Invalid backup format');
}
}
// Clear existing data if not merging
if (!merge && overwrite) {
await this.clear({ entities: true, relations: true });
}
// Restore entities
for (const entity of backup.entities) {
try {
const noun = {
id: entity.id,
vector: entity.vector || new Array(384).fill(0), // Default vector if missing
connections: new Map(),
level: 0,
metadata: {
...entity.metadata,
noun: entity.type,
service: entity.service
}
};
// Check if entity exists when merging
if (merge) {
const existing = await this.storage.getNoun(entity.id);
if (existing && !overwrite) {
continue; // Skip existing entities unless overwriting
}
}
await this.storage.saveNoun(noun);
}
catch (error) {
console.error(`Failed to restore entity ${entity.id}:`, error);
}
}
// Restore relations
for (const relation of backup.relations) {
try {
// Get source and target entities to compute relation vector
const sourceNoun = await this.storage.getNoun(relation.from);
const targetNoun = await this.storage.getNoun(relation.to);
if (!sourceNoun || !targetNoun) {
console.warn(`Skipping relation ${relation.id}: missing entities`);
continue;
}
// Compute relation vector as average of source and target
const relationVector = sourceNoun.vector.map((v, i) => (v + targetNoun.vector[i]) / 2);
const verb = {
id: relation.id,
vector: relationVector,
sourceId: relation.from,
targetId: relation.to,
source: sourceNoun.metadata?.noun || NounType.Thing,
target: targetNoun.metadata?.noun || NounType.Thing,
verb: relation.type,
type: relation.type,
weight: relation.weight,
metadata: relation.metadata,
createdAt: Date.now()
};
// Check if relation exists when merging
if (merge) {
const existing = await this.storage.getVerb(relation.id);
if (existing && !overwrite) {
continue;
}
}
await this.storage.saveVerb(verb);
}
catch (error) {
console.error(`Failed to restore relation ${relation.id}:`, error);
}
}
}
/**
* Clear data
*/
async clear(params = {}) {
const { entities = true, relations = true, config = false } = params;
if (entities) {
// Clear all entities
const nounsResult = await this.storage.getNouns({
pagination: { limit: 1000000 }
});
for (const noun of nounsResult.items) {
await this.storage.deleteNoun(noun.id);
}
// Also clear the HNSW index if available
if (this.brain?.index?.clear) {
this.brain.index.clear();
}
// Clear metadata index if available
if (this.brain?.metadataIndex) {
await this.brain.metadataIndex.rebuild(); // Rebuild empty index
}
}
if (relations) {
// Clear all relations
const verbsResult = await this.storage.getVerbs({
pagination: { limit: 1000000 }
});
for (const verb of verbsResult.items) {
await this.storage.deleteVerb(verb.id);
}
}
if (config) {
// Clear configuration would be handled by ConfigAPI
// For now, skip this
}
}
/**
* Import data from various formats
*/
async import(params) {
const { data, format, mapping = {}, batchSize = 100, validate = true } = params;
const result = {
successful: 0,
failed: 0,
errors: [],
duration: 0
};
const startTime = Date.now();
try {
// ALWAYS use neural import for proper type matching
const { UniversalImportAPI } = await import('./UniversalImportAPI.js');
const universalImport = new UniversalImportAPI(this.brain);
await universalImport.init();
// Convert to ImportSource format
const neuralResult = await universalImport.import({
type: 'object',
data,
format: format || 'json',
metadata: { mapping, batchSize, validate }
});
// Convert neural result to ImportResult format
result.successful = neuralResult.stats.entitiesCreated;
result.failed = 0; // Neural import always succeeds with best match
result.duration = neuralResult.stats.processingTimeMs;
// Log relationships created
if (neuralResult.stats.relationshipsCreated > 0) {
console.log(`Neural import also created ${neuralResult.stats.relationshipsCreated} relationships`);
}
return result;
}
catch (error) {
// Fallback to legacy import ONLY if neural import fails to load
console.warn('Neural import failed, using legacy import:', error);
let items = [];
// Parse data based on format
switch (format) {
case 'json':
items = Array.isArray(data) ? data : [data];
break;
case 'csv':
// CSV parsing would go here
// For now, assume data is already parsed
items = data;
break;
case 'parquet':
// Parquet parsing would go here
throw new Error('Parquet format not yet implemented');
default:
throw new Error(`Unsupported format: ${format}`);
}
// Process items in batches
for (let i = 0; i < items.length; i += batchSize) {
const batch = items.slice(i, i + batchSize);
for (const item of batch) {
try {
// Apply field mapping
const mapped = this.applyMapping(item, mapping);
// Validate if requested
if (validate) {
this.validateImportItem(mapped);
}
// Save as entity
const noun = {
id: mapped.id || this.generateId(),
vector: mapped.vector || new Array(384).fill(0),
connections: new Map(),
level: 0,
metadata: mapped
};
await this.storage.saveNoun(noun);
result.successful++;
}
catch (error) {
result.failed++;
result.errors.push({
item,
error: error.message
});
}
}
}
result.duration = Date.now() - startTime;
return result;
}
}
/**
* Export data to various formats
*/
async export(params = {}) {
const { format = 'json', filter = {}, includeVectors = false } = params;
// Get filtered entities
const nounsResult = await this.storage.getNouns({
pagination: { limit: 1000000 }
});
let entities = nounsResult.items;
// Apply filters
if (filter.type) {
const types = Array.isArray(filter.type) ? filter.type : [filter.type];
entities = entities.filter(e => types.includes(e.metadata?.noun));
}
if (filter.service) {
entities = entities.filter(e => e.metadata?.service === filter.service);
}
if (filter.where) {
entities = entities.filter(e => this.matchesFilter(e.metadata, filter.where));
}
// Format data based on export format
switch (format) {
case 'json':
return entities.map(e => ({
id: e.id,
vector: includeVectors ? e.vector : undefined,
...e.metadata
}));
case 'csv':
// Convert to CSV format
// For now, return simplified format
return this.convertToCSV(entities);
case 'parquet':
throw new Error('Parquet export not yet implemented');
default:
throw new Error(`Unsupported export format: ${format}`);
}
}
/**
* Get storage statistics
*/
async getStats() {
const nounsResult = await this.storage.getNouns({
pagination: { limit: 1 }
});
const verbsResult = await this.storage.getVerbs({
pagination: { limit: 1 }
});
const firstNoun = nounsResult.items[0];
return {
entities: nounsResult.totalCount || nounsResult.items.length,
relations: verbsResult.totalCount || verbsResult.items.length,
vectorDimensions: firstNoun?.vector?.length
};
}
// Helper methods
applyMapping(item, mapping) {
const mapped = {};
for (const [key, value] of Object.entries(item)) {
const mappedKey = mapping[key] || key;
mapped[mappedKey] = value;
}
return mapped;
}
validateImportItem(item) {
// Basic validation
if (!item || typeof item !== 'object') {
throw new Error('Invalid item: must be an object');
}
// Could add more validation here
}
matchesFilter(metadata, filter) {
for (const [key, value] of Object.entries(filter)) {
if (metadata[key] !== value) {
return false;
}
}
return true;
}
convertToCSV(entities) {
if (entities.length === 0)
return '';
// Get all unique keys from metadata
const keys = new Set();
for (const entity of entities) {
if (entity.metadata) {
Object.keys(entity.metadata).forEach(k => keys.add(k));
}
}
// Create CSV header
const headers = ['id', ...Array.from(keys)];
const rows = [headers.join(',')];
// Add data rows
for (const entity of entities) {
const row = [entity.id];
for (const key of keys) {
const value = entity.metadata?.[key] || '';
// Escape values that contain commas
const escaped = String(value).includes(',')
? `"${String(value).replace(/"/g, '""')}"`
: String(value);
row.push(escaped);
}
rows.push(row.join(','));
}
return rows.join('\n');
}
generateId() {
return `import_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
}
}
//# sourceMappingURL=DataAPI.js.map

1
.recovery-workspace/dist-backup-20250910-141917/api/DataAPI.js.map Normal file
View file

File diff suppressed because one or more lines are too long

50
.recovery-workspace/dist-backup-20250910-141917/api/SecurityAPI.d.ts vendored Normal file
View file

@ -0,0 +1,50 @@
/**
* Security API for Brainy 3.0
* Provides encryption, decryption, hashing, and secure storage
*/
export declare class SecurityAPI {
private config?;
private encryptionKey?;
constructor(config?: {
encryptionKey?: string;
} | undefined);
/**
* Encrypt data using AES-256-CBC
*/
encrypt(data: string): Promise<string>;
/**
* Decrypt data encrypted with encrypt()
*/
decrypt(encryptedData: string): Promise<string>;
/**
* Create a one-way hash of data (for passwords, etc)
*/
hash(data: string, algorithm?: 'sha256' | 'sha512'): Promise<string>;
/**
* Compare data with a hash (for password verification)
*/
compare(data: string, hash: string, algorithm?: 'sha256' | 'sha512'): Promise<boolean>;
/**
* Generate a secure random token
*/
generateToken(bytes?: number): Promise<string>;
/**
* Derive a key from a password using PBKDF2
* Note: Simplified version using hash instead of PBKDF2 which may not be available
*/
deriveKey(password: string, salt?: string, iterations?: number): Promise<{
key: string;
salt: string;
}>;
/**
* Sign data with HMAC
*/
sign(data: string, secret?: string): Promise<string>;
/**
* Verify HMAC signature
*/
verify(data: string, signature: string, secret: string): Promise<boolean>;
private hexToBytes;
private bytesToHex;
private constantTimeCompare;
}

139
.recovery-workspace/dist-backup-20250910-141917/api/SecurityAPI.js Normal file
View file

@ -0,0 +1,139 @@
/**
* Security API for Brainy 3.0
* Provides encryption, decryption, hashing, and secure storage
*/
export class SecurityAPI {
constructor(config) {
this.config = config;
if (config?.encryptionKey) {
// Use provided key (must be 32 bytes hex string)
this.encryptionKey = this.hexToBytes(config.encryptionKey);
}
}
/**
* Encrypt data using AES-256-CBC
*/
async encrypt(data) {
const crypto = await import('../universal/crypto.js');
// Generate or use existing key
const key = this.encryptionKey || crypto.randomBytes(32);
const iv = crypto.randomBytes(16);
const cipher = crypto.createCipheriv('aes-256-cbc', key, iv);
let encrypted = cipher.update(data, 'utf8', 'hex');
encrypted += cipher.final('hex');
// Package encrypted data with metadata
// In production, store keys separately in a key management service
return JSON.stringify({
encrypted,
key: this.bytesToHex(key),
iv: this.bytesToHex(iv),
algorithm: 'aes-256-cbc',
timestamp: Date.now()
});
}
/**
* Decrypt data encrypted with encrypt()
*/
async decrypt(encryptedData) {
const crypto = await import('../universal/crypto.js');
try {
const parsed = JSON.parse(encryptedData);
const { encrypted, key: keyHex, iv: ivHex, algorithm } = parsed;
if (algorithm && algorithm !== 'aes-256-cbc') {
throw new Error(`Unsupported encryption algorithm: ${algorithm}`);
}
const key = this.hexToBytes(keyHex);
const iv = this.hexToBytes(ivHex);
const decipher = crypto.createDecipheriv('aes-256-cbc', key, iv);
let decrypted = decipher.update(encrypted, 'hex', 'utf8');
decrypted += decipher.final('utf8');
return decrypted;
}
catch (error) {
throw new Error(`Decryption failed: ${error.message}`);
}
}
/**
* Create a one-way hash of data (for passwords, etc)
*/
async hash(data, algorithm = 'sha256') {
const crypto = await import('../universal/crypto.js');
const hash = crypto.createHash(algorithm);
hash.update(data);
return hash.digest('hex');
}
/**
* Compare data with a hash (for password verification)
*/
async compare(data, hash, algorithm = 'sha256') {
const dataHash = await this.hash(data, algorithm);
return this.constantTimeCompare(dataHash, hash);
}
/**
* Generate a secure random token
*/
async generateToken(bytes = 32) {
const crypto = await import('../universal/crypto.js');
const buffer = crypto.randomBytes(bytes);
return this.bytesToHex(buffer);
}
/**
* Derive a key from a password using PBKDF2
* Note: Simplified version using hash instead of PBKDF2 which may not be available
*/
async deriveKey(password, salt, iterations = 100000) {
const crypto = await import('../universal/crypto.js');
const actualSalt = salt || this.bytesToHex(crypto.randomBytes(32));
// Simplified key derivation using repeated hashing
// In production, use a proper PBKDF2 implementation
let derived = password + actualSalt;
for (let i = 0; i < Math.min(iterations, 1000); i++) {
const hash = crypto.createHash('sha256');
hash.update(derived);
derived = hash.digest('hex');
}
return {
key: derived,
salt: actualSalt
};
}
/**
* Sign data with HMAC
*/
async sign(data, secret) {
const crypto = await import('../universal/crypto.js');
const actualSecret = secret || (await this.generateToken());
const hmac = crypto.createHmac('sha256', actualSecret);
hmac.update(data);
return hmac.digest('hex');
}
/**
* Verify HMAC signature
*/
async verify(data, signature, secret) {
const expectedSignature = await this.sign(data, secret);
return this.constantTimeCompare(signature, expectedSignature);
}
// Helper methods
hexToBytes(hex) {
const matches = hex.match(/.{1,2}/g);
if (!matches)
throw new Error('Invalid hex string');
return new Uint8Array(matches.map(byte => parseInt(byte, 16)));
}
bytesToHex(bytes) {
return Array.from(bytes)
.map(b => b.toString(16).padStart(2, '0'))
.join('');
}
constantTimeCompare(a, b) {
if (a.length !== b.length)
return false;
let result = 0;
for (let i = 0; i < a.length; i++) {
result |= a.charCodeAt(i) ^ b.charCodeAt(i);
}
return result === 0;
}
}
//# sourceMappingURL=SecurityAPI.js.map

1
.recovery-workspace/dist-backup-20250910-141917/api/SecurityAPI.js.map Normal file
View file

@ -0,0 +1 @@
{"version":3,"file":"SecurityAPI.js","sourceRoot":"","sources":["../../src/api/SecurityAPI.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,MAAM,OAAO,WAAW;IAGtB,YAAoB,MAAmC;QAAnC,WAAM,GAAN,MAAM,CAA6B;QACrD,IAAI,MAAM,EAAE,aAAa,EAAE,CAAC;YAC1B,iDAAiD;YACjD,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,aAAa,CAAC,CAAA;QAC5D,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO,CAAC,IAAY;QACxB,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,wBAAwB,CAAC,CAAA;QAErD,+BAA+B;QAC/B,MAAM,GAAG,GAAG,IAAI,CAAC,aAAa,IAAI,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC,CAAA;QACxD,MAAM,EAAE,GAAG,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC,CAAA;QAEjC,MAAM,MAAM,GAAG,MAAM,CAAC,cAAc,CAAC,aAAa,EAAE,GAAG,EAAE,EAAE,CAAC,CAAA;QAC5D,IAAI,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,CAAA;QAClD,SAAS,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;QAEhC,uCAAuC;QACvC,mEAAmE;QACnE,OAAO,IAAI,CAAC,SAAS,CAAC;YACpB,SAAS;YACT,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;YACzB,EAAE,EAAE,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;YACvB,SAAS,EAAE,aAAa;YACxB,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;SACtB,CAAC,CAAA;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO,CAAC,aAAqB;QACjC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,wBAAwB,CAAC,CAAA;QAErD,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAA;YACxC,MAAM,EAAE,SAAS,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,MAAM,CAAA;YAE/D,IAAI,SAAS,IAAI,SAAS,KAAK,aAAa,EAAE,CAAC;gBAC7C,MAAM,IAAI,KAAK,CAAC,qCAAqC,SAAS,EAAE,CAAC,CAAA;YACnE,CAAC;YAED,MAAM,GAAG,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAA;YACnC,MAAM,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAA;YAEjC,MAAM,QAAQ,GAAG,MAAM,CAAC,gBAAgB,CAAC,aAAa,EAAE,GAAG,EAAE,EAAE,CAAC,CAAA;YAChE,IAAI,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC,SAAS,EAAE,KAAK,EAAE,MAAM,CAAC,CAAA;YACzD,SAAS,IAAI,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;YAEnC,OAAO,SAAS,CAAA;QAClB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,sBAAuB,KAAe,CAAC,OAAO,EAAE,CAAC,CAAA;QACnE,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI,CAAC,IAAY,EAAE,YAAiC,QAAQ;QAChE,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,wBAAwB,CAAC,CAAA;QACrD,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,CAAA;QACzC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;QACjB,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;IAC3B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO,CAAC,IAAY,EAAE,IAAY,EAAE,YAAiC,QAAQ;QACjF,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC,CAAA;QACjD,OAAO,IAAI,CAAC,mBAAmB,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAA;IACjD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa,CAAC,QAAgB,EAAE;QACpC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,wBAAwB,CAAC,CAAA;QACrD,MAAM,MAAM,GAAG,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,CAAA;QACxC,OAAO,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAA;IAChC,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,SAAS,CAAC,QAAgB,EAAE,IAAa,EAAE,aAAqB,MAAM;QAI1E,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,wBAAwB,CAAC,CAAA;QACrD,MAAM,UAAU,GAAG,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAA;QAElE,mDAAmD;QACnD,oDAAoD;QACpD,IAAI,OAAO,GAAG,QAAQ,GAAG,UAAU,CAAA;QACnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YACpD,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAA;YACxC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;YACpB,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QAC9B,CAAC;QAED,OAAO;YACL,GAAG,EAAE,OAAO;YACZ,IAAI,EAAE,UAAU;SACjB,CAAA;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI,CAAC,IAAY,EAAE,MAAe;QACtC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,wBAAwB,CAAC,CAAA;QACrD,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC,CAAA;QAC3D,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAA;QACtD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;QACjB,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;IAC3B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,IAAY,EAAE,SAAiB,EAAE,MAAc;QAC1D,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;QACvD,OAAO,IAAI,CAAC,mBAAmB,CAAC,SAAS,EAAE,iBAAiB,CAAC,CAAA;IAC/D,CAAC;IAED,iBAAiB;IAET,UAAU,CAAC,GAAW;QAC5B,MAAM,OAAO,GAAG,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;QACpC,IAAI,CAAC,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAA;QACnD,OAAO,IAAI,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,CAAA;IAChE,CAAC;IAEO,UAAU,CAAC,KAA0B;QAC3C,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC;aACrB,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;aACzC,IAAI,CAAC,EAAE,CAAC,CAAA;IACb,CAAC;IAEO,mBAAmB,CAAC,CAAS,EAAE,CAAS;QAC9C,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM;YAAE,OAAO,KAAK,CAAA;QAEvC,IAAI,MAAM,GAAG,CAAC,CAAA;QACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAClC,MAAM,IAAI,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;QAC7C,CAAC;QACD,OAAO,MAAM,KAAK,CAAC,CAAA;IACrB,CAAC;CACF"}

134
.recovery-workspace/dist-backup-20250910-141917/api/UniversalImportAPI.d.ts vendored Normal file
View file

@ -0,0 +1,134 @@
/**
* Universal Neural Import API
*
* ALWAYS uses neural matching to map ANY data to our strict NounTypes and VerbTypes
* Never falls back to rules - neural matching is MANDATORY
*
* Handles:
* - Strings (text, JSON, CSV, YAML, Markdown)
* - Files (local paths, any format)
* - URLs (web pages, APIs, documents)
* - Objects (structured data)
* - Binary data (images, PDFs via extraction)
*/
import { NounType, VerbType } from '../types/graphTypes.js';
import { Vector } from '../coreTypes.js';
import type { Brainy } from '../brainy.js';
export interface ImportSource {
type: 'string' | 'file' | 'url' | 'object' | 'binary';
data: any;
format?: string;
metadata?: any;
}
export interface NeuralImportResult {
entities: Array<{
id: string;
type: NounType;
data: any;
vector: Vector;
confidence: number;
metadata: any;
}>;
relationships: Array<{
id: string;
from: string;
to: string;
type: VerbType;
weight: number;
confidence: number;
metadata?: any;
}>;
stats: {
totalProcessed: number;
entitiesCreated: number;
relationshipsCreated: number;
averageConfidence: number;
processingTimeMs: number;
};
}
export declare class UniversalImportAPI {
private brain;
private typeMatcher;
private neuralImport;
private embedCache;
constructor(brain: Brainy<any>);
/**
* Initialize the neural import system
*/
init(): Promise<void>;
/**
* Universal import - handles ANY data source
* ALWAYS uses neural matching, NEVER falls back
*/
import(source: ImportSource | string | any): Promise<NeuralImportResult>;
/**
* Import from URL - fetches and processes
*/
importFromURL(url: string): Promise<NeuralImportResult>;
/**
* Import from file - reads and processes
* Note: In browser environment, use File API instead
*/
importFromFile(filePath: string): Promise<NeuralImportResult>;
/**
* Normalize any input to ImportSource
*/
private normalizeSource;
/**
* Extract structured data from source
*/
private extractData;
/**
* Extract data from URL
*/
private extractFromURL;
/**
* Extract data from file
*/
private extractFromFile;
/**
* Extract data from string based on format
*/
private extractFromString;
/**
* Extract from binary data (images, PDFs, etc)
*/
private extractFromBinary;
/**
* Extract entities from plain text
*/
private extractFromText;
/**
* Neural processing - CORE of the system
* ALWAYS uses embeddings and neural matching
*/
private neuralProcess;
/**
* Generate embedding for any data
*/
private generateEmbedding;
/**
* Convert any data to text for embedding
*/
private dataToText;
/**
* Detect relationships using neural matching
*/
private detectNeuralRelationships;
/**
* Check if a field looks like a reference
*/
private looksLikeReference;
/**
* Store processed data in brain
*/
private storeInBrain;
private detectFormat;
private parseCSV;
private parseYAML;
private parseMarkdown;
private parseHTML;
private generateId;
private simpleHash;
private hashBinary;
}

610
.recovery-workspace/dist-backup-20250910-141917/api/UniversalImportAPI.js Normal file
View file

@ -0,0 +1,610 @@
/**
* Universal Neural Import API
*
* ALWAYS uses neural matching to map ANY data to our strict NounTypes and VerbTypes
* Never falls back to rules - neural matching is MANDATORY
*
* Handles:
* - Strings (text, JSON, CSV, YAML, Markdown)
* - Files (local paths, any format)
* - URLs (web pages, APIs, documents)
* - Objects (structured data)
* - Binary data (images, PDFs via extraction)
*/
import { getBrainyTypes } from '../augmentations/typeMatching/brainyTypes.js';
import { NeuralImportAugmentation } from '../augmentations/neuralImport.js';
export class UniversalImportAPI {
constructor(brain) {
this.embedCache = new Map();
this.brain = brain;
this.neuralImport = new NeuralImportAugmentation({
confidenceThreshold: 0.0, // Accept ALL confidence levels - never reject
enableWeights: true,
skipDuplicates: false // Process everything
});
}
/**
* Initialize the neural import system
*/
async init() {
this.typeMatcher = await getBrainyTypes();
// Neural import initializes itself
}
/**
* Universal import - handles ANY data source
* ALWAYS uses neural matching, NEVER falls back
*/
async import(source) {
const startTime = Date.now();
// Normalize source
const normalizedSource = this.normalizeSource(source);
// Extract data based on source type
const extractedData = await this.extractData(normalizedSource);
// Neural processing - MANDATORY
const neuralResults = await this.neuralProcess(extractedData);
// Store in brain
const result = await this.storeInBrain(neuralResults);
result.stats.processingTimeMs = Date.now() - startTime;
return result;
}
/**
* Import from URL - fetches and processes
*/
async importFromURL(url) {
const response = await fetch(url);
const contentType = response.headers.get('content-type') || 'text/plain';
let data;
if (contentType.includes('json')) {
data = await response.json();
}
else if (contentType.includes('text') || contentType.includes('html')) {
data = await response.text();
}
else {
// Binary data
const buffer = await response.arrayBuffer();
data = new Uint8Array(buffer);
}
return this.import({
type: 'url',
data,
format: contentType,
metadata: { url, fetchedAt: Date.now() }
});
}
/**
* Import from file - reads and processes
* Note: In browser environment, use File API instead
*/
async importFromFile(filePath) {
// For file imports, the caller should read the file and pass content
// This is a placeholder that treats the path as a reference
const ext = filePath.split('.').pop()?.toLowerCase() || 'txt';
return this.import({
type: 'file',
data: filePath, // Path as reference
format: ext,
metadata: {
path: filePath,
importedAt: Date.now()
}
});
}
/**
* Normalize any input to ImportSource
*/
normalizeSource(source) {
// Already normalized
if (source && typeof source === 'object' && 'type' in source && 'data' in source) {
return source;
}
// String input
if (typeof source === 'string') {
// Check if it's a URL
if (source.startsWith('http://') || source.startsWith('https://')) {
return { type: 'url', data: source };
}
// Check if it looks like a file path
if (source.includes('/') || source.includes('\\') || source.includes('.')) {
// Assume it's a file path reference
return { type: 'file', data: source };
}
// Treat as raw string data
return { type: 'string', data: source };
}
// Object/Array input
if (typeof source === 'object') {
return { type: 'object', data: source };
}
// Default to string
return { type: 'string', data: String(source) };
}
/**
* Extract structured data from source
*/
async extractData(source) {
switch (source.type) {
case 'url':
// URL is in data field, need to fetch
return this.extractFromURL(source.data);
case 'file':
// File path is in data field, need to read
return this.extractFromFile(source.data);
case 'string':
return this.extractFromString(source.data, source.format);
case 'object':
return Array.isArray(source.data) ? source.data : [source.data];
case 'binary':
return this.extractFromBinary(source.data, source.format);
default:
// Unknown type, treat as object
return [source.data];
}
}
/**
* Extract data from URL
*/
async extractFromURL(url) {
const result = await this.importFromURL(url);
return result.entities.map(e => e.data);
}
/**
* Extract data from file
*/
async extractFromFile(filePath) {
const result = await this.importFromFile(filePath);
return result.entities.map(e => e.data);
}
/**
* Extract data from string based on format
*/
extractFromString(data, format) {
// Try to detect format if not provided
const detectedFormat = format || this.detectFormat(data);
switch (detectedFormat) {
case 'json':
try {
const parsed = JSON.parse(data);
return Array.isArray(parsed) ? parsed : [parsed];
}
catch {
// Not valid JSON, treat as text
return this.extractFromText(data);
}
case 'csv':
return this.parseCSV(data);
case 'yaml':
case 'yml':
return this.parseYAML(data);
case 'markdown':
case 'md':
return this.parseMarkdown(data);
case 'xml':
case 'html':
return this.parseHTML(data);
default:
return this.extractFromText(data);
}
}
/**
* Extract from binary data (images, PDFs, etc)
*/
async extractFromBinary(data, format) {
// For now, create a single entity representing the binary data
// In production, would use OCR, image recognition, PDF extraction, etc.
return [{
type: 'binary',
format: format || 'unknown',
size: data.length,
hash: await this.hashBinary(data),
extractedAt: Date.now()
}];
}
/**
* Extract entities from plain text
*/
extractFromText(text) {
// Split into meaningful chunks
const chunks = [];
// Split by paragraphs
const paragraphs = text.split(/\n\n+/);
for (const para of paragraphs) {
if (para.trim()) {
chunks.push({
text: para.trim(),
type: 'paragraph',
length: para.length
});
}
}
// If no paragraphs, split by sentences
if (chunks.length === 0) {
const sentences = text.match(/[^.!?]+[.!?]+/g) || [text];
for (const sentence of sentences) {
if (sentence.trim()) {
chunks.push({
text: sentence.trim(),
type: 'sentence',
length: sentence.length
});
}
}
}
return chunks;
}
/**
* Neural processing - CORE of the system
* ALWAYS uses embeddings and neural matching
*/
async neuralProcess(data) {
const entities = new Map();
const relationships = new Map();
for (const item of data) {
// Generate embedding for the item
const embedding = await this.generateEmbedding(item);
// Neural type matching - MANDATORY
const nounMatch = await this.typeMatcher.matchNounType(item);
// Never reject based on confidence - we ALWAYS accept the best match
const entityId = this.generateId(item);
entities.set(entityId, {
id: entityId,
type: nounMatch.type, // Always use the neural match
data: item,
vector: embedding,
confidence: nounMatch.confidence,
metadata: {
...item,
_neuralMatch: nounMatch,
_importedAt: Date.now()
}
});
// Detect relationships using neural matching
await this.detectNeuralRelationships(item, entityId, entities, relationships);
}
return { entities, relationships };
}
/**
* Generate embedding for any data
*/
async generateEmbedding(data) {
// Convert to string for embedding
const text = this.dataToText(data);
// Check cache
if (this.embedCache.has(text)) {
return this.embedCache.get(text);
}
// Generate new embedding
const embedding = await this.brain.embed(text);
// Cache it
this.embedCache.set(text, embedding);
return embedding;
}
/**
* Convert any data to text for embedding
*/
dataToText(data) {
if (typeof data === 'string')
return data;
if (typeof data === 'object') {
// Extract meaningful text from object
const parts = [];
// Priority fields
const priorityFields = ['name', 'title', 'description', 'text', 'content', 'label', 'value'];
for (const field of priorityFields) {
if (data[field]) {
parts.push(String(data[field]));
}
}
// Add other fields
for (const [key, value] of Object.entries(data)) {
if (!priorityFields.includes(key) && value) {
if (typeof value === 'string' || typeof value === 'number') {
parts.push(`${key}: ${value}`);
}
}
}
return parts.join(' ');
}
return JSON.stringify(data);
}
/**
* Detect relationships using neural matching
*/
async detectNeuralRelationships(item, sourceId, entities, relationships) {
if (typeof item !== 'object')
return;
// Look for references to other entities
for (const [key, value] of Object.entries(item)) {
// Check if this looks like a reference
if (this.looksLikeReference(key, value)) {
// Find or predict target entity
const targetId = String(value);
// Neural verb type matching
const verbMatch = await this.typeMatcher.matchVerbType(item, // source object
{ id: targetId }, // target (we may not have full data)
key // field name as context
);
// Always create relationship with neural match
const relationId = `${sourceId}_${verbMatch.type}_${targetId}`;
relationships.set(relationId, {
id: relationId,
from: sourceId,
to: targetId,
type: verbMatch.type,
weight: verbMatch.confidence, // Use confidence as weight
confidence: verbMatch.confidence,
metadata: {
field: key,
_neuralMatch: verbMatch,
_importedAt: Date.now()
}
});
}
// Handle arrays of references
if (Array.isArray(value)) {
for (const item of value) {
if (this.looksLikeReference(key, item)) {
const targetId = String(item);
const verbMatch = await this.typeMatcher.matchVerbType(item, { id: targetId }, key);
const relationId = `${sourceId}_${verbMatch.type}_${targetId}`;
relationships.set(relationId, {
id: relationId,
from: sourceId,
to: targetId,
type: verbMatch.type,
weight: verbMatch.confidence,
confidence: verbMatch.confidence,
metadata: {
field: key,
array: true,
_neuralMatch: verbMatch,
_importedAt: Date.now()
}
});
}
}
}
}
}
/**
* Check if a field looks like a reference
*/
looksLikeReference(key, value) {
// Field name patterns that suggest references
const refPatterns = [
/[Ii]d$/, // ends with Id or id
/_id$/, // ends with _id
/^parent/i, // starts with parent
/^child/i, // starts with child
/^related/i, // starts with related
/^ref/i, // starts with ref
/^link/i, // starts with link
/^target/i, // starts with target
/^source/i, // starts with source
];
// Check if field name matches patterns
const fieldLooksLikeRef = refPatterns.some(pattern => pattern.test(key));
// Check if value looks like an ID
const valueLooksLikeId = (typeof value === 'string' ||
typeof value === 'number') && String(value).length > 0;
return fieldLooksLikeRef && valueLooksLikeId;
}
/**
* Store processed data in brain
*/
async storeInBrain(neuralResults) {
const result = {
entities: [],
relationships: [],
stats: {
totalProcessed: neuralResults.entities.size + neuralResults.relationships.size,
entitiesCreated: 0,
relationshipsCreated: 0,
averageConfidence: 0,
processingTimeMs: 0
}
};
let totalConfidence = 0;
// Store entities
for (const entity of neuralResults.entities.values()) {
const id = await this.brain.add({
data: entity.data,
type: entity.type,
metadata: entity.metadata,
vector: entity.vector,
writeOnly: true // Fast mode since we already have vectors
});
// Update entity ID for relationship mapping
entity.id = id;
result.entities.push({
...entity,
id
});
result.stats.entitiesCreated++;
totalConfidence += entity.confidence;
}
// Store relationships
for (const relation of neuralResults.relationships.values()) {
// Map to actual entity IDs
const sourceEntity = Array.from(neuralResults.entities.values())
.find(e => e.id === relation.from);
const targetEntity = Array.from(neuralResults.entities.values())
.find(e => e.id === relation.to);
if (sourceEntity && targetEntity) {
const id = await this.brain.relate({
from: sourceEntity.id,
to: targetEntity.id,
type: relation.type,
weight: relation.weight,
metadata: relation.metadata,
writeOnly: true
});
result.relationships.push({
...relation,
id,
from: sourceEntity.id,
to: targetEntity.id
});
result.stats.relationshipsCreated++;
totalConfidence += relation.confidence;
}
}
// Calculate average confidence
const totalItems = result.stats.entitiesCreated + result.stats.relationshipsCreated;
result.stats.averageConfidence = totalItems > 0 ? totalConfidence / totalItems : 0;
return result;
}
// Helper methods for parsing different formats
detectFormat(data) {
const trimmed = data.trim();
// JSON
if ((trimmed.startsWith('{') && trimmed.endsWith('}')) ||
(trimmed.startsWith('[') && trimmed.endsWith(']'))) {
return 'json';
}
// CSV (has commas and newlines)
if (trimmed.includes(',') && trimmed.includes('\n')) {
return 'csv';
}
// YAML (has colons and indentation)
if (trimmed.includes(':') && (trimmed.includes('\n ') || trimmed.includes('\n\t'))) {
return 'yaml';
}
// Markdown (has headers)
if (trimmed.includes('#') || trimmed.includes('```')) {
return 'markdown';
}
// HTML/XML
if (trimmed.includes('<') && trimmed.includes('>')) {
return trimmed.toLowerCase().includes('<!doctype html') ? 'html' : 'xml';
}
return 'text';
}
parseCSV(data) {
// Reuse the CSV parser from neural import
const lines = data.split('\n').filter(l => l.trim());
if (lines.length === 0)
return [];
const headers = lines[0].split(',').map(h => h.trim());
const results = [];
for (let i = 1; i < lines.length; i++) {
const values = lines[i].split(',').map(v => v.trim());
const obj = {};
headers.forEach((header, index) => {
obj[header] = values[index] || '';
});
results.push(obj);
}
return results;
}
parseYAML(data) {
// Simple YAML parser
const results = [];
const lines = data.split('\n');
let current = null;
for (const line of lines) {
const trimmed = line.trim();
if (!trimmed || trimmed.startsWith('#'))
continue;
if (trimmed.startsWith('- ')) {
// Array item
const value = trimmed.substring(2);
if (!current) {
results.push(value);
}
else {
if (!current._items)
current._items = [];
current._items.push(value);
}
}
else if (trimmed.includes(':')) {
// Key-value
const [key, ...valueParts] = trimmed.split(':');
const value = valueParts.join(':').trim();
if (!current) {
current = {};
results.push(current);
}
current[key.trim()] = value;
}
}
return results.length > 0 ? results : [{ text: data }];
}
parseMarkdown(data) {
const results = [];
const lines = data.split('\n');
let current = null;
let inCodeBlock = false;
for (const line of lines) {
if (line.startsWith('```')) {
inCodeBlock = !inCodeBlock;
if (inCodeBlock && current) {
current.code = '';
}
continue;
}
if (inCodeBlock && current) {
current.code += line + '\n';
}
else if (line.startsWith('#')) {
// Header
const level = line.match(/^#+/)?.[0].length || 1;
const text = line.replace(/^#+\s*/, '');
current = {
type: 'heading',
level,
text
};
results.push(current);
}
else if (line.trim()) {
// Paragraph
if (!current || current.type !== 'paragraph') {
current = {
type: 'paragraph',
text: ''
};
results.push(current);
}
current.text += line + ' ';
}
}
return results;
}
parseHTML(data) {
// Simple HTML text extraction
const text = data
.replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, '') // Remove scripts
.replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, '') // Remove styles
.replace(/<[^>]+>/g, ' ') // Remove tags
.replace(/\s+/g, ' ') // Normalize whitespace
.trim();
return this.extractFromText(text);
}
generateId(data) {
// Generate deterministic ID based on content
const text = this.dataToText(data);
const hash = this.simpleHash(text);
return `import_${hash}_${Date.now()}`;
}
simpleHash(text) {
let hash = 0;
for (let i = 0; i < text.length; i++) {
const char = text.charCodeAt(i);
hash = ((hash << 5) - hash) + char;
hash = hash & hash;
}
return Math.abs(hash).toString(36);
}
async hashBinary(data) {
// Simple binary hash
let hash = 0;
for (let i = 0; i < Math.min(data.length, 1000); i++) {
hash = ((hash << 5) - hash) + data[i];
hash = hash & hash;
}
return Math.abs(hash).toString(36);
}
}
//# sourceMappingURL=UniversalImportAPI.js.map

1
.recovery-workspace/dist-backup-20250910-141917/api/UniversalImportAPI.js.map Normal file
View file

File diff suppressed because one or more lines are too long