/** * 🧠 Brainy 3.0 - The Future of Neural Databases * * Beautiful, Professional, Planet-Scale, Fun to Use * NO STUBS, NO MOCKS, REAL IMPLEMENTATION */ import { v4 as uuidv4 } from './universal/uuid.js' import { HNSWIndex } from './hnsw/hnswIndex.js' import { HNSWIndexOptimized } from './hnsw/hnswIndexOptimized.js' import { createStorage } from './storage/storageFactory.js' import { StorageAdapter, Vector, DistanceFunction, EmbeddingFunction, GraphVerb } from './coreTypes.js' import { defaultEmbeddingFunction, cosineDistance } from './utils/index.js' import { validateNounType, validateVerbType } from './utils/typeValidation.js' import { matchesMetadataFilter } from './utils/metadataFilter.js' import { AugmentationRegistry, AugmentationContext } from './augmentations/brainyAugmentation.js' import { createDefaultAugmentations } from './augmentations/defaultAugmentations.js' import { ImprovedNeuralAPI } from './neural/improvedNeuralAPI.js' import { NaturalLanguageProcessor } from './neural/naturalLanguageProcessor.js' import { TripleIntelligenceSystem } from './triple/TripleIntelligenceSystem.js' import { MetadataIndexManager } from './utils/metadataIndex.js' import { Entity, Relation, Result, AddParams, UpdateParams, RelateParams, FindParams, SimilarParams, GetRelationsParams, AddManyParams, DeleteManyParams, BatchResult, BrainyConfig } from './types/brainy.types.js' import { NounType, VerbType } from './types/graphTypes.js' /** * The main Brainy class - Clean, Beautiful, Powerful * REAL IMPLEMENTATION - No stubs, no mocks */ export class Brainy { // Core components private index!: HNSWIndex | HNSWIndexOptimized private storage!: StorageAdapter private embedder: EmbeddingFunction private distance: DistanceFunction private augmentationRegistry: AugmentationRegistry private config: Required // Sub-APIs (lazy-loaded) private _neural?: ImprovedNeuralAPI private _nlp?: NaturalLanguageProcessor private _tripleIntelligence?: TripleIntelligenceSystem private _metadataIndex?: MetadataIndexManager // State private initialized = false private dimensions?: number constructor(config?: BrainyConfig) { // Normalize configuration with defaults this.config = this.normalizeConfig(config) // Setup core components this.distance = cosineDistance this.embedder = this.setupEmbedder() this.augmentationRegistry = this.setupAugmentations() // Index and storage are initialized in init() because they may need each other } /** * Initialize Brainy - MUST be called before use * @param overrides Optional configuration overrides for init */ async init(overrides?: Partial): Promise { if (this.initialized) { return } // Apply any init-time configuration overrides if (overrides) { const { dimensions, ...configOverrides } = overrides this.config = { ...this.config, ...configOverrides, storage: { ...this.config.storage, ...configOverrides.storage }, model: { ...this.config.model, ...configOverrides.model }, index: { ...this.config.index, ...configOverrides.index }, augmentations: { ...this.config.augmentations, ...configOverrides.augmentations } } // Set dimensions if provided if (dimensions) { this.dimensions = dimensions } } try { // Setup and initialize storage this.storage = await this.setupStorage() await this.storage.init() // Setup index now that we have storage this.index = this.setupIndex() // Initialize augmentations await this.augmentationRegistry.initializeAll({ brain: this, storage: this.storage, config: this.config, log: (message: string, level = 'info') => { // Simple logging for now if (level === 'error') { console.error(message) } else if (level === 'warn') { console.warn(message) } else { console.log(message) } } }) // Warm up if configured if (this.config.warmup) { await this.warmup() } this.initialized = true } catch (error) { throw new Error(`Failed to initialize Brainy: ${error}`) } } /** * Ensure Brainy is initialized */ private async ensureInitialized(): Promise { if (!this.initialized) { throw new Error('Brainy not initialized. Call init() first.') } } // ============= CORE CRUD OPERATIONS ============= /** * Add an entity to the database */ async add(params: AddParams): Promise { await this.ensureInitialized() // Validate parameters if (!params.data && !params.vector) { throw new Error('Either data or vector is required') } validateNounType(params.type) // Generate ID if not provided const id = params.id || uuidv4() // Get or compute vector const vector = params.vector || (await this.embed(params.data)) // Ensure dimensions are set if (!this.dimensions) { this.dimensions = vector.length } else if (vector.length !== this.dimensions) { throw new Error( `Vector dimension mismatch: expected ${this.dimensions}, got ${vector.length}` ) } // Execute through augmentation pipeline return this.augmentationRegistry.execute('add', params, async () => { // Add to index await this.index.addItem({ id, vector }) // Save to storage await this.storage.saveNoun({ id, vector, connections: new Map(), level: 0, metadata: { ...(typeof params.data === 'object' && params.data !== null ? params.data : {}), ...params.metadata, noun: params.type, service: params.service, createdAt: Date.now() } }) return id }) } /** * Get an entity by ID */ async get(id: string): Promise | null> { await this.ensureInitialized() return this.augmentationRegistry.execute('get', { id }, async () => { // Get from storage const noun = await this.storage.getNoun(id) if (!noun) { return null } // Extract metadata - separate user metadata from system metadata const { noun: nounType, service, createdAt, updatedAt, ...userMetadata } = noun.metadata || {} // Construct entity from noun const entity: Entity = { id: noun.id, vector: noun.vector, type: (nounType as NounType) || NounType.Thing, metadata: userMetadata as T, service: service as string, createdAt: (createdAt as number) || Date.now(), updatedAt: updatedAt as number } return entity }) } /** * Update an entity */ async update(params: UpdateParams): Promise { await this.ensureInitialized() if (!params.id) { throw new Error('ID is required for update') } return this.augmentationRegistry.execute('update', params, async () => { // Get existing entity const existing = await this.get(params.id) if (!existing) { throw new Error(`Entity ${params.id} not found`) } // Update vector if data changed let vector = existing.vector if (params.data) { vector = params.vector || (await this.embed(params.data)) // Update in index (remove and re-add since no update method) await this.index.removeItem(params.id) await this.index.addItem({ id: params.id, vector }) } // Always update the noun with new metadata const newMetadata = params.merge !== false ? { ...existing.metadata, ...params.metadata } : params.metadata || existing.metadata // Merge data objects if both old and new are objects const dataFields = typeof params.data === 'object' && params.data !== null ? params.data : {} await this.storage.saveNoun({ id: params.id, vector, connections: new Map(), level: 0, metadata: { ...newMetadata, ...dataFields, noun: params.type || existing.type, service: existing.service, createdAt: existing.createdAt, updatedAt: Date.now() } }) }) } /** * Delete an entity */ async delete(id: string): Promise { await this.ensureInitialized() return this.augmentationRegistry.execute('delete', { id }, async () => { // Remove from index await this.index.removeItem(id) // Delete from storage await this.storage.deleteNoun(id) // Delete metadata (if it exists as separate) try { await this.storage.saveMetadata(id, null as any) // Clear metadata } catch { // Ignore if not supported } // Delete related verbs const verbs = await this.storage.getVerbsBySource(id) const targetVerbs = await this.storage.getVerbsByTarget(id) const allVerbs = [...verbs, ...targetVerbs] for (const verb of allVerbs) { await this.storage.deleteVerb(verb.id) } }) } // ============= RELATIONSHIP OPERATIONS ============= /** * Create a relationship between entities */ async relate(params: RelateParams): Promise { await this.ensureInitialized() // Validate parameters if (!params.from || !params.to) { throw new Error('Both from and to are required') } validateVerbType(params.type) // Verify entities exist const fromEntity = await this.get(params.from) const toEntity = await this.get(params.to) if (!fromEntity) { throw new Error(`Source entity ${params.from} not found`) } if (!toEntity) { throw new Error(`Target entity ${params.to} not found`) } // Generate ID const id = uuidv4() // Compute relationship vector (average of entities) const relationVector = fromEntity.vector.map( (v, i) => (v + toEntity.vector[i]) / 2 ) return this.augmentationRegistry.execute('relate', params, async () => { // Save to storage const verb: GraphVerb = { id, vector: relationVector, sourceId: params.from, targetId: params.to, source: fromEntity.type, target: toEntity.type, verb: params.type, type: params.type, weight: params.weight ?? 1.0, metadata: params.metadata as any, createdAt: Date.now() } as any await this.storage.saveVerb(verb) // Create bidirectional if requested if (params.bidirectional) { const reverseId = uuidv4() const reverseVerb: GraphVerb = { ...verb, id: reverseId, sourceId: params.to, targetId: params.from, source: toEntity.type, target: fromEntity.type } as any await this.storage.saveVerb(reverseVerb) } return id }) } /** * Delete a relationship */ async unrelate(id: string): Promise { await this.ensureInitialized() return this.augmentationRegistry.execute('unrelate', { id }, async () => { await this.storage.deleteVerb(id) }) } /** * Get relationships */ async getRelations( params: GetRelationsParams = {} ): Promise[]> { await this.ensureInitialized() const relations: Relation[] = [] if (params.from) { const verbs = await this.storage.getVerbsBySource(params.from) relations.push(...this.verbsToRelations(verbs)) } if (params.to) { const verbs = await this.storage.getVerbsByTarget(params.to) relations.push(...this.verbsToRelations(verbs)) } // Filter by type let filtered = relations if (params.type) { const types = Array.isArray(params.type) ? params.type : [params.type] filtered = relations.filter((r) => types.includes(r.type)) } // Filter by service if (params.service) { filtered = filtered.filter((r) => r.service === params.service) } // Apply pagination const limit = params.limit || 100 const offset = params.offset || 0 return filtered.slice(offset, offset + limit) } // ============= SEARCH & DISCOVERY ============= /** * Unified find method - supports natural language and structured queries */ async find(query: string | FindParams): Promise[]> { await this.ensureInitialized() // Parse natural language queries const params: FindParams = typeof query === 'string' ? await this.parseNaturalQuery(query) : query return this.augmentationRegistry.execute('find', params, async () => { let results: Result[] = [] // Vector search if (params.query || params.vector) { const vector = params.vector || (await this.embed(params.query!)) const limit = params.limit || 10 // Search index - returns array of [id, score] tuples const searchResults = await this.index.search(vector, limit * 2) // Get extra for filtering // Hydrate results for (const [id, score] of searchResults) { const entity = await this.get(id) if (entity) { results.push({ id, score, entity }) } } } // Proximity search if (params.near) { const nearEntity = await this.get(params.near.id) if (nearEntity) { const nearResults = await this.index.search( nearEntity.vector, params.limit || 10 ) for (const [id, score] of nearResults) { if (score >= (params.near.threshold || 0.7)) { const entity = await this.get(id) if (entity) { results.push({ id, score, entity }) } } } } } // Apply O(log n) metadata filtering using MetadataIndexManager if (params.where || params.type || params.service) { const metadataIndex = await this.getMetadataIndex() // Build filter object for metadata index const filter: any = {} if (params.where) Object.assign(filter, params.where) if (params.type) { const types = Array.isArray(params.type) ? params.type : [params.type] filter.noun = types.length === 1 ? types[0] : { anyOf: types } } if (params.service) filter.service = params.service const filteredIds = await metadataIndex.getIdsForFilter(filter) // Only keep results that match the metadata filter const filteredIdSet = new Set(filteredIds) results = results.filter((r) => filteredIdSet.has(r.id)) } // Graph constraints if (params.connected) { results = await this.applyGraphConstraints(results, params.connected) } // Sort by score and limit results.sort((a, b) => b.score - a.score) const limit = params.limit || 10 const offset = params.offset || 0 return results.slice(offset, offset + limit) }) } /** * Find similar entities */ async similar(params: SimilarParams): Promise[]> { await this.ensureInitialized() // Get target vector let targetVector: Vector if (typeof params.to === 'string') { const entity = await this.get(params.to) if (!entity) { throw new Error(`Entity ${params.to} not found`) } targetVector = entity.vector } else if (Array.isArray(params.to)) { targetVector = params.to as Vector } else { targetVector = (params.to as Entity).vector } // Use find with vector return this.find({ vector: targetVector, limit: params.limit, type: params.type, where: params.where, service: params.service }) } // ============= BATCH OPERATIONS ============= /** * Add multiple entities */ async addMany(params: AddManyParams): Promise> { await this.ensureInitialized() const result: BatchResult = { successful: [], failed: [], total: params.items.length, duration: 0 } const startTime = Date.now() const chunkSize = params.chunkSize || 100 // Process in chunks for (let i = 0; i < params.items.length; i += chunkSize) { const chunk = params.items.slice(i, i + chunkSize) const promises = chunk.map(async (item) => { try { const id = await this.add(item) result.successful.push(id) } catch (error) { result.failed.push({ item, error: (error as Error).message }) if (!params.continueOnError) { throw error } } }) if (params.parallel !== false) { await Promise.allSettled(promises) } else { for (const promise of promises) { await promise } } // Report progress if (params.onProgress) { params.onProgress( result.successful.length + result.failed.length, result.total ) } } result.duration = Date.now() - startTime return result } /** * Delete multiple entities */ async deleteMany(params: DeleteManyParams): Promise> { await this.ensureInitialized() // Determine what to delete let idsToDelete: string[] = [] if (params.ids) { idsToDelete = params.ids } else if (params.type || params.where) { // Find entities to delete const entities = await this.find({ type: params.type, where: params.where, limit: params.limit || 1000 }) idsToDelete = entities.map((e) => e.id) } const result: BatchResult = { successful: [], failed: [], total: idsToDelete.length, duration: 0 } const startTime = Date.now() for (const id of idsToDelete) { try { await this.delete(id) result.successful.push(id) } catch (error) { result.failed.push({ item: id, error: (error as Error).message }) } if (params.onProgress) { params.onProgress( result.successful.length + result.failed.length, result.total ) } } result.duration = Date.now() - startTime return result } /** * Update multiple entities with batch processing */ async updateMany(params: { items: UpdateParams[] chunkSize?: number parallel?: boolean continueOnError?: boolean onProgress?: (completed: number, total: number) => void }): Promise> { await this.ensureInitialized() const result: BatchResult = { successful: [], failed: [], total: params.items.length, duration: 0 } const startTime = Date.now() const chunkSize = params.chunkSize || 100 // Process in chunks for (let i = 0; i < params.items.length; i += chunkSize) { const chunk = params.items.slice(i, i + chunkSize) const promises = chunk.map(async (item, chunkIndex) => { try { await this.update(item) result.successful.push(item.id) } catch (error) { result.failed.push({ item, error: (error as Error).message }) if (!params.continueOnError) { throw error } } }) if (params.parallel !== false) { await Promise.allSettled(promises) } else { for (const promise of promises) { await promise } } // Report progress if (params.onProgress) { params.onProgress( result.successful.length + result.failed.length, result.total ) } } result.duration = Date.now() - startTime return result } /** * Clear all data from the database */ async clear(): Promise { await this.ensureInitialized() return this.augmentationRegistry.execute('clear', {}, async () => { // Clear storage await this.storage.clear() // Reset index if ('clear' in this.index && typeof this.index.clear === 'function') { await this.index.clear() } else { // Recreate index if no clear method this.index = this.setupIndex() } // Reset dimensions this.dimensions = undefined // Clear any cached sub-APIs this._neural = undefined this._nlp = undefined this._tripleIntelligence = undefined }) } // ============= SUB-APIS ============= /** * Neural API - Advanced AI operations */ neural(): ImprovedNeuralAPI { if (!this._neural) { this._neural = new ImprovedNeuralAPI(this as any) } return this._neural } /** * Natural Language Processing API */ nlp(): NaturalLanguageProcessor { if (!this._nlp) { this._nlp = new NaturalLanguageProcessor(this) } return this._nlp } /** * Data Management API - backup, restore, import, export */ async data() { const { DataAPI } = await import('./api/DataAPI.js') return new DataAPI( this.storage, (id: string) => this.get(id), undefined, // No getRelation method yet this ) } /** * Get Triple Intelligence System * Advanced pattern recognition and relationship analysis */ getTripleIntelligence(): TripleIntelligenceSystem { if (!this._tripleIntelligence) { // For now, pass minimal parameters to avoid errors // This will be properly initialized when needed this._tripleIntelligence = new TripleIntelligenceSystem( null as any, // metadataIndex this.index as any, null as any, // graphIndex async (text: string) => this.embedder(text), this.storage ) } return this._tripleIntelligence } /** * Get Metadata Index Manager for O(log n) filtering */ private async getMetadataIndex(): Promise { if (!this._metadataIndex) { this._metadataIndex = new MetadataIndexManager(this.storage) } return this._metadataIndex } /** * Create a streaming pipeline */ stream() { const { Pipeline } = require('./streaming/pipeline.js') return new Pipeline(this) } /** * Get insights about the data */ async insights(): Promise<{ entities: number relationships: number types: Record services: string[] density: number }> { await this.ensureInitialized() // Get all entities count - use getNouns with high limit const entitiesResult = await this.storage.getNouns({ pagination: { limit: 10000 } }) const entities = entitiesResult.totalCount || entitiesResult.items.length // Get relationships count - use getVerbs with high limit const verbsResult = await this.storage.getVerbs({ pagination: { limit: 10000 } }) const relationships = verbsResult.totalCount || verbsResult.items.length // Count by type const types: Record = {} for (const entity of entitiesResult.items) { const type = (entity.metadata?.noun as string) || 'unknown' types[type] = (types[type] || 0) + 1 } // Get unique services const services = [...new Set(entitiesResult.items.map((e: any) => e.metadata?.service).filter(Boolean))] as string[] // Calculate density (relationships per entity) const density = entities > 0 ? relationships / entities : 0 return { entities, relationships, types, services, density } } /** * Augmentations API - Clean and simple */ get augmentations() { return { list: () => this.augmentationRegistry.getAll().map(a => a.name), get: (name: string) => this.augmentationRegistry.getAll().find(a => a.name === name), has: (name: string) => this.augmentationRegistry.getAll().some(a => a.name === name) } } // ============= HELPER METHODS ============= /** * Parse natural language query */ private async parseNaturalQuery(query: string): Promise> { if (!this._nlp) { this._nlp = new NaturalLanguageProcessor(this as any) } const parsed = await this._nlp.processNaturalQuery(query) return parsed as FindParams } /** * Apply graph constraints to results */ private async applyGraphConstraints( results: Result[], constraints: any ): Promise[]> { // Filter by graph connections if (constraints.to || constraints.from) { const filtered: Result[] = [] for (const result of results) { if (constraints.to) { const verbs = await this.storage.getVerbsBySource(result.id) const hasConnection = verbs.some(v => v.targetId === constraints.to) if (hasConnection) { filtered.push(result) } } if (constraints.from) { const verbs = await this.storage.getVerbsByTarget(result.id) const hasConnection = verbs.some(v => v.sourceId === constraints.from) if (hasConnection) { filtered.push(result) } } } return filtered } return results } /** * Convert verbs to relations */ private verbsToRelations(verbs: GraphVerb[]): Relation[] { return verbs.map((v) => ({ id: v.id, from: v.sourceId, to: v.targetId, type: (v.verb || v.type) as VerbType, weight: v.weight, metadata: v.metadata, service: v.metadata?.service as string, createdAt: typeof v.createdAt === 'number' ? v.createdAt : Date.now() })) } /** * Embed data into vector */ private async embed(data: any): Promise { return this.embedder(data) } /** * Warm up the system */ private async warmup(): Promise { // Warm up embedder await this.embed('warmup') } /** * Setup embedder */ private setupEmbedder(): EmbeddingFunction { // Custom model loading removed - not implemented // Only 'fast' and 'accurate' model types are supported return defaultEmbeddingFunction } /** * Setup storage */ private async setupStorage(): Promise { const storage = await createStorage({ type: this.config.storage?.type || 'memory', ...this.config.storage?.options }) return storage } /** * Setup index */ private setupIndex(): HNSWIndex | HNSWIndexOptimized { const indexConfig = { ...this.config.index, distanceFunction: this.distance } // Use optimized index for larger datasets if (this.config.storage?.type !== 'memory') { return new HNSWIndexOptimized(indexConfig, this.distance, this.storage) } return new HNSWIndex(indexConfig as any) } /** * Setup augmentations */ private setupAugmentations(): AugmentationRegistry { const registry = new AugmentationRegistry() // Register default augmentations const defaults = createDefaultAugmentations(this.config.augmentations) for (const aug of defaults) { registry.register(aug) } return registry } /** * Normalize and validate configuration */ private normalizeConfig(config?: BrainyConfig): Required { // Validate storage configuration if (config?.storage?.type && !['memory', 'filesystem', 'opfs', 'remote'].includes(config.storage.type)) { throw new Error(`Invalid storage type: ${config.storage.type}. Must be one of: memory, filesystem, opfs, remote`) } // Validate model configuration if (config?.model?.type && !['fast', 'accurate', 'custom'].includes(config.model.type)) { throw new Error(`Invalid model type: ${config.model.type}. Must be one of: fast, accurate, custom`) } // Validate numeric configurations if (config?.index?.m && (config.index.m < 1 || config.index.m > 128)) { throw new Error(`Invalid index m parameter: ${config.index.m}. Must be between 1 and 128`) } if (config?.index?.efConstruction && (config.index.efConstruction < 1 || config.index.efConstruction > 1000)) { throw new Error(`Invalid index efConstruction: ${config.index.efConstruction}. Must be between 1 and 1000`) } if (config?.index?.efSearch && (config.index.efSearch < 1 || config.index.efSearch > 1000)) { throw new Error(`Invalid index efSearch: ${config.index.efSearch}. Must be between 1 and 1000`) } return { storage: config?.storage || { type: 'memory' }, model: config?.model || { type: 'fast' }, index: config?.index || {}, cache: config?.cache ?? true, augmentations: config?.augmentations || {}, warmup: config?.warmup ?? false, realtime: config?.realtime ?? false, multiTenancy: config?.multiTenancy ?? false, telemetry: config?.telemetry ?? false } } /** * Close and cleanup */ async close(): Promise { // Shutdown augmentations const augs = this.augmentationRegistry.getAll() for (const aug of augs) { if ('shutdown' in aug && typeof aug.shutdown === 'function') { await aug.shutdown() } } // Storage doesn't have close in current interface // We'll just mark as not initialized this.initialized = false } } // Re-export types for convenience export * from './types/brainy.types.js' export { NounType, VerbType } from './types/graphTypes.js'