diff --git a/docs/PLUGINS.md b/docs/PLUGINS.md new file mode 100644 index 00000000..48ed1b78 --- /dev/null +++ b/docs/PLUGINS.md @@ -0,0 +1,386 @@ +# Plugin Development Guide + +Brainy has a plugin system that allows third-party packages to replace internal subsystems with custom implementations. This is how `@soulcraft/cortex` provides native Rust acceleration, and it's the same system available to any developer. + +## Architecture Overview + +Brainy's plugin system uses **named providers** — string keys mapped to implementations. During `init()`, brainy: + +1. Auto-detects installed plugin packages via dynamic `import()` +2. Activates each plugin, passing a `BrainyPluginContext` +3. The plugin calls `context.registerProvider(key, implementation)` for each subsystem it provides +4. Brainy checks each provider key and wires the implementation into its internal pipeline + +If no plugin provides a given key, brainy uses its built-in JavaScript implementation. This means brainy works perfectly standalone — plugins only enhance performance or add capabilities. + +## Creating a Plugin + +### 1. Implement the `BrainyPlugin` interface + +```typescript +import type { BrainyPlugin, BrainyPluginContext } from '@soulcraft/brainy/plugin' + +const myPlugin: BrainyPlugin = { + name: 'my-brainy-plugin', // Must be unique (typically your npm package name) + + async activate(context: BrainyPluginContext): Promise { + // Register your providers here + context.registerProvider('distance', myFastDistanceFunction) + + // Return true if activation succeeded, false to skip + return true + }, + + async deactivate(): Promise { + // Optional cleanup when brainy.close() is called + } +} + +export default myPlugin +``` + +### 2. Package exports + +Your package must export the plugin as the default export so brainy's auto-detection works: + +```typescript +// index.ts +export { default } from './plugin.js' +``` + +### 3. Registration + +**Auto-detection:** Brainy auto-detects plugins by package name. Pass your package name in the brainy config: + +```typescript +const brain = new Brainy({ + plugins: ['my-brainy-plugin'] +}) +await brain.init() +``` + +**Manual registration:** For plugins not installed as npm packages: + +```typescript +import { Brainy } from '@soulcraft/brainy' +import myPlugin from './my-plugin.js' + +const brain = new Brainy() +brain.pluginRegistry.register(myPlugin) +await brain.init() +``` + +## Provider Keys Reference + +Each key has a specific expected signature. Brainy checks for these during `init()` and wires them into the appropriate code paths. + +### Core Providers + +#### `distance` +**Type:** `(a: number[], b: number[]) => number` + +Replaces the default cosine distance function used in HNSW search and neural APIs. This is the highest-impact single provider — it's called for every vector comparison. + +```typescript +context.registerProvider('distance', (a: number[], b: number[]): number => { + // Your SIMD-accelerated or GPU distance calculation + return myFastCosineDistance(a, b) +}) +``` + +#### `embeddings` +**Type:** `(text: string | string[]) => Promise` + +Replaces the built-in WASM embedding engine. Called for every `brain.add()`, `brain.update()`, and `brain.find()` operation that involves text. + +```typescript +context.registerProvider('embeddings', async (text: string | string[]) => { + if (Array.isArray(text)) { + return myEngine.embedBatch(text) + } + return myEngine.embed(text) +}) +``` + +#### `embedBatch` +**Type:** `(texts: string[]) => Promise` + +Dedicated batch embedding provider. When registered, brainy uses this for bulk operations (import, reindex, batch add) instead of calling the `embeddings` provider N times. This enables true single-forward-pass batch processing. + +Priority order for batch operations: +1. `embedBatch` provider (single forward pass — fastest) +2. `embeddings` provider with `Promise.all()` (N individual calls) +3. Built-in WASM batch API (fallback) + +```typescript +context.registerProvider('embedBatch', async (texts: string[]) => { + // Process all texts in a single forward pass + return myEngine.batchEmbed(texts) +}) +``` + +### Index Providers + +#### `hnsw` +**Type:** `(config: object, distanceFunction: Function, options: object) => HNSWIndex-compatible` + +Factory function that creates an HNSW index instance. The returned object must implement the `HNSWIndex` public API: + +- `addItem(item: { id: string, vector: number[] }): Promise` +- `search(queryVector: number[], k: number, filter?, options?): Promise>` +- `removeItem(id: string): Promise` +- `size(): number` +- `clear(): void` +- `flush(): Promise` +- `rebuild(options?): Promise` +- `getDirtyNodeCount(): number` +- `getPersistMode(): 'immediate' | 'deferred'` +- `getEntryPointId(): string | null` +- `getMaxLevel(): number` +- `getDimension(): number | null` +- `getConfig(): object` +- `getDistanceFunction(): Function` +- `enableCOW(parent): void` +- `setUseParallelization(boolean): void` + +For type-aware indexes (separate graph per noun type), also implement: +- `getIndexForType(type: string): HNSWIndex` (duck-typed detection) +- `search(queryVector, k, type?, filter?, options?): Promise>` + +```typescript +context.registerProvider('hnsw', (config, distanceFn, options) => { + return new MyNativeHNSWIndex(config, distanceFn, options) +}) +``` + +#### `metadataIndex` +**Type:** `(storage: StorageAdapter) => MetadataIndexManager-compatible` + +Factory function that creates a metadata index. The returned object must implement the `MetadataIndexManager` interface including `init()`, `addEntity()`, `removeEntity()`, `query()`, `flush()`, `clear()`, etc. + +```typescript +context.registerProvider('metadataIndex', (storage) => { + return new MyNativeMetadataIndex(storage) +}) +``` + +#### `graphIndex` +**Type:** `(storage: StorageAdapter) => GraphAdjacencyIndex-compatible` + +Factory function that creates a graph adjacency index for relationship tracking (verbs/triples). Must implement the `GraphAdjacencyIndex` interface including `addVerb()`, `getVerbsBySource()`, `getVerbsByTarget()`, `flush()`, etc. + +```typescript +context.registerProvider('graphIndex', (storage) => { + return new MyNativeGraphIndex(storage) +}) +``` + +### Utility Providers + +#### `cache` +**Type:** `UnifiedCache` + +Replaces the global `UnifiedCache` singleton used for VFS path resolution, semantic caching, and HNSW vector caching. Must implement the `UnifiedCache` interface (available from `@soulcraft/brainy/internals`). + +```typescript +import type { UnifiedCache } from '@soulcraft/brainy/internals' + +context.registerProvider('cache', myNativeCache) +``` + +#### `entityIdMapper` +**Type:** `(storage: StorageAdapter) => EntityIdMapper-compatible` + +Factory for bidirectional UUID ↔ integer mapping used by roaring bitmaps. Must implement `getOrAssign()`, `getUuid()`, `getInt()`, `has()`, `remove()`, `flush()`, `clear()`. + +#### `roaring` +**Type:** `RoaringBitmap32 class` + +Replacement for the roaring bitmap implementation. Used internally by the metadata index for set operations. Must be API-compatible with `roaring-wasm`. + +#### `msgpack` +**Type:** `{ encode: (data: any) => Buffer, decode: (buffer: Buffer) => any }` + +Native msgpack encode/decode for SSTable serialization. + +## Storage Adapter Plugins + +Plugins can register custom storage backends that users reference by name. + +### Implementing a Storage Adapter + +```typescript +import type { StorageAdapterFactory } from '@soulcraft/brainy/plugin' +import type { StorageAdapter } from '@soulcraft/brainy' + +class MyStorageAdapter implements StorageAdapter { + async init(): Promise { /* ... */ } + async saveNoun(noun: HNSWNoun): Promise { /* ... */ } + async getNoun(id: string): Promise { /* ... */ } + async deleteNoun(id: string): Promise { /* ... */ } + // ... implement all StorageAdapter methods +} +``` + +### Registering a Storage Adapter + +```typescript +context.registerProvider('storage:my-backend', { + name: 'my-backend', + create: (config: Record) => { + return new MyStorageAdapter(config) + } +} satisfies StorageAdapterFactory) +``` + +Users can then use your storage: + +```typescript +const brain = new Brainy({ storage: 'my-backend', myBackendOption: 'value' }) +``` + +## Import Paths + +Brainy provides three entry points for plugin developers: + +| Import Path | Contents | Stability | +|-------------|----------|-----------| +| `@soulcraft/brainy` | Public API, types, StorageAdapter | Stable (semver) | +| `@soulcraft/brainy/plugin` | BrainyPlugin, BrainyPluginContext, StorageAdapterFactory | Stable (semver) | +| `@soulcraft/brainy/internals` | UnifiedCache, EntityIdMapper, logger utilities | Internal (may change between minor versions) | + +## Diagnostics + +Brainy provides a `diagnostics()` method to verify plugin wiring: + +```typescript +const brain = new Brainy() +await brain.init() + +const diag = brain.diagnostics() +console.log(diag) +// { +// version: '7.14.0', +// plugins: { active: ['my-plugin'], count: 1 }, +// providers: { +// metadataIndex: { source: 'default' }, +// graphIndex: { source: 'default' }, +// embeddings: { source: 'plugin' }, +// embedBatch: { source: 'plugin' }, +// distance: { source: 'plugin' }, +// hnsw: { source: 'default' }, +// ... +// }, +// indexes: { +// hnsw: { size: 0, type: 'TypeAwareHNSWIndex' }, +// metadata: { type: 'MetadataIndexManager', initialized: true }, +// graph: { type: 'GraphAdjacencyIndex', initialized: true, wiredToStorage: true } +// } +// } +``` + +The CLI also supports diagnostics: + +```bash +brainy diagnostics +``` + +### Init-Time Summary + +When a plugin is active, brainy automatically logs a provider summary after `init()`: + +``` +[brainy] Plugin activated: @soulcraft/cortex +[brainy] Providers: 8/10 native (@soulcraft/cortex) | default: hnsw, cache +``` + +This tells you at a glance how many subsystems are accelerated and which ones are falling back to JavaScript. The log respects `config.silent`. + +### Fail-Fast for Production + +Use `requireProviders()` after `init()` to guarantee specific providers are plugin-supplied. This prevents silent fallback to JavaScript in deployments where you expect native acceleration: + +```typescript +const brain = new Brainy() +await brain.init() + +// Throws immediately if any of these are using JS fallback +brain.requireProviders(['distance', 'embeddings', 'metadataIndex', 'graphIndex']) +``` + +If a required provider is missing, the error message tells you exactly what's wrong: + +``` +[brainy] Required providers using JS fallback: graphIndex. +Active plugins: @soulcraft/cortex. +These providers must be supplied by a plugin for this deployment. +Check plugin installation, license, and native module availability. +``` + +This is the recommended pattern for production deployments with paid plugins — fail at startup rather than silently degrading performance. + +## Complete Example: Distance Acceleration Plugin + +A minimal but useful plugin that provides SIMD-accelerated distance calculations: + +```typescript +// simd-distance-plugin/src/plugin.ts +import type { BrainyPlugin, BrainyPluginContext } from '@soulcraft/brainy/plugin' + +// Hypothetical native module +import { simdCosineDistance } from './native.js' + +const simdDistancePlugin: BrainyPlugin = { + name: 'brainy-simd-distance', + + async activate(context: BrainyPluginContext): Promise { + // Check if SIMD is available on this platform + if (!checkSimdSupport()) { + console.log('[simd-distance] SIMD not available, skipping') + return false // Don't activate — brainy uses JS fallback + } + + context.registerProvider('distance', simdCosineDistance) + return true + } +} + +export default simdDistancePlugin +``` + +```json +// simd-distance-plugin/package.json +{ + "name": "brainy-simd-distance", + "main": "./dist/plugin.js", + "types": "./dist/plugin.d.ts", + "peerDependencies": { + "@soulcraft/brainy": ">=7.0.0" + } +} +``` + +Usage: + +```typescript +import { Brainy } from '@soulcraft/brainy' + +const brain = new Brainy({ plugins: ['brainy-simd-distance'] }) +await brain.init() + +// Verify it's active +const diag = brain.diagnostics() +console.log(diag.providers.distance) // { source: 'plugin' } +``` + +## Design Principles + +1. **Brainy works perfectly without plugins.** Every provider has a JavaScript fallback. Plugins only improve performance or add capabilities. + +2. **Provider keys are string-based.** The plugin system is not coupled to any specific plugin. Any package can register any provider. + +3. **Clean separation.** Plugins access brainy through the documented `BrainyPluginContext` interface. No direct access to internal classes is needed. + +4. **Fail-safe activation.** If a plugin throws during `activate()`, brainy logs a warning and continues with defaults. A broken plugin never prevents brainy from working. + +5. **Lifecycle management.** `deactivate()` is called during `brainy.close()` for resource cleanup. Native resources, connections, and file handles should be released here. diff --git a/src/brainy.ts b/src/brainy.ts index 52e446e1..e8192aa4 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -109,6 +109,20 @@ const STOPWORDS = new Set([ 'what', 'which', 'who', 'whom', 'whose', 'am' ]) +/** + * Result type for brain.diagnostics() + */ +export interface DiagnosticsResult { + version: string + plugins: { active: string[], count: number } + providers: Record + indexes: { + hnsw: { size: number, type: string } + metadata: { type: string, initialized: boolean } + graph: { type: string, initialized: boolean, wiredToStorage: boolean } + } +} + /** * The main Brainy class - Clean, Beautiful, Powerful * REAL IMPLEMENTATION - No stubs, no mocks @@ -122,6 +136,7 @@ export class Brainy implements BrainyInterface { // Core components private index!: HNSWIndex | TypeAwareHNSWIndex + private indexIsTypeAware = false // true when index supports addItem(item, type) / search(v, k, type) private storage!: BaseStorage private metadataIndex!: MetadataIndexManager private graphIndex!: GraphAdjacencyIndex @@ -292,18 +307,10 @@ export class Brainy implements BrainyInterface { this.distance = nativeDistance } - // Provider: HNSW index factory - const hnswFactory = this.pluginRegistry.getProvider<(config: any, distance: DistanceFunction, options: any) => any>('hnsw') - if (hnswFactory) { - const persistMode = this.resolveHNSWPersistMode() - this.index = hnswFactory( - { ...this.config.index, distanceFunction: this.distance }, - this.distance, - { storage: this.storage, persistMode } - ) - } else { - this.index = this.setupIndex() - } + // Provider: HNSW index factory (plugin or JS fallback) + this.index = this.createIndex() + this.indexIsTypeAware = this.index instanceof TypeAwareHNSWIndex + || typeof (this.index as any).getIndexForType === 'function' // Provider: metadata index factory const metadataFactory = this.pluginRegistry.getProvider<(storage: StorageAdapter) => any>('metadataIndex') @@ -315,6 +322,7 @@ export class Brainy implements BrainyInterface { const graphFactory = this.pluginRegistry.getProvider<(storage: StorageAdapter) => any>('graphIndex') if (graphFactory) { this.graphIndex = graphFactory(this.storage) + this.storage.setGraphIndex(this.graphIndex) await this.metadataIndex.init() } else { const [, graphIndex] = await Promise.all([ @@ -350,6 +358,23 @@ export class Brainy implements BrainyInterface { }) } + // Log provider summary after all wiring is complete + // Shows developers exactly what's native vs falling back to JS + if (this.pluginRegistry.hasActivePlugins() && !this.config.silent) { + const wellKnownKeys = [ + 'metadataIndex', 'graphIndex', 'entityIdMapper', 'cache', + 'hnsw', 'roaring', 'embeddings', 'embedBatch', 'distance', 'msgpack' + ] + const native = wellKnownKeys.filter(k => this.pluginRegistry.hasProvider(k)) + const fallback = wellKnownKeys.filter(k => !this.pluginRegistry.hasProvider(k)) + const plugins = this.pluginRegistry.getActivePlugins().join(', ') + if (fallback.length === 0) { + console.log(`[brainy] Providers: ${native.length}/${wellKnownKeys.length} native (${plugins})`) + } else { + console.log(`[brainy] Providers: ${native.length}/${wellKnownKeys.length} native (${plugins}) | default: ${fallback.join(', ')}`) + } + } + // Mark as initialized BEFORE VFS init // VFS.init() needs brain to be marked initialized to call brain methods this.initialized = true @@ -365,10 +390,10 @@ export class Brainy implements BrainyInterface { // instead of lazily on first embed() call. This moves the 90-140 second // WASM compilation to container startup rather than first request. // Recommended for: Cloud Run, Lambda, Fargate, Kubernetes - if (this.config.eagerEmbeddings) { - console.log('🚀 Eager embedding initialization enabled...') + if (this.config.eagerEmbeddings && !this.pluginRegistry.hasProvider('embeddings')) { + console.log('Eager embedding initialization enabled...') await embeddingManager.init() - console.log('✅ Embedding engine ready') + console.log('Embedding engine ready') } // Integration Hub initialization @@ -730,10 +755,10 @@ export class Brainy implements BrainyInterface { ) // Operation 3: Add to HNSW index (after entity saved) - if (this.index instanceof TypeAwareHNSWIndex) { + if (this.indexIsTypeAware) { tx.addOperation( new AddToTypeAwareHNSWOperation( - this.index, + this.index as any, id, vector, params.type as any @@ -741,7 +766,7 @@ export class Brainy implements BrainyInterface { ) } else { tx.addOperation( - new AddToHNSWOperation(this.index, id, vector) + new AddToHNSWOperation(this.index as any, id, vector) ) } @@ -1157,10 +1182,10 @@ export class Brainy implements BrainyInterface { // Operation 3-4: Update HNSW index (remove and re-add if reindexing needed) if (needsReindexing) { - if (this.index instanceof TypeAwareHNSWIndex) { + if (this.indexIsTypeAware) { tx.addOperation( new RemoveFromTypeAwareHNSWOperation( - this.index, + this.index as any, params.id, existing.vector, existing.type as any @@ -1168,7 +1193,7 @@ export class Brainy implements BrainyInterface { ) tx.addOperation( new AddToTypeAwareHNSWOperation( - this.index, + this.index as any, params.id, vector, newType as any @@ -1176,10 +1201,10 @@ export class Brainy implements BrainyInterface { ) } else { tx.addOperation( - new RemoveFromHNSWOperation(this.index, params.id, existing.vector) + new RemoveFromHNSWOperation(this.index as any, params.id, existing.vector) ) tx.addOperation( - new AddToHNSWOperation(this.index, params.id, vector) + new AddToHNSWOperation(this.index as any, params.id, vector) ) } } @@ -1240,18 +1265,18 @@ export class Brainy implements BrainyInterface { await this.transactionManager.executeTransaction(async (tx) => { // Operation 1: Remove from vector index if (noun && metadata) { - if (this.index instanceof TypeAwareHNSWIndex && metadata.noun) { + if (this.indexIsTypeAware && metadata.noun) { tx.addOperation( new RemoveFromTypeAwareHNSWOperation( - this.index, + this.index as any, id, noun.vector, metadata.noun as any ) ) - } else if (this.index instanceof HNSWIndex) { + } else { tx.addOperation( - new RemoveFromHNSWOperation(this.index, id, noun.vector) + new RemoveFromHNSWOperation(this.index as any, id, noun.vector) ) } } @@ -2505,18 +2530,18 @@ export class Brainy implements BrainyInterface { // Add delete operations to transaction if (noun && metadata) { - if (this.index instanceof TypeAwareHNSWIndex && metadata.noun) { + if (this.indexIsTypeAware && metadata.noun) { tx.addOperation( new RemoveFromTypeAwareHNSWOperation( - this.index, + this.index as any, id, noun.vector, metadata.noun as any ) ) - } else if (this.index instanceof HNSWIndex) { + } else { tx.addOperation( - new RemoveFromHNSWOperation(this.index, id, noun.vector) + new RemoveFromHNSWOperation(this.index as any, id, noun.vector) ) } } @@ -2752,13 +2777,15 @@ export class Brainy implements BrainyInterface { if ('clear' in this.index && typeof this.index.clear === 'function') { await this.index.clear() } else { - // Recreate index if no clear method - this.index = this.setupIndex() + // Recreate index using plugin factory when available + this.index = this.createIndex() } // Recreate metadata index to clear cached data - // Bug: Metadata index cache was not being cleared, causing find() with type filters to return stale data - this.metadataIndex = new MetadataIndexManager(this.storage) + const clearMetadataFactory = this.pluginRegistry.getProvider<(storage: StorageAdapter) => any>('metadataIndex') + this.metadataIndex = clearMetadataFactory + ? clearMetadataFactory(this.storage) + : new MetadataIndexManager(this.storage) await this.metadataIndex.init() // Reset dimensions @@ -2919,8 +2946,9 @@ export class Brainy implements BrainyInterface { clone.storage.currentBranch = branchName // isInitialized inherited from prototype - // Shallow copy HNSW index (INSTANT - just copies Map references) - clone.index = this.setupIndex() + // Create HNSW index (uses plugin factory when available) + clone.index = this.createIndex() + clone.indexIsTypeAware = this.indexIsTypeAware // Enable COW (handle both HNSWIndex and TypeAwareHNSWIndex) if ('enableCOW' in clone.index && typeof clone.index.enableCOW === 'function') { @@ -2928,7 +2956,10 @@ export class Brainy implements BrainyInterface { } // Fast rebuild for small indexes from COW storage (Metadata/Graph are fast) - clone.metadataIndex = new MetadataIndexManager(clone.storage) + const cloneMetadataFactory = this.pluginRegistry.getProvider<(storage: StorageAdapter) => any>('metadataIndex') + clone.metadataIndex = cloneMetadataFactory + ? cloneMetadataFactory(clone.storage) + : new MetadataIndexManager(clone.storage) await clone.metadataIndex.init() // GraphAdjacencyIndex SINGLETON pattern for fork() @@ -2937,7 +2968,13 @@ export class Brainy implements BrainyInterface { // create a fresh instance for the clone's branch. ;(clone.storage as any).graphIndex = undefined ;(clone.storage as any).graphIndexPromise = undefined - clone.graphIndex = await (clone.storage as any).getGraphIndex() + const cloneGraphFactory = this.pluginRegistry.getProvider<(storage: StorageAdapter) => any>('graphIndex') + if (cloneGraphFactory) { + clone.graphIndex = cloneGraphFactory(clone.storage) + clone.storage.setGraphIndex(clone.graphIndex) + } else { + clone.graphIndex = await (clone.storage as any).getGraphIndex() + } // getGraphIndex() will rebuild automatically if data exists (via _initializeGraphIndex) // Mark as initialized @@ -3021,15 +3058,24 @@ export class Brainy implements BrainyInterface { // Fix: Reload indexes from new branch WITHOUT recreating storage // Previous implementation called init() which recreated storage, losing currentBranch - this.index = this.setupIndex() - this.metadataIndex = new (MetadataIndexManager as any)(this.storage) + this.index = this.createIndex() + const checkoutMetadataFactory = this.pluginRegistry.getProvider<(storage: StorageAdapter) => any>('metadataIndex') + this.metadataIndex = checkoutMetadataFactory + ? checkoutMetadataFactory(this.storage) + : new MetadataIndexManager(this.storage) await this.metadataIndex.init() // GraphAdjacencyIndex SINGLETON pattern for checkout() // Invalidate the old graphIndex (it has data from the old branch) // and get a fresh instance for the new branch ;(this.storage as any).invalidateGraphIndex() - this.graphIndex = await (this.storage as any).getGraphIndex() + const checkoutGraphFactory = this.pluginRegistry.getProvider<(storage: StorageAdapter) => any>('graphIndex') + if (checkoutGraphFactory) { + this.graphIndex = checkoutGraphFactory(this.storage) + this.storage.setGraphIndex(this.graphIndex) + } else { + this.graphIndex = await (this.storage as any).getGraphIndex() + } // Reset lazy loading state when switching branches // Indexes contain data from previous branch, must rebuild for new branch @@ -4264,6 +4310,94 @@ export class Brainy implements BrainyInterface { } } + /** + * Plugin and provider diagnostics — shows what's active and how subsystems are wired. + * + * @example + * ```typescript + * const diag = brain.diagnostics() + * console.log(diag.providers) // { hnsw: { source: 'plugin' }, ... } + * console.log(diag.indexes.graph.wiredToStorage) // true + * ``` + */ + diagnostics(): DiagnosticsResult { + const wellKnownKeys = [ + 'metadataIndex', 'graphIndex', 'entityIdMapper', 'cache', + 'hnsw', 'roaring', 'embeddings', 'embedBatch', 'distance', 'msgpack' + ] as const + + const providers: Record = {} + for (const key of wellKnownKeys) { + providers[key] = { + source: this.pluginRegistry.hasProvider(key) ? 'plugin' : 'default' + } + } + + const hnswSize = this.index.size() + const metadataInitialized = !!this.metadataIndex + const graphInitialized = !!this.graphIndex + const storageGraphIndex = (this.storage as any).graphIndex + + return { + version: getBrainyVersion(), + plugins: { + active: this.pluginRegistry.getActivePlugins(), + count: this.pluginRegistry.getActivePlugins().length + }, + providers, + indexes: { + hnsw: { + size: hnswSize, + type: this.index.constructor.name + }, + metadata: { + type: this.metadataIndex?.constructor.name || 'none', + initialized: metadataInitialized + }, + graph: { + type: this.graphIndex?.constructor.name || 'none', + initialized: graphInitialized, + wiredToStorage: graphInitialized && storageGraphIndex === this.graphIndex + } + } + } + } + + /** + * Assert that specific providers are supplied by a plugin (not using JS fallback). + * + * Call after init() in production to fail fast if a paid plugin (e.g. cortex) + * isn't providing the expected acceleration. Throws if any listed key is using + * the default JavaScript implementation. + * + * @param keys - Provider keys that MUST come from a plugin + * @throws Error listing which providers are falling back to defaults + * + * @example + * ```typescript + * const brain = new Brainy() + * await brain.init() + * + * // Fail fast if cortex isn't providing these + * brain.requireProviders(['distance', 'embeddings', 'metadataIndex', 'graphIndex']) + * ``` + */ + requireProviders(keys: string[]): void { + const missing = keys.filter(k => !this.pluginRegistry.hasProvider(k)) + if (missing.length > 0) { + const active = this.pluginRegistry.getActivePlugins() + const pluginInfo = active.length > 0 + ? `Active plugins: ${active.join(', ')}` + : 'No plugins active' + throw new Error( + `[brainy] Required providers using JS fallback: ${missing.join(', ')}. ` + + `${pluginInfo}. ` + + `These providers must be supplied by a plugin for this deployment. ` + + `Check plugin installation, license, and native module availability.` + ) + } + } + /** * Efficient Pagination API - Production-scale pagination using index-first approach * Automatically optimizes based on query type and applies pagination at the index level @@ -4655,7 +4789,16 @@ export class Brainy implements BrainyInterface { return [] } - // Use native WASM batch API: single forward pass instead of N individual calls + // Plugin provides native batch embedding — single forward pass for all texts + const batchProvider = this.pluginRegistry.getProvider<(texts: string[]) => Promise>('embedBatch') + if (batchProvider) { + return batchProvider(texts) + } + // Plugin provides single-text embedding engine — map through it + if (this.pluginRegistry.hasProvider('embeddings')) { + return Promise.all(texts.map(t => this.embedder(t))) + } + // Default: WASM batch API (single forward pass, more efficient than N calls) return await embeddingManager.embedBatch(texts, options) } @@ -5481,9 +5624,9 @@ export class Brainy implements BrainyInterface { const vector = params.vector || (await this.embed(params.query!)) const limit = params.limit || 10 - // Phase 2: Pass type for TypeAwareHNSWIndex (10x faster for type-specific queries) - const searchResults = this.index instanceof TypeAwareHNSWIndex - ? await this.index.search(vector, limit * 2, params.type as any) + // Pass type for type-aware indexes (10x faster for type-specific queries) + const searchResults: [string, number][] = this.indexIsTypeAware && params.type + ? await (this.index as any).search(vector, limit * 2, params.type as any) : await this.index.search(vector, limit * 2) // Batch-load entities for 10-50x faster cloud storage performance @@ -5512,9 +5655,9 @@ export class Brainy implements BrainyInterface { const nearEntity = await this.get(params.near.id) if (!nearEntity) return [] - // Phase 2: Pass type for TypeAwareHNSWIndex - const nearResults = this.index instanceof TypeAwareHNSWIndex - ? await this.index.search(nearEntity.vector, params.limit || 10, params.type as any) + // Pass type for type-aware indexes + const nearResults: [string, number][] = this.indexIsTypeAware && params.type + ? await (this.index as any).search(nearEntity.vector, params.limit || 10, params.type as any) : await this.index.search(nearEntity.vector, params.limit || 10) // Filter by threshold first to minimize batch fetch @@ -6068,11 +6211,16 @@ export class Brainy implements BrainyInterface { throw new Error('Brain must be initialized before warming up embeddings. Call init() first.') } - console.log('🚀 Warming up embedding engine...') + // Plugin-provided embeddings are already ready (native, no WASM warmup needed) + if (this.pluginRegistry.hasProvider('embeddings')) { + return + } + + console.log('Warming up embedding engine...') const start = Date.now() await embeddingManager.init() const elapsed = Date.now() - start - console.log(`✅ Embedding engine ready in ${elapsed}ms`) + console.log(`Embedding engine ready in ${elapsed}ms`) } /** @@ -6081,6 +6229,10 @@ export class Brainy implements BrainyInterface { * @returns true if embedding engine is ready for immediate use */ isEmbeddingReady(): boolean { + // Plugin-provided embeddings are always ready (native, no WASM init required) + if (this.pluginRegistry.hasProvider('embeddings')) { + return true + } return embeddingManager.isInitialized() } @@ -6177,6 +6329,23 @@ export class Brainy implements BrainyInterface { return new HNSWIndex(indexConfig as any, this.distance, { persistMode }) } + /** + * Create an HNSW index, using plugin factory when available. + * Shared by init(), fork(), checkout(), and clear() to avoid duplication. + */ + private createIndex(): HNSWIndex | TypeAwareHNSWIndex { + const hnswFactory = this.pluginRegistry.getProvider<(config: any, distance: DistanceFunction, options: any) => any>('hnsw') + if (hnswFactory) { + const persistMode = this.resolveHNSWPersistMode() + return hnswFactory( + { ...this.config.index, distanceFunction: this.distance }, + this.distance, + { storage: this.storage, persistMode } + ) + } + return this.setupIndex() + } + /** * Resolve HNSW persistence mode. * Extracted so both setupIndex() and the HNSW plugin factory path can use it. diff --git a/src/cli/commands/core.ts b/src/cli/commands/core.ts index 9f9f2947..2766e3ea 100644 --- a/src/cli/commands/core.ts +++ b/src/cli/commands/core.ts @@ -918,5 +918,55 @@ export const coreCommands = { console.error(chalk.red(error.message)) process.exit(1) } + }, + + /** + * Show plugin and provider diagnostics + */ + async diagnostics(options: CoreOptions) { + try { + const brain = getBrainy() + await brain.init() + + const diag = brain.diagnostics() + + if (options.json) { + formatOutput(diag, options) + return + } + + console.log(chalk.bold('\nBrainy Diagnostics')) + console.log(chalk.dim(`Version: ${diag.version}`)) + console.log() + + // Plugins + console.log(chalk.bold('Plugins:')) + if (diag.plugins.count === 0) { + console.log(chalk.dim(' (none active)')) + } else { + for (const name of diag.plugins.active) { + console.log(chalk.green(` ✓ ${name}`)) + } + } + console.log() + + // Providers + console.log(chalk.bold('Providers:')) + for (const [key, info] of Object.entries(diag.providers)) { + const icon = info.source === 'plugin' ? chalk.green('✓ plugin') : chalk.dim('default') + console.log(` ${key.padEnd(16)} ${icon}`) + } + console.log() + + // Indexes + console.log(chalk.bold('Indexes:')) + console.log(` HNSW: ${diag.indexes.hnsw.type} (${diag.indexes.hnsw.size} vectors)`) + console.log(` Metadata: ${diag.indexes.metadata.type} (initialized: ${diag.indexes.metadata.initialized})`) + console.log(` Graph: ${diag.indexes.graph.type} (initialized: ${diag.indexes.graph.initialized}, wired: ${diag.indexes.graph.wiredToStorage})`) + console.log() + } catch (error: any) { + console.error(chalk.red('Diagnostics failed: ' + error.message)) + process.exit(1) + } } } \ No newline at end of file diff --git a/src/cli/index.ts b/src/cli/index.ts index 9894830d..9f1b09eb 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -174,6 +174,14 @@ program .option('-f, --format ', 'Output format (json|csv|jsonl)', 'json') .action(coreCommands.export) +program + .command('diagnostics') + .alias('diag') + .description('Show plugin and provider diagnostics') + .option('--json', 'Output as JSON') + .option('--pretty', 'Pretty print JSON') + .action(coreCommands.diagnostics) + // ===== Neural Commands ===== program diff --git a/src/index.ts b/src/index.ts index da8171da..cfa93062 100644 --- a/src/index.ts +++ b/src/index.ts @@ -14,6 +14,9 @@ import { Brainy } from './brainy.js' export { Brainy } +// Export diagnostics result type +export type { DiagnosticsResult } from './brainy.js' + // Export Brainy configuration and types export type { BrainyConfig, diff --git a/src/neural/improvedNeuralAPI.ts b/src/neural/improvedNeuralAPI.ts index 9e439312..839b3828 100644 --- a/src/neural/improvedNeuralAPI.ts +++ b/src/neural/improvedNeuralAPI.ts @@ -92,18 +92,20 @@ interface ItemWithMetadata { export class ImprovedNeuralAPI { private brain: any // Brainy instance private config: NeuralAPIConfig - + private distanceFn: (a: Vector, b: Vector) => number + // Caching for performance private similarityCache = new Map() private clusterCache = new Map() private hierarchyCache = new Map() private neighborsCache = new Map() - + // Performance tracking private performanceMetrics = new Map() - + constructor(brain: any, config: NeuralAPIConfig = {}) { this.brain = brain + this.distanceFn = brain.distance || cosineDistance this.config = { cacheSize: 1000, defaultAlgorithm: 'auto', @@ -114,7 +116,7 @@ export class ImprovedNeuralAPI { streamingBatchSize: 100, ...config } - + this._initializeCleanupTimer() } @@ -1458,7 +1460,7 @@ export class ImprovedNeuralAPI { switch (metric) { case 'cosine': - score = 1 - cosineDistance(v1, v2) + score = 1 - this.distanceFn(v1, v2) break case 'euclidean': score = 1 / (1 + euclideanDistance(v1, v2)) @@ -1467,7 +1469,7 @@ export class ImprovedNeuralAPI { score = 1 / (1 + this._manhattanDistance(v1, v2)) break default: - score = 1 - cosineDistance(v1, v2) + score = 1 - this.distanceFn(v1, v2) } if (options.detailed) { @@ -3047,7 +3049,7 @@ export class ImprovedNeuralAPI { continue } - const similarity = 1 - cosineDistance( + const similarity = 1 - this.distanceFn( Array.from(c1.centroid) as number[], Array.from(c2.centroid) as number[] ) diff --git a/src/neural/neuralAPI.ts b/src/neural/neuralAPI.ts index 7ba11921..ca7d9c5a 100644 --- a/src/neural/neuralAPI.ts +++ b/src/neural/neuralAPI.ts @@ -132,12 +132,14 @@ export interface LODConfig { */ export class NeuralAPI { private brain: any // Brainy instance + private distanceFn: (a: Vector, b: Vector) => number private similarityCache: Map = new Map() private clusterCache: Map = new Map() // Enhanced for enterprise private hierarchyCache: Map = new Map() - + constructor(brain: any) { this.brain = brain + this.distanceFn = brain.distance || cosineDistance } // ===== SMART USER-FRIENDLY API ===== @@ -471,7 +473,7 @@ export class NeuralAPI { } // Calculate similarity - const score = cosineDistance(itemA.vector, itemB.vector) + const score = this.distanceFn(itemA.vector, itemB.vector) this.similarityCache.set(cacheKey, score) @@ -498,7 +500,7 @@ export class NeuralAPI { } private async similarityByVector(vectorA: Vector, vectorB: Vector, options?: SimilarityOptions): Promise { - const score = cosineDistance(vectorA, vectorB) + const score = this.distanceFn(vectorA, vectorB) if (options?.explain) { return { @@ -610,7 +612,7 @@ export class NeuralAPI { // Find minimum distance to existing sample let minDistance = Infinity for (const selected of sample) { - const distance = cosineDistance(candidate.vector, selected.vector) + const distance = this.distanceFn(candidate.vector, selected.vector) minDistance = Math.min(minDistance, distance) } @@ -652,7 +654,7 @@ export class NeuralAPI { let bestDistance = Infinity for (let c = 0; c < k; c++) { - const distance = cosineDistance(item.vector, centroids[c]) + const distance = this.distanceFn(item.vector, centroids[c]) if (distance < bestDistance) { bestDistance = distance bestCluster = c @@ -679,7 +681,7 @@ export class NeuralAPI { let bestDistance = Infinity for (let cc = 0; cc < k; cc++) { - const distance = cosineDistance(item.vector, centroids[cc]) + const distance = this.distanceFn(item.vector, centroids[cc]) if (distance < bestDistance) { bestDistance = distance bestCluster = cc @@ -750,7 +752,7 @@ export class NeuralAPI { let merged = false for (let i = 0; i < result.length; i++) { - const similarity = cosineDistance(result[i].centroid, batchCluster.centroid) + const similarity = this.distanceFn(result[i].centroid, batchCluster.centroid) if (similarity > 0.8) { // Merge clusters diff --git a/src/neural/signals/VerbEmbeddingSignal.ts b/src/neural/signals/VerbEmbeddingSignal.ts index fdb086de..282a26fa 100644 --- a/src/neural/signals/VerbEmbeddingSignal.ts +++ b/src/neural/signals/VerbEmbeddingSignal.ts @@ -64,6 +64,7 @@ interface HistoricalEntry { */ export class VerbEmbeddingSignal { private brain: Brainy + private distanceFn: (a: Vector, b: Vector) => number private options: Required // Pre-computed verb type embeddings (loaded once at startup) @@ -88,6 +89,7 @@ export class VerbEmbeddingSignal { constructor(brain: Brainy, options?: VerbEmbeddingSignalOptions) { this.brain = brain + this.distanceFn = (brain as any).distance || cosineDistance this.options = { minConfidence: options?.minConfidence ?? 0.60, minSimilarity: options?.minSimilarity ?? 0.55, @@ -142,7 +144,7 @@ export class VerbEmbeddingSignal { const similarities: Array<{ type: VerbType; similarity: number }> = [] for (const [verbType, typeEmbedding] of this.verbTypeEmbeddings) { - const distance = cosineDistance(embedding, typeEmbedding) + const distance = this.distanceFn(embedding, typeEmbedding) const similarity = 1 - distance // Convert distance to similarity similarities.push({ type: verbType, similarity }) } diff --git a/src/plugin.ts b/src/plugin.ts index b4b4e2be..4ed44443 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -43,7 +43,8 @@ export interface BrainyPluginContext { * - 'cache' — UnifiedCache replacement * - 'hnsw' — HNSWIndex replacement * - 'roaring' — RoaringBitmap32 replacement - * - 'embeddings' — Embedding engine replacement + * - 'embeddings' — Embedding engine replacement (single text) + * - 'embedBatch' — Batch embedding engine (texts[] → vectors[]) * - 'distance' — Distance function overrides * - 'msgpack' — Msgpack encode/decode * diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts index bf8c9569..3b93118c 100644 --- a/src/storage/baseStorage.ts +++ b/src/storage/baseStorage.ts @@ -379,6 +379,16 @@ export abstract class BaseStorage extends BaseStorageAdapter { } } + /** + * Set the graph index instance (used by Brainy to wire plugin-provided graph indexes). + * This ensures getVerbsBySource() uses the fast GraphAdjacencyIndex path + * instead of falling back to O(n) shard iteration. + */ + public setGraphIndex(index: GraphAdjacencyIndex): void { + this.graphIndex = index + this.graphIndexPromise = Promise.resolve(index) + } + /** * Ensure the storage adapter is initialized */ diff --git a/src/vfs/PathResolver.ts b/src/vfs/PathResolver.ts index 3c8ad897..9d3ae742 100644 --- a/src/vfs/PathResolver.ts +++ b/src/vfs/PathResolver.ts @@ -201,9 +201,8 @@ export class PathResolver { * Falls back to graph traversal if MetadataIndex unavailable */ private async resolveWithMetadataIndex(path: string): Promise { - // Access MetadataIndexManager from brain's storage - const storage = (this.brain as any).storage - const metadataIndex = storage?.metadataIndex + // Access MetadataIndexManager from brain instance (not storage — metadataIndex lives on Brainy) + const metadataIndex = (this.brain as any).metadataIndex if (!metadataIndex) { // MetadataIndex not available, use graph traversal diff --git a/src/vfs/semantic/SemanticPathResolver.ts b/src/vfs/semantic/SemanticPathResolver.ts index db6b2249..9b51c36a 100644 --- a/src/vfs/semantic/SemanticPathResolver.ts +++ b/src/vfs/semantic/SemanticPathResolver.ts @@ -17,7 +17,7 @@ import { PathResolver } from '../PathResolver.js' import { VFSEntity, VFSError, VFSErrorCode } from '../types.js' import { SemanticPathParser, ParsedSemanticPath } from './SemanticPathParser.js' import { ProjectionRegistry } from './ProjectionRegistry.js' -import { UnifiedCache } from '../../utils/unifiedCache.js' +import { getGlobalCache, UnifiedCache } from '../../utils/unifiedCache.js' /** * Semantic Path Resolver @@ -44,12 +44,8 @@ export class SemanticPathResolver { this.registry = registry this.parser = new SemanticPathParser() - // Use Brainy's UnifiedCache for semantic path caching - // Zero-config: Uses 2GB default from UnifiedCache - this.cache = new UnifiedCache({ - enableRequestCoalescing: true, - enableFairnessCheck: true - }) + // Use global UnifiedCache (picks up plugin-provided cache when available) + this.cache = getGlobalCache() // Create traditional path resolver (uses its own optimized cache with defaults) this.pathResolver = new PathResolver(brain, rootEntityId)