diff --git a/README.md b/README.md index 1c85d92f..d309cc7a 100644 --- a/README.md +++ b/README.md @@ -574,6 +574,14 @@ const stats = brain.getCacheStats() // Performance insights **[📖 Capacity Planning Guide →](docs/operations/capacity-planning.md)** +### Native Acceleration (Optional) + +Install `@soulcraft/brainy-cortex` for Rust-powered native acceleration: SIMD distance calculations, native metadata/graph indexes, CRoaring bitmaps, and Candle ML embeddings. Auto-detected — zero configuration required. + +```bash +npm install @soulcraft/brainy-cortex +``` + --- ## Benchmarks @@ -710,6 +718,8 @@ This comprehensive guide includes: ## Requirements +> **Deprecation Notice:** Browser support (OPFS storage, Web Workers, WASM embeddings) is deprecated in v7.10.0 and will be removed in v8.0.0. Brainy v8+ will be server-only (Node.js >= 22, Bun >= 1.3.0). Migrate browser deployments to a server-side API. + **Bun 1.0+** (recommended) or **Node.js 22 LTS** ```bash diff --git a/package.json b/package.json index a2b2cfe0..45d5e96e 100644 --- a/package.json +++ b/package.json @@ -51,6 +51,14 @@ "./neural/SmartRelationshipExtractor": { "import": "./dist/neural/SmartRelationshipExtractor.js", "types": "./dist/neural/SmartRelationshipExtractor.d.ts" + }, + "./plugin": { + "import": "./dist/plugin.js", + "types": "./dist/plugin.d.ts" + }, + "./internals": { + "import": "./dist/internals.js", + "types": "./dist/internals.d.ts" } }, "browser": { diff --git a/src/brainy.ts b/src/brainy.ts index 234edf70..06269d37 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -34,6 +34,8 @@ import { BlobStorage } from './storage/cow/BlobStorage.js' import { NULL_HASH } from './storage/cow/constants.js' import { createPipeline } from './streaming/pipeline.js' import { configureLogger, LogLevel } from './utils/logger.js' +import { setGlobalCache } from './utils/unifiedCache.js' +import type { UnifiedCache } from './utils/unifiedCache.js' import { PluginRegistry } from './plugin.js' import type { BrainyPlugin, BrainyPluginContext } from './plugin.js' import { TransactionManager } from './transaction/TransactionManager.js' @@ -262,7 +264,11 @@ export class Brainy implements BrainyInterface { } try { - // Setup and initialize storage + // Auto-detect and activate plugins BEFORE storage setup + // so plugin-provided storage factories (e.g., mmap-filesystem) are available + await this.loadPlugins() + + // Setup and initialize storage (checks plugin storage factories first) this.storage = await this.setupStorage() await this.storage.init() @@ -273,20 +279,55 @@ export class Brainy implements BrainyInterface { (this.storage as any).enableCOWLightweight((this.config.storage as any)?.branch || 'main') } - // Auto-detect and activate plugins (e.g., @soulcraft/brainy-cortex) - await this.loadPlugins() + // Provider: embeddings (reassign embedder if plugin provides one) + const embeddingProvider = this.pluginRegistry.getProvider('embeddings') + if (embeddingProvider) { + this.embedder = embeddingProvider + } - // Setup index now that we have storage - this.index = this.setupIndex() + // Provider: cache (replace global singleton before any consumer uses it) + const cacheProvider = this.pluginRegistry.getProvider('cache') + if (cacheProvider) { + setGlobalCache(cacheProvider) + } - // Initialize metadata index and graph index in parallel - // Both only need storage — they are independent of each other - this.metadataIndex = new MetadataIndexManager(this.storage) - const [, graphIndex] = await Promise.all([ - this.metadataIndex.init(), - (this.storage as any).getGraphIndex() - ]) - this.graphIndex = graphIndex + // Provider: distance function (resolve BEFORE setupIndex — index uses this.distance) + const nativeDistance = this.pluginRegistry.getProvider('distance') + if (nativeDistance) { + this.distance = nativeDistance + } + + // Provider: HNSW index factory + const hnswFactory = this.pluginRegistry.getProvider<(config: any, distance: DistanceFunction, options: any) => any>('hnsw') + if (hnswFactory) { + const persistMode = this.resolveHNSWPersistMode() + this.index = hnswFactory( + { ...this.config.index, distanceFunction: this.distance }, + this.distance, + { storage: this.storage, persistMode } + ) + } else { + this.index = this.setupIndex() + } + + // Provider: metadata index factory + const metadataFactory = this.pluginRegistry.getProvider<(storage: StorageAdapter) => any>('metadataIndex') + this.metadataIndex = metadataFactory + ? metadataFactory(this.storage) + : new MetadataIndexManager(this.storage) + + // Provider: graph index factory + const graphFactory = this.pluginRegistry.getProvider<(storage: StorageAdapter) => any>('graphIndex') + if (graphFactory) { + this.graphIndex = graphFactory(this.storage) + await this.metadataIndex.init() + } else { + const [, graphIndex] = await Promise.all([ + this.metadataIndex.init(), + (this.storage as any).getGraphIndex() + ]) + this.graphIndex = graphIndex + } // Rebuild indexes if needed for existing data await this.rebuildIndexesIfNeeded() @@ -6140,9 +6181,18 @@ export class Brainy implements BrainyInterface { * Setup storage */ private async setupStorage(): Promise { - // Pass the entire storage config object to createStorage - // This ensures all storage-specific configs (gcsNativeStorage, s3Storage, etc.) are passed through - const storage = await createStorage(this.config.storage as any) + const storageConfig = (this.config.storage || {}) as Record + const storageType = (storageConfig.type as string) || 'auto' + + // Check plugin-provided storage factories (e.g., 'mmap-filesystem' from cortex) + const pluginFactory = this.pluginRegistry.getStorageFactory(storageType) + if (pluginFactory) { + const adapter = await pluginFactory.create(storageConfig) + return adapter as BaseStorage + } + + // Fall through to built-in storage types + const storage = await createStorage(storageConfig as any) return storage as BaseStorage } @@ -6189,23 +6239,9 @@ export class Brainy implements BrainyInterface { distanceFunction: this.distance } - // Determine persist mode (user config > smart default) - // Fixed to use getStorageType() for reliable detection - let persistMode: 'immediate' | 'deferred' = this.config.hnswPersistMode || 'immediate' + const persistMode = this.resolveHNSWPersistMode() - // Smart default: Use deferred mode for cloud storage adapters - if (!this.config.hnswPersistMode) { - // FIX: Use instance-based detection as fallback - // Previously this.config.storage.type was often undefined after storage creation, - // causing cloud storage to incorrectly use 'immediate' mode (50-100x slower) - const storageType = this.config.storage?.type || this.getStorageType() - const cloudStorageTypes = ['gcs', 's3', 'r2', 'azure'] - if (cloudStorageTypes.includes(storageType)) { - persistMode = 'deferred' - } - } - - // Phase 2: Use TypeAwareHNSWIndex for billion-scale optimization + // Use TypeAwareHNSWIndex for non-memory storage (billion-scale optimization) const detectedStorageType = this.config.storage?.type || this.getStorageType() if (detectedStorageType !== 'memory') { return new TypeAwareHNSWIndex(indexConfig, this.distance, { @@ -6218,6 +6254,27 @@ export class Brainy implements BrainyInterface { return new HNSWIndex(indexConfig as any, this.distance, { persistMode }) } + /** + * Resolve HNSW persistence mode. + * Extracted so both setupIndex() and the HNSW plugin factory path can use it. + * + * User config > smart default: + * - Cloud storage (GCS/S3/R2/Azure): 'deferred' for 30-50x faster adds + * - Local storage (FileSystem/Memory/OPFS): 'immediate' (already fast) + */ + private resolveHNSWPersistMode(): 'immediate' | 'deferred' { + let persistMode: 'immediate' | 'deferred' = this.config.hnswPersistMode || 'immediate' + + if (!this.config.hnswPersistMode) { + const storageType = this.config.storage?.type || this.getStorageType() + if (['gcs', 's3', 'r2', 'azure'].includes(storageType)) { + persistMode = 'deferred' + } + } + + return persistMode + } + /** * Setup augmentations */ diff --git a/src/embeddings/EmbeddingManager.ts b/src/embeddings/EmbeddingManager.ts index 51cbc6bf..d30702f7 100644 --- a/src/embeddings/EmbeddingManager.ts +++ b/src/embeddings/EmbeddingManager.ts @@ -14,6 +14,7 @@ import { Vector, EmbeddingFunction } from '../coreTypes.js' import { WASMEmbeddingEngine } from './wasm/index.js' +import { isBrowser } from '../utils/environment.js' // Types export type ModelPrecision = 'q8' | 'fp32' @@ -115,6 +116,9 @@ export class EmbeddingManager { const startTime = Date.now() try { + if (isBrowser()) { + console.warn('[brainy] Browser WASM embedding engine is deprecated and will be removed in v8.0. Use Node.js/Bun with native embeddings (@soulcraft/brainy-cortex) instead.') + } // Initialize WASM engine (handles all model loading) await this.engine.initialize() diff --git a/src/internals.ts b/src/internals.ts new file mode 100644 index 00000000..d9fe1140 --- /dev/null +++ b/src/internals.ts @@ -0,0 +1,11 @@ +/** + * Internal utilities for first-party plugins. + * NOT part of the public API — may change between minor versions. + */ +export { getGlobalCache, setGlobalCache, clearGlobalCache, UnifiedCache } from './utils/unifiedCache.js' +export type { UnifiedCacheConfig, CacheItem } from './utils/unifiedCache.js' +export { prodLog, createModuleLogger } from './utils/logger.js' +export { FieldTypeInference, FieldType } from './utils/fieldTypeInference.js' +export type { FieldTypeInfo } from './utils/fieldTypeInference.js' +export { EntityIdMapper } from './utils/entityIdMapper.js' +export type { EntityIdMapperOptions, EntityIdMapperData } from './utils/entityIdMapper.js' diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts index 1bc97436..ea68f85d 100644 --- a/src/storage/adapters/fileSystemStorage.ts +++ b/src/storage/adapters/fileSystemStorage.ts @@ -78,7 +78,7 @@ export class FileSystemStorage extends BaseStorage { private readonly SHARDING_DEPTH = 1 as const private readonly MAX_SHARDS = 256 // Hex range: 00-ff private cachedShardingDepth: number = this.SHARDING_DEPTH // Always use fixed depth - private rootDir: string + protected rootDir: string private nounsDir!: string private verbsDir!: string private metadataDir!: string diff --git a/src/storage/adapters/opfsStorage.ts b/src/storage/adapters/opfsStorage.ts index 7139d82a..db1710ad 100644 --- a/src/storage/adapters/opfsStorage.ts +++ b/src/storage/adapters/opfsStorage.ts @@ -80,6 +80,7 @@ export class OPFSStorage extends BaseStorage { constructor() { super() + console.warn('[brainy] OPFS storage is deprecated and will be removed in v8.0. Use Node.js/Bun with filesystem or mmap-filesystem storage instead.') // Check if OPFS is available this.isAvailable = typeof navigator !== 'undefined' && diff --git a/src/storage/storageFactory.ts b/src/storage/storageFactory.ts index 07ac43e7..7b075c92 100644 --- a/src/storage/storageFactory.ts +++ b/src/storage/storageFactory.ts @@ -480,6 +480,7 @@ export async function createStorage( return memStorage case 'opfs': { + console.warn('[brainy] OPFS storage is deprecated and will be removed in v8.0. Migrate to Node.js/Bun with filesystem or mmap-filesystem storage.') // Check if OPFS is available const opfsStorage = new OPFSStorage() if (opfsStorage.isOPFSAvailable()) { @@ -839,6 +840,7 @@ export async function createStorage( // Next, try OPFS (browser only) if (isBrowser()) { + console.warn('[brainy] Browser environment detected. Browser support is deprecated and will be removed in v8.0. Migrate to Node.js/Bun.') const opfsStorage = new OPFSStorage() if (opfsStorage.isOPFSAvailable()) { console.log('Using OPFS storage (auto-detected) + TypeAware wrapper') diff --git a/src/utils/environment.ts b/src/utils/environment.ts index c818cb62..021e983a 100644 --- a/src/utils/environment.ts +++ b/src/utils/environment.ts @@ -4,6 +4,7 @@ /** * Check if code is running in a browser environment + * @deprecated Browser support is deprecated and will be removed in v8.0. Use Node.js or Bun instead. */ export function isBrowser(): boolean { return typeof window !== 'undefined' && typeof document !== 'undefined' @@ -39,6 +40,7 @@ export function isWebWorker(): boolean { /** * Check if Web Workers are available in the current environment + * @deprecated Browser support is deprecated and will be removed in v8.0. Use Node.js Worker Threads instead. */ export function areWebWorkersAvailable(): boolean { return isBrowser() && typeof Worker !== 'undefined' diff --git a/src/utils/unifiedCache.ts b/src/utils/unifiedCache.ts index c91b6ad4..dde561ac 100644 --- a/src/utils/unifiedCache.ts +++ b/src/utils/unifiedCache.ts @@ -602,6 +602,10 @@ export function getGlobalCache(config?: UnifiedCacheConfig): UnifiedCache { return globalCache } +export function setGlobalCache(cache: UnifiedCache): void { + globalCache = cache +} + export function clearGlobalCache(): void { if (globalCache) { globalCache.clear() diff --git a/tests/unit/plugin.test.ts b/tests/unit/plugin.test.ts new file mode 100644 index 00000000..74ff9956 --- /dev/null +++ b/tests/unit/plugin.test.ts @@ -0,0 +1,324 @@ +/** + * Plugin System Integration Tests + * + * Tests the PluginRegistry lifecycle, provider resolution in init(), + * fallback behavior, and error handling. + */ + +import { describe, it, expect, beforeEach, vi } from 'vitest' +import { PluginRegistry } from '../../src/plugin.js' +import type { BrainyPlugin, BrainyPluginContext } from '../../src/plugin.js' +import { Brainy } from '../../src/brainy.js' +import { NounType } from '../../src/types/graphTypes.js' +import { MetadataIndexManager } from '../../src/utils/metadataIndex.js' +import { setGlobalCache, getGlobalCache, clearGlobalCache, UnifiedCache } from '../../src/utils/unifiedCache.js' + +describe('PluginRegistry', () => { + let registry: PluginRegistry + + beforeEach(() => { + registry = new PluginRegistry() + }) + + it('should register and retrieve a plugin', () => { + const plugin: BrainyPlugin = { + name: 'test-plugin', + activate: async () => true + } + + registry.register(plugin) + expect(registry.getActivePlugins()).toEqual([]) + }) + + it('should activate a plugin and expose providers', async () => { + const plugin: BrainyPlugin = { + name: 'test-plugin', + activate: async (ctx) => { + ctx.registerProvider('distance', () => 0.42) + return true + } + } + + registry.register(plugin) + const context: BrainyPluginContext = { + registerProvider: (key, impl) => registry.registerProvider(key, impl), + version: '7.9.3' + } + + const activated = await registry.activateAll(context) + expect(activated).toEqual(['test-plugin']) + expect(registry.getActivePlugins()).toEqual(['test-plugin']) + expect(registry.hasProvider('distance')).toBe(true) + + const distFn = registry.getProvider<() => number>('distance') + expect(distFn!()).toBe(0.42) + }) + + it('should skip plugin that returns false', async () => { + const plugin: BrainyPlugin = { + name: 'unavailable-plugin', + activate: async () => false + } + + registry.register(plugin) + const context: BrainyPluginContext = { + registerProvider: (key, impl) => registry.registerProvider(key, impl), + version: '7.9.3' + } + + const activated = await registry.activateAll(context) + expect(activated).toEqual([]) + expect(registry.getActivePlugins()).toEqual([]) + }) + + it('should handle plugin that throws during activate', async () => { + const plugin: BrainyPlugin = { + name: 'broken-plugin', + activate: async () => { + throw new Error('activation failed') + } + } + + registry.register(plugin) + const context: BrainyPluginContext = { + registerProvider: (key, impl) => registry.registerProvider(key, impl), + version: '7.9.3' + } + + // Should not throw — graceful fallback + const activated = await registry.activateAll(context) + expect(activated).toEqual([]) + expect(registry.getActivePlugins()).toEqual([]) + }) + + it('should deactivate plugins', async () => { + const deactivateFn = vi.fn() + const plugin: BrainyPlugin = { + name: 'test-plugin', + activate: async () => true, + deactivate: deactivateFn + } + + registry.register(plugin) + const context: BrainyPluginContext = { + registerProvider: (key, impl) => registry.registerProvider(key, impl), + version: '7.9.3' + } + + await registry.activateAll(context) + expect(registry.getActivePlugins()).toEqual(['test-plugin']) + + await registry.deactivateAll() + expect(registry.getActivePlugins()).toEqual([]) + expect(deactivateFn).toHaveBeenCalledOnce() + }) + + it('should not activate same plugin twice', async () => { + let activateCount = 0 + const plugin: BrainyPlugin = { + name: 'once-plugin', + activate: async () => { + activateCount++ + return true + } + } + + registry.register(plugin) + const context: BrainyPluginContext = { + registerProvider: (key, impl) => registry.registerProvider(key, impl), + version: '7.9.3' + } + + await registry.activateAll(context) + await registry.activateAll(context) + expect(activateCount).toBe(1) + }) + + it('should register multiple providers from one plugin', async () => { + const plugin: BrainyPlugin = { + name: 'multi-provider', + activate: async (ctx) => { + ctx.registerProvider('distance', () => 0.5) + ctx.registerProvider('msgpack', { encode: () => null, decode: () => null }) + ctx.registerProvider('roaring', class FakeRoaring {}) + return true + } + } + + registry.register(plugin) + const context: BrainyPluginContext = { + registerProvider: (key, impl) => registry.registerProvider(key, impl), + version: '7.9.3' + } + + await registry.activateAll(context) + expect(registry.hasProvider('distance')).toBe(true) + expect(registry.hasProvider('msgpack')).toBe(true) + expect(registry.hasProvider('roaring')).toBe(true) + expect(registry.hasProvider('hnsw')).toBe(false) + }) + + it('should handle storage adapter factory registration', () => { + registry.registerProvider('storage:redis', { + name: 'redis', + create: () => ({} as any) + }) + + const factory = registry.getStorageFactory('redis') + expect(factory).toBeDefined() + expect(factory!.name).toBe('redis') + }) +}) + +describe('setGlobalCache', () => { + beforeEach(() => { + clearGlobalCache() + }) + + it('should replace the global cache singleton', () => { + // Get default cache + const defaultCache = getGlobalCache() + expect(defaultCache).toBeInstanceOf(UnifiedCache) + + // Replace with a new one + const customCache = new UnifiedCache({ maxSize: 1024 * 1024 }) + setGlobalCache(customCache) + + // Verify replacement + const retrieved = getGlobalCache() + expect(retrieved).toBe(customCache) + expect(retrieved).not.toBe(defaultCache) + + // Clean up + clearGlobalCache() + }) +}) + +describe('Brainy plugin integration', () => { + it('should use distance provider from plugin', async () => { + const mockPlugin: BrainyPlugin = { + name: 'test-distance', + activate: async (ctx) => { + ctx.registerProvider('distance', (a: number[], b: number[]) => { + // Simple euclidean for testing + let sum = 0 + for (let i = 0; i < a.length; i++) { + sum += (a[i] - b[i]) ** 2 + } + return Math.sqrt(sum) + }) + return true + } + } + + const brain = new Brainy({ + storage: { type: 'memory' }, + silent: true, + augmentations: { cache: false, metrics: false, display: false, monitoring: false } + }) + brain.use(mockPlugin) + await brain.init() + + expect(brain.getActivePlugins()).toContain('test-distance') + await brain.close() + }) + + it('should verify metadataIndex factory provider is called', async () => { + let factoryCalled = false + + const mockPlugin: BrainyPlugin = { + name: 'test-metadata-check', + activate: async (ctx) => { + // Register a factory — we only verify it gets called, + // not that the mock works as a full MetadataIndexManager replacement. + // The real NativeMetadataIndex in cortex implements the full interface. + ctx.registerProvider('metadataIndex', (storage: any) => { + factoryCalled = true + // Return a real MetadataIndexManager so init() works end-to-end + return new MetadataIndexManager(storage) + }) + return true + } + } + + const brain = new Brainy({ + storage: { type: 'memory' }, + silent: true, + augmentations: { cache: false, metrics: false, display: false, monitoring: false } + }) + brain.use(mockPlugin) + await brain.init() + + expect(factoryCalled).toBe(true) + expect(brain.getActivePlugins()).toContain('test-metadata-check') + await brain.close() + }) + + it('should fall back to TS implementations when plugin returns false', async () => { + const mockPlugin: BrainyPlugin = { + name: 'unavailable-native', + activate: async () => false + } + + const brain = new Brainy({ + storage: { type: 'memory' }, + silent: true, + augmentations: { cache: false, metrics: false, display: false, monitoring: false } + }) + brain.use(mockPlugin) + await brain.init() + + // Should initialize normally with TS fallbacks + expect(brain.getActivePlugins()).toEqual([]) + + // Should still work with TS implementations + const id = await brain.add({ + data: { name: 'test' }, + type: NounType.Concept, + metadata: { tag: 'unit-test' } + }) + expect(id).toBeTypeOf('string') + + await brain.close() + }) + + it('should fall back gracefully when plugin throws', async () => { + const mockPlugin: BrainyPlugin = { + name: 'crashing-plugin', + activate: async () => { + throw new Error('native module not found') + } + } + + const brain = new Brainy({ + storage: { type: 'memory' }, + silent: true, + augmentations: { cache: false, metrics: false, display: false, monitoring: false } + }) + brain.use(mockPlugin) + + // Should not throw — graceful fallback to TS + await brain.init() + expect(brain.getActivePlugins()).toEqual([]) + + const id = await brain.add({ + data: { name: 'test' }, + type: NounType.Concept, + metadata: { tag: 'unit-test' } + }) + expect(id).toBeTypeOf('string') + + await brain.close() + }) + + it('should use() return this for chaining', () => { + const plugin: BrainyPlugin = { + name: 'chain-test', + activate: async () => true + } + + const brain = new Brainy({ storage: { type: 'memory' } }) + const result = brain.use(plugin) + expect(result).toBe(brain) + }) +})