feat: wire plugin system with provider resolution, storage factories, and browser deprecation

- Wire PluginRegistry into Brainy init() with provider resolution for distance,
  metadataIndex, graphIndex, embeddings, roaring, msgpack, and storage adapters
- Add setupStorage() factory that resolves storage:* providers from plugins before
  falling back to built-in createStorage()
- Export internals API (setGlobalCache, UnifiedCache, EntityIdMapper, etc.) for
  cortex plugin consumption
- Add plugin.test.ts verifying registration, activation, and provider resolution
- Deprecate browser support (OPFS, Web Workers, WASM embeddings) with warnings
  in preparation for v8.0 server-only release
- FileSystemStorage: fix setupStorage resolution for mmap-filesystem provider
This commit is contained in:
David Snelling 2026-01-31 12:02:13 -08:00
parent 25912b5728
commit 1513e297ef
11 changed files with 456 additions and 33 deletions

View file

@ -34,6 +34,8 @@ import { BlobStorage } from './storage/cow/BlobStorage.js'
import { NULL_HASH } from './storage/cow/constants.js'
import { createPipeline } from './streaming/pipeline.js'
import { configureLogger, LogLevel } from './utils/logger.js'
import { setGlobalCache } from './utils/unifiedCache.js'
import type { UnifiedCache } from './utils/unifiedCache.js'
import { PluginRegistry } from './plugin.js'
import type { BrainyPlugin, BrainyPluginContext } from './plugin.js'
import { TransactionManager } from './transaction/TransactionManager.js'
@ -262,7 +264,11 @@ export class Brainy<T = any> implements BrainyInterface<T> {
}
try {
// Setup and initialize storage
// Auto-detect and activate plugins BEFORE storage setup
// so plugin-provided storage factories (e.g., mmap-filesystem) are available
await this.loadPlugins()
// Setup and initialize storage (checks plugin storage factories first)
this.storage = await this.setupStorage()
await this.storage.init()
@ -273,20 +279,55 @@ export class Brainy<T = any> implements BrainyInterface<T> {
(this.storage as any).enableCOWLightweight((this.config.storage as any)?.branch || 'main')
}
// Auto-detect and activate plugins (e.g., @soulcraft/brainy-cortex)
await this.loadPlugins()
// Provider: embeddings (reassign embedder if plugin provides one)
const embeddingProvider = this.pluginRegistry.getProvider<EmbeddingFunction>('embeddings')
if (embeddingProvider) {
this.embedder = embeddingProvider
}
// Setup index now that we have storage
this.index = this.setupIndex()
// Provider: cache (replace global singleton before any consumer uses it)
const cacheProvider = this.pluginRegistry.getProvider<UnifiedCache>('cache')
if (cacheProvider) {
setGlobalCache(cacheProvider)
}
// Initialize metadata index and graph index in parallel
// Both only need storage — they are independent of each other
this.metadataIndex = new MetadataIndexManager(this.storage)
const [, graphIndex] = await Promise.all([
this.metadataIndex.init(),
(this.storage as any).getGraphIndex()
])
this.graphIndex = graphIndex
// Provider: distance function (resolve BEFORE setupIndex — index uses this.distance)
const nativeDistance = this.pluginRegistry.getProvider<DistanceFunction>('distance')
if (nativeDistance) {
this.distance = nativeDistance
}
// Provider: HNSW index factory
const hnswFactory = this.pluginRegistry.getProvider<(config: any, distance: DistanceFunction, options: any) => any>('hnsw')
if (hnswFactory) {
const persistMode = this.resolveHNSWPersistMode()
this.index = hnswFactory(
{ ...this.config.index, distanceFunction: this.distance },
this.distance,
{ storage: this.storage, persistMode }
)
} else {
this.index = this.setupIndex()
}
// Provider: metadata index factory
const metadataFactory = this.pluginRegistry.getProvider<(storage: StorageAdapter) => any>('metadataIndex')
this.metadataIndex = metadataFactory
? metadataFactory(this.storage)
: new MetadataIndexManager(this.storage)
// Provider: graph index factory
const graphFactory = this.pluginRegistry.getProvider<(storage: StorageAdapter) => any>('graphIndex')
if (graphFactory) {
this.graphIndex = graphFactory(this.storage)
await this.metadataIndex.init()
} else {
const [, graphIndex] = await Promise.all([
this.metadataIndex.init(),
(this.storage as any).getGraphIndex()
])
this.graphIndex = graphIndex
}
// Rebuild indexes if needed for existing data
await this.rebuildIndexesIfNeeded()
@ -6140,9 +6181,18 @@ export class Brainy<T = any> implements BrainyInterface<T> {
* Setup storage
*/
private async setupStorage(): Promise<BaseStorage> {
// Pass the entire storage config object to createStorage
// This ensures all storage-specific configs (gcsNativeStorage, s3Storage, etc.) are passed through
const storage = await createStorage(this.config.storage as any)
const storageConfig = (this.config.storage || {}) as Record<string, unknown>
const storageType = (storageConfig.type as string) || 'auto'
// Check plugin-provided storage factories (e.g., 'mmap-filesystem' from cortex)
const pluginFactory = this.pluginRegistry.getStorageFactory(storageType)
if (pluginFactory) {
const adapter = await pluginFactory.create(storageConfig)
return adapter as BaseStorage
}
// Fall through to built-in storage types
const storage = await createStorage(storageConfig as any)
return storage as BaseStorage
}
@ -6189,23 +6239,9 @@ export class Brainy<T = any> implements BrainyInterface<T> {
distanceFunction: this.distance
}
// Determine persist mode (user config > smart default)
// Fixed to use getStorageType() for reliable detection
let persistMode: 'immediate' | 'deferred' = this.config.hnswPersistMode || 'immediate'
const persistMode = this.resolveHNSWPersistMode()
// Smart default: Use deferred mode for cloud storage adapters
if (!this.config.hnswPersistMode) {
// FIX: Use instance-based detection as fallback
// Previously this.config.storage.type was often undefined after storage creation,
// causing cloud storage to incorrectly use 'immediate' mode (50-100x slower)
const storageType = this.config.storage?.type || this.getStorageType()
const cloudStorageTypes = ['gcs', 's3', 'r2', 'azure']
if (cloudStorageTypes.includes(storageType)) {
persistMode = 'deferred'
}
}
// Phase 2: Use TypeAwareHNSWIndex for billion-scale optimization
// Use TypeAwareHNSWIndex for non-memory storage (billion-scale optimization)
const detectedStorageType = this.config.storage?.type || this.getStorageType()
if (detectedStorageType !== 'memory') {
return new TypeAwareHNSWIndex(indexConfig, this.distance, {
@ -6218,6 +6254,27 @@ export class Brainy<T = any> implements BrainyInterface<T> {
return new HNSWIndex(indexConfig as any, this.distance, { persistMode })
}
/**
* Resolve HNSW persistence mode.
* Extracted so both setupIndex() and the HNSW plugin factory path can use it.
*
* User config > smart default:
* - Cloud storage (GCS/S3/R2/Azure): 'deferred' for 30-50x faster adds
* - Local storage (FileSystem/Memory/OPFS): 'immediate' (already fast)
*/
private resolveHNSWPersistMode(): 'immediate' | 'deferred' {
let persistMode: 'immediate' | 'deferred' = this.config.hnswPersistMode || 'immediate'
if (!this.config.hnswPersistMode) {
const storageType = this.config.storage?.type || this.getStorageType()
if (['gcs', 's3', 'r2', 'azure'].includes(storageType)) {
persistMode = 'deferred'
}
}
return persistMode
}
/**
* Setup augmentations
*/

View file

@ -14,6 +14,7 @@
import { Vector, EmbeddingFunction } from '../coreTypes.js'
import { WASMEmbeddingEngine } from './wasm/index.js'
import { isBrowser } from '../utils/environment.js'
// Types
export type ModelPrecision = 'q8' | 'fp32'
@ -115,6 +116,9 @@ export class EmbeddingManager {
const startTime = Date.now()
try {
if (isBrowser()) {
console.warn('[brainy] Browser WASM embedding engine is deprecated and will be removed in v8.0. Use Node.js/Bun with native embeddings (@soulcraft/brainy-cortex) instead.')
}
// Initialize WASM engine (handles all model loading)
await this.engine.initialize()

11
src/internals.ts Normal file
View file

@ -0,0 +1,11 @@
/**
* Internal utilities for first-party plugins.
* NOT part of the public API may change between minor versions.
*/
export { getGlobalCache, setGlobalCache, clearGlobalCache, UnifiedCache } from './utils/unifiedCache.js'
export type { UnifiedCacheConfig, CacheItem } from './utils/unifiedCache.js'
export { prodLog, createModuleLogger } from './utils/logger.js'
export { FieldTypeInference, FieldType } from './utils/fieldTypeInference.js'
export type { FieldTypeInfo } from './utils/fieldTypeInference.js'
export { EntityIdMapper } from './utils/entityIdMapper.js'
export type { EntityIdMapperOptions, EntityIdMapperData } from './utils/entityIdMapper.js'

View file

@ -78,7 +78,7 @@ export class FileSystemStorage extends BaseStorage {
private readonly SHARDING_DEPTH = 1 as const
private readonly MAX_SHARDS = 256 // Hex range: 00-ff
private cachedShardingDepth: number = this.SHARDING_DEPTH // Always use fixed depth
private rootDir: string
protected rootDir: string
private nounsDir!: string
private verbsDir!: string
private metadataDir!: string

View file

@ -80,6 +80,7 @@ export class OPFSStorage extends BaseStorage {
constructor() {
super()
console.warn('[brainy] OPFS storage is deprecated and will be removed in v8.0. Use Node.js/Bun with filesystem or mmap-filesystem storage instead.')
// Check if OPFS is available
this.isAvailable =
typeof navigator !== 'undefined' &&

View file

@ -480,6 +480,7 @@ export async function createStorage(
return memStorage
case 'opfs': {
console.warn('[brainy] OPFS storage is deprecated and will be removed in v8.0. Migrate to Node.js/Bun with filesystem or mmap-filesystem storage.')
// Check if OPFS is available
const opfsStorage = new OPFSStorage()
if (opfsStorage.isOPFSAvailable()) {
@ -839,6 +840,7 @@ export async function createStorage(
// Next, try OPFS (browser only)
if (isBrowser()) {
console.warn('[brainy] Browser environment detected. Browser support is deprecated and will be removed in v8.0. Migrate to Node.js/Bun.')
const opfsStorage = new OPFSStorage()
if (opfsStorage.isOPFSAvailable()) {
console.log('Using OPFS storage (auto-detected) + TypeAware wrapper')

View file

@ -4,6 +4,7 @@
/**
* Check if code is running in a browser environment
* @deprecated Browser support is deprecated and will be removed in v8.0. Use Node.js or Bun instead.
*/
export function isBrowser(): boolean {
return typeof window !== 'undefined' && typeof document !== 'undefined'
@ -39,6 +40,7 @@ export function isWebWorker(): boolean {
/**
* Check if Web Workers are available in the current environment
* @deprecated Browser support is deprecated and will be removed in v8.0. Use Node.js Worker Threads instead.
*/
export function areWebWorkersAvailable(): boolean {
return isBrowser() && typeof Worker !== 'undefined'

View file

@ -602,6 +602,10 @@ export function getGlobalCache(config?: UnifiedCacheConfig): UnifiedCache {
return globalCache
}
export function setGlobalCache(cache: UnifiedCache): void {
globalCache = cache
}
export function clearGlobalCache(): void {
if (globalCache) {
globalCache.clear()