feat: wire plugin system with provider resolution, storage factories, and browser deprecation
- Wire PluginRegistry into Brainy init() with provider resolution for distance, metadataIndex, graphIndex, embeddings, roaring, msgpack, and storage adapters - Add setupStorage() factory that resolves storage:* providers from plugins before falling back to built-in createStorage() - Export internals API (setGlobalCache, UnifiedCache, EntityIdMapper, etc.) for cortex plugin consumption - Add plugin.test.ts verifying registration, activation, and provider resolution - Deprecate browser support (OPFS, Web Workers, WASM embeddings) with warnings in preparation for v8.0 server-only release - FileSystemStorage: fix setupStorage resolution for mmap-filesystem provider
This commit is contained in:
parent
25912b5728
commit
1513e297ef
11 changed files with 456 additions and 33 deletions
121
src/brainy.ts
121
src/brainy.ts
|
|
@ -34,6 +34,8 @@ import { BlobStorage } from './storage/cow/BlobStorage.js'
|
|||
import { NULL_HASH } from './storage/cow/constants.js'
|
||||
import { createPipeline } from './streaming/pipeline.js'
|
||||
import { configureLogger, LogLevel } from './utils/logger.js'
|
||||
import { setGlobalCache } from './utils/unifiedCache.js'
|
||||
import type { UnifiedCache } from './utils/unifiedCache.js'
|
||||
import { PluginRegistry } from './plugin.js'
|
||||
import type { BrainyPlugin, BrainyPluginContext } from './plugin.js'
|
||||
import { TransactionManager } from './transaction/TransactionManager.js'
|
||||
|
|
@ -262,7 +264,11 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
}
|
||||
|
||||
try {
|
||||
// Setup and initialize storage
|
||||
// Auto-detect and activate plugins BEFORE storage setup
|
||||
// so plugin-provided storage factories (e.g., mmap-filesystem) are available
|
||||
await this.loadPlugins()
|
||||
|
||||
// Setup and initialize storage (checks plugin storage factories first)
|
||||
this.storage = await this.setupStorage()
|
||||
await this.storage.init()
|
||||
|
||||
|
|
@ -273,20 +279,55 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
(this.storage as any).enableCOWLightweight((this.config.storage as any)?.branch || 'main')
|
||||
}
|
||||
|
||||
// Auto-detect and activate plugins (e.g., @soulcraft/brainy-cortex)
|
||||
await this.loadPlugins()
|
||||
// Provider: embeddings (reassign embedder if plugin provides one)
|
||||
const embeddingProvider = this.pluginRegistry.getProvider<EmbeddingFunction>('embeddings')
|
||||
if (embeddingProvider) {
|
||||
this.embedder = embeddingProvider
|
||||
}
|
||||
|
||||
// Setup index now that we have storage
|
||||
this.index = this.setupIndex()
|
||||
// Provider: cache (replace global singleton before any consumer uses it)
|
||||
const cacheProvider = this.pluginRegistry.getProvider<UnifiedCache>('cache')
|
||||
if (cacheProvider) {
|
||||
setGlobalCache(cacheProvider)
|
||||
}
|
||||
|
||||
// Initialize metadata index and graph index in parallel
|
||||
// Both only need storage — they are independent of each other
|
||||
this.metadataIndex = new MetadataIndexManager(this.storage)
|
||||
const [, graphIndex] = await Promise.all([
|
||||
this.metadataIndex.init(),
|
||||
(this.storage as any).getGraphIndex()
|
||||
])
|
||||
this.graphIndex = graphIndex
|
||||
// Provider: distance function (resolve BEFORE setupIndex — index uses this.distance)
|
||||
const nativeDistance = this.pluginRegistry.getProvider<DistanceFunction>('distance')
|
||||
if (nativeDistance) {
|
||||
this.distance = nativeDistance
|
||||
}
|
||||
|
||||
// Provider: HNSW index factory
|
||||
const hnswFactory = this.pluginRegistry.getProvider<(config: any, distance: DistanceFunction, options: any) => any>('hnsw')
|
||||
if (hnswFactory) {
|
||||
const persistMode = this.resolveHNSWPersistMode()
|
||||
this.index = hnswFactory(
|
||||
{ ...this.config.index, distanceFunction: this.distance },
|
||||
this.distance,
|
||||
{ storage: this.storage, persistMode }
|
||||
)
|
||||
} else {
|
||||
this.index = this.setupIndex()
|
||||
}
|
||||
|
||||
// Provider: metadata index factory
|
||||
const metadataFactory = this.pluginRegistry.getProvider<(storage: StorageAdapter) => any>('metadataIndex')
|
||||
this.metadataIndex = metadataFactory
|
||||
? metadataFactory(this.storage)
|
||||
: new MetadataIndexManager(this.storage)
|
||||
|
||||
// Provider: graph index factory
|
||||
const graphFactory = this.pluginRegistry.getProvider<(storage: StorageAdapter) => any>('graphIndex')
|
||||
if (graphFactory) {
|
||||
this.graphIndex = graphFactory(this.storage)
|
||||
await this.metadataIndex.init()
|
||||
} else {
|
||||
const [, graphIndex] = await Promise.all([
|
||||
this.metadataIndex.init(),
|
||||
(this.storage as any).getGraphIndex()
|
||||
])
|
||||
this.graphIndex = graphIndex
|
||||
}
|
||||
|
||||
// Rebuild indexes if needed for existing data
|
||||
await this.rebuildIndexesIfNeeded()
|
||||
|
|
@ -6140,9 +6181,18 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
* Setup storage
|
||||
*/
|
||||
private async setupStorage(): Promise<BaseStorage> {
|
||||
// Pass the entire storage config object to createStorage
|
||||
// This ensures all storage-specific configs (gcsNativeStorage, s3Storage, etc.) are passed through
|
||||
const storage = await createStorage(this.config.storage as any)
|
||||
const storageConfig = (this.config.storage || {}) as Record<string, unknown>
|
||||
const storageType = (storageConfig.type as string) || 'auto'
|
||||
|
||||
// Check plugin-provided storage factories (e.g., 'mmap-filesystem' from cortex)
|
||||
const pluginFactory = this.pluginRegistry.getStorageFactory(storageType)
|
||||
if (pluginFactory) {
|
||||
const adapter = await pluginFactory.create(storageConfig)
|
||||
return adapter as BaseStorage
|
||||
}
|
||||
|
||||
// Fall through to built-in storage types
|
||||
const storage = await createStorage(storageConfig as any)
|
||||
return storage as BaseStorage
|
||||
}
|
||||
|
||||
|
|
@ -6189,23 +6239,9 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
distanceFunction: this.distance
|
||||
}
|
||||
|
||||
// Determine persist mode (user config > smart default)
|
||||
// Fixed to use getStorageType() for reliable detection
|
||||
let persistMode: 'immediate' | 'deferred' = this.config.hnswPersistMode || 'immediate'
|
||||
const persistMode = this.resolveHNSWPersistMode()
|
||||
|
||||
// Smart default: Use deferred mode for cloud storage adapters
|
||||
if (!this.config.hnswPersistMode) {
|
||||
// FIX: Use instance-based detection as fallback
|
||||
// Previously this.config.storage.type was often undefined after storage creation,
|
||||
// causing cloud storage to incorrectly use 'immediate' mode (50-100x slower)
|
||||
const storageType = this.config.storage?.type || this.getStorageType()
|
||||
const cloudStorageTypes = ['gcs', 's3', 'r2', 'azure']
|
||||
if (cloudStorageTypes.includes(storageType)) {
|
||||
persistMode = 'deferred'
|
||||
}
|
||||
}
|
||||
|
||||
// Phase 2: Use TypeAwareHNSWIndex for billion-scale optimization
|
||||
// Use TypeAwareHNSWIndex for non-memory storage (billion-scale optimization)
|
||||
const detectedStorageType = this.config.storage?.type || this.getStorageType()
|
||||
if (detectedStorageType !== 'memory') {
|
||||
return new TypeAwareHNSWIndex(indexConfig, this.distance, {
|
||||
|
|
@ -6218,6 +6254,27 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
return new HNSWIndex(indexConfig as any, this.distance, { persistMode })
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve HNSW persistence mode.
|
||||
* Extracted so both setupIndex() and the HNSW plugin factory path can use it.
|
||||
*
|
||||
* User config > smart default:
|
||||
* - Cloud storage (GCS/S3/R2/Azure): 'deferred' for 30-50x faster adds
|
||||
* - Local storage (FileSystem/Memory/OPFS): 'immediate' (already fast)
|
||||
*/
|
||||
private resolveHNSWPersistMode(): 'immediate' | 'deferred' {
|
||||
let persistMode: 'immediate' | 'deferred' = this.config.hnswPersistMode || 'immediate'
|
||||
|
||||
if (!this.config.hnswPersistMode) {
|
||||
const storageType = this.config.storage?.type || this.getStorageType()
|
||||
if (['gcs', 's3', 'r2', 'azure'].includes(storageType)) {
|
||||
persistMode = 'deferred'
|
||||
}
|
||||
}
|
||||
|
||||
return persistMode
|
||||
}
|
||||
|
||||
/**
|
||||
* Setup augmentations
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@
|
|||
|
||||
import { Vector, EmbeddingFunction } from '../coreTypes.js'
|
||||
import { WASMEmbeddingEngine } from './wasm/index.js'
|
||||
import { isBrowser } from '../utils/environment.js'
|
||||
|
||||
// Types
|
||||
export type ModelPrecision = 'q8' | 'fp32'
|
||||
|
|
@ -115,6 +116,9 @@ export class EmbeddingManager {
|
|||
const startTime = Date.now()
|
||||
|
||||
try {
|
||||
if (isBrowser()) {
|
||||
console.warn('[brainy] Browser WASM embedding engine is deprecated and will be removed in v8.0. Use Node.js/Bun with native embeddings (@soulcraft/brainy-cortex) instead.')
|
||||
}
|
||||
// Initialize WASM engine (handles all model loading)
|
||||
await this.engine.initialize()
|
||||
|
||||
|
|
|
|||
11
src/internals.ts
Normal file
11
src/internals.ts
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
/**
|
||||
* Internal utilities for first-party plugins.
|
||||
* NOT part of the public API — may change between minor versions.
|
||||
*/
|
||||
export { getGlobalCache, setGlobalCache, clearGlobalCache, UnifiedCache } from './utils/unifiedCache.js'
|
||||
export type { UnifiedCacheConfig, CacheItem } from './utils/unifiedCache.js'
|
||||
export { prodLog, createModuleLogger } from './utils/logger.js'
|
||||
export { FieldTypeInference, FieldType } from './utils/fieldTypeInference.js'
|
||||
export type { FieldTypeInfo } from './utils/fieldTypeInference.js'
|
||||
export { EntityIdMapper } from './utils/entityIdMapper.js'
|
||||
export type { EntityIdMapperOptions, EntityIdMapperData } from './utils/entityIdMapper.js'
|
||||
|
|
@ -78,7 +78,7 @@ export class FileSystemStorage extends BaseStorage {
|
|||
private readonly SHARDING_DEPTH = 1 as const
|
||||
private readonly MAX_SHARDS = 256 // Hex range: 00-ff
|
||||
private cachedShardingDepth: number = this.SHARDING_DEPTH // Always use fixed depth
|
||||
private rootDir: string
|
||||
protected rootDir: string
|
||||
private nounsDir!: string
|
||||
private verbsDir!: string
|
||||
private metadataDir!: string
|
||||
|
|
|
|||
|
|
@ -80,6 +80,7 @@ export class OPFSStorage extends BaseStorage {
|
|||
|
||||
constructor() {
|
||||
super()
|
||||
console.warn('[brainy] OPFS storage is deprecated and will be removed in v8.0. Use Node.js/Bun with filesystem or mmap-filesystem storage instead.')
|
||||
// Check if OPFS is available
|
||||
this.isAvailable =
|
||||
typeof navigator !== 'undefined' &&
|
||||
|
|
|
|||
|
|
@ -480,6 +480,7 @@ export async function createStorage(
|
|||
return memStorage
|
||||
|
||||
case 'opfs': {
|
||||
console.warn('[brainy] OPFS storage is deprecated and will be removed in v8.0. Migrate to Node.js/Bun with filesystem or mmap-filesystem storage.')
|
||||
// Check if OPFS is available
|
||||
const opfsStorage = new OPFSStorage()
|
||||
if (opfsStorage.isOPFSAvailable()) {
|
||||
|
|
@ -839,6 +840,7 @@ export async function createStorage(
|
|||
|
||||
// Next, try OPFS (browser only)
|
||||
if (isBrowser()) {
|
||||
console.warn('[brainy] Browser environment detected. Browser support is deprecated and will be removed in v8.0. Migrate to Node.js/Bun.')
|
||||
const opfsStorage = new OPFSStorage()
|
||||
if (opfsStorage.isOPFSAvailable()) {
|
||||
console.log('Using OPFS storage (auto-detected) + TypeAware wrapper')
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@
|
|||
|
||||
/**
|
||||
* Check if code is running in a browser environment
|
||||
* @deprecated Browser support is deprecated and will be removed in v8.0. Use Node.js or Bun instead.
|
||||
*/
|
||||
export function isBrowser(): boolean {
|
||||
return typeof window !== 'undefined' && typeof document !== 'undefined'
|
||||
|
|
@ -39,6 +40,7 @@ export function isWebWorker(): boolean {
|
|||
|
||||
/**
|
||||
* Check if Web Workers are available in the current environment
|
||||
* @deprecated Browser support is deprecated and will be removed in v8.0. Use Node.js Worker Threads instead.
|
||||
*/
|
||||
export function areWebWorkersAvailable(): boolean {
|
||||
return isBrowser() && typeof Worker !== 'undefined'
|
||||
|
|
|
|||
|
|
@ -602,6 +602,10 @@ export function getGlobalCache(config?: UnifiedCacheConfig): UnifiedCache {
|
|||
return globalCache
|
||||
}
|
||||
|
||||
export function setGlobalCache(cache: UnifiedCache): void {
|
||||
globalCache = cache
|
||||
}
|
||||
|
||||
export function clearGlobalCache(): void {
|
||||
if (globalCache) {
|
||||
globalCache.clear()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue