feat: harden plugin system wiring and add developer diagnostics
Fix critical wiring bugs that prevented plugin-provided implementations from being used at runtime. All CRUD operations, fork/checkout/clear, batch embedding, neural APIs, and VFS path resolution now properly dispatch through the plugin registry. Changes: - Wire graphIndex to storage for getVerbsBySource() fast path - Replace instanceof checks with duck-typing (indexIsTypeAware flag) so plugin HNSW indexes work in add/update/delete/search - Add createIndex() shared helper for plugin HNSW factory - Fix fork/checkout/clear to use plugin factories for metadataIndex, graphIndex, and HNSW instead of hardcoding JS constructors - Add three-tier embedBatch priority: embedBatch > embeddings > WASM - Skip WASM warmup/eagerEmbeddings when plugin provides embeddings - Fix PathResolver metadataIndex access (was looking on storage) - Use global UnifiedCache in SemanticPathResolver - Wire plugin distance function through neural APIs - Add diagnostics() method and CLI command for provider inspection - Add requireProviders() for production fail-fast assertions - Add init-time provider summary log - Add plugin developer documentation (docs/PLUGINS.md) - Export DiagnosticsResult type
This commit is contained in:
parent
3a21bf62b7
commit
401e300ff2
12 changed files with 705 additions and 77 deletions
271
src/brainy.ts
271
src/brainy.ts
|
|
@ -109,6 +109,20 @@ const STOPWORDS = new Set([
|
|||
'what', 'which', 'who', 'whom', 'whose', 'am'
|
||||
])
|
||||
|
||||
/**
|
||||
* Result type for brain.diagnostics()
|
||||
*/
|
||||
export interface DiagnosticsResult {
|
||||
version: string
|
||||
plugins: { active: string[], count: number }
|
||||
providers: Record<string, { source: 'plugin' | 'default' }>
|
||||
indexes: {
|
||||
hnsw: { size: number, type: string }
|
||||
metadata: { type: string, initialized: boolean }
|
||||
graph: { type: string, initialized: boolean, wiredToStorage: boolean }
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The main Brainy class - Clean, Beautiful, Powerful
|
||||
* REAL IMPLEMENTATION - No stubs, no mocks
|
||||
|
|
@ -122,6 +136,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
|
||||
// Core components
|
||||
private index!: HNSWIndex | TypeAwareHNSWIndex
|
||||
private indexIsTypeAware = false // true when index supports addItem(item, type) / search(v, k, type)
|
||||
private storage!: BaseStorage
|
||||
private metadataIndex!: MetadataIndexManager
|
||||
private graphIndex!: GraphAdjacencyIndex
|
||||
|
|
@ -292,18 +307,10 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
this.distance = nativeDistance
|
||||
}
|
||||
|
||||
// Provider: HNSW index factory
|
||||
const hnswFactory = this.pluginRegistry.getProvider<(config: any, distance: DistanceFunction, options: any) => any>('hnsw')
|
||||
if (hnswFactory) {
|
||||
const persistMode = this.resolveHNSWPersistMode()
|
||||
this.index = hnswFactory(
|
||||
{ ...this.config.index, distanceFunction: this.distance },
|
||||
this.distance,
|
||||
{ storage: this.storage, persistMode }
|
||||
)
|
||||
} else {
|
||||
this.index = this.setupIndex()
|
||||
}
|
||||
// Provider: HNSW index factory (plugin or JS fallback)
|
||||
this.index = this.createIndex()
|
||||
this.indexIsTypeAware = this.index instanceof TypeAwareHNSWIndex
|
||||
|| typeof (this.index as any).getIndexForType === 'function'
|
||||
|
||||
// Provider: metadata index factory
|
||||
const metadataFactory = this.pluginRegistry.getProvider<(storage: StorageAdapter) => any>('metadataIndex')
|
||||
|
|
@ -315,6 +322,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
const graphFactory = this.pluginRegistry.getProvider<(storage: StorageAdapter) => any>('graphIndex')
|
||||
if (graphFactory) {
|
||||
this.graphIndex = graphFactory(this.storage)
|
||||
this.storage.setGraphIndex(this.graphIndex)
|
||||
await this.metadataIndex.init()
|
||||
} else {
|
||||
const [, graphIndex] = await Promise.all([
|
||||
|
|
@ -350,6 +358,23 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
})
|
||||
}
|
||||
|
||||
// Log provider summary after all wiring is complete
|
||||
// Shows developers exactly what's native vs falling back to JS
|
||||
if (this.pluginRegistry.hasActivePlugins() && !this.config.silent) {
|
||||
const wellKnownKeys = [
|
||||
'metadataIndex', 'graphIndex', 'entityIdMapper', 'cache',
|
||||
'hnsw', 'roaring', 'embeddings', 'embedBatch', 'distance', 'msgpack'
|
||||
]
|
||||
const native = wellKnownKeys.filter(k => this.pluginRegistry.hasProvider(k))
|
||||
const fallback = wellKnownKeys.filter(k => !this.pluginRegistry.hasProvider(k))
|
||||
const plugins = this.pluginRegistry.getActivePlugins().join(', ')
|
||||
if (fallback.length === 0) {
|
||||
console.log(`[brainy] Providers: ${native.length}/${wellKnownKeys.length} native (${plugins})`)
|
||||
} else {
|
||||
console.log(`[brainy] Providers: ${native.length}/${wellKnownKeys.length} native (${plugins}) | default: ${fallback.join(', ')}`)
|
||||
}
|
||||
}
|
||||
|
||||
// Mark as initialized BEFORE VFS init
|
||||
// VFS.init() needs brain to be marked initialized to call brain methods
|
||||
this.initialized = true
|
||||
|
|
@ -365,10 +390,10 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
// instead of lazily on first embed() call. This moves the 90-140 second
|
||||
// WASM compilation to container startup rather than first request.
|
||||
// Recommended for: Cloud Run, Lambda, Fargate, Kubernetes
|
||||
if (this.config.eagerEmbeddings) {
|
||||
console.log('🚀 Eager embedding initialization enabled...')
|
||||
if (this.config.eagerEmbeddings && !this.pluginRegistry.hasProvider('embeddings')) {
|
||||
console.log('Eager embedding initialization enabled...')
|
||||
await embeddingManager.init()
|
||||
console.log('✅ Embedding engine ready')
|
||||
console.log('Embedding engine ready')
|
||||
}
|
||||
|
||||
// Integration Hub initialization
|
||||
|
|
@ -730,10 +755,10 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
)
|
||||
|
||||
// Operation 3: Add to HNSW index (after entity saved)
|
||||
if (this.index instanceof TypeAwareHNSWIndex) {
|
||||
if (this.indexIsTypeAware) {
|
||||
tx.addOperation(
|
||||
new AddToTypeAwareHNSWOperation(
|
||||
this.index,
|
||||
this.index as any,
|
||||
id,
|
||||
vector,
|
||||
params.type as any
|
||||
|
|
@ -741,7 +766,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
)
|
||||
} else {
|
||||
tx.addOperation(
|
||||
new AddToHNSWOperation(this.index, id, vector)
|
||||
new AddToHNSWOperation(this.index as any, id, vector)
|
||||
)
|
||||
}
|
||||
|
||||
|
|
@ -1157,10 +1182,10 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
|
||||
// Operation 3-4: Update HNSW index (remove and re-add if reindexing needed)
|
||||
if (needsReindexing) {
|
||||
if (this.index instanceof TypeAwareHNSWIndex) {
|
||||
if (this.indexIsTypeAware) {
|
||||
tx.addOperation(
|
||||
new RemoveFromTypeAwareHNSWOperation(
|
||||
this.index,
|
||||
this.index as any,
|
||||
params.id,
|
||||
existing.vector,
|
||||
existing.type as any
|
||||
|
|
@ -1168,7 +1193,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
)
|
||||
tx.addOperation(
|
||||
new AddToTypeAwareHNSWOperation(
|
||||
this.index,
|
||||
this.index as any,
|
||||
params.id,
|
||||
vector,
|
||||
newType as any
|
||||
|
|
@ -1176,10 +1201,10 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
)
|
||||
} else {
|
||||
tx.addOperation(
|
||||
new RemoveFromHNSWOperation(this.index, params.id, existing.vector)
|
||||
new RemoveFromHNSWOperation(this.index as any, params.id, existing.vector)
|
||||
)
|
||||
tx.addOperation(
|
||||
new AddToHNSWOperation(this.index, params.id, vector)
|
||||
new AddToHNSWOperation(this.index as any, params.id, vector)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
|
@ -1240,18 +1265,18 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
await this.transactionManager.executeTransaction(async (tx) => {
|
||||
// Operation 1: Remove from vector index
|
||||
if (noun && metadata) {
|
||||
if (this.index instanceof TypeAwareHNSWIndex && metadata.noun) {
|
||||
if (this.indexIsTypeAware && metadata.noun) {
|
||||
tx.addOperation(
|
||||
new RemoveFromTypeAwareHNSWOperation(
|
||||
this.index,
|
||||
this.index as any,
|
||||
id,
|
||||
noun.vector,
|
||||
metadata.noun as any
|
||||
)
|
||||
)
|
||||
} else if (this.index instanceof HNSWIndex) {
|
||||
} else {
|
||||
tx.addOperation(
|
||||
new RemoveFromHNSWOperation(this.index, id, noun.vector)
|
||||
new RemoveFromHNSWOperation(this.index as any, id, noun.vector)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
|
@ -2505,18 +2530,18 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
|
||||
// Add delete operations to transaction
|
||||
if (noun && metadata) {
|
||||
if (this.index instanceof TypeAwareHNSWIndex && metadata.noun) {
|
||||
if (this.indexIsTypeAware && metadata.noun) {
|
||||
tx.addOperation(
|
||||
new RemoveFromTypeAwareHNSWOperation(
|
||||
this.index,
|
||||
this.index as any,
|
||||
id,
|
||||
noun.vector,
|
||||
metadata.noun as any
|
||||
)
|
||||
)
|
||||
} else if (this.index instanceof HNSWIndex) {
|
||||
} else {
|
||||
tx.addOperation(
|
||||
new RemoveFromHNSWOperation(this.index, id, noun.vector)
|
||||
new RemoveFromHNSWOperation(this.index as any, id, noun.vector)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
|
@ -2752,13 +2777,15 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
if ('clear' in this.index && typeof this.index.clear === 'function') {
|
||||
await this.index.clear()
|
||||
} else {
|
||||
// Recreate index if no clear method
|
||||
this.index = this.setupIndex()
|
||||
// Recreate index using plugin factory when available
|
||||
this.index = this.createIndex()
|
||||
}
|
||||
|
||||
// Recreate metadata index to clear cached data
|
||||
// Bug: Metadata index cache was not being cleared, causing find() with type filters to return stale data
|
||||
this.metadataIndex = new MetadataIndexManager(this.storage)
|
||||
const clearMetadataFactory = this.pluginRegistry.getProvider<(storage: StorageAdapter) => any>('metadataIndex')
|
||||
this.metadataIndex = clearMetadataFactory
|
||||
? clearMetadataFactory(this.storage)
|
||||
: new MetadataIndexManager(this.storage)
|
||||
await this.metadataIndex.init()
|
||||
|
||||
// Reset dimensions
|
||||
|
|
@ -2919,8 +2946,9 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
clone.storage.currentBranch = branchName
|
||||
// isInitialized inherited from prototype
|
||||
|
||||
// Shallow copy HNSW index (INSTANT - just copies Map references)
|
||||
clone.index = this.setupIndex()
|
||||
// Create HNSW index (uses plugin factory when available)
|
||||
clone.index = this.createIndex()
|
||||
clone.indexIsTypeAware = this.indexIsTypeAware
|
||||
|
||||
// Enable COW (handle both HNSWIndex and TypeAwareHNSWIndex)
|
||||
if ('enableCOW' in clone.index && typeof clone.index.enableCOW === 'function') {
|
||||
|
|
@ -2928,7 +2956,10 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
}
|
||||
|
||||
// Fast rebuild for small indexes from COW storage (Metadata/Graph are fast)
|
||||
clone.metadataIndex = new MetadataIndexManager(clone.storage)
|
||||
const cloneMetadataFactory = this.pluginRegistry.getProvider<(storage: StorageAdapter) => any>('metadataIndex')
|
||||
clone.metadataIndex = cloneMetadataFactory
|
||||
? cloneMetadataFactory(clone.storage)
|
||||
: new MetadataIndexManager(clone.storage)
|
||||
await clone.metadataIndex.init()
|
||||
|
||||
// GraphAdjacencyIndex SINGLETON pattern for fork()
|
||||
|
|
@ -2937,7 +2968,13 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
// create a fresh instance for the clone's branch.
|
||||
;(clone.storage as any).graphIndex = undefined
|
||||
;(clone.storage as any).graphIndexPromise = undefined
|
||||
clone.graphIndex = await (clone.storage as any).getGraphIndex()
|
||||
const cloneGraphFactory = this.pluginRegistry.getProvider<(storage: StorageAdapter) => any>('graphIndex')
|
||||
if (cloneGraphFactory) {
|
||||
clone.graphIndex = cloneGraphFactory(clone.storage)
|
||||
clone.storage.setGraphIndex(clone.graphIndex)
|
||||
} else {
|
||||
clone.graphIndex = await (clone.storage as any).getGraphIndex()
|
||||
}
|
||||
// getGraphIndex() will rebuild automatically if data exists (via _initializeGraphIndex)
|
||||
|
||||
// Mark as initialized
|
||||
|
|
@ -3021,15 +3058,24 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
|
||||
// Fix: Reload indexes from new branch WITHOUT recreating storage
|
||||
// Previous implementation called init() which recreated storage, losing currentBranch
|
||||
this.index = this.setupIndex()
|
||||
this.metadataIndex = new (MetadataIndexManager as any)(this.storage)
|
||||
this.index = this.createIndex()
|
||||
const checkoutMetadataFactory = this.pluginRegistry.getProvider<(storage: StorageAdapter) => any>('metadataIndex')
|
||||
this.metadataIndex = checkoutMetadataFactory
|
||||
? checkoutMetadataFactory(this.storage)
|
||||
: new MetadataIndexManager(this.storage)
|
||||
await this.metadataIndex.init()
|
||||
|
||||
// GraphAdjacencyIndex SINGLETON pattern for checkout()
|
||||
// Invalidate the old graphIndex (it has data from the old branch)
|
||||
// and get a fresh instance for the new branch
|
||||
;(this.storage as any).invalidateGraphIndex()
|
||||
this.graphIndex = await (this.storage as any).getGraphIndex()
|
||||
const checkoutGraphFactory = this.pluginRegistry.getProvider<(storage: StorageAdapter) => any>('graphIndex')
|
||||
if (checkoutGraphFactory) {
|
||||
this.graphIndex = checkoutGraphFactory(this.storage)
|
||||
this.storage.setGraphIndex(this.graphIndex)
|
||||
} else {
|
||||
this.graphIndex = await (this.storage as any).getGraphIndex()
|
||||
}
|
||||
|
||||
// Reset lazy loading state when switching branches
|
||||
// Indexes contain data from previous branch, must rebuild for new branch
|
||||
|
|
@ -4264,6 +4310,94 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Plugin and provider diagnostics — shows what's active and how subsystems are wired.
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const diag = brain.diagnostics()
|
||||
* console.log(diag.providers) // { hnsw: { source: 'plugin' }, ... }
|
||||
* console.log(diag.indexes.graph.wiredToStorage) // true
|
||||
* ```
|
||||
*/
|
||||
diagnostics(): DiagnosticsResult {
|
||||
const wellKnownKeys = [
|
||||
'metadataIndex', 'graphIndex', 'entityIdMapper', 'cache',
|
||||
'hnsw', 'roaring', 'embeddings', 'embedBatch', 'distance', 'msgpack'
|
||||
] as const
|
||||
|
||||
const providers: Record<string, { source: 'plugin' | 'default' }> = {}
|
||||
for (const key of wellKnownKeys) {
|
||||
providers[key] = {
|
||||
source: this.pluginRegistry.hasProvider(key) ? 'plugin' : 'default'
|
||||
}
|
||||
}
|
||||
|
||||
const hnswSize = this.index.size()
|
||||
const metadataInitialized = !!this.metadataIndex
|
||||
const graphInitialized = !!this.graphIndex
|
||||
const storageGraphIndex = (this.storage as any).graphIndex
|
||||
|
||||
return {
|
||||
version: getBrainyVersion(),
|
||||
plugins: {
|
||||
active: this.pluginRegistry.getActivePlugins(),
|
||||
count: this.pluginRegistry.getActivePlugins().length
|
||||
},
|
||||
providers,
|
||||
indexes: {
|
||||
hnsw: {
|
||||
size: hnswSize,
|
||||
type: this.index.constructor.name
|
||||
},
|
||||
metadata: {
|
||||
type: this.metadataIndex?.constructor.name || 'none',
|
||||
initialized: metadataInitialized
|
||||
},
|
||||
graph: {
|
||||
type: this.graphIndex?.constructor.name || 'none',
|
||||
initialized: graphInitialized,
|
||||
wiredToStorage: graphInitialized && storageGraphIndex === this.graphIndex
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Assert that specific providers are supplied by a plugin (not using JS fallback).
|
||||
*
|
||||
* Call after init() in production to fail fast if a paid plugin (e.g. cortex)
|
||||
* isn't providing the expected acceleration. Throws if any listed key is using
|
||||
* the default JavaScript implementation.
|
||||
*
|
||||
* @param keys - Provider keys that MUST come from a plugin
|
||||
* @throws Error listing which providers are falling back to defaults
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const brain = new Brainy()
|
||||
* await brain.init()
|
||||
*
|
||||
* // Fail fast if cortex isn't providing these
|
||||
* brain.requireProviders(['distance', 'embeddings', 'metadataIndex', 'graphIndex'])
|
||||
* ```
|
||||
*/
|
||||
requireProviders(keys: string[]): void {
|
||||
const missing = keys.filter(k => !this.pluginRegistry.hasProvider(k))
|
||||
if (missing.length > 0) {
|
||||
const active = this.pluginRegistry.getActivePlugins()
|
||||
const pluginInfo = active.length > 0
|
||||
? `Active plugins: ${active.join(', ')}`
|
||||
: 'No plugins active'
|
||||
throw new Error(
|
||||
`[brainy] Required providers using JS fallback: ${missing.join(', ')}. ` +
|
||||
`${pluginInfo}. ` +
|
||||
`These providers must be supplied by a plugin for this deployment. ` +
|
||||
`Check plugin installation, license, and native module availability.`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Efficient Pagination API - Production-scale pagination using index-first approach
|
||||
* Automatically optimizes based on query type and applies pagination at the index level
|
||||
|
|
@ -4655,7 +4789,16 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
return []
|
||||
}
|
||||
|
||||
// Use native WASM batch API: single forward pass instead of N individual calls
|
||||
// Plugin provides native batch embedding — single forward pass for all texts
|
||||
const batchProvider = this.pluginRegistry.getProvider<(texts: string[]) => Promise<number[][]>>('embedBatch')
|
||||
if (batchProvider) {
|
||||
return batchProvider(texts)
|
||||
}
|
||||
// Plugin provides single-text embedding engine — map through it
|
||||
if (this.pluginRegistry.hasProvider('embeddings')) {
|
||||
return Promise.all(texts.map(t => this.embedder(t)))
|
||||
}
|
||||
// Default: WASM batch API (single forward pass, more efficient than N calls)
|
||||
return await embeddingManager.embedBatch(texts, options)
|
||||
}
|
||||
|
||||
|
|
@ -5481,9 +5624,9 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
const vector = params.vector || (await this.embed(params.query!))
|
||||
const limit = params.limit || 10
|
||||
|
||||
// Phase 2: Pass type for TypeAwareHNSWIndex (10x faster for type-specific queries)
|
||||
const searchResults = this.index instanceof TypeAwareHNSWIndex
|
||||
? await this.index.search(vector, limit * 2, params.type as any)
|
||||
// Pass type for type-aware indexes (10x faster for type-specific queries)
|
||||
const searchResults: [string, number][] = this.indexIsTypeAware && params.type
|
||||
? await (this.index as any).search(vector, limit * 2, params.type as any)
|
||||
: await this.index.search(vector, limit * 2)
|
||||
|
||||
// Batch-load entities for 10-50x faster cloud storage performance
|
||||
|
|
@ -5512,9 +5655,9 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
const nearEntity = await this.get(params.near.id)
|
||||
if (!nearEntity) return []
|
||||
|
||||
// Phase 2: Pass type for TypeAwareHNSWIndex
|
||||
const nearResults = this.index instanceof TypeAwareHNSWIndex
|
||||
? await this.index.search(nearEntity.vector, params.limit || 10, params.type as any)
|
||||
// Pass type for type-aware indexes
|
||||
const nearResults: [string, number][] = this.indexIsTypeAware && params.type
|
||||
? await (this.index as any).search(nearEntity.vector, params.limit || 10, params.type as any)
|
||||
: await this.index.search(nearEntity.vector, params.limit || 10)
|
||||
|
||||
// Filter by threshold first to minimize batch fetch
|
||||
|
|
@ -6068,11 +6211,16 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
throw new Error('Brain must be initialized before warming up embeddings. Call init() first.')
|
||||
}
|
||||
|
||||
console.log('🚀 Warming up embedding engine...')
|
||||
// Plugin-provided embeddings are already ready (native, no WASM warmup needed)
|
||||
if (this.pluginRegistry.hasProvider('embeddings')) {
|
||||
return
|
||||
}
|
||||
|
||||
console.log('Warming up embedding engine...')
|
||||
const start = Date.now()
|
||||
await embeddingManager.init()
|
||||
const elapsed = Date.now() - start
|
||||
console.log(`✅ Embedding engine ready in ${elapsed}ms`)
|
||||
console.log(`Embedding engine ready in ${elapsed}ms`)
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -6081,6 +6229,10 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
* @returns true if embedding engine is ready for immediate use
|
||||
*/
|
||||
isEmbeddingReady(): boolean {
|
||||
// Plugin-provided embeddings are always ready (native, no WASM init required)
|
||||
if (this.pluginRegistry.hasProvider('embeddings')) {
|
||||
return true
|
||||
}
|
||||
return embeddingManager.isInitialized()
|
||||
}
|
||||
|
||||
|
|
@ -6177,6 +6329,23 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
return new HNSWIndex(indexConfig as any, this.distance, { persistMode })
|
||||
}
|
||||
|
||||
/**
|
||||
* Create an HNSW index, using plugin factory when available.
|
||||
* Shared by init(), fork(), checkout(), and clear() to avoid duplication.
|
||||
*/
|
||||
private createIndex(): HNSWIndex | TypeAwareHNSWIndex {
|
||||
const hnswFactory = this.pluginRegistry.getProvider<(config: any, distance: DistanceFunction, options: any) => any>('hnsw')
|
||||
if (hnswFactory) {
|
||||
const persistMode = this.resolveHNSWPersistMode()
|
||||
return hnswFactory(
|
||||
{ ...this.config.index, distanceFunction: this.distance },
|
||||
this.distance,
|
||||
{ storage: this.storage, persistMode }
|
||||
)
|
||||
}
|
||||
return this.setupIndex()
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve HNSW persistence mode.
|
||||
* Extracted so both setupIndex() and the HNSW plugin factory path can use it.
|
||||
|
|
|
|||
|
|
@ -918,5 +918,55 @@ export const coreCommands = {
|
|||
console.error(chalk.red(error.message))
|
||||
process.exit(1)
|
||||
}
|
||||
},
|
||||
|
||||
/**
|
||||
* Show plugin and provider diagnostics
|
||||
*/
|
||||
async diagnostics(options: CoreOptions) {
|
||||
try {
|
||||
const brain = getBrainy()
|
||||
await brain.init()
|
||||
|
||||
const diag = brain.diagnostics()
|
||||
|
||||
if (options.json) {
|
||||
formatOutput(diag, options)
|
||||
return
|
||||
}
|
||||
|
||||
console.log(chalk.bold('\nBrainy Diagnostics'))
|
||||
console.log(chalk.dim(`Version: ${diag.version}`))
|
||||
console.log()
|
||||
|
||||
// Plugins
|
||||
console.log(chalk.bold('Plugins:'))
|
||||
if (diag.plugins.count === 0) {
|
||||
console.log(chalk.dim(' (none active)'))
|
||||
} else {
|
||||
for (const name of diag.plugins.active) {
|
||||
console.log(chalk.green(` ✓ ${name}`))
|
||||
}
|
||||
}
|
||||
console.log()
|
||||
|
||||
// Providers
|
||||
console.log(chalk.bold('Providers:'))
|
||||
for (const [key, info] of Object.entries(diag.providers)) {
|
||||
const icon = info.source === 'plugin' ? chalk.green('✓ plugin') : chalk.dim('default')
|
||||
console.log(` ${key.padEnd(16)} ${icon}`)
|
||||
}
|
||||
console.log()
|
||||
|
||||
// Indexes
|
||||
console.log(chalk.bold('Indexes:'))
|
||||
console.log(` HNSW: ${diag.indexes.hnsw.type} (${diag.indexes.hnsw.size} vectors)`)
|
||||
console.log(` Metadata: ${diag.indexes.metadata.type} (initialized: ${diag.indexes.metadata.initialized})`)
|
||||
console.log(` Graph: ${diag.indexes.graph.type} (initialized: ${diag.indexes.graph.initialized}, wired: ${diag.indexes.graph.wiredToStorage})`)
|
||||
console.log()
|
||||
} catch (error: any) {
|
||||
console.error(chalk.red('Diagnostics failed: ' + error.message))
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -174,6 +174,14 @@ program
|
|||
.option('-f, --format <format>', 'Output format (json|csv|jsonl)', 'json')
|
||||
.action(coreCommands.export)
|
||||
|
||||
program
|
||||
.command('diagnostics')
|
||||
.alias('diag')
|
||||
.description('Show plugin and provider diagnostics')
|
||||
.option('--json', 'Output as JSON')
|
||||
.option('--pretty', 'Pretty print JSON')
|
||||
.action(coreCommands.diagnostics)
|
||||
|
||||
// ===== Neural Commands =====
|
||||
|
||||
program
|
||||
|
|
|
|||
|
|
@ -14,6 +14,9 @@ import { Brainy } from './brainy.js'
|
|||
|
||||
export { Brainy }
|
||||
|
||||
// Export diagnostics result type
|
||||
export type { DiagnosticsResult } from './brainy.js'
|
||||
|
||||
// Export Brainy configuration and types
|
||||
export type {
|
||||
BrainyConfig,
|
||||
|
|
|
|||
|
|
@ -92,18 +92,20 @@ interface ItemWithMetadata {
|
|||
export class ImprovedNeuralAPI {
|
||||
private brain: any // Brainy instance
|
||||
private config: NeuralAPIConfig
|
||||
|
||||
private distanceFn: (a: Vector, b: Vector) => number
|
||||
|
||||
// Caching for performance
|
||||
private similarityCache = new Map<string, number | SimilarityResult>()
|
||||
private clusterCache = new Map<string, ClusteringResult>()
|
||||
private hierarchyCache = new Map<string, SemanticHierarchy>()
|
||||
private neighborsCache = new Map<string, NeighborsResult>()
|
||||
|
||||
|
||||
// Performance tracking
|
||||
private performanceMetrics = new Map<string, PerformanceMetrics[]>()
|
||||
|
||||
|
||||
constructor(brain: any, config: NeuralAPIConfig = {}) {
|
||||
this.brain = brain
|
||||
this.distanceFn = brain.distance || cosineDistance
|
||||
this.config = {
|
||||
cacheSize: 1000,
|
||||
defaultAlgorithm: 'auto',
|
||||
|
|
@ -114,7 +116,7 @@ export class ImprovedNeuralAPI {
|
|||
streamingBatchSize: 100,
|
||||
...config
|
||||
}
|
||||
|
||||
|
||||
this._initializeCleanupTimer()
|
||||
}
|
||||
|
||||
|
|
@ -1458,7 +1460,7 @@ export class ImprovedNeuralAPI {
|
|||
|
||||
switch (metric) {
|
||||
case 'cosine':
|
||||
score = 1 - cosineDistance(v1, v2)
|
||||
score = 1 - this.distanceFn(v1, v2)
|
||||
break
|
||||
case 'euclidean':
|
||||
score = 1 / (1 + euclideanDistance(v1, v2))
|
||||
|
|
@ -1467,7 +1469,7 @@ export class ImprovedNeuralAPI {
|
|||
score = 1 / (1 + this._manhattanDistance(v1, v2))
|
||||
break
|
||||
default:
|
||||
score = 1 - cosineDistance(v1, v2)
|
||||
score = 1 - this.distanceFn(v1, v2)
|
||||
}
|
||||
|
||||
if (options.detailed) {
|
||||
|
|
@ -3047,7 +3049,7 @@ export class ImprovedNeuralAPI {
|
|||
continue
|
||||
}
|
||||
|
||||
const similarity = 1 - cosineDistance(
|
||||
const similarity = 1 - this.distanceFn(
|
||||
Array.from(c1.centroid) as number[],
|
||||
Array.from(c2.centroid) as number[]
|
||||
)
|
||||
|
|
|
|||
|
|
@ -132,12 +132,14 @@ export interface LODConfig {
|
|||
*/
|
||||
export class NeuralAPI {
|
||||
private brain: any // Brainy instance
|
||||
private distanceFn: (a: Vector, b: Vector) => number
|
||||
private similarityCache: Map<string, number> = new Map()
|
||||
private clusterCache: Map<string, any> = new Map() // Enhanced for enterprise
|
||||
private hierarchyCache: Map<string, SemanticHierarchy> = new Map()
|
||||
|
||||
|
||||
constructor(brain: any) {
|
||||
this.brain = brain
|
||||
this.distanceFn = brain.distance || cosineDistance
|
||||
}
|
||||
|
||||
// ===== SMART USER-FRIENDLY API =====
|
||||
|
|
@ -471,7 +473,7 @@ export class NeuralAPI {
|
|||
}
|
||||
|
||||
// Calculate similarity
|
||||
const score = cosineDistance(itemA.vector, itemB.vector)
|
||||
const score = this.distanceFn(itemA.vector, itemB.vector)
|
||||
|
||||
this.similarityCache.set(cacheKey, score)
|
||||
|
||||
|
|
@ -498,7 +500,7 @@ export class NeuralAPI {
|
|||
}
|
||||
|
||||
private async similarityByVector(vectorA: Vector, vectorB: Vector, options?: SimilarityOptions): Promise<number | SimilarityResult> {
|
||||
const score = cosineDistance(vectorA, vectorB)
|
||||
const score = this.distanceFn(vectorA, vectorB)
|
||||
|
||||
if (options?.explain) {
|
||||
return {
|
||||
|
|
@ -610,7 +612,7 @@ export class NeuralAPI {
|
|||
// Find minimum distance to existing sample
|
||||
let minDistance = Infinity
|
||||
for (const selected of sample) {
|
||||
const distance = cosineDistance(candidate.vector, selected.vector)
|
||||
const distance = this.distanceFn(candidate.vector, selected.vector)
|
||||
minDistance = Math.min(minDistance, distance)
|
||||
}
|
||||
|
||||
|
|
@ -652,7 +654,7 @@ export class NeuralAPI {
|
|||
let bestDistance = Infinity
|
||||
|
||||
for (let c = 0; c < k; c++) {
|
||||
const distance = cosineDistance(item.vector, centroids[c])
|
||||
const distance = this.distanceFn(item.vector, centroids[c])
|
||||
if (distance < bestDistance) {
|
||||
bestDistance = distance
|
||||
bestCluster = c
|
||||
|
|
@ -679,7 +681,7 @@ export class NeuralAPI {
|
|||
let bestDistance = Infinity
|
||||
|
||||
for (let cc = 0; cc < k; cc++) {
|
||||
const distance = cosineDistance(item.vector, centroids[cc])
|
||||
const distance = this.distanceFn(item.vector, centroids[cc])
|
||||
if (distance < bestDistance) {
|
||||
bestDistance = distance
|
||||
bestCluster = cc
|
||||
|
|
@ -750,7 +752,7 @@ export class NeuralAPI {
|
|||
let merged = false
|
||||
|
||||
for (let i = 0; i < result.length; i++) {
|
||||
const similarity = cosineDistance(result[i].centroid, batchCluster.centroid)
|
||||
const similarity = this.distanceFn(result[i].centroid, batchCluster.centroid)
|
||||
|
||||
if (similarity > 0.8) {
|
||||
// Merge clusters
|
||||
|
|
|
|||
|
|
@ -64,6 +64,7 @@ interface HistoricalEntry {
|
|||
*/
|
||||
export class VerbEmbeddingSignal {
|
||||
private brain: Brainy
|
||||
private distanceFn: (a: Vector, b: Vector) => number
|
||||
private options: Required<VerbEmbeddingSignalOptions>
|
||||
|
||||
// Pre-computed verb type embeddings (loaded once at startup)
|
||||
|
|
@ -88,6 +89,7 @@ export class VerbEmbeddingSignal {
|
|||
|
||||
constructor(brain: Brainy, options?: VerbEmbeddingSignalOptions) {
|
||||
this.brain = brain
|
||||
this.distanceFn = (brain as any).distance || cosineDistance
|
||||
this.options = {
|
||||
minConfidence: options?.minConfidence ?? 0.60,
|
||||
minSimilarity: options?.minSimilarity ?? 0.55,
|
||||
|
|
@ -142,7 +144,7 @@ export class VerbEmbeddingSignal {
|
|||
const similarities: Array<{ type: VerbType; similarity: number }> = []
|
||||
|
||||
for (const [verbType, typeEmbedding] of this.verbTypeEmbeddings) {
|
||||
const distance = cosineDistance(embedding, typeEmbedding)
|
||||
const distance = this.distanceFn(embedding, typeEmbedding)
|
||||
const similarity = 1 - distance // Convert distance to similarity
|
||||
similarities.push({ type: verbType, similarity })
|
||||
}
|
||||
|
|
|
|||
|
|
@ -43,7 +43,8 @@ export interface BrainyPluginContext {
|
|||
* - 'cache' — UnifiedCache replacement
|
||||
* - 'hnsw' — HNSWIndex replacement
|
||||
* - 'roaring' — RoaringBitmap32 replacement
|
||||
* - 'embeddings' — Embedding engine replacement
|
||||
* - 'embeddings' — Embedding engine replacement (single text)
|
||||
* - 'embedBatch' — Batch embedding engine (texts[] → vectors[])
|
||||
* - 'distance' — Distance function overrides
|
||||
* - 'msgpack' — Msgpack encode/decode
|
||||
*
|
||||
|
|
|
|||
|
|
@ -379,6 +379,16 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the graph index instance (used by Brainy to wire plugin-provided graph indexes).
|
||||
* This ensures getVerbsBySource() uses the fast GraphAdjacencyIndex path
|
||||
* instead of falling back to O(n) shard iteration.
|
||||
*/
|
||||
public setGraphIndex(index: GraphAdjacencyIndex): void {
|
||||
this.graphIndex = index
|
||||
this.graphIndexPromise = Promise.resolve(index)
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure the storage adapter is initialized
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -201,9 +201,8 @@ export class PathResolver {
|
|||
* Falls back to graph traversal if MetadataIndex unavailable
|
||||
*/
|
||||
private async resolveWithMetadataIndex(path: string): Promise<string> {
|
||||
// Access MetadataIndexManager from brain's storage
|
||||
const storage = (this.brain as any).storage
|
||||
const metadataIndex = storage?.metadataIndex
|
||||
// Access MetadataIndexManager from brain instance (not storage — metadataIndex lives on Brainy)
|
||||
const metadataIndex = (this.brain as any).metadataIndex
|
||||
|
||||
if (!metadataIndex) {
|
||||
// MetadataIndex not available, use graph traversal
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ import { PathResolver } from '../PathResolver.js'
|
|||
import { VFSEntity, VFSError, VFSErrorCode } from '../types.js'
|
||||
import { SemanticPathParser, ParsedSemanticPath } from './SemanticPathParser.js'
|
||||
import { ProjectionRegistry } from './ProjectionRegistry.js'
|
||||
import { UnifiedCache } from '../../utils/unifiedCache.js'
|
||||
import { getGlobalCache, UnifiedCache } from '../../utils/unifiedCache.js'
|
||||
|
||||
/**
|
||||
* Semantic Path Resolver
|
||||
|
|
@ -44,12 +44,8 @@ export class SemanticPathResolver {
|
|||
this.registry = registry
|
||||
this.parser = new SemanticPathParser()
|
||||
|
||||
// Use Brainy's UnifiedCache for semantic path caching
|
||||
// Zero-config: Uses 2GB default from UnifiedCache
|
||||
this.cache = new UnifiedCache({
|
||||
enableRequestCoalescing: true,
|
||||
enableFairnessCheck: true
|
||||
})
|
||||
// Use global UnifiedCache (picks up plugin-provided cache when available)
|
||||
this.cache = getGlobalCache()
|
||||
|
||||
// Create traditional path resolver (uses its own optimized cache with defaults)
|
||||
this.pathResolver = new PathResolver(brain, rootEntityId)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue