feat: implement zero-config system with Node.js 22 compatibility
- Add comprehensive zero-config preset system (production, development, minimal) - Implement intelligent auto-configuration for models and storage - Add Node.js version enforcement for ONNX Runtime stability - Force single-threaded ONNX operations to prevent V8 HandleScope crashes - Create extensible configuration architecture - Add 14 distributed system presets for enterprise deployments - Include detailed documentation and migration guides BREAKING CHANGE: Now requires Node.js 22.x LTS for optimal stability
This commit is contained in:
parent
4d60384755
commit
5f862bad98
20 changed files with 3718 additions and 71 deletions
335
src/config/extensibleConfig.ts
Normal file
335
src/config/extensibleConfig.ts
Normal file
|
|
@ -0,0 +1,335 @@
|
|||
/**
|
||||
* Extensible Configuration System
|
||||
* Allows augmentations to register new storage types, presets, and configurations
|
||||
*/
|
||||
|
||||
import { StorageOption, PresetName, PresetConfig, ModelPrecision, DistributedRole, PresetCategory } from './distributedPresets.js'
|
||||
import { StorageConfigResult } from './storageAutoConfig.js'
|
||||
|
||||
/**
|
||||
* Storage provider registration interface
|
||||
*/
|
||||
export interface StorageProvider {
|
||||
type: string // e.g., 'redis', 'mongodb', 'postgres'
|
||||
name: string
|
||||
description: string
|
||||
|
||||
// Detection function - returns true if this storage should be auto-selected
|
||||
detect: () => Promise<boolean>
|
||||
|
||||
// Configuration builder
|
||||
getConfig: () => Promise<any>
|
||||
|
||||
// Priority for auto-detection (higher = checked first)
|
||||
priority?: number
|
||||
|
||||
// Required environment variables or packages
|
||||
requirements?: {
|
||||
env?: string[]
|
||||
packages?: string[]
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Preset extension interface
|
||||
*/
|
||||
export interface PresetExtension {
|
||||
name: string
|
||||
config: PresetConfig
|
||||
override?: boolean // Override existing preset
|
||||
}
|
||||
|
||||
/**
|
||||
* Global registry for extensions
|
||||
*/
|
||||
class ConfigurationRegistry {
|
||||
private static instance: ConfigurationRegistry
|
||||
|
||||
// Registered storage providers
|
||||
private storageProviders: Map<string, StorageProvider> = new Map()
|
||||
|
||||
// Registered preset extensions
|
||||
private presetExtensions: Map<string, PresetExtension> = new Map()
|
||||
|
||||
// Custom auto-detection hooks
|
||||
private autoDetectHooks: Array<() => Promise<any>> = []
|
||||
|
||||
private constructor() {
|
||||
// Initialize with built-in providers
|
||||
this.registerBuiltInProviders()
|
||||
}
|
||||
|
||||
static getInstance(): ConfigurationRegistry {
|
||||
if (!ConfigurationRegistry.instance) {
|
||||
ConfigurationRegistry.instance = new ConfigurationRegistry()
|
||||
}
|
||||
return ConfigurationRegistry.instance
|
||||
}
|
||||
|
||||
/**
|
||||
* Register a new storage provider
|
||||
* This is how augmentations add new storage types
|
||||
*/
|
||||
registerStorageProvider(provider: StorageProvider): void {
|
||||
console.log(`📦 Registering storage provider: ${provider.type} (${provider.name})`)
|
||||
this.storageProviders.set(provider.type, provider)
|
||||
}
|
||||
|
||||
/**
|
||||
* Register a new preset
|
||||
*/
|
||||
registerPreset(name: string, extension: PresetExtension): void {
|
||||
console.log(`🎨 Registering preset: ${name}`)
|
||||
this.presetExtensions.set(name, extension)
|
||||
}
|
||||
|
||||
/**
|
||||
* Register an auto-detection hook
|
||||
*/
|
||||
registerAutoDetectHook(hook: () => Promise<any>): void {
|
||||
this.autoDetectHooks.push(hook)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all registered storage providers
|
||||
*/
|
||||
getStorageProviders(): StorageProvider[] {
|
||||
return Array.from(this.storageProviders.values())
|
||||
.sort((a, b) => (b.priority || 0) - (a.priority || 0))
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all registered presets (built-in + extensions)
|
||||
*/
|
||||
getAllPresets(): Map<string, PresetConfig> {
|
||||
// Start with built-in presets
|
||||
const allPresets = new Map<string, PresetConfig>()
|
||||
|
||||
// Note: Would import from distributedPresets-new.ts
|
||||
// Add extended presets
|
||||
for (const [name, extension] of this.presetExtensions) {
|
||||
if (extension.override || !allPresets.has(name)) {
|
||||
allPresets.set(name, extension.config)
|
||||
}
|
||||
}
|
||||
|
||||
return allPresets
|
||||
}
|
||||
|
||||
/**
|
||||
* Auto-detect storage including extensions
|
||||
*/
|
||||
async autoDetectStorage(): Promise<StorageConfigResult> {
|
||||
// Check registered providers first (in priority order)
|
||||
for (const provider of this.getStorageProviders()) {
|
||||
try {
|
||||
if (await provider.detect()) {
|
||||
const config = await provider.getConfig()
|
||||
return {
|
||||
type: provider.type as any,
|
||||
config,
|
||||
reason: `Auto-detected ${provider.name}`,
|
||||
autoSelected: true
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn(`Failed to detect ${provider.type}:`, error)
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback to built-in detection
|
||||
const { autoDetectStorage } = await import('./storageAutoConfig.js')
|
||||
return autoDetectStorage()
|
||||
}
|
||||
|
||||
/**
|
||||
* Register built-in providers
|
||||
*/
|
||||
private registerBuiltInProviders(): void {
|
||||
// These would be the built-in ones, but could be overridden
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Example: Redis storage provider registration
|
||||
* This would be in the redis augmentation package
|
||||
*/
|
||||
export const redisStorageProvider: StorageProvider = {
|
||||
type: 'redis',
|
||||
name: 'Redis Storage',
|
||||
description: 'High-performance in-memory data store',
|
||||
priority: 10, // Check before filesystem
|
||||
|
||||
requirements: {
|
||||
env: ['REDIS_URL', 'REDIS_HOST'],
|
||||
packages: ['redis', 'ioredis']
|
||||
},
|
||||
|
||||
async detect(): Promise<boolean> {
|
||||
// Check for Redis connection info
|
||||
if (process.env.REDIS_URL || process.env.REDIS_HOST) {
|
||||
try {
|
||||
// Try to connect to Redis (dynamic import for optional dependency)
|
||||
const redis: any = await new Function('return import("ioredis")')().catch(() => null)
|
||||
if (!redis) return false
|
||||
|
||||
const client = new redis.default(process.env.REDIS_URL)
|
||||
await client.ping()
|
||||
await client.quit()
|
||||
return true
|
||||
} catch {
|
||||
// Redis not available
|
||||
}
|
||||
}
|
||||
return false
|
||||
},
|
||||
|
||||
async getConfig(): Promise<any> {
|
||||
return {
|
||||
type: 'redis',
|
||||
redisStorage: {
|
||||
url: process.env.REDIS_URL || `redis://${process.env.REDIS_HOST}:${process.env.REDIS_PORT || 6379}`,
|
||||
prefix: process.env.REDIS_PREFIX || 'brainy:',
|
||||
ttl: process.env.REDIS_TTL ? parseInt(process.env.REDIS_TTL) : undefined
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Example: MongoDB storage provider
|
||||
*/
|
||||
export const mongoStorageProvider: StorageProvider = {
|
||||
type: 'mongodb',
|
||||
name: 'MongoDB Storage',
|
||||
description: 'Document database for complex data',
|
||||
priority: 8,
|
||||
|
||||
requirements: {
|
||||
env: ['MONGODB_URI', 'MONGO_URL'],
|
||||
packages: ['mongodb']
|
||||
},
|
||||
|
||||
async detect(): Promise<boolean> {
|
||||
if (process.env.MONGODB_URI || process.env.MONGO_URL) {
|
||||
try {
|
||||
const mongodb: any = await new Function('return import("mongodb")')().catch(() => null)
|
||||
if (!mongodb) return false
|
||||
|
||||
const client = new mongodb.MongoClient(process.env.MONGODB_URI || process.env.MONGO_URL!)
|
||||
await client.connect()
|
||||
await client.close()
|
||||
return true
|
||||
} catch {
|
||||
// MongoDB not available
|
||||
}
|
||||
}
|
||||
return false
|
||||
},
|
||||
|
||||
async getConfig(): Promise<any> {
|
||||
return {
|
||||
type: 'mongodb',
|
||||
mongoStorage: {
|
||||
uri: process.env.MONGODB_URI || process.env.MONGO_URL,
|
||||
database: process.env.MONGO_DATABASE || 'brainy',
|
||||
collection: process.env.MONGO_COLLECTION || 'vectors'
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Example: PostgreSQL with pgvector extension
|
||||
*/
|
||||
export const postgresStorageProvider: StorageProvider = {
|
||||
type: 'postgres',
|
||||
name: 'PostgreSQL Storage',
|
||||
description: 'PostgreSQL with pgvector for scalable vector search',
|
||||
priority: 9,
|
||||
|
||||
requirements: {
|
||||
env: ['DATABASE_URL', 'POSTGRES_URL'],
|
||||
packages: ['pg', 'pgvector']
|
||||
},
|
||||
|
||||
async detect(): Promise<boolean> {
|
||||
const url = process.env.DATABASE_URL || process.env.POSTGRES_URL
|
||||
if (url && url.includes('postgres')) {
|
||||
try {
|
||||
const pg: any = await new Function('return import("pg")')().catch(() => null)
|
||||
if (!pg) return false
|
||||
|
||||
const client = new pg.Client({ connectionString: url })
|
||||
await client.connect()
|
||||
|
||||
// Check for pgvector extension
|
||||
const result = await client.query(
|
||||
"SELECT * FROM pg_extension WHERE extname = 'vector'"
|
||||
)
|
||||
await client.end()
|
||||
|
||||
return result.rows.length > 0
|
||||
} catch {
|
||||
// PostgreSQL not available or pgvector not installed
|
||||
}
|
||||
}
|
||||
return false
|
||||
},
|
||||
|
||||
async getConfig(): Promise<any> {
|
||||
return {
|
||||
type: 'postgres',
|
||||
postgresStorage: {
|
||||
connectionString: process.env.DATABASE_URL || process.env.POSTGRES_URL,
|
||||
table: process.env.POSTGRES_TABLE || 'brainy_vectors',
|
||||
schema: process.env.POSTGRES_SCHEMA || 'public'
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* How an augmentation would register its storage provider
|
||||
*/
|
||||
export function registerStorageAugmentation(provider: StorageProvider): void {
|
||||
const registry = ConfigurationRegistry.getInstance()
|
||||
registry.registerStorageProvider(provider)
|
||||
}
|
||||
|
||||
/**
|
||||
* How to register a new preset
|
||||
*/
|
||||
export function registerPresetAugmentation(name: string, config: PresetConfig): void {
|
||||
const registry = ConfigurationRegistry.getInstance()
|
||||
registry.registerPreset(name, {
|
||||
name,
|
||||
config,
|
||||
override: false
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Example preset for Redis-based caching service
|
||||
*/
|
||||
export const redisCachePreset: PresetConfig = {
|
||||
storage: 'redis' as any, // Extended storage type
|
||||
model: ModelPrecision.Q8,
|
||||
features: ['core', 'cache', 'search'],
|
||||
distributed: true,
|
||||
role: DistributedRole.READER,
|
||||
readOnly: true,
|
||||
cache: {
|
||||
hotCacheMaxSize: 50000, // Large Redis cache
|
||||
autoTune: true
|
||||
},
|
||||
description: 'Redis-backed caching layer',
|
||||
category: PresetCategory.SERVICE
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the configuration registry
|
||||
*/
|
||||
export function getConfigRegistry(): ConfigurationRegistry {
|
||||
return ConfigurationRegistry.getInstance()
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue