soulcraft/brainy
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

feat: implement zero-config system with Node.js 22 compatibility

Browse source 
- Add comprehensive zero-config preset system (production, development, minimal)
- Implement intelligent auto-configuration for models and storage
- Add Node.js version enforcement for ONNX Runtime stability
- Force single-threaded ONNX operations to prevent V8 HandleScope crashes
- Create extensible configuration architecture
- Add 14 distributed system presets for enterprise deployments
- Include detailed documentation and migration guides

BREAKING CHANGE: Now requires Node.js 22.x LTS for optimal stability
This commit is contained in:
David Snelling 2025-08-29 15:39:07 -07:00
parent 4d60384755
commit 5f862bad98
20 changed files with 3718 additions and 71 deletions
Download patch file Download diff file Expand all files Collapse all files

142
src/brainyData.ts
View file

@ -34,6 +34,7 @@ import {
} from './utils/index.js'
import { getAugmentationVersion } from './utils/version.js'
import { matchesMetadataFilter } from './utils/metadataFilter.js'
import { enforceNodeVersion } from './utils/nodeVersionCheck.js'
import { MetadataIndexManager, MetadataIndexConfig } from './utils/metadataIndex.js'
import {
createNamespacedMetadata,
@ -545,6 +546,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
private allowDirectReads: boolean
private storageConfig: BrainyDataConfig['storage'] = {}
private config: BrainyDataConfig
private rawConfig: any // Raw config input for zero-config processing
private useOptimizedIndex: boolean = false
private _dimensions: number
private loggingConfig: BrainyDataConfig['logging'] = { verbose: true }
@ -664,21 +666,37 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
/**
* Create a new vector database
* @param config - Zero-config string ('production', 'development', 'minimal'),
* simplified config object, or legacy full config
*/
constructor(config: BrainyDataConfig = {}) {
// Store config
this.config = config
constructor(config: BrainyDataConfig | string | any = {}) {
// Enforce Node.js version requirement for ONNX stability
if (typeof process !== 'undefined' && process.version) {
enforceNodeVersion()
}
// Store raw config for processing in init()
this.rawConfig = config
// For now, process as legacy config if it's an object
// The actual zero-config processing will happen in init() since it's async
if (typeof config === 'object') {
this.config = config
} else {
// String preset or simplified config - use minimal defaults for now
this.config = {}
}
// Set dimensions to fixed value of 384 (all-MiniLM-L6-v2 dimension)
this._dimensions = 384
// Set distance function
this.distanceFunction = config.distanceFunction || cosineDistance
this.distanceFunction = this.config.distanceFunction || cosineDistance
// Always use the optimized HNSW index implementation
// Configure HNSW with disk-based storage when a storage adapter is provided
const hnswConfig = config.hnsw || {}
if (config.storageAdapter) {
const hnswConfig = this.config.hnsw || {}
if (this.config.storageAdapter) {
hnswConfig.useDiskBasedIndex = true
}
@ -690,41 +708,41 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
this.useOptimizedIndex = false
// Set storage if provided, otherwise it will be initialized in init()
this.storage = config.storageAdapter || null
this.storage = this.config.storageAdapter || null
// Store logging configuration
if (config.logging !== undefined) {
if (this.config.logging !== undefined) {
this.loggingConfig = {
...this.loggingConfig,
...config.logging
...this.config.logging
}
}
// Set embedding function if provided, otherwise create one with the appropriate verbose setting
if (config.embeddingFunction) {
this.embeddingFunction = config.embeddingFunction
if (this.config.embeddingFunction) {
this.embeddingFunction = this.config.embeddingFunction
} else {
this.embeddingFunction = defaultEmbeddingFunction
}
// Set persistent storage request flag
this.requestPersistentStorage =
config.storage?.requestPersistentStorage || false
this.config.storage?.requestPersistentStorage || false
// Set read-only flag
this.readOnly = config.readOnly || false
this.readOnly = this.config.readOnly || false
// Set frozen flag (defaults to false to allow optimizations in readOnly mode)
this.frozen = config.frozen || false
this.frozen = this.config.frozen || false
// Set lazy loading in read-only mode flag
this.lazyLoadInReadOnlyMode = config.lazyLoadInReadOnlyMode || false
this.lazyLoadInReadOnlyMode = this.config.lazyLoadInReadOnlyMode || false
// Set write-only flag
this.writeOnly = config.writeOnly || false
this.writeOnly = this.config.writeOnly || false
// Set allowDirectReads flag
this.allowDirectReads = config.allowDirectReads || false
this.allowDirectReads = this.config.allowDirectReads || false
// Validate that readOnly and writeOnly are not both true
if (this.readOnly && this.writeOnly) {
@ -732,27 +750,27 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
}
// Set default service name if provided
if (config.defaultService) {
this.defaultService = config.defaultService
if (this.config.defaultService) {
this.defaultService = this.config.defaultService
}
// Store storage configuration for later use in init()
this.storageConfig = config.storage || {}
this.storageConfig = this.config.storage || {}
// Store timeout and retry configuration
this.timeoutConfig = config.timeouts || {}
this.retryConfig = config.retryPolicy || {}
this.timeoutConfig = this.config.timeouts || {}
this.retryConfig = this.config.retryPolicy || {}
// Store remote server configuration if provided
if (config.remoteServer) {
this.remoteServerConfig = config.remoteServer
if (this.config.remoteServer) {
this.remoteServerConfig = this.config.remoteServer
}
// Initialize real-time update configuration if provided
if (config.realtimeUpdates) {
if (this.config.realtimeUpdates) {
this.realtimeUpdateConfig = {
...this.realtimeUpdateConfig,
...config.realtimeUpdates
...this.config.realtimeUpdates
}
}
@ -774,23 +792,23 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
}
// Override defaults with user-provided configuration if available
if (config.cache) {
if (this.config.cache) {
this.cacheConfig = {
...this.cacheConfig,
...config.cache
...this.config.cache
}
}
// Store distributed configuration
if (config.distributed) {
if (typeof config.distributed === 'boolean') {
if (this.config.distributed) {
if (typeof this.config.distributed === 'boolean') {
// Auto-mode enabled
this.distributedConfig = {
enabled: true
}
} else {
// Explicit configuration
this.distributedConfig = config.distributed
this.distributedConfig = this.config.distributed
}
}
@ -951,17 +969,29 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
}
}
// Check if specific storage is configured
// Check if specific storage is configured (legacy and new formats)
if (storageOptions.s3Storage || storageOptions.r2Storage ||
storageOptions.gcsStorage || storageOptions.forceMemoryStorage ||
storageOptions.forceFileSystemStorage) {
// Create storage from config
const { createStorage } = await import('./storage/storageFactory.js')
this.storage = await createStorage(storageOptions as any)
storageOptions.forceFileSystemStorage ||
typeof storageOptions === 'string') {
// Wrap in augmentation for consistency
const wrapper = new DynamicStorageAugmentation(this.storage)
this.augmentations.register(wrapper)
// Handle string storage types (new zero-config)
if (typeof storageOptions === 'string') {
const { createAutoStorageAugmentation } = await import('./augmentations/storageAugmentations.js')
// For now, use auto-detection - TODO: extend to support preferred types
const autoAug = await createAutoStorageAugmentation({
rootDirectory: './brainy-data'
})
this.augmentations.register(autoAug)
} else {
// Legacy object config
const { createStorage } = await import('./storage/storageFactory.js')
this.storage = await createStorage(storageOptions as any)
// Wrap in augmentation for consistency
const wrapper = new DynamicStorageAugmentation(this.storage)
this.augmentations.register(wrapper)
}
} else {
// Zero-config: auto-select based on environment
const autoAug = await createAutoStorageAugmentation({
@ -1609,6 +1639,40 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
this.isInitializing = true
// Process zero-config if needed
if (this.rawConfig !== undefined) {
try {
const { applyZeroConfig } = await import('./config/index.js')
const processedConfig = await applyZeroConfig(this.rawConfig)
// Apply processed config if it's different from raw
if (processedConfig !== this.rawConfig) {
// Log if verbose
if (processedConfig.logging?.verbose) {
console.log('🤖 Zero-config applied successfully')
}
// Update config with processed values
this.config = processedConfig
// Update relevant properties from processed config
this.storageConfig = processedConfig.storage || {}
this.loggingConfig = processedConfig.logging || { verbose: false }
// Update embedding function if precision was specified
if (processedConfig.embeddingOptions?.precision) {
const { createEmbeddingFunctionWithPrecision } = await import('./config/index.js')
this.embeddingFunction = await createEmbeddingFunctionWithPrecision(
processedConfig.embeddingOptions.precision
)
}
}
} catch (error) {
console.warn('Zero-config processing failed, using defaults:', error)
// Continue with existing config
}
}
// CRITICAL: Initialize universal memory manager ONLY for default embedding function
// This preserves custom embedding functions (like test mocks)
if (typeof this.embeddingFunction === 'function' && this.embeddingFunction === defaultEmbeddingFunction) {

362
src/config/distributedPresets.ts Normal file
View file

@ -0,0 +1,362 @@
/**
* Extended Distributed Configuration Presets
* Common patterns for distributed and multi-service architectures
* All strongly typed with enums for compile-time safety
*/
/**
* Strongly typed enum for preset names
*/
export enum PresetName {
// Basic presets
PRODUCTION = 'production',
DEVELOPMENT = 'development',
MINIMAL = 'minimal',
ZERO = 'zero',
// Distributed presets
WRITER = 'writer',
READER = 'reader',
// Service-specific presets
INGESTION_SERVICE = 'ingestion-service',
SEARCH_API = 'search-api',
ANALYTICS_SERVICE = 'analytics-service',
EDGE_CACHE = 'edge-cache',
BATCH_PROCESSOR = 'batch-processor',
STREAMING_SERVICE = 'streaming-service',
ML_TRAINING = 'ml-training',
SIDECAR = 'sidecar'
}
/**
* Preset categories for organization
*/
export enum PresetCategory {
BASIC = 'basic',
DISTRIBUTED = 'distributed',
SERVICE = 'service'
}
/**
* Model precision options
*/
export enum ModelPrecision {
FP32 = 'fp32',
Q8 = 'q8',
AUTO = 'auto',
FAST = 'fast',
SMALL = 'small'
}
/**
* Storage options
*/
export enum StorageOption {
AUTO = 'auto',
MEMORY = 'memory',
DISK = 'disk',
CLOUD = 'cloud'
}
/**
* Feature set options
*/
export enum FeatureSet {
MINIMAL = 'minimal',
DEFAULT = 'default',
FULL = 'full',
CUSTOM = 'custom' // For custom feature arrays
}
/**
* Distributed role options
*/
export enum DistributedRole {
WRITER = 'writer',
READER = 'reader',
HYBRID = 'hybrid'
}
/**
* Preset configuration interface
*/
export interface PresetConfig {
storage: StorageOption
model: ModelPrecision
features: FeatureSet | string[]
distributed: boolean
role?: DistributedRole
readOnly?: boolean
writeOnly?: boolean
allowDirectReads?: boolean
lazyLoadInReadOnlyMode?: boolean
cache?: {
hotCacheMaxSize?: number
autoTune?: boolean
batchSize?: number
}
verbose?: boolean
description: string
category: PresetCategory
}
/**
* Strongly typed preset configurations
*/
export const PRESET_CONFIGS: Readonly<Record<PresetName, PresetConfig>> = {
// Basic presets
[PresetName.PRODUCTION]: {
storage: StorageOption.DISK,
model: ModelPrecision.AUTO,
features: FeatureSet.DEFAULT,
distributed: false,
verbose: false,
description: 'Optimized for production use',
category: PresetCategory.BASIC
},
[PresetName.DEVELOPMENT]: {
storage: StorageOption.MEMORY,
model: ModelPrecision.FP32,
features: FeatureSet.FULL,
distributed: false,
verbose: true,
description: 'Optimized for development with verbose logging',
category: PresetCategory.BASIC
},
[PresetName.MINIMAL]: {
storage: StorageOption.MEMORY,
model: ModelPrecision.Q8,
features: FeatureSet.MINIMAL,
distributed: false,
verbose: false,
description: 'Minimal footprint configuration',
category: PresetCategory.BASIC
},
[PresetName.ZERO]: {
storage: StorageOption.AUTO,
model: ModelPrecision.AUTO,
features: FeatureSet.DEFAULT,
distributed: false,
verbose: false,
description: 'True zero configuration with auto-detection',
category: PresetCategory.BASIC
},
// Distributed basic presets
[PresetName.WRITER]: {
storage: StorageOption.AUTO,
model: ModelPrecision.AUTO,
features: FeatureSet.MINIMAL,
distributed: true,
role: DistributedRole.WRITER,
writeOnly: true,
allowDirectReads: true,
verbose: false,
description: 'Write-only instance for distributed setups',
category: PresetCategory.DISTRIBUTED
},
[PresetName.READER]: {
storage: StorageOption.AUTO,
model: ModelPrecision.AUTO,
features: FeatureSet.DEFAULT,
distributed: true,
role: DistributedRole.READER,
readOnly: true,
lazyLoadInReadOnlyMode: true,
verbose: false,
description: 'Read-only instance for distributed setups',
category: PresetCategory.DISTRIBUTED
},
// Service-specific presets
[PresetName.INGESTION_SERVICE]: {
storage: StorageOption.CLOUD,
model: ModelPrecision.Q8,
features: ['core', 'batch-processing', 'entity-registry'],
distributed: true,
role: DistributedRole.WRITER,
writeOnly: true,
allowDirectReads: true,
verbose: false,
description: 'High-throughput data ingestion service',
category: PresetCategory.SERVICE
},
[PresetName.SEARCH_API]: {
storage: StorageOption.CLOUD,
model: ModelPrecision.FP32,
features: ['core', 'search', 'cache', 'triple-intelligence'],
distributed: true,
role: DistributedRole.READER,
readOnly: true,
lazyLoadInReadOnlyMode: true,
cache: {
hotCacheMaxSize: 10000,
autoTune: true
},
verbose: false,
description: 'Low-latency search API service',
category: PresetCategory.SERVICE
},
[PresetName.ANALYTICS_SERVICE]: {
storage: StorageOption.CLOUD,
model: ModelPrecision.AUTO,
features: ['core', 'search', 'metrics', 'monitoring'],
distributed: true,
role: DistributedRole.HYBRID,
verbose: false,
description: 'Analytics and data processing service',
category: PresetCategory.SERVICE
},
[PresetName.EDGE_CACHE]: {
storage: StorageOption.AUTO,
model: ModelPrecision.Q8,
features: ['core', 'search', 'cache'],
distributed: true,
role: DistributedRole.READER,
readOnly: true,
lazyLoadInReadOnlyMode: true,
cache: {
hotCacheMaxSize: 1000,
autoTune: false
},
verbose: false,
description: 'Edge location cache with minimal footprint',
category: PresetCategory.SERVICE
},
[PresetName.BATCH_PROCESSOR]: {
storage: StorageOption.CLOUD,
model: ModelPrecision.Q8,
features: ['core', 'batch-processing', 'neural-api'],
distributed: true,
role: DistributedRole.HYBRID,
cache: {
hotCacheMaxSize: 5000,
batchSize: 500
},
verbose: false,
description: 'Batch processing and bulk operations',
category: PresetCategory.SERVICE
},
[PresetName.STREAMING_SERVICE]: {
storage: StorageOption.CLOUD,
model: ModelPrecision.Q8,
features: ['core', 'batch-processing', 'wal'],
distributed: true,
role: DistributedRole.WRITER,
writeOnly: true,
allowDirectReads: false,
verbose: false,
description: 'Real-time data streaming service',
category: PresetCategory.SERVICE
},
[PresetName.ML_TRAINING]: {
storage: StorageOption.CLOUD,
model: ModelPrecision.FP32,
features: FeatureSet.FULL,
distributed: true,
role: DistributedRole.HYBRID,
cache: {
hotCacheMaxSize: 20000,
autoTune: true
},
verbose: true,
description: 'Machine learning training service',
category: PresetCategory.SERVICE
},
[PresetName.SIDECAR]: {
storage: StorageOption.AUTO,
model: ModelPrecision.Q8,
features: FeatureSet.MINIMAL,
distributed: false,
verbose: false,
description: 'Lightweight sidecar for microservices',
category: PresetCategory.SERVICE
}
} as const
/**
* Type-safe preset getter
*/
export function getPreset(name: PresetName): PresetConfig {
return PRESET_CONFIGS[name]
}
/**
* Check if a string is a valid preset name
*/
export function isValidPreset(name: string): name is PresetName {
return Object.values(PresetName).includes(name as PresetName)
}
/**
* Get presets by category
*/
export function getPresetsByCategory(category: PresetCategory): PresetName[] {
return Object.entries(PRESET_CONFIGS)
.filter(([_, config]) => config.category === category)
.map(([name]) => name as PresetName)
}
/**
* Get all preset names
*/
export function getAllPresetNames(): PresetName[] {
return Object.values(PresetName)
}
/**
* Get preset description
*/
export function getPresetDescription(name: PresetName): string {
return PRESET_CONFIGS[name].description
}
/**
* Convert preset config to Brainy config
*/
export function presetToBrainyConfig(preset: PresetConfig): any {
const config: any = {
storage: preset.storage,
model: preset.model,
verbose: preset.verbose
}
// Handle features
if (Array.isArray(preset.features)) {
config.features = preset.features
} else {
config.features = preset.features // Will be expanded by processZeroConfig
}
// Handle distributed settings
if (preset.distributed) {
config.distributed = {
enabled: true,
role: preset.role
}
if (preset.readOnly) config.readOnly = true
if (preset.writeOnly) config.writeOnly = true
if (preset.allowDirectReads) config.allowDirectReads = true
if (preset.lazyLoadInReadOnlyMode) config.lazyLoadInReadOnlyMode = true
}
// Handle cache settings
if (preset.cache) {
config.cache = preset.cache
}
return config
}

335
src/config/extensibleConfig.ts Normal file
View file

@ -0,0 +1,335 @@
/**
* Extensible Configuration System
* Allows augmentations to register new storage types, presets, and configurations
*/
import { StorageOption, PresetName, PresetConfig, ModelPrecision, DistributedRole, PresetCategory } from './distributedPresets.js'
import { StorageConfigResult } from './storageAutoConfig.js'
/**
* Storage provider registration interface
*/
export interface StorageProvider {
type: string // e.g., 'redis', 'mongodb', 'postgres'
name: string
description: string
// Detection function - returns true if this storage should be auto-selected
detect: () => Promise<boolean>
// Configuration builder
getConfig: () => Promise<any>
// Priority for auto-detection (higher = checked first)
priority?: number
// Required environment variables or packages
requirements?: {
env?: string[]
packages?: string[]
}
}
/**
* Preset extension interface
*/
export interface PresetExtension {
name: string
config: PresetConfig
override?: boolean // Override existing preset
}
/**
* Global registry for extensions
*/
class ConfigurationRegistry {
private static instance: ConfigurationRegistry
// Registered storage providers
private storageProviders: Map<string, StorageProvider> = new Map()
// Registered preset extensions
private presetExtensions: Map<string, PresetExtension> = new Map()
// Custom auto-detection hooks
private autoDetectHooks: Array<() => Promise<any>> = []
private constructor() {
// Initialize with built-in providers
this.registerBuiltInProviders()
}
static getInstance(): ConfigurationRegistry {
if (!ConfigurationRegistry.instance) {
ConfigurationRegistry.instance = new ConfigurationRegistry()
}
return ConfigurationRegistry.instance
}
/**
* Register a new storage provider
* This is how augmentations add new storage types
*/
registerStorageProvider(provider: StorageProvider): void {
console.log(`📦 Registering storage provider: ${provider.type} (${provider.name})`)
this.storageProviders.set(provider.type, provider)
}
/**
* Register a new preset
*/
registerPreset(name: string, extension: PresetExtension): void {
console.log(`🎨 Registering preset: ${name}`)
this.presetExtensions.set(name, extension)
}
/**
* Register an auto-detection hook
*/
registerAutoDetectHook(hook: () => Promise<any>): void {
this.autoDetectHooks.push(hook)
}
/**
* Get all registered storage providers
*/
getStorageProviders(): StorageProvider[] {
return Array.from(this.storageProviders.values())
.sort((a, b) => (b.priority || 0) - (a.priority || 0))
}
/**
* Get all registered presets (built-in + extensions)
*/
getAllPresets(): Map<string, PresetConfig> {
// Start with built-in presets
const allPresets = new Map<string, PresetConfig>()
// Note: Would import from distributedPresets-new.ts
// Add extended presets
for (const [name, extension] of this.presetExtensions) {
if (extension.override || !allPresets.has(name)) {
allPresets.set(name, extension.config)
}
}
return allPresets
}
/**
* Auto-detect storage including extensions
*/
async autoDetectStorage(): Promise<StorageConfigResult> {
// Check registered providers first (in priority order)
for (const provider of this.getStorageProviders()) {
try {
if (await provider.detect()) {
const config = await provider.getConfig()
return {
type: provider.type as any,
config,
reason: `Auto-detected ${provider.name}`,
autoSelected: true
}
}
} catch (error) {
console.warn(`Failed to detect ${provider.type}:`, error)
}
}
// Fallback to built-in detection
const { autoDetectStorage } = await import('./storageAutoConfig.js')
return autoDetectStorage()
}
/**
* Register built-in providers
*/
private registerBuiltInProviders(): void {
// These would be the built-in ones, but could be overridden
}
}
/**
* Example: Redis storage provider registration
* This would be in the redis augmentation package
*/
export const redisStorageProvider: StorageProvider = {
type: 'redis',
name: 'Redis Storage',
description: 'High-performance in-memory data store',
priority: 10, // Check before filesystem
requirements: {
env: ['REDIS_URL', 'REDIS_HOST'],
packages: ['redis', 'ioredis']
},
async detect(): Promise<boolean> {
// Check for Redis connection info
if (process.env.REDIS_URL || process.env.REDIS_HOST) {
try {
// Try to connect to Redis (dynamic import for optional dependency)
const redis: any = await new Function('return import("ioredis")')().catch(() => null)
if (!redis) return false
const client = new redis.default(process.env.REDIS_URL)
await client.ping()
await client.quit()
return true
} catch {
// Redis not available
}
}
return false
},
async getConfig(): Promise<any> {
return {
type: 'redis',
redisStorage: {
url: process.env.REDIS_URL || `redis://${process.env.REDIS_HOST}:${process.env.REDIS_PORT || 6379}`,
prefix: process.env.REDIS_PREFIX || 'brainy:',
ttl: process.env.REDIS_TTL ? parseInt(process.env.REDIS_TTL) : undefined
}
}
}
}
/**
* Example: MongoDB storage provider
*/
export const mongoStorageProvider: StorageProvider = {
type: 'mongodb',
name: 'MongoDB Storage',
description: 'Document database for complex data',
priority: 8,
requirements: {
env: ['MONGODB_URI', 'MONGO_URL'],
packages: ['mongodb']
},
async detect(): Promise<boolean> {
if (process.env.MONGODB_URI || process.env.MONGO_URL) {
try {
const mongodb: any = await new Function('return import("mongodb")')().catch(() => null)
if (!mongodb) return false
const client = new mongodb.MongoClient(process.env.MONGODB_URI || process.env.MONGO_URL!)
await client.connect()
await client.close()
return true
} catch {
// MongoDB not available
}
}
return false
},
async getConfig(): Promise<any> {
return {
type: 'mongodb',
mongoStorage: {
uri: process.env.MONGODB_URI || process.env.MONGO_URL,
database: process.env.MONGO_DATABASE || 'brainy',
collection: process.env.MONGO_COLLECTION || 'vectors'
}
}
}
}
/**
* Example: PostgreSQL with pgvector extension
*/
export const postgresStorageProvider: StorageProvider = {
type: 'postgres',
name: 'PostgreSQL Storage',
description: 'PostgreSQL with pgvector for scalable vector search',
priority: 9,
requirements: {
env: ['DATABASE_URL', 'POSTGRES_URL'],
packages: ['pg', 'pgvector']
},
async detect(): Promise<boolean> {
const url = process.env.DATABASE_URL || process.env.POSTGRES_URL
if (url && url.includes('postgres')) {
try {
const pg: any = await new Function('return import("pg")')().catch(() => null)
if (!pg) return false
const client = new pg.Client({ connectionString: url })
await client.connect()
// Check for pgvector extension
const result = await client.query(
"SELECT * FROM pg_extension WHERE extname = 'vector'"
)
await client.end()
return result.rows.length > 0
} catch {
// PostgreSQL not available or pgvector not installed
}
}
return false
},
async getConfig(): Promise<any> {
return {
type: 'postgres',
postgresStorage: {
connectionString: process.env.DATABASE_URL || process.env.POSTGRES_URL,
table: process.env.POSTGRES_TABLE || 'brainy_vectors',
schema: process.env.POSTGRES_SCHEMA || 'public'
}
}
}
}
/**
* How an augmentation would register its storage provider
*/
export function registerStorageAugmentation(provider: StorageProvider): void {
const registry = ConfigurationRegistry.getInstance()
registry.registerStorageProvider(provider)
}
/**
* How to register a new preset
*/
export function registerPresetAugmentation(name: string, config: PresetConfig): void {
const registry = ConfigurationRegistry.getInstance()
registry.registerPreset(name, {
name,
config,
override: false
})
}
/**
* Example preset for Redis-based caching service
*/
export const redisCachePreset: PresetConfig = {
storage: 'redis' as any, // Extended storage type
model: ModelPrecision.Q8,
features: ['core', 'cache', 'search'],
distributed: true,
role: DistributedRole.READER,
readOnly: true,
cache: {
hotCacheMaxSize: 50000, // Large Redis cache
autoTune: true
},
description: 'Redis-backed caching layer',
category: PresetCategory.SERVICE
}
/**
* Get the configuration registry
*/
export function getConfigRegistry(): ConfigurationRegistry {
return ConfigurationRegistry.getInstance()
}

83
src/config/index.ts Normal file
View file

@ -0,0 +1,83 @@
/**
* Zero-Configuration System
* Main entry point for all auto-configuration features
*/
// Model configuration
export {
autoSelectModelPrecision,
ModelPrecision as ModelPrecisionType, // Avoid conflict
ModelPreset,
shouldAutoDownloadModels,
getModelPath,
logModelConfig
} from './modelAutoConfig.js'
// Storage configuration
export {
autoDetectStorage,
StorageType,
StoragePreset,
StorageConfigResult,
logStorageConfig,
// Backward compatibility types
type StorageTypeString,
type StoragePresetString
} from './storageAutoConfig.js'
// Shared configuration for multi-instance
export {
SharedConfig,
SharedConfigManager
} from './sharedConfigManager.js'
// Main zero-config processor
export {
BrainyZeroConfig,
processZeroConfig,
createEmbeddingFunctionWithPrecision
} from './zeroConfig.js'
// Strongly-typed presets and enums
export {
PresetName,
PresetCategory,
ModelPrecision,
StorageOption,
FeatureSet,
DistributedRole,
PresetConfig,
PRESET_CONFIGS,
getPreset,
isValidPreset,
getPresetsByCategory,
getAllPresetNames,
getPresetDescription,
presetToBrainyConfig
} from './distributedPresets.js'
// Extensible configuration
export {
StorageProvider,
registerStorageAugmentation,
registerPresetAugmentation,
getConfigRegistry
} from './extensibleConfig.js'
/**
* Main zero-config processor
* This is what BrainyData will call
*/
export async function applyZeroConfig(input?: string | any): Promise<any> {
// Handle legacy config (full object) by detecting known legacy properties
if (input && typeof input === 'object' &&
(input.storage?.forceMemoryStorage || input.storage?.forceFileSystemStorage || input.storage?.s3Storage)) {
// This is a legacy config object - pass through unchanged
console.log('📦 Using legacy configuration format')
return input
}
// Process as zero-config (includes new object format)
const { processZeroConfig } = await import('./zeroConfig.js')
return processZeroConfig(input)
}

228
src/config/modelAutoConfig.ts Normal file
View file

@ -0,0 +1,228 @@
/**
* Model Configuration Auto-Selection
* Intelligently selects model precision based on environment
* while allowing manual override
*/
import { isBrowser, isNode } from '../utils/environment.js'
export type ModelPrecision = 'fp32' | 'q8'
export type ModelPreset = 'fast' | 'small' | 'auto'
interface ModelConfigResult {
precision: ModelPrecision
reason: string
autoSelected: boolean
}
/**
* Auto-select model precision based on environment and resources
* @param override - Manual override: 'fp32', 'q8', 'fast' (fp32), 'small' (q8), or 'auto'
*/
export function autoSelectModelPrecision(override?: ModelPrecision | ModelPreset): ModelConfigResult {
// Handle direct precision override
if (override === 'fp32' || override === 'q8') {
return {
precision: override,
reason: `Manually specified: ${override}`,
autoSelected: false
}
}
// Handle preset overrides
if (override === 'fast') {
return {
precision: 'fp32',
reason: 'Preset: fast (fp32 for best quality)',
autoSelected: false
}
}
if (override === 'small') {
return {
precision: 'q8',
reason: 'Preset: small (q8 for reduced size)',
autoSelected: false
}
}
// Auto-selection logic
return autoDetectBestPrecision()
}
/**
* Automatically detect the best model precision for the environment
*/
function autoDetectBestPrecision(): ModelConfigResult {
// Browser environment - use Q8 for smaller download/memory
if (isBrowser()) {
return {
precision: 'q8',
reason: 'Browser environment detected - using Q8 for smaller size',
autoSelected: true
}
}
// Serverless environments - use Q8 for faster cold starts
if (isServerlessEnvironment()) {
return {
precision: 'q8',
reason: 'Serverless environment detected - using Q8 for faster cold starts',
autoSelected: true
}
}
// Check available memory
const memoryMB = getAvailableMemoryMB()
if (memoryMB < 512) {
return {
precision: 'q8',
reason: `Low memory detected (${memoryMB}MB) - using Q8`,
autoSelected: true
}
}
// Development environment - use FP32 for best quality
if (process.env.NODE_ENV === 'development') {
return {
precision: 'fp32',
reason: 'Development environment - using FP32 for best quality',
autoSelected: true
}
}
// Production with adequate memory - use FP32
if (memoryMB >= 2048) {
return {
precision: 'fp32',
reason: `Adequate memory (${memoryMB}MB) - using FP32 for best quality`,
autoSelected: true
}
}
// Default to Q8 for moderate memory environments
return {
precision: 'q8',
reason: `Moderate memory (${memoryMB}MB) - using Q8 for balance`,
autoSelected: true
}
}
/**
* Check if running in a serverless environment
*/
function isServerlessEnvironment(): boolean {
if (!isNode()) return false
return !!(
process.env.AWS_LAMBDA_FUNCTION_NAME ||
process.env.VERCEL ||
process.env.NETLIFY ||
process.env.CLOUDFLARE_WORKERS ||
process.env.FUNCTIONS_WORKER_RUNTIME ||
process.env.K_SERVICE // Google Cloud Run
)
}
/**
* Get available memory in MB
*/
function getAvailableMemoryMB(): number {
if (isBrowser()) {
// @ts-ignore - navigator.deviceMemory is experimental
if (navigator.deviceMemory) {
// @ts-ignore
return navigator.deviceMemory * 1024 // Device memory in GB
}
return 256 // Conservative default for browsers
}
if (isNode()) {
try {
// Try to get memory info synchronously for Node.js
// This will be available in Node.js environments
if (typeof process !== 'undefined' && process.memoryUsage) {
// Use RSS (Resident Set Size) as a proxy for available memory
const rss = process.memoryUsage().rss
// Assume we can use up to 4GB or 50% more than current usage
return Math.min(4096, Math.floor(rss / (1024 * 1024) * 1.5))
}
} catch {
// Fall through to default
}
return 1024 // Default 1GB for Node.js
}
return 512 // Conservative default
}
/**
* Convenience function to check if models need to be downloaded
* This replaces the need for BRAINY_ALLOW_REMOTE_MODELS
*/
export function shouldAutoDownloadModels(): boolean {
// Always allow downloads unless explicitly disabled
// This eliminates the need for BRAINY_ALLOW_REMOTE_MODELS
const explicitlyDisabled = process.env.BRAINY_ALLOW_REMOTE_MODELS === 'false'
if (explicitlyDisabled) {
console.warn('Model downloads disabled via BRAINY_ALLOW_REMOTE_MODELS=false')
return false
}
// In production, always allow downloads for seamless operation
if (process.env.NODE_ENV === 'production') {
return true
}
// In development, allow downloads with a one-time notice
if (process.env.NODE_ENV === 'development') {
return true
}
// Default: allow downloads
return true
}
/**
* Get the model path with intelligent defaults
* This replaces the need for BRAINY_MODELS_PATH env var
*/
export function getModelPath(): string {
// Check if user explicitly set a path (keeping this for advanced users)
if (process.env.BRAINY_MODELS_PATH) {
return process.env.BRAINY_MODELS_PATH
}
// Browser - use cache API or IndexedDB (handled by transformers.js)
if (isBrowser()) {
return 'browser-cache'
}
// Serverless - use /tmp for ephemeral storage
if (isServerlessEnvironment()) {
return '/tmp/.brainy/models'
}
// Node.js - use home directory for persistent storage
if (isNode()) {
// Use process.env.HOME as a fallback
const homeDir = process.env.HOME || process.env.USERPROFILE || '~'
return `${homeDir}/.brainy/models`
}
// Fallback
return './.brainy/models'
}
/**
* Log model configuration decision (only in verbose mode)
*/
export function logModelConfig(config: ModelConfigResult, verbose: boolean = false): void {
if (!verbose && process.env.NODE_ENV === 'production') {
return // Silent in production unless verbose
}
const icon = config.autoSelected ? '🤖' : '👤'
console.log(`${icon} Model: ${config.precision.toUpperCase()} - ${config.reason}`)
}

266
src/config/sharedConfigManager.ts Normal file
View file

@ -0,0 +1,266 @@
/**
* Shared Configuration Manager
* Ensures configuration consistency across multiple instances using shared storage
*/
import { ModelPrecision } from './modelAutoConfig.js'
import { StorageType } from './storageAutoConfig.js'
export interface SharedConfig {
// Critical parameters that MUST match across instances
version: string
precision: ModelPrecision
dimensions: number
hnswM: number
hnswEfConstruction: number
distanceFunction: string
createdAt: string
lastUpdated: string
// Informational (can differ between instances)
instanceCount?: number
lastAccessedBy?: string
}
/**
* Manages configuration consistency for shared storage
*/
export class SharedConfigManager {
private static CONFIG_FILE = '.brainy/config.json'
/**
* Load or create shared configuration
* When connecting to existing data, this OVERRIDES auto-configuration!
*/
static async loadOrCreateSharedConfig(
storage: any,
localConfig: any
): Promise<{ config: any; warnings: string[] }> {
const warnings: string[] = []
try {
// Check if we're using shared storage
if (!this.isSharedStorage(localConfig.storageType)) {
// Local storage - use local config
return { config: localConfig, warnings: [] }
}
// Try to load existing configuration from shared storage
const existingConfig = await this.loadConfigFromStorage(storage)
if (existingConfig) {
// EXISTING SHARED DATA - Must use its configuration!
console.log('📁 Found existing shared data configuration')
// Check for critical mismatches
const mismatches = this.checkCriticalMismatches(localConfig, existingConfig)
if (mismatches.length > 0) {
console.warn('⚠️ Configuration override required for shared storage:')
mismatches.forEach(m => {
console.warn(` - ${m.param}: ${m.local} → ${m.shared} (using shared)`)
warnings.push(`${m.param} overridden: ${m.local} → ${m.shared}`)
})
}
// Override critical parameters with shared values
const mergedConfig = this.mergeWithSharedConfig(localConfig, existingConfig)
// Update last accessed
await this.updateAccessInfo(storage, existingConfig)
return { config: mergedConfig, warnings }
} else {
// NEW SHARED STORAGE - Save our configuration
console.log('📝 Initializing new shared storage with configuration')
const sharedConfig = this.createSharedConfig(localConfig)
await this.saveConfigToStorage(storage, sharedConfig)
return { config: localConfig, warnings: [] }
}
} catch (error) {
console.error('Failed to manage shared configuration:', error)
warnings.push('Could not verify shared configuration - proceeding with caution')
return { config: localConfig, warnings }
}
}
/**
* Check if storage type is shared (multi-instance)
*/
private static isSharedStorage(storageType: StorageType): boolean {
return ['s3', 'gcs', 'r2'].includes(storageType)
}
/**
* Load configuration from shared storage
*/
private static async loadConfigFromStorage(storage: any): Promise<SharedConfig | null> {
try {
const configData = await storage.get(this.CONFIG_FILE)
if (!configData) return null
const config = JSON.parse(configData)
return config as SharedConfig
} catch (error) {
// Config doesn't exist yet
return null
}
}
/**
* Save configuration to shared storage
*/
private static async saveConfigToStorage(storage: any, config: SharedConfig): Promise<void> {
try {
await storage.set(this.CONFIG_FILE, JSON.stringify(config, null, 2))
} catch (error) {
console.error('Failed to save shared configuration:', error)
throw new Error('Cannot initialize shared storage without saving configuration')
}
}
/**
* Create shared configuration from local config
*/
private static createSharedConfig(localConfig: any): SharedConfig {
return {
version: '2.10.0', // Brainy version
precision: localConfig.embeddingOptions?.precision || 'fp32',
dimensions: 384, // Fixed for all-MiniLM-L6-v2
hnswM: localConfig.hnsw?.M || 16,
hnswEfConstruction: localConfig.hnsw?.efConstruction || 200,
distanceFunction: localConfig.distanceFunction || 'cosine',
createdAt: new Date().toISOString(),
lastUpdated: new Date().toISOString(),
instanceCount: 1,
lastAccessedBy: this.getInstanceIdentifier()
}
}
/**
* Check for critical configuration mismatches
*/
private static checkCriticalMismatches(
localConfig: any,
sharedConfig: SharedConfig
): Array<{ param: string; local: any; shared: any }> {
const mismatches: Array<{ param: string; local: any; shared: any }> = []
// Model precision - CRITICAL!
const localPrecision = localConfig.embeddingOptions?.precision || 'fp32'
if (localPrecision !== sharedConfig.precision) {
mismatches.push({
param: 'Model Precision',
local: localPrecision,
shared: sharedConfig.precision
})
}
// HNSW parameters - Important for index consistency
const localM = localConfig.hnsw?.M || 16
if (localM !== sharedConfig.hnswM) {
mismatches.push({
param: 'HNSW M',
local: localM,
shared: sharedConfig.hnswM
})
}
const localEf = localConfig.hnsw?.efConstruction || 200
if (localEf !== sharedConfig.hnswEfConstruction) {
mismatches.push({
param: 'HNSW efConstruction',
local: localEf,
shared: sharedConfig.hnswEfConstruction
})
}
// Distance function
const localDistance = localConfig.distanceFunction || 'cosine'
if (localDistance !== sharedConfig.distanceFunction) {
mismatches.push({
param: 'Distance Function',
local: localDistance,
shared: sharedConfig.distanceFunction
})
}
return mismatches
}
/**
* Merge local config with shared config (shared takes precedence for critical params)
*/
private static mergeWithSharedConfig(localConfig: any, sharedConfig: SharedConfig): any {
return {
...localConfig,
// Override critical parameters with shared values
embeddingOptions: {
...localConfig.embeddingOptions,
precision: sharedConfig.precision // MUST use shared precision!
},
hnsw: {
...localConfig.hnsw,
M: sharedConfig.hnswM,
efConstruction: sharedConfig.hnswEfConstruction
},
distanceFunction: sharedConfig.distanceFunction,
// Add metadata about shared configuration
_sharedConfig: {
loaded: true,
version: sharedConfig.version,
createdAt: sharedConfig.createdAt,
precision: sharedConfig.precision
}
}
}
/**
* Update access information in shared config
*/
private static async updateAccessInfo(storage: any, config: SharedConfig): Promise<void> {
try {
config.lastUpdated = new Date().toISOString()
config.instanceCount = (config.instanceCount || 0) + 1
config.lastAccessedBy = this.getInstanceIdentifier()
await this.saveConfigToStorage(storage, config)
} catch {
// Non-critical - don't fail if we can't update access info
}
}
/**
* Get unique identifier for this instance
*/
private static getInstanceIdentifier(): string {
if (process.env.HOSTNAME) return process.env.HOSTNAME
if (process.env.CONTAINER_ID) return process.env.CONTAINER_ID
if (process.env.K_SERVICE) return process.env.K_SERVICE
if (process.env.AWS_LAMBDA_FUNCTION_NAME) return process.env.AWS_LAMBDA_FUNCTION_NAME
// Generate a random identifier
return `instance-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`
}
/**
* Validate that a configuration is compatible with shared data
*/
static validateCompatibility(config1: SharedConfig, config2: SharedConfig): boolean {
return (
config1.precision === config2.precision &&
config1.dimensions === config2.dimensions &&
config1.hnswM === config2.hnswM &&
config1.hnswEfConstruction === config2.hnswEfConstruction &&
config1.distanceFunction === config2.distanceFunction
)
}
}

376
src/config/storageAutoConfig.ts Normal file
View file

@ -0,0 +1,376 @@
/**
* Storage Configuration Auto-Detection
* Intelligently selects storage based on environment and available services
*/
import { isBrowser, isNode } from '../utils/environment.js'
/**
* Low-level storage implementation types
*/
export enum StorageType {
MEMORY = 'memory',
FILESYSTEM = 'filesystem',
OPFS = 'opfs',
S3 = 's3',
GCS = 'gcs',
R2 = 'r2'
}
/**
* High-level storage presets (maps to StorageType)
*/
export enum StoragePreset {
AUTO = 'auto',
MEMORY = 'memory',
DISK = 'disk',
CLOUD = 'cloud'
}
// Backward compatibility type aliases
export type StorageTypeString = 'memory' | 'filesystem' | 'opfs' | 's3' | 'gcs' | 'r2'
export type StoragePresetString = 'auto' | 'memory' | 'disk' | 'cloud'
export interface StorageConfigResult {
type: StorageType | StorageTypeString // Support both enum and string for compatibility
config: any
reason: string
autoSelected: boolean
}
/**
* Auto-detect the best storage configuration
* @param override - Manual override: specific type or preset
*/
export async function autoDetectStorage(
override?: StorageType | StoragePreset | StorageTypeString | StoragePresetString | any
): Promise<StorageConfigResult> {
// Handle direct storage config object
if (override && typeof override === 'object') {
return {
type: override.type || 'memory',
config: override,
reason: 'Manually configured storage',
autoSelected: false
}
}
// Handle storage type override (enum values or strings)
if (override && Object.values(StorageType).includes(override as StorageType)) {
return {
type: override as StorageType,
config: override,
reason: `Manually specified: ${override}`,
autoSelected: false
}
}
// Handle presets (both enum and string values)
if (override === StoragePreset.MEMORY || override === 'memory') {
return {
type: StorageType.MEMORY,
config: StorageType.MEMORY,
reason: 'Preset: memory storage',
autoSelected: false
}
}
if (override === StoragePreset.DISK || override === 'disk') {
const diskType = isBrowser() ? StorageType.OPFS : StorageType.FILESYSTEM
return {
type: diskType,
config: diskType,
reason: `Preset: disk storage (${diskType})`,
autoSelected: false
}
}
if (override === StoragePreset.CLOUD || override === 'cloud') {
const cloudStorage = await detectCloudStorage()
if (cloudStorage) {
return {
...cloudStorage,
reason: `Preset: cloud storage (${cloudStorage.type})`,
autoSelected: false
}
}
// Fallback to disk if no cloud storage detected
const diskType = isBrowser() ? StorageType.OPFS : StorageType.FILESYSTEM
return {
type: diskType,
config: diskType,
reason: 'Preset: cloud (none detected, using disk)',
autoSelected: false
}
}
// Auto-detection logic
return await autoDetectBestStorage()
}
/**
* Automatically detect the best storage option
*/
async function autoDetectBestStorage(): Promise<StorageConfigResult> {
// Check for cloud storage first (highest priority in production)
const cloudStorage = await detectCloudStorage()
if (cloudStorage && process.env.NODE_ENV === 'production') {
return {
...cloudStorage,
reason: `Auto-detected ${cloudStorage.type} in production`,
autoSelected: true
}
}
// Browser environment
if (isBrowser()) {
// Check for OPFS support
if (await hasOPFSSupport()) {
return {
type: StorageType.OPFS,
config: { requestPersistentStorage: true },
reason: 'Browser with OPFS support detected',
autoSelected: true
}
}
// Fallback to memory for browsers without OPFS
return {
type: StorageType.MEMORY,
config: StorageType.MEMORY,
reason: 'Browser without OPFS - using memory',
autoSelected: true
}
}
// Serverless environment - prefer memory or cloud
if (isServerlessEnvironment()) {
if (cloudStorage) {
return {
...cloudStorage,
reason: `Serverless with ${cloudStorage.type} detected`,
autoSelected: true
}
}
return {
type: StorageType.MEMORY,
config: StorageType.MEMORY,
reason: 'Serverless environment - using memory',
autoSelected: true
}
}
// Node.js environment - use filesystem
if (isNode()) {
const dataPath = await findBestDataPath()
return {
type: StorageType.FILESYSTEM,
config: StorageType.FILESYSTEM,
reason: `Node.js environment - using filesystem at ${dataPath}`,
autoSelected: true
}
}
// Fallback to memory
return {
type: StorageType.MEMORY,
config: StorageType.MEMORY,
reason: 'Default fallback - using memory',
autoSelected: true
}
}
/**
* Detect cloud storage from environment variables
*/
async function detectCloudStorage(): Promise<{ type: StorageType; config: any } | null> {
// AWS S3 Detection
if (hasAWSConfig()) {
return {
type: StorageType.S3,
config: {
s3Storage: {
bucketName: process.env.AWS_BUCKET || process.env.S3_BUCKET_NAME || 'brainy-data',
region: process.env.AWS_REGION || process.env.AWS_DEFAULT_REGION || 'us-east-1',
// Credentials will be picked up by AWS SDK automatically
}
}
}
}
// Google Cloud Storage Detection
if (hasGCPConfig()) {
return {
type: StorageType.GCS,
config: {
gcsStorage: {
bucketName: process.env.GCS_BUCKET || process.env.GOOGLE_STORAGE_BUCKET || 'brainy-data',
// Credentials will be picked up by GCP SDK automatically
}
}
}
}
// Cloudflare R2 Detection
if (hasR2Config()) {
return {
type: StorageType.R2,
config: {
r2Storage: {
bucketName: process.env.R2_BUCKET || 'brainy-data',
accountId: process.env.CLOUDFLARE_ACCOUNT_ID,
accessKeyId: process.env.R2_ACCESS_KEY_ID,
secretAccessKey: process.env.R2_SECRET_ACCESS_KEY
}
}
}
}
return null
}
/**
* Check if AWS S3 is configured
*/
function hasAWSConfig(): boolean {
return !!(
// Explicit S3 bucket
process.env.AWS_BUCKET ||
process.env.S3_BUCKET_NAME ||
// AWS credentials (SDK will find them)
process.env.AWS_ACCESS_KEY_ID ||
process.env.AWS_PROFILE ||
// AWS environment indicators
process.env.AWS_EXECUTION_ENV ||
process.env.AWS_LAMBDA_FUNCTION_NAME ||
process.env.ECS_CONTAINER_METADATA_URI
)
}
/**
* Check if Google Cloud Storage is configured
*/
function hasGCPConfig(): boolean {
return !!(
// Explicit GCS bucket
process.env.GCS_BUCKET ||
process.env.GOOGLE_STORAGE_BUCKET ||
// GCP credentials
process.env.GOOGLE_APPLICATION_CREDENTIALS ||
// GCP environment indicators
process.env.GOOGLE_CLOUD_PROJECT ||
process.env.K_SERVICE ||
process.env.GAE_SERVICE
)
}
/**
* Check if Cloudflare R2 is configured
*/
function hasR2Config(): boolean {
return !!(
process.env.R2_BUCKET ||
(process.env.CLOUDFLARE_ACCOUNT_ID && process.env.R2_ACCESS_KEY_ID)
)
}
/**
* Check if running in serverless environment
*/
function isServerlessEnvironment(): boolean {
if (!isNode()) return false
return !!(
process.env.AWS_LAMBDA_FUNCTION_NAME ||
process.env.VERCEL ||
process.env.NETLIFY ||
process.env.CLOUDFLARE_WORKERS ||
process.env.FUNCTIONS_WORKER_RUNTIME ||
process.env.K_SERVICE ||
process.env.RAILWAY_ENVIRONMENT ||
process.env.FLY_APP_NAME
)
}
/**
* Check for OPFS support in browser
*/
async function hasOPFSSupport(): Promise<boolean> {
if (!isBrowser()) return false
try {
return 'storage' in navigator &&
'getDirectory' in navigator.storage
} catch {
return false
}
}
/**
* Find the best path for filesystem storage
*/
async function findBestDataPath(): Promise<string> {
if (!isNode()) return './brainy-data'
const homeDir = process.env.HOME || process.env.USERPROFILE || '~'
const tempDir = process.env.TMPDIR || process.env.TEMP || '/tmp'
const candidates = [
// User-specified path
process.env.BRAINY_DATA_PATH,
// Current directory
'./brainy-data',
// Home directory
`${homeDir}/.brainy/data`,
// Temp directory (last resort)
`${tempDir}/brainy-data`
].filter(Boolean) as string[]
// Find first writable directory
for (const candidate of candidates) {
if (await isWritable(candidate)) {
return candidate
}
}
// Default fallback
return candidates[1] // ./brainy-data
}
/**
* Check if a directory is writable
*/
async function isWritable(dirPath: string): Promise<boolean> {
if (!isNode()) return false
try {
// Dynamic import fs for Node.js
const { promises: fs } = await import('fs')
const path = await import('path')
// Try to create directory if it doesn't exist
await fs.mkdir(dirPath, { recursive: true })
// Try to write a test file
const testFile = path.join(dirPath, '.write-test')
await fs.writeFile(testFile, 'test')
await fs.unlink(testFile)
return true
} catch {
return false
}
}
// Legacy getStorageConfig function removed - now using simple string types
/**
* Log storage configuration decision
*/
export function logStorageConfig(config: StorageConfigResult, verbose: boolean = false): void {
if (!verbose && process.env.NODE_ENV === 'production') {
return // Silent in production unless verbose
}
const icon = config.autoSelected ? '🤖' : '👤'
console.log(`${icon} Storage: ${config.type.toUpperCase()} - ${config.reason}`)
}

390
src/config/zeroConfig.ts Normal file
View file

@ -0,0 +1,390 @@
/**
* Zero-Configuration System for Brainy
* Provides intelligent defaults while preserving full control
*/
import { autoSelectModelPrecision, ModelPrecision, ModelPreset, getModelPath, shouldAutoDownloadModels } from './modelAutoConfig.js'
import { autoDetectStorage, StorageType, StoragePreset } from './storageAutoConfig.js'
import { AutoConfiguration } from '../utils/autoConfiguration.js'
/**
* Simplified configuration interface
* Everything is optional - zero config by default!
*/
export interface BrainyZeroConfig {
/**
* Configuration preset for common scenarios
* - 'production': Optimized for production (disk storage, auto model, default features)
* - 'development': Optimized for development (memory storage, fp32, verbose logging)
* - 'minimal': Minimal footprint (memory storage, q8, minimal features)
* - 'zero': True zero config (all auto-detected)
* - 'writer': Write-only instance for distributed setups (no index loading)
* - 'reader': Read-only instance for distributed setups (no write operations)
*/
mode?: 'production' | 'development' | 'minimal' | 'zero' | 'writer' | 'reader'
/**
* Model precision configuration
* - 'fp32': Full precision (best quality, larger size)
* - 'q8': Quantized 8-bit (smaller size, slightly lower quality)
* - 'fast': Alias for fp32
* - 'small': Alias for q8
* - 'auto': Auto-detect based on environment (default)
*/
model?: ModelPrecision | ModelPreset
/**
* Storage configuration
* - 'memory': In-memory only (no persistence)
* - 'disk': Local disk (filesystem or OPFS)
* - 'cloud': Cloud storage (S3/GCS/R2 if configured)
* - 'auto': Auto-detect best option (default)
* - Object: Custom storage configuration
*/
storage?: StorageType | StoragePreset | any
/**
* Feature set configuration
* - 'minimal': Core features only (fastest startup)
* - 'default': Standard features (balanced)
* - 'full': All features enabled (most capable)
* - Array: Specific features to enable
*/
features?: 'minimal' | 'default' | 'full' | string[]
/**
* Logging verbosity
* - true: Show configuration decisions and progress
* - false: Silent operation (default in production)
*/
verbose?: boolean
/**
* Advanced configuration (escape hatch for power users)
* Any additional configuration can be passed here
*/
advanced?: any
}
/**
* Configuration presets for common scenarios
*/
const PRESETS = {
production: {
storage: 'disk' as const,
model: 'auto' as const,
features: 'default' as const,
verbose: false
},
development: {
storage: 'memory' as const,
model: 'fp32' as const,
features: 'full' as const,
verbose: true
},
minimal: {
storage: 'memory' as const,
model: 'q8' as const,
features: 'minimal' as const,
verbose: false
},
zero: {
storage: 'auto' as const,
model: 'auto' as const,
features: 'default' as const,
verbose: false
},
writer: {
storage: 'auto' as const,
model: 'auto' as const,
features: 'minimal' as const,
verbose: false,
// Writer-specific settings
distributed: true,
role: 'writer' as const,
writeOnly: true,
allowDirectReads: true // Allow deduplication checks
},
reader: {
storage: 'auto' as const,
model: 'auto' as const,
features: 'default' as const,
verbose: false,
// Reader-specific settings
distributed: true,
role: 'reader' as const,
readOnly: true,
lazyLoadInReadOnlyMode: true // Optimize for search
}
}
/**
* Feature sets configuration
*/
const FEATURE_SETS = {
minimal: [
'core',
'search',
'storage'
],
default: [
'core',
'search',
'storage',
'cache',
'metadata-index',
'batch-processing',
'entity-registry',
'request-deduplicator'
],
full: [
'core',
'search',
'storage',
'cache',
'metadata-index',
'batch-processing',
'entity-registry',
'request-deduplicator',
'connection-pool',
'wal',
'monitoring',
'metrics',
'intelligent-verb-scoring',
'triple-intelligence',
'neural-api'
]
}
/**
* Process zero-config input into full configuration
*/
export async function processZeroConfig(input?: string | BrainyZeroConfig): Promise<any> {
let config: BrainyZeroConfig = {}
// Handle string shorthand (preset name)
if (typeof input === 'string') {
if (input in PRESETS) {
config = { mode: input as any }
} else {
throw new Error(`Unknown preset: ${input}. Valid presets: ${Object.keys(PRESETS).join(', ')}`)
}
} else if (input) {
config = input
}
// Apply preset if specified
if (config.mode && config.mode in PRESETS) {
const preset = PRESETS[config.mode]
config = {
...preset,
...config,
// Preserve explicit overrides
model: config.model ?? preset.model,
storage: config.storage ?? preset.storage,
features: config.features ?? preset.features,
verbose: config.verbose ?? preset.verbose
}
}
// Auto-detect environment if not in preset mode
const environment = detectEnvironmentMode()
// Process model configuration
const modelConfig = autoSelectModelPrecision(config.model)
// Process storage configuration
const storageConfig = await autoDetectStorage(config.storage)
// Process features configuration
const features = processFeatures(config.features)
// Get auto-configuration recommendations
const autoConfig = await AutoConfiguration.getInstance().detectAndConfigure({
expectedDataSize: estimateDataSize(environment),
s3Available: storageConfig.type === 's3',
memoryBudget: undefined // Let it auto-detect
})
// Determine verbosity
const verbose = config.verbose ?? (process.env.NODE_ENV === 'development')
// Log configuration decisions if verbose
if (verbose) {
logConfigurationSummary({
mode: config.mode || 'auto',
model: modelConfig,
storage: storageConfig,
features: features,
environment: environment,
autoConfig: autoConfig
})
}
// Build final configuration
const finalConfig: any = {
// Model configuration
embeddingFunction: undefined, // Will be created with correct precision
embeddingOptions: {
precision: modelConfig.precision,
modelPath: getModelPath(),
allowRemoteDownload: shouldAutoDownloadModels()
},
// Storage configuration
storage: storageConfig.config,
storageType: storageConfig.type,
// HNSW configuration from auto-config
hnsw: {
M: autoConfig.recommendedConfig.enablePartitioning ? 32 : 16,
efConstruction: autoConfig.recommendedConfig.enablePartitioning ? 400 : 200,
maxDatasetSize: autoConfig.recommendedConfig.expectedDatasetSize,
partitioning: autoConfig.recommendedConfig.enablePartitioning,
maxNodesPerPartition: autoConfig.recommendedConfig.maxNodesPerPartition
},
// Cache configuration from auto-config
cache: {
autoTune: true,
hotCacheMaxSize: Math.floor(autoConfig.recommendedConfig.maxMemoryUsage / (1024 * 1024 * 10)), // 10% of memory budget
batchSize: autoConfig.recommendedConfig.enablePartitioning ? 100 : 50
},
// Features configuration
enabledFeatures: features,
// Metadata index configuration
metadataIndex: features.includes('metadata-index') ? {
enabled: true,
autoRebuild: true
} : undefined,
// Intelligent verb scoring
intelligentVerbScoring: features.includes('intelligent-verb-scoring') ? {
enabled: true
} : undefined,
// Logging configuration
logging: {
verbose: verbose
},
// Performance flags from auto-config
optimizations: autoConfig.optimizationFlags,
// Advanced overrides (if any)
...config.advanced
}
// Apply distributed preset settings if applicable
if (config.mode === 'writer' || config.mode === 'reader') {
const presetSettings = PRESETS[config.mode] as any // Cast to any since we know these presets have additional properties
// Apply distributed-specific settings
finalConfig.distributed = presetSettings.distributed
finalConfig.readOnly = presetSettings.readOnly || false
finalConfig.writeOnly = presetSettings.writeOnly || false
finalConfig.allowDirectReads = presetSettings.allowDirectReads || false
finalConfig.lazyLoadInReadOnlyMode = presetSettings.lazyLoadInReadOnlyMode || false
// Set distributed role in distributed config
if (finalConfig.distributed) {
finalConfig.distributed = {
enabled: true,
role: presetSettings.role
}
}
// Log distributed mode if verbose
if (verbose) {
console.log(`📡 Distributed mode: ${config.mode.toUpperCase()}`)
console.log(` Role: ${presetSettings.role}`)
console.log(` Read-only: ${finalConfig.readOnly}`)
console.log(` Write-only: ${finalConfig.writeOnly}`)
}
}
return finalConfig
}
/**
* Detect environment mode if not specified
*/
function detectEnvironmentMode(): 'production' | 'development' | 'unknown' {
if (process.env.NODE_ENV === 'production') return 'production'
if (process.env.NODE_ENV === 'development') return 'development'
if (process.env.NODE_ENV === 'test') return 'development'
// Check for CI environments
if (process.env.CI || process.env.GITHUB_ACTIONS) return 'production'
// Check for production indicators
if (process.env.VERCEL_ENV === 'production' ||
process.env.NETLIFY_ENV === 'production' ||
process.env.RAILWAY_ENVIRONMENT === 'production') {
return 'production'
}
return 'unknown'
}
/**
* Process features configuration
*/
function processFeatures(features?: 'minimal' | 'default' | 'full' | string[]): string[] {
if (Array.isArray(features)) {
return features
}
if (features && features in FEATURE_SETS) {
return FEATURE_SETS[features]
}
// Default based on environment
const env = detectEnvironmentMode()
if (env === 'production') return FEATURE_SETS.default
if (env === 'development') return FEATURE_SETS.full
return FEATURE_SETS.default
}
/**
* Estimate dataset size based on environment
*/
function estimateDataSize(environment: string): number {
switch (environment) {
case 'production': return 100000
case 'development': return 10000
default: return 50000
}
}
/**
* Log configuration summary
*/
function logConfigurationSummary(config: any): void {
console.log('\n🧠 Brainy Zero-Config Summary')
console.log('================================')
console.log(`Mode: ${config.mode}`)
console.log(`Environment: ${config.environment}`)
console.log(`Model: ${config.model.precision.toUpperCase()} (${config.model.reason})`)
console.log(`Storage: ${config.storage.type.toUpperCase()} (${config.storage.reason})`)
console.log(`Features: ${config.features.length} enabled`)
console.log(`Memory Budget: ${Math.floor(config.autoConfig.recommendedConfig.maxMemoryUsage / (1024 * 1024))}MB`)
console.log(`Expected Dataset: ${config.autoConfig.recommendedConfig.expectedDatasetSize.toLocaleString()} items`)
console.log('================================\n')
}
/**
* Create embedding function with specified precision
* This ensures the model precision is respected
*/
export async function createEmbeddingFunctionWithPrecision(precision: ModelPrecision): Promise<any> {
const { createEmbeddingFunction } = await import('../utils/embedding.js')
// Create embedding function with specified precision
return createEmbeddingFunction({
precision: precision,
verbose: false // Silent by default in zero-config
})
}

28
src/index.ts
View file

@ -15,6 +15,34 @@ import { BrainyData, BrainyDataConfig } from './brainyData.js'
export { BrainyData }
export type { BrainyDataConfig }
// Export zero-configuration types and enums
export {
// Preset names
PresetName,
// Model configuration
ModelPrecision,
// Storage configuration
StorageOption,
// Feature configuration
FeatureSet,
// Distributed roles
DistributedRole,
// Categories
PresetCategory,
// Config type
BrainyZeroConfig,
// Extensibility
StorageProvider,
registerStorageAugmentation,
registerPresetAugmentation,
// Preset utilities
getPreset,
isValidPreset,
getPresetsByCategory,
getAllPresetNames,
getPresetDescription
} from './config/index.js'
// Export Cortex (the orchestrator)
export {
Cortex,

4
src/scripts/precomputePatternEmbeddings.ts
View file

@ -28,8 +28,8 @@ async function precomputeEmbeddings() {
// Initialize Brainy with minimal config
const brain = new BrainyData({
storage: { forceMemoryStorage: true },
logging: { verbose: false }
storage: 'memory',
verbose: false
})
await brain.init()

19
src/utils/embedding.ts
View file

@ -17,9 +17,10 @@ import { pipeline, env } from '@huggingface/transformers'
if (typeof process !== 'undefined' && process.env) {
process.env.ORT_DISABLE_MEMORY_ARENA = '1'
process.env.ORT_DISABLE_MEMORY_PATTERN = '1'
// Also limit ONNX thread count for more predictable memory usage
process.env.ORT_INTRA_OP_NUM_THREADS = '2'
process.env.ORT_INTER_OP_NUM_THREADS = '2'
// Force single-threaded operation for maximum stability (Node.js 24 compatibility)
process.env.ORT_INTRA_OP_NUM_THREADS = '1' // Single thread for operators
process.env.ORT_INTER_OP_NUM_THREADS = '1' // Single thread for sessions
process.env.ORT_NUM_THREADS = '1' // Additional safety override
}
/**
@ -111,7 +112,7 @@ export class TransformerEmbedding implements EmbeddingModel {
// 1. Explicit option takes highest priority
localFilesOnly = options.localFilesOnly
} else if (process.env.BRAINY_ALLOW_REMOTE_MODELS === 'false') {
// 2. Environment variable explicitly disables remote models
// 2. Environment variable explicitly disables remote models (legacy support)
localFilesOnly = true
} else if (process.env.NODE_ENV === 'development') {
// 3. Development mode allows remote models
@ -313,9 +314,9 @@ export class TransformerEmbedding implements EmbeddingModel {
session_options: {
enableCpuMemArena: false, // Disable pre-allocated memory arena
enableMemPattern: false, // Disable memory pattern optimization
interOpNumThreads: 2, // Limit thread count
intraOpNumThreads: 2, // Limit parallelism
graphOptimizationLevel: 'all'
interOpNumThreads: 1, // Force single thread for V8 stability
intraOpNumThreads: 1, // Force single thread for V8 stability
graphOptimizationLevel: 'disabled' // Disable threading optimizations
}
}
@ -367,8 +368,8 @@ export class TransformerEmbedding implements EmbeddingModel {
// Both local and remote failed - throw comprehensive error
const errorMsg = `Failed to load embedding model "${this.options.model}". ` +
`Local models not found and remote download failed. ` +
`To fix: 1) Set BRAINY_ALLOW_REMOTE_MODELS=true, ` +
`2) Run "npm run download-models", or ` +
`To fix: 1) Run "npm run download-models", ` +
`2) Check your internet connection, or ` +
`3) Use a custom embedding function.`
throw new Error(errorMsg)
}

81
src/utils/nodeVersionCheck.ts Normal file
View file

@ -0,0 +1,81 @@
/**
* Node.js Version Compatibility Check
*
* Brainy requires Node.js 22.x LTS for maximum stability with ONNX Runtime.
* This prevents V8 HandleScope locking issues in worker threads.
*/
export interface VersionInfo {
current: string
major: number
isSupported: boolean
recommendation: string
}
/**
* Check if the current Node.js version is supported
*/
export function checkNodeVersion(): VersionInfo {
const nodeVersion = process.version
const majorVersion = parseInt(nodeVersion.split('.')[0].substring(1))
const versionInfo: VersionInfo = {
current: nodeVersion,
major: majorVersion,
isSupported: majorVersion === 22,
recommendation: 'Node.js 22.x LTS'
}
return versionInfo
}
/**
* Enforce Node.js version requirement with helpful error messaging
*/
export function enforceNodeVersion(): void {
const versionInfo = checkNodeVersion()
if (!versionInfo.isSupported) {
const errorMessage = [
'🚨 BRAINY COMPATIBILITY ERROR',
'━'.repeat(50),
`❌ Current Node.js: ${versionInfo.current}`,
`✅ Required: ${versionInfo.recommendation}`,
'',
'💡 Quick Fix:',
' nvm install 22 && nvm use 22',
' npm install',
'',
'📖 Why Node.js 22?',
' • Maximum ONNX Runtime stability',
' • Prevents V8 threading crashes',
' • Optimal zero-config performance',
'',
'🔗 More info: https://github.com/soulcraftlabs/brainy#node-version',
'━'.repeat(50)
].join('\n')
throw new Error(errorMessage)
}
}
/**
* Soft warning for version issues (non-blocking)
*/
export function warnNodeVersion(): boolean {
const versionInfo = checkNodeVersion()
if (!versionInfo.isSupported) {
console.warn([
'⚠️ BRAINY VERSION WARNING',
` Current: ${versionInfo.current}`,
` Recommended: ${versionInfo.recommendation}`,
' Consider upgrading for best stability',
''
].join('\n'))
return false
}
return true
}