chore: recovery checkpoint - v3.0 API successfully recovered
CRITICAL CHECKPOINT - DO NOT PUSH TO GITHUB Recovery Status: - Successfully recovered brainy.ts from compiled JavaScript - All core v3.0 API methods functional (add, get, update, delete, relate, find, etc.) - Neural subsystem intact (562KB embedded patterns, NLP working) - Augmentation pipeline operational (20+ augmentations) - HNSW clustering system complete - Triple Intelligence compiled (needs constructor fix) - Test suite validates functionality Changes preserved: - 898 files with changes from last 3 days - 144,475 insertions - All augmentation improvements - All test coverage enhancements - Complete v3.0 feature set This is a LOCAL checkpoint only - contains recovered work after corruption incident. Created backup in .backups/brainy-full-20250910-151314.tar.gz Branch: recovery-checkpoint-20250910-151433 Date: Wed Sep 10 03:18:04 PM PDT 2025
This commit is contained in:
parent
f65455fb22
commit
8ff382ca3b
895 changed files with 143654 additions and 28268 deletions
118
.recovery-workspace/dist-backup-20250910-141917/config/distributedPresets-new.d.ts
vendored
Normal file
118
.recovery-workspace/dist-backup-20250910-141917/config/distributedPresets-new.d.ts
vendored
Normal file
|
|
@ -0,0 +1,118 @@
|
|||
/**
|
||||
* Extended Distributed Configuration Presets
|
||||
* Common patterns for distributed and multi-service architectures
|
||||
* All strongly typed with enums for compile-time safety
|
||||
*/
|
||||
/**
|
||||
* Strongly typed enum for preset names
|
||||
*/
|
||||
export declare enum PresetName {
|
||||
PRODUCTION = "production",
|
||||
DEVELOPMENT = "development",
|
||||
MINIMAL = "minimal",
|
||||
ZERO = "zero",
|
||||
WRITER = "writer",
|
||||
READER = "reader",
|
||||
INGESTION_SERVICE = "ingestion-service",
|
||||
SEARCH_API = "search-api",
|
||||
ANALYTICS_SERVICE = "analytics-service",
|
||||
EDGE_CACHE = "edge-cache",
|
||||
BATCH_PROCESSOR = "batch-processor",
|
||||
STREAMING_SERVICE = "streaming-service",
|
||||
ML_TRAINING = "ml-training",
|
||||
SIDECAR = "sidecar"
|
||||
}
|
||||
/**
|
||||
* Preset categories for organization
|
||||
*/
|
||||
export declare enum PresetCategory {
|
||||
BASIC = "basic",
|
||||
DISTRIBUTED = "distributed",
|
||||
SERVICE = "service"
|
||||
}
|
||||
/**
|
||||
* Model precision options
|
||||
*/
|
||||
export declare enum ModelPrecision {
|
||||
FP32 = "fp32",
|
||||
Q8 = "q8",
|
||||
AUTO = "auto",
|
||||
FAST = "fast",
|
||||
SMALL = "small"
|
||||
}
|
||||
/**
|
||||
* Storage options
|
||||
*/
|
||||
export declare enum StorageOption {
|
||||
AUTO = "auto",
|
||||
MEMORY = "memory",
|
||||
DISK = "disk",
|
||||
CLOUD = "cloud"
|
||||
}
|
||||
/**
|
||||
* Feature set options
|
||||
*/
|
||||
export declare enum FeatureSet {
|
||||
MINIMAL = "minimal",
|
||||
DEFAULT = "default",
|
||||
FULL = "full",
|
||||
CUSTOM = "custom"
|
||||
}
|
||||
/**
|
||||
* Distributed role options
|
||||
*/
|
||||
export declare enum DistributedRole {
|
||||
WRITER = "writer",
|
||||
READER = "reader",
|
||||
HYBRID = "hybrid"
|
||||
}
|
||||
/**
|
||||
* Preset configuration interface
|
||||
*/
|
||||
export interface PresetConfig {
|
||||
storage: StorageOption;
|
||||
model: ModelPrecision;
|
||||
features: FeatureSet | string[];
|
||||
distributed: boolean;
|
||||
role?: DistributedRole;
|
||||
readOnly?: boolean;
|
||||
writeOnly?: boolean;
|
||||
allowDirectReads?: boolean;
|
||||
lazyLoadInReadOnlyMode?: boolean;
|
||||
cache?: {
|
||||
hotCacheMaxSize?: number;
|
||||
autoTune?: boolean;
|
||||
batchSize?: number;
|
||||
};
|
||||
verbose?: boolean;
|
||||
description: string;
|
||||
category: PresetCategory;
|
||||
}
|
||||
/**
|
||||
* Strongly typed preset configurations
|
||||
*/
|
||||
export declare const PRESET_CONFIGS: Readonly<Record<PresetName, PresetConfig>>;
|
||||
/**
|
||||
* Type-safe preset getter
|
||||
*/
|
||||
export declare function getPreset(name: PresetName): PresetConfig;
|
||||
/**
|
||||
* Check if a string is a valid preset name
|
||||
*/
|
||||
export declare function isValidPreset(name: string): name is PresetName;
|
||||
/**
|
||||
* Get presets by category
|
||||
*/
|
||||
export declare function getPresetsByCategory(category: PresetCategory): PresetName[];
|
||||
/**
|
||||
* Get all preset names
|
||||
*/
|
||||
export declare function getAllPresetNames(): PresetName[];
|
||||
/**
|
||||
* Get preset description
|
||||
*/
|
||||
export declare function getPresetDescription(name: PresetName): string;
|
||||
/**
|
||||
* Convert preset config to Brainy config
|
||||
*/
|
||||
export declare function presetToBrainyConfig(preset: PresetConfig): any;
|
||||
|
|
@ -0,0 +1,318 @@
|
|||
/**
|
||||
* Extended Distributed Configuration Presets
|
||||
* Common patterns for distributed and multi-service architectures
|
||||
* All strongly typed with enums for compile-time safety
|
||||
*/
|
||||
/**
|
||||
* Strongly typed enum for preset names
|
||||
*/
|
||||
export var PresetName;
|
||||
(function (PresetName) {
|
||||
// Basic presets
|
||||
PresetName["PRODUCTION"] = "production";
|
||||
PresetName["DEVELOPMENT"] = "development";
|
||||
PresetName["MINIMAL"] = "minimal";
|
||||
PresetName["ZERO"] = "zero";
|
||||
// Distributed presets
|
||||
PresetName["WRITER"] = "writer";
|
||||
PresetName["READER"] = "reader";
|
||||
// Service-specific presets
|
||||
PresetName["INGESTION_SERVICE"] = "ingestion-service";
|
||||
PresetName["SEARCH_API"] = "search-api";
|
||||
PresetName["ANALYTICS_SERVICE"] = "analytics-service";
|
||||
PresetName["EDGE_CACHE"] = "edge-cache";
|
||||
PresetName["BATCH_PROCESSOR"] = "batch-processor";
|
||||
PresetName["STREAMING_SERVICE"] = "streaming-service";
|
||||
PresetName["ML_TRAINING"] = "ml-training";
|
||||
PresetName["SIDECAR"] = "sidecar";
|
||||
})(PresetName || (PresetName = {}));
|
||||
/**
|
||||
* Preset categories for organization
|
||||
*/
|
||||
export var PresetCategory;
|
||||
(function (PresetCategory) {
|
||||
PresetCategory["BASIC"] = "basic";
|
||||
PresetCategory["DISTRIBUTED"] = "distributed";
|
||||
PresetCategory["SERVICE"] = "service";
|
||||
})(PresetCategory || (PresetCategory = {}));
|
||||
/**
|
||||
* Model precision options
|
||||
*/
|
||||
export var ModelPrecision;
|
||||
(function (ModelPrecision) {
|
||||
ModelPrecision["FP32"] = "fp32";
|
||||
ModelPrecision["Q8"] = "q8";
|
||||
ModelPrecision["AUTO"] = "auto";
|
||||
ModelPrecision["FAST"] = "fast";
|
||||
ModelPrecision["SMALL"] = "small";
|
||||
})(ModelPrecision || (ModelPrecision = {}));
|
||||
/**
|
||||
* Storage options
|
||||
*/
|
||||
export var StorageOption;
|
||||
(function (StorageOption) {
|
||||
StorageOption["AUTO"] = "auto";
|
||||
StorageOption["MEMORY"] = "memory";
|
||||
StorageOption["DISK"] = "disk";
|
||||
StorageOption["CLOUD"] = "cloud";
|
||||
})(StorageOption || (StorageOption = {}));
|
||||
/**
|
||||
* Feature set options
|
||||
*/
|
||||
export var FeatureSet;
|
||||
(function (FeatureSet) {
|
||||
FeatureSet["MINIMAL"] = "minimal";
|
||||
FeatureSet["DEFAULT"] = "default";
|
||||
FeatureSet["FULL"] = "full";
|
||||
FeatureSet["CUSTOM"] = "custom"; // For custom feature arrays
|
||||
})(FeatureSet || (FeatureSet = {}));
|
||||
/**
|
||||
* Distributed role options
|
||||
*/
|
||||
export var DistributedRole;
|
||||
(function (DistributedRole) {
|
||||
DistributedRole["WRITER"] = "writer";
|
||||
DistributedRole["READER"] = "reader";
|
||||
DistributedRole["HYBRID"] = "hybrid";
|
||||
})(DistributedRole || (DistributedRole = {}));
|
||||
/**
|
||||
* Strongly typed preset configurations
|
||||
*/
|
||||
export const PRESET_CONFIGS = {
|
||||
// Basic presets
|
||||
[PresetName.PRODUCTION]: {
|
||||
storage: StorageOption.DISK,
|
||||
model: ModelPrecision.AUTO,
|
||||
features: FeatureSet.DEFAULT,
|
||||
distributed: false,
|
||||
verbose: false,
|
||||
description: 'Optimized for production use',
|
||||
category: PresetCategory.BASIC
|
||||
},
|
||||
[PresetName.DEVELOPMENT]: {
|
||||
storage: StorageOption.MEMORY,
|
||||
model: ModelPrecision.FP32,
|
||||
features: FeatureSet.FULL,
|
||||
distributed: false,
|
||||
verbose: true,
|
||||
description: 'Optimized for development with verbose logging',
|
||||
category: PresetCategory.BASIC
|
||||
},
|
||||
[PresetName.MINIMAL]: {
|
||||
storage: StorageOption.MEMORY,
|
||||
model: ModelPrecision.Q8,
|
||||
features: FeatureSet.MINIMAL,
|
||||
distributed: false,
|
||||
verbose: false,
|
||||
description: 'Minimal footprint configuration',
|
||||
category: PresetCategory.BASIC
|
||||
},
|
||||
[PresetName.ZERO]: {
|
||||
storage: StorageOption.AUTO,
|
||||
model: ModelPrecision.AUTO,
|
||||
features: FeatureSet.DEFAULT,
|
||||
distributed: false,
|
||||
verbose: false,
|
||||
description: 'True zero configuration with auto-detection',
|
||||
category: PresetCategory.BASIC
|
||||
},
|
||||
// Distributed basic presets
|
||||
[PresetName.WRITER]: {
|
||||
storage: StorageOption.AUTO,
|
||||
model: ModelPrecision.AUTO,
|
||||
features: FeatureSet.MINIMAL,
|
||||
distributed: true,
|
||||
role: DistributedRole.WRITER,
|
||||
writeOnly: true,
|
||||
allowDirectReads: true,
|
||||
verbose: false,
|
||||
description: 'Write-only instance for distributed setups',
|
||||
category: PresetCategory.DISTRIBUTED
|
||||
},
|
||||
[PresetName.READER]: {
|
||||
storage: StorageOption.AUTO,
|
||||
model: ModelPrecision.AUTO,
|
||||
features: FeatureSet.DEFAULT,
|
||||
distributed: true,
|
||||
role: DistributedRole.READER,
|
||||
readOnly: true,
|
||||
lazyLoadInReadOnlyMode: true,
|
||||
verbose: false,
|
||||
description: 'Read-only instance for distributed setups',
|
||||
category: PresetCategory.DISTRIBUTED
|
||||
},
|
||||
// Service-specific presets
|
||||
[PresetName.INGESTION_SERVICE]: {
|
||||
storage: StorageOption.CLOUD,
|
||||
model: ModelPrecision.Q8,
|
||||
features: ['core', 'batch-processing', 'entity-registry'],
|
||||
distributed: true,
|
||||
role: DistributedRole.WRITER,
|
||||
writeOnly: true,
|
||||
allowDirectReads: true,
|
||||
verbose: false,
|
||||
description: 'High-throughput data ingestion service',
|
||||
category: PresetCategory.SERVICE
|
||||
},
|
||||
[PresetName.SEARCH_API]: {
|
||||
storage: StorageOption.CLOUD,
|
||||
model: ModelPrecision.FP32,
|
||||
features: ['core', 'search', 'cache', 'triple-intelligence'],
|
||||
distributed: true,
|
||||
role: DistributedRole.READER,
|
||||
readOnly: true,
|
||||
lazyLoadInReadOnlyMode: true,
|
||||
cache: {
|
||||
hotCacheMaxSize: 10000,
|
||||
autoTune: true
|
||||
},
|
||||
verbose: false,
|
||||
description: 'Low-latency search API service',
|
||||
category: PresetCategory.SERVICE
|
||||
},
|
||||
[PresetName.ANALYTICS_SERVICE]: {
|
||||
storage: StorageOption.CLOUD,
|
||||
model: ModelPrecision.AUTO,
|
||||
features: ['core', 'search', 'metrics', 'monitoring'],
|
||||
distributed: true,
|
||||
role: DistributedRole.HYBRID,
|
||||
verbose: false,
|
||||
description: 'Analytics and data processing service',
|
||||
category: PresetCategory.SERVICE
|
||||
},
|
||||
[PresetName.EDGE_CACHE]: {
|
||||
storage: StorageOption.AUTO,
|
||||
model: ModelPrecision.Q8,
|
||||
features: ['core', 'search', 'cache'],
|
||||
distributed: true,
|
||||
role: DistributedRole.READER,
|
||||
readOnly: true,
|
||||
lazyLoadInReadOnlyMode: true,
|
||||
cache: {
|
||||
hotCacheMaxSize: 1000,
|
||||
autoTune: false
|
||||
},
|
||||
verbose: false,
|
||||
description: 'Edge location cache with minimal footprint',
|
||||
category: PresetCategory.SERVICE
|
||||
},
|
||||
[PresetName.BATCH_PROCESSOR]: {
|
||||
storage: StorageOption.CLOUD,
|
||||
model: ModelPrecision.Q8,
|
||||
features: ['core', 'batch-processing', 'neural-api'],
|
||||
distributed: true,
|
||||
role: DistributedRole.HYBRID,
|
||||
cache: {
|
||||
hotCacheMaxSize: 5000,
|
||||
batchSize: 500
|
||||
},
|
||||
verbose: false,
|
||||
description: 'Batch processing and bulk operations',
|
||||
category: PresetCategory.SERVICE
|
||||
},
|
||||
[PresetName.STREAMING_SERVICE]: {
|
||||
storage: StorageOption.CLOUD,
|
||||
model: ModelPrecision.Q8,
|
||||
features: ['core', 'batch-processing', 'wal'],
|
||||
distributed: true,
|
||||
role: DistributedRole.WRITER,
|
||||
writeOnly: true,
|
||||
allowDirectReads: false,
|
||||
verbose: false,
|
||||
description: 'Real-time data streaming service',
|
||||
category: PresetCategory.SERVICE
|
||||
},
|
||||
[PresetName.ML_TRAINING]: {
|
||||
storage: StorageOption.CLOUD,
|
||||
model: ModelPrecision.FP32,
|
||||
features: FeatureSet.FULL,
|
||||
distributed: true,
|
||||
role: DistributedRole.HYBRID,
|
||||
cache: {
|
||||
hotCacheMaxSize: 20000,
|
||||
autoTune: true
|
||||
},
|
||||
verbose: true,
|
||||
description: 'Machine learning training service',
|
||||
category: PresetCategory.SERVICE
|
||||
},
|
||||
[PresetName.SIDECAR]: {
|
||||
storage: StorageOption.AUTO,
|
||||
model: ModelPrecision.Q8,
|
||||
features: FeatureSet.MINIMAL,
|
||||
distributed: false,
|
||||
verbose: false,
|
||||
description: 'Lightweight sidecar for microservices',
|
||||
category: PresetCategory.SERVICE
|
||||
}
|
||||
};
|
||||
/**
|
||||
* Type-safe preset getter
|
||||
*/
|
||||
export function getPreset(name) {
|
||||
return PRESET_CONFIGS[name];
|
||||
}
|
||||
/**
|
||||
* Check if a string is a valid preset name
|
||||
*/
|
||||
export function isValidPreset(name) {
|
||||
return Object.values(PresetName).includes(name);
|
||||
}
|
||||
/**
|
||||
* Get presets by category
|
||||
*/
|
||||
export function getPresetsByCategory(category) {
|
||||
return Object.entries(PRESET_CONFIGS)
|
||||
.filter(([_, config]) => config.category === category)
|
||||
.map(([name]) => name);
|
||||
}
|
||||
/**
|
||||
* Get all preset names
|
||||
*/
|
||||
export function getAllPresetNames() {
|
||||
return Object.values(PresetName);
|
||||
}
|
||||
/**
|
||||
* Get preset description
|
||||
*/
|
||||
export function getPresetDescription(name) {
|
||||
return PRESET_CONFIGS[name].description;
|
||||
}
|
||||
/**
|
||||
* Convert preset config to Brainy config
|
||||
*/
|
||||
export function presetToBrainyConfig(preset) {
|
||||
const config = {
|
||||
storage: preset.storage,
|
||||
model: preset.model,
|
||||
verbose: preset.verbose
|
||||
};
|
||||
// Handle features
|
||||
if (Array.isArray(preset.features)) {
|
||||
config.features = preset.features;
|
||||
}
|
||||
else {
|
||||
config.features = preset.features; // Will be expanded by processZeroConfig
|
||||
}
|
||||
// Handle distributed settings
|
||||
if (preset.distributed) {
|
||||
config.distributed = {
|
||||
enabled: true,
|
||||
role: preset.role
|
||||
};
|
||||
if (preset.readOnly)
|
||||
config.readOnly = true;
|
||||
if (preset.writeOnly)
|
||||
config.writeOnly = true;
|
||||
if (preset.allowDirectReads)
|
||||
config.allowDirectReads = true;
|
||||
if (preset.lazyLoadInReadOnlyMode)
|
||||
config.lazyLoadInReadOnlyMode = true;
|
||||
}
|
||||
// Handle cache settings
|
||||
if (preset.cache) {
|
||||
config.cache = preset.cache;
|
||||
}
|
||||
return config;
|
||||
}
|
||||
//# sourceMappingURL=distributedPresets-new.js.map
|
||||
File diff suppressed because one or more lines are too long
118
.recovery-workspace/dist-backup-20250910-141917/config/distributedPresets.d.ts
vendored
Normal file
118
.recovery-workspace/dist-backup-20250910-141917/config/distributedPresets.d.ts
vendored
Normal file
|
|
@ -0,0 +1,118 @@
|
|||
/**
|
||||
* Extended Distributed Configuration Presets
|
||||
* Common patterns for distributed and multi-service architectures
|
||||
* All strongly typed with enums for compile-time safety
|
||||
*/
|
||||
/**
|
||||
* Strongly typed enum for preset names
|
||||
*/
|
||||
export declare enum PresetName {
|
||||
PRODUCTION = "production",
|
||||
DEVELOPMENT = "development",
|
||||
MINIMAL = "minimal",
|
||||
ZERO = "zero",
|
||||
WRITER = "writer",
|
||||
READER = "reader",
|
||||
INGESTION_SERVICE = "ingestion-service",
|
||||
SEARCH_API = "search-api",
|
||||
ANALYTICS_SERVICE = "analytics-service",
|
||||
EDGE_CACHE = "edge-cache",
|
||||
BATCH_PROCESSOR = "batch-processor",
|
||||
STREAMING_SERVICE = "streaming-service",
|
||||
ML_TRAINING = "ml-training",
|
||||
SIDECAR = "sidecar"
|
||||
}
|
||||
/**
|
||||
* Preset categories for organization
|
||||
*/
|
||||
export declare enum PresetCategory {
|
||||
BASIC = "basic",
|
||||
DISTRIBUTED = "distributed",
|
||||
SERVICE = "service"
|
||||
}
|
||||
/**
|
||||
* Model precision options
|
||||
*/
|
||||
export declare enum ModelPrecision {
|
||||
FP32 = "fp32",
|
||||
Q8 = "q8",
|
||||
AUTO = "auto",
|
||||
FAST = "fast",
|
||||
SMALL = "small"
|
||||
}
|
||||
/**
|
||||
* Storage options
|
||||
*/
|
||||
export declare enum StorageOption {
|
||||
AUTO = "auto",
|
||||
MEMORY = "memory",
|
||||
DISK = "disk",
|
||||
CLOUD = "cloud"
|
||||
}
|
||||
/**
|
||||
* Feature set options
|
||||
*/
|
||||
export declare enum FeatureSet {
|
||||
MINIMAL = "minimal",
|
||||
DEFAULT = "default",
|
||||
FULL = "full",
|
||||
CUSTOM = "custom"
|
||||
}
|
||||
/**
|
||||
* Distributed role options
|
||||
*/
|
||||
export declare enum DistributedRole {
|
||||
WRITER = "writer",
|
||||
READER = "reader",
|
||||
HYBRID = "hybrid"
|
||||
}
|
||||
/**
|
||||
* Preset configuration interface
|
||||
*/
|
||||
export interface PresetConfig {
|
||||
storage: StorageOption;
|
||||
model: ModelPrecision;
|
||||
features: FeatureSet | string[];
|
||||
distributed: boolean;
|
||||
role?: DistributedRole;
|
||||
readOnly?: boolean;
|
||||
writeOnly?: boolean;
|
||||
allowDirectReads?: boolean;
|
||||
lazyLoadInReadOnlyMode?: boolean;
|
||||
cache?: {
|
||||
hotCacheMaxSize?: number;
|
||||
autoTune?: boolean;
|
||||
batchSize?: number;
|
||||
};
|
||||
verbose?: boolean;
|
||||
description: string;
|
||||
category: PresetCategory;
|
||||
}
|
||||
/**
|
||||
* Strongly typed preset configurations
|
||||
*/
|
||||
export declare const PRESET_CONFIGS: Readonly<Record<PresetName, PresetConfig>>;
|
||||
/**
|
||||
* Type-safe preset getter
|
||||
*/
|
||||
export declare function getPreset(name: PresetName): PresetConfig;
|
||||
/**
|
||||
* Check if a string is a valid preset name
|
||||
*/
|
||||
export declare function isValidPreset(name: string): name is PresetName;
|
||||
/**
|
||||
* Get presets by category
|
||||
*/
|
||||
export declare function getPresetsByCategory(category: PresetCategory): PresetName[];
|
||||
/**
|
||||
* Get all preset names
|
||||
*/
|
||||
export declare function getAllPresetNames(): PresetName[];
|
||||
/**
|
||||
* Get preset description
|
||||
*/
|
||||
export declare function getPresetDescription(name: PresetName): string;
|
||||
/**
|
||||
* Convert preset config to Brainy config
|
||||
*/
|
||||
export declare function presetToBrainyConfig(preset: PresetConfig): any;
|
||||
|
|
@ -0,0 +1,318 @@
|
|||
/**
|
||||
* Extended Distributed Configuration Presets
|
||||
* Common patterns for distributed and multi-service architectures
|
||||
* All strongly typed with enums for compile-time safety
|
||||
*/
|
||||
/**
|
||||
* Strongly typed enum for preset names
|
||||
*/
|
||||
export var PresetName;
|
||||
(function (PresetName) {
|
||||
// Basic presets
|
||||
PresetName["PRODUCTION"] = "production";
|
||||
PresetName["DEVELOPMENT"] = "development";
|
||||
PresetName["MINIMAL"] = "minimal";
|
||||
PresetName["ZERO"] = "zero";
|
||||
// Distributed presets
|
||||
PresetName["WRITER"] = "writer";
|
||||
PresetName["READER"] = "reader";
|
||||
// Service-specific presets
|
||||
PresetName["INGESTION_SERVICE"] = "ingestion-service";
|
||||
PresetName["SEARCH_API"] = "search-api";
|
||||
PresetName["ANALYTICS_SERVICE"] = "analytics-service";
|
||||
PresetName["EDGE_CACHE"] = "edge-cache";
|
||||
PresetName["BATCH_PROCESSOR"] = "batch-processor";
|
||||
PresetName["STREAMING_SERVICE"] = "streaming-service";
|
||||
PresetName["ML_TRAINING"] = "ml-training";
|
||||
PresetName["SIDECAR"] = "sidecar";
|
||||
})(PresetName || (PresetName = {}));
|
||||
/**
|
||||
* Preset categories for organization
|
||||
*/
|
||||
export var PresetCategory;
|
||||
(function (PresetCategory) {
|
||||
PresetCategory["BASIC"] = "basic";
|
||||
PresetCategory["DISTRIBUTED"] = "distributed";
|
||||
PresetCategory["SERVICE"] = "service";
|
||||
})(PresetCategory || (PresetCategory = {}));
|
||||
/**
|
||||
* Model precision options
|
||||
*/
|
||||
export var ModelPrecision;
|
||||
(function (ModelPrecision) {
|
||||
ModelPrecision["FP32"] = "fp32";
|
||||
ModelPrecision["Q8"] = "q8";
|
||||
ModelPrecision["AUTO"] = "auto";
|
||||
ModelPrecision["FAST"] = "fast";
|
||||
ModelPrecision["SMALL"] = "small";
|
||||
})(ModelPrecision || (ModelPrecision = {}));
|
||||
/**
|
||||
* Storage options
|
||||
*/
|
||||
export var StorageOption;
|
||||
(function (StorageOption) {
|
||||
StorageOption["AUTO"] = "auto";
|
||||
StorageOption["MEMORY"] = "memory";
|
||||
StorageOption["DISK"] = "disk";
|
||||
StorageOption["CLOUD"] = "cloud";
|
||||
})(StorageOption || (StorageOption = {}));
|
||||
/**
|
||||
* Feature set options
|
||||
*/
|
||||
export var FeatureSet;
|
||||
(function (FeatureSet) {
|
||||
FeatureSet["MINIMAL"] = "minimal";
|
||||
FeatureSet["DEFAULT"] = "default";
|
||||
FeatureSet["FULL"] = "full";
|
||||
FeatureSet["CUSTOM"] = "custom"; // For custom feature arrays
|
||||
})(FeatureSet || (FeatureSet = {}));
|
||||
/**
|
||||
* Distributed role options
|
||||
*/
|
||||
export var DistributedRole;
|
||||
(function (DistributedRole) {
|
||||
DistributedRole["WRITER"] = "writer";
|
||||
DistributedRole["READER"] = "reader";
|
||||
DistributedRole["HYBRID"] = "hybrid";
|
||||
})(DistributedRole || (DistributedRole = {}));
|
||||
/**
|
||||
* Strongly typed preset configurations
|
||||
*/
|
||||
export const PRESET_CONFIGS = {
|
||||
// Basic presets
|
||||
[PresetName.PRODUCTION]: {
|
||||
storage: StorageOption.DISK,
|
||||
model: ModelPrecision.AUTO,
|
||||
features: FeatureSet.DEFAULT,
|
||||
distributed: false,
|
||||
verbose: false,
|
||||
description: 'Optimized for production use',
|
||||
category: PresetCategory.BASIC
|
||||
},
|
||||
[PresetName.DEVELOPMENT]: {
|
||||
storage: StorageOption.MEMORY,
|
||||
model: ModelPrecision.FP32,
|
||||
features: FeatureSet.FULL,
|
||||
distributed: false,
|
||||
verbose: true,
|
||||
description: 'Optimized for development with verbose logging',
|
||||
category: PresetCategory.BASIC
|
||||
},
|
||||
[PresetName.MINIMAL]: {
|
||||
storage: StorageOption.MEMORY,
|
||||
model: ModelPrecision.Q8,
|
||||
features: FeatureSet.MINIMAL,
|
||||
distributed: false,
|
||||
verbose: false,
|
||||
description: 'Minimal footprint configuration',
|
||||
category: PresetCategory.BASIC
|
||||
},
|
||||
[PresetName.ZERO]: {
|
||||
storage: StorageOption.AUTO,
|
||||
model: ModelPrecision.AUTO,
|
||||
features: FeatureSet.DEFAULT,
|
||||
distributed: false,
|
||||
verbose: false,
|
||||
description: 'True zero configuration with auto-detection',
|
||||
category: PresetCategory.BASIC
|
||||
},
|
||||
// Distributed basic presets
|
||||
[PresetName.WRITER]: {
|
||||
storage: StorageOption.AUTO,
|
||||
model: ModelPrecision.AUTO,
|
||||
features: FeatureSet.MINIMAL,
|
||||
distributed: true,
|
||||
role: DistributedRole.WRITER,
|
||||
writeOnly: true,
|
||||
allowDirectReads: true,
|
||||
verbose: false,
|
||||
description: 'Write-only instance for distributed setups',
|
||||
category: PresetCategory.DISTRIBUTED
|
||||
},
|
||||
[PresetName.READER]: {
|
||||
storage: StorageOption.AUTO,
|
||||
model: ModelPrecision.AUTO,
|
||||
features: FeatureSet.DEFAULT,
|
||||
distributed: true,
|
||||
role: DistributedRole.READER,
|
||||
readOnly: true,
|
||||
lazyLoadInReadOnlyMode: true,
|
||||
verbose: false,
|
||||
description: 'Read-only instance for distributed setups',
|
||||
category: PresetCategory.DISTRIBUTED
|
||||
},
|
||||
// Service-specific presets
|
||||
[PresetName.INGESTION_SERVICE]: {
|
||||
storage: StorageOption.CLOUD,
|
||||
model: ModelPrecision.Q8,
|
||||
features: ['core', 'batch-processing', 'entity-registry'],
|
||||
distributed: true,
|
||||
role: DistributedRole.WRITER,
|
||||
writeOnly: true,
|
||||
allowDirectReads: true,
|
||||
verbose: false,
|
||||
description: 'High-throughput data ingestion service',
|
||||
category: PresetCategory.SERVICE
|
||||
},
|
||||
[PresetName.SEARCH_API]: {
|
||||
storage: StorageOption.CLOUD,
|
||||
model: ModelPrecision.FP32,
|
||||
features: ['core', 'search', 'cache', 'triple-intelligence'],
|
||||
distributed: true,
|
||||
role: DistributedRole.READER,
|
||||
readOnly: true,
|
||||
lazyLoadInReadOnlyMode: true,
|
||||
cache: {
|
||||
hotCacheMaxSize: 10000,
|
||||
autoTune: true
|
||||
},
|
||||
verbose: false,
|
||||
description: 'Low-latency search API service',
|
||||
category: PresetCategory.SERVICE
|
||||
},
|
||||
[PresetName.ANALYTICS_SERVICE]: {
|
||||
storage: StorageOption.CLOUD,
|
||||
model: ModelPrecision.AUTO,
|
||||
features: ['core', 'search', 'metrics', 'monitoring'],
|
||||
distributed: true,
|
||||
role: DistributedRole.HYBRID,
|
||||
verbose: false,
|
||||
description: 'Analytics and data processing service',
|
||||
category: PresetCategory.SERVICE
|
||||
},
|
||||
[PresetName.EDGE_CACHE]: {
|
||||
storage: StorageOption.AUTO,
|
||||
model: ModelPrecision.Q8,
|
||||
features: ['core', 'search', 'cache'],
|
||||
distributed: true,
|
||||
role: DistributedRole.READER,
|
||||
readOnly: true,
|
||||
lazyLoadInReadOnlyMode: true,
|
||||
cache: {
|
||||
hotCacheMaxSize: 1000,
|
||||
autoTune: false
|
||||
},
|
||||
verbose: false,
|
||||
description: 'Edge location cache with minimal footprint',
|
||||
category: PresetCategory.SERVICE
|
||||
},
|
||||
[PresetName.BATCH_PROCESSOR]: {
|
||||
storage: StorageOption.CLOUD,
|
||||
model: ModelPrecision.Q8,
|
||||
features: ['core', 'batch-processing', 'neural-api'],
|
||||
distributed: true,
|
||||
role: DistributedRole.HYBRID,
|
||||
cache: {
|
||||
hotCacheMaxSize: 5000,
|
||||
batchSize: 500
|
||||
},
|
||||
verbose: false,
|
||||
description: 'Batch processing and bulk operations',
|
||||
category: PresetCategory.SERVICE
|
||||
},
|
||||
[PresetName.STREAMING_SERVICE]: {
|
||||
storage: StorageOption.CLOUD,
|
||||
model: ModelPrecision.Q8,
|
||||
features: ['core', 'batch-processing', 'wal'],
|
||||
distributed: true,
|
||||
role: DistributedRole.WRITER,
|
||||
writeOnly: true,
|
||||
allowDirectReads: false,
|
||||
verbose: false,
|
||||
description: 'Real-time data streaming service',
|
||||
category: PresetCategory.SERVICE
|
||||
},
|
||||
[PresetName.ML_TRAINING]: {
|
||||
storage: StorageOption.CLOUD,
|
||||
model: ModelPrecision.FP32,
|
||||
features: FeatureSet.FULL,
|
||||
distributed: true,
|
||||
role: DistributedRole.HYBRID,
|
||||
cache: {
|
||||
hotCacheMaxSize: 20000,
|
||||
autoTune: true
|
||||
},
|
||||
verbose: true,
|
||||
description: 'Machine learning training service',
|
||||
category: PresetCategory.SERVICE
|
||||
},
|
||||
[PresetName.SIDECAR]: {
|
||||
storage: StorageOption.AUTO,
|
||||
model: ModelPrecision.Q8,
|
||||
features: FeatureSet.MINIMAL,
|
||||
distributed: false,
|
||||
verbose: false,
|
||||
description: 'Lightweight sidecar for microservices',
|
||||
category: PresetCategory.SERVICE
|
||||
}
|
||||
};
|
||||
/**
|
||||
* Type-safe preset getter
|
||||
*/
|
||||
export function getPreset(name) {
|
||||
return PRESET_CONFIGS[name];
|
||||
}
|
||||
/**
|
||||
* Check if a string is a valid preset name
|
||||
*/
|
||||
export function isValidPreset(name) {
|
||||
return Object.values(PresetName).includes(name);
|
||||
}
|
||||
/**
|
||||
* Get presets by category
|
||||
*/
|
||||
export function getPresetsByCategory(category) {
|
||||
return Object.entries(PRESET_CONFIGS)
|
||||
.filter(([_, config]) => config.category === category)
|
||||
.map(([name]) => name);
|
||||
}
|
||||
/**
|
||||
* Get all preset names
|
||||
*/
|
||||
export function getAllPresetNames() {
|
||||
return Object.values(PresetName);
|
||||
}
|
||||
/**
|
||||
* Get preset description
|
||||
*/
|
||||
export function getPresetDescription(name) {
|
||||
return PRESET_CONFIGS[name].description;
|
||||
}
|
||||
/**
|
||||
* Convert preset config to Brainy config
|
||||
*/
|
||||
export function presetToBrainyConfig(preset) {
|
||||
const config = {
|
||||
storage: preset.storage,
|
||||
model: preset.model,
|
||||
verbose: preset.verbose
|
||||
};
|
||||
// Handle features
|
||||
if (Array.isArray(preset.features)) {
|
||||
config.features = preset.features;
|
||||
}
|
||||
else {
|
||||
config.features = preset.features; // Will be expanded by processZeroConfig
|
||||
}
|
||||
// Handle distributed settings
|
||||
if (preset.distributed) {
|
||||
config.distributed = {
|
||||
enabled: true,
|
||||
role: preset.role
|
||||
};
|
||||
if (preset.readOnly)
|
||||
config.readOnly = true;
|
||||
if (preset.writeOnly)
|
||||
config.writeOnly = true;
|
||||
if (preset.allowDirectReads)
|
||||
config.allowDirectReads = true;
|
||||
if (preset.lazyLoadInReadOnlyMode)
|
||||
config.lazyLoadInReadOnlyMode = true;
|
||||
}
|
||||
// Handle cache settings
|
||||
if (preset.cache) {
|
||||
config.cache = preset.cache;
|
||||
}
|
||||
return config;
|
||||
}
|
||||
//# sourceMappingURL=distributedPresets.js.map
|
||||
File diff suppressed because one or more lines are too long
99
.recovery-workspace/dist-backup-20250910-141917/config/extensibleConfig.d.ts
vendored
Normal file
99
.recovery-workspace/dist-backup-20250910-141917/config/extensibleConfig.d.ts
vendored
Normal file
|
|
@ -0,0 +1,99 @@
|
|||
/**
|
||||
* Extensible Configuration System
|
||||
* Allows augmentations to register new storage types, presets, and configurations
|
||||
*/
|
||||
import { PresetConfig } from './distributedPresets.js';
|
||||
import { StorageConfigResult } from './storageAutoConfig.js';
|
||||
/**
|
||||
* Storage provider registration interface
|
||||
*/
|
||||
export interface StorageProvider {
|
||||
type: string;
|
||||
name: string;
|
||||
description: string;
|
||||
detect: () => Promise<boolean>;
|
||||
getConfig: () => Promise<any>;
|
||||
priority?: number;
|
||||
requirements?: {
|
||||
env?: string[];
|
||||
packages?: string[];
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Preset extension interface
|
||||
*/
|
||||
export interface PresetExtension {
|
||||
name: string;
|
||||
config: PresetConfig;
|
||||
override?: boolean;
|
||||
}
|
||||
/**
|
||||
* Global registry for extensions
|
||||
*/
|
||||
declare class ConfigurationRegistry {
|
||||
private static instance;
|
||||
private storageProviders;
|
||||
private presetExtensions;
|
||||
private autoDetectHooks;
|
||||
private constructor();
|
||||
static getInstance(): ConfigurationRegistry;
|
||||
/**
|
||||
* Register a new storage provider
|
||||
* This is how augmentations add new storage types
|
||||
*/
|
||||
registerStorageProvider(provider: StorageProvider): void;
|
||||
/**
|
||||
* Register a new preset
|
||||
*/
|
||||
registerPreset(name: string, extension: PresetExtension): void;
|
||||
/**
|
||||
* Register an auto-detection hook
|
||||
*/
|
||||
registerAutoDetectHook(hook: () => Promise<any>): void;
|
||||
/**
|
||||
* Get all registered storage providers
|
||||
*/
|
||||
getStorageProviders(): StorageProvider[];
|
||||
/**
|
||||
* Get all registered presets (built-in + extensions)
|
||||
*/
|
||||
getAllPresets(): Map<string, PresetConfig>;
|
||||
/**
|
||||
* Auto-detect storage including extensions
|
||||
*/
|
||||
autoDetectStorage(): Promise<StorageConfigResult>;
|
||||
/**
|
||||
* Register built-in providers
|
||||
*/
|
||||
private registerBuiltInProviders;
|
||||
}
|
||||
/**
|
||||
* Example: Redis storage provider registration
|
||||
* This would be in the redis augmentation package
|
||||
*/
|
||||
export declare const redisStorageProvider: StorageProvider;
|
||||
/**
|
||||
* Example: MongoDB storage provider
|
||||
*/
|
||||
export declare const mongoStorageProvider: StorageProvider;
|
||||
/**
|
||||
* Example: PostgreSQL with pgvector extension
|
||||
*/
|
||||
export declare const postgresStorageProvider: StorageProvider;
|
||||
/**
|
||||
* How an augmentation would register its storage provider
|
||||
*/
|
||||
export declare function registerStorageAugmentation(provider: StorageProvider): void;
|
||||
/**
|
||||
* How to register a new preset
|
||||
*/
|
||||
export declare function registerPresetAugmentation(name: string, config: PresetConfig): void;
|
||||
/**
|
||||
* Example preset for Redis-based caching service
|
||||
*/
|
||||
export declare const redisCachePreset: PresetConfig;
|
||||
/**
|
||||
* Get the configuration registry
|
||||
*/
|
||||
export declare function getConfigRegistry(): ConfigurationRegistry;
|
||||
export {};
|
||||
|
|
@ -0,0 +1,268 @@
|
|||
/**
|
||||
* Extensible Configuration System
|
||||
* Allows augmentations to register new storage types, presets, and configurations
|
||||
*/
|
||||
import { ModelPrecision, DistributedRole, PresetCategory } from './distributedPresets.js';
|
||||
/**
|
||||
* Global registry for extensions
|
||||
*/
|
||||
class ConfigurationRegistry {
|
||||
constructor() {
|
||||
// Registered storage providers
|
||||
this.storageProviders = new Map();
|
||||
// Registered preset extensions
|
||||
this.presetExtensions = new Map();
|
||||
// Custom auto-detection hooks
|
||||
this.autoDetectHooks = [];
|
||||
// Initialize with built-in providers
|
||||
this.registerBuiltInProviders();
|
||||
}
|
||||
static getInstance() {
|
||||
if (!ConfigurationRegistry.instance) {
|
||||
ConfigurationRegistry.instance = new ConfigurationRegistry();
|
||||
}
|
||||
return ConfigurationRegistry.instance;
|
||||
}
|
||||
/**
|
||||
* Register a new storage provider
|
||||
* This is how augmentations add new storage types
|
||||
*/
|
||||
registerStorageProvider(provider) {
|
||||
console.log(`📦 Registering storage provider: ${provider.type} (${provider.name})`);
|
||||
this.storageProviders.set(provider.type, provider);
|
||||
}
|
||||
/**
|
||||
* Register a new preset
|
||||
*/
|
||||
registerPreset(name, extension) {
|
||||
console.log(`🎨 Registering preset: ${name}`);
|
||||
this.presetExtensions.set(name, extension);
|
||||
}
|
||||
/**
|
||||
* Register an auto-detection hook
|
||||
*/
|
||||
registerAutoDetectHook(hook) {
|
||||
this.autoDetectHooks.push(hook);
|
||||
}
|
||||
/**
|
||||
* Get all registered storage providers
|
||||
*/
|
||||
getStorageProviders() {
|
||||
return Array.from(this.storageProviders.values())
|
||||
.sort((a, b) => (b.priority || 0) - (a.priority || 0));
|
||||
}
|
||||
/**
|
||||
* Get all registered presets (built-in + extensions)
|
||||
*/
|
||||
getAllPresets() {
|
||||
// Start with built-in presets
|
||||
const allPresets = new Map();
|
||||
// Note: Would import from distributedPresets-new.ts
|
||||
// Add extended presets
|
||||
for (const [name, extension] of this.presetExtensions) {
|
||||
if (extension.override || !allPresets.has(name)) {
|
||||
allPresets.set(name, extension.config);
|
||||
}
|
||||
}
|
||||
return allPresets;
|
||||
}
|
||||
/**
|
||||
* Auto-detect storage including extensions
|
||||
*/
|
||||
async autoDetectStorage() {
|
||||
// Check registered providers first (in priority order)
|
||||
for (const provider of this.getStorageProviders()) {
|
||||
try {
|
||||
if (await provider.detect()) {
|
||||
const config = await provider.getConfig();
|
||||
return {
|
||||
type: provider.type,
|
||||
config,
|
||||
reason: `Auto-detected ${provider.name}`,
|
||||
autoSelected: true
|
||||
};
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
console.warn(`Failed to detect ${provider.type}:`, error);
|
||||
}
|
||||
}
|
||||
// Fallback to built-in detection
|
||||
const { autoDetectStorage } = await import('./storageAutoConfig.js');
|
||||
return autoDetectStorage();
|
||||
}
|
||||
/**
|
||||
* Register built-in providers
|
||||
*/
|
||||
registerBuiltInProviders() {
|
||||
// These would be the built-in ones, but could be overridden
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Example: Redis storage provider registration
|
||||
* This would be in the redis augmentation package
|
||||
*/
|
||||
export const redisStorageProvider = {
|
||||
type: 'redis',
|
||||
name: 'Redis Storage',
|
||||
description: 'High-performance in-memory data store',
|
||||
priority: 10, // Check before filesystem
|
||||
requirements: {
|
||||
env: ['REDIS_URL', 'REDIS_HOST'],
|
||||
packages: ['redis', 'ioredis']
|
||||
},
|
||||
async detect() {
|
||||
// Check for Redis connection info
|
||||
if (process.env.REDIS_URL || process.env.REDIS_HOST) {
|
||||
try {
|
||||
// Try to connect to Redis (dynamic import for optional dependency)
|
||||
const redis = await new Function('return import("ioredis")')().catch(() => null);
|
||||
if (!redis)
|
||||
return false;
|
||||
const client = new redis.default(process.env.REDIS_URL);
|
||||
await client.ping();
|
||||
await client.quit();
|
||||
return true;
|
||||
}
|
||||
catch {
|
||||
// Redis not available
|
||||
}
|
||||
}
|
||||
return false;
|
||||
},
|
||||
async getConfig() {
|
||||
return {
|
||||
type: 'redis',
|
||||
redisStorage: {
|
||||
url: process.env.REDIS_URL || `redis://${process.env.REDIS_HOST}:${process.env.REDIS_PORT || 6379}`,
|
||||
prefix: process.env.REDIS_PREFIX || 'brainy:',
|
||||
ttl: process.env.REDIS_TTL ? parseInt(process.env.REDIS_TTL) : undefined
|
||||
}
|
||||
};
|
||||
}
|
||||
};
|
||||
/**
|
||||
* Example: MongoDB storage provider
|
||||
*/
|
||||
export const mongoStorageProvider = {
|
||||
type: 'mongodb',
|
||||
name: 'MongoDB Storage',
|
||||
description: 'Document database for complex data',
|
||||
priority: 8,
|
||||
requirements: {
|
||||
env: ['MONGODB_URI', 'MONGO_URL'],
|
||||
packages: ['mongodb']
|
||||
},
|
||||
async detect() {
|
||||
if (process.env.MONGODB_URI || process.env.MONGO_URL) {
|
||||
try {
|
||||
const mongodb = await new Function('return import("mongodb")')().catch(() => null);
|
||||
if (!mongodb)
|
||||
return false;
|
||||
const client = new mongodb.MongoClient(process.env.MONGODB_URI || process.env.MONGO_URL);
|
||||
await client.connect();
|
||||
await client.close();
|
||||
return true;
|
||||
}
|
||||
catch {
|
||||
// MongoDB not available
|
||||
}
|
||||
}
|
||||
return false;
|
||||
},
|
||||
async getConfig() {
|
||||
return {
|
||||
type: 'mongodb',
|
||||
mongoStorage: {
|
||||
uri: process.env.MONGODB_URI || process.env.MONGO_URL,
|
||||
database: process.env.MONGO_DATABASE || 'brainy',
|
||||
collection: process.env.MONGO_COLLECTION || 'vectors'
|
||||
}
|
||||
};
|
||||
}
|
||||
};
|
||||
/**
|
||||
* Example: PostgreSQL with pgvector extension
|
||||
*/
|
||||
export const postgresStorageProvider = {
|
||||
type: 'postgres',
|
||||
name: 'PostgreSQL Storage',
|
||||
description: 'PostgreSQL with pgvector for scalable vector search',
|
||||
priority: 9,
|
||||
requirements: {
|
||||
env: ['DATABASE_URL', 'POSTGRES_URL'],
|
||||
packages: ['pg', 'pgvector']
|
||||
},
|
||||
async detect() {
|
||||
const url = process.env.DATABASE_URL || process.env.POSTGRES_URL;
|
||||
if (url && url.includes('postgres')) {
|
||||
try {
|
||||
const pg = await new Function('return import("pg")')().catch(() => null);
|
||||
if (!pg)
|
||||
return false;
|
||||
const client = new pg.Client({ connectionString: url });
|
||||
await client.connect();
|
||||
// Check for pgvector extension
|
||||
const result = await client.query("SELECT * FROM pg_extension WHERE extname = 'vector'");
|
||||
await client.end();
|
||||
return result.rows.length > 0;
|
||||
}
|
||||
catch {
|
||||
// PostgreSQL not available or pgvector not installed
|
||||
}
|
||||
}
|
||||
return false;
|
||||
},
|
||||
async getConfig() {
|
||||
return {
|
||||
type: 'postgres',
|
||||
postgresStorage: {
|
||||
connectionString: process.env.DATABASE_URL || process.env.POSTGRES_URL,
|
||||
table: process.env.POSTGRES_TABLE || 'brainy_vectors',
|
||||
schema: process.env.POSTGRES_SCHEMA || 'public'
|
||||
}
|
||||
};
|
||||
}
|
||||
};
|
||||
/**
|
||||
* How an augmentation would register its storage provider
|
||||
*/
|
||||
export function registerStorageAugmentation(provider) {
|
||||
const registry = ConfigurationRegistry.getInstance();
|
||||
registry.registerStorageProvider(provider);
|
||||
}
|
||||
/**
|
||||
* How to register a new preset
|
||||
*/
|
||||
export function registerPresetAugmentation(name, config) {
|
||||
const registry = ConfigurationRegistry.getInstance();
|
||||
registry.registerPreset(name, {
|
||||
name,
|
||||
config,
|
||||
override: false
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Example preset for Redis-based caching service
|
||||
*/
|
||||
export const redisCachePreset = {
|
||||
storage: 'redis', // Extended storage type
|
||||
model: ModelPrecision.Q8,
|
||||
features: ['core', 'cache', 'search'],
|
||||
distributed: true,
|
||||
role: DistributedRole.READER,
|
||||
readOnly: true,
|
||||
cache: {
|
||||
hotCacheMaxSize: 50000, // Large Redis cache
|
||||
autoTune: true
|
||||
},
|
||||
description: 'Redis-backed caching layer',
|
||||
category: PresetCategory.SERVICE
|
||||
};
|
||||
/**
|
||||
* Get the configuration registry
|
||||
*/
|
||||
export function getConfigRegistry() {
|
||||
return ConfigurationRegistry.getInstance();
|
||||
}
|
||||
//# sourceMappingURL=extensibleConfig.js.map
|
||||
File diff suppressed because one or more lines are too long
17
.recovery-workspace/dist-backup-20250910-141917/config/index.d.ts
vendored
Normal file
17
.recovery-workspace/dist-backup-20250910-141917/config/index.d.ts
vendored
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
/**
|
||||
* Zero-Configuration System
|
||||
* Main entry point for all auto-configuration features
|
||||
*/
|
||||
export { autoSelectModelPrecision, ModelPrecision as ModelPrecisionType, // Avoid conflict
|
||||
ModelPreset, shouldAutoDownloadModels, getModelPath, logModelConfig } from './modelAutoConfig.js';
|
||||
export { ModelPrecisionManager, getModelPrecision, setModelPrecision, lockModelPrecision, validateModelPrecision } from './modelPrecisionManager.js';
|
||||
export { autoDetectStorage, StorageType, StoragePreset, StorageConfigResult, logStorageConfig, type StorageTypeString, type StoragePresetString } from './storageAutoConfig.js';
|
||||
export { SharedConfig, SharedConfigManager } from './sharedConfigManager.js';
|
||||
export { BrainyZeroConfig, processZeroConfig, createEmbeddingFunctionWithPrecision } from './zeroConfig.js';
|
||||
export { PresetName, PresetCategory, ModelPrecision, StorageOption, FeatureSet, DistributedRole, PresetConfig, PRESET_CONFIGS, getPreset, isValidPreset, getPresetsByCategory, getAllPresetNames, getPresetDescription, presetToBrainyConfig } from './distributedPresets.js';
|
||||
export { StorageProvider, registerStorageAugmentation, registerPresetAugmentation, getConfigRegistry } from './extensibleConfig.js';
|
||||
/**
|
||||
* Main zero-config processor
|
||||
* This is what Brainy will call
|
||||
*/
|
||||
export declare function applyZeroConfig(input?: string | any): Promise<any>;
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
/**
|
||||
* Zero-Configuration System
|
||||
* Main entry point for all auto-configuration features
|
||||
*/
|
||||
// Model configuration
|
||||
export { autoSelectModelPrecision, shouldAutoDownloadModels, getModelPath, logModelConfig } from './modelAutoConfig.js';
|
||||
// Model precision manager
|
||||
export { ModelPrecisionManager, getModelPrecision, setModelPrecision, lockModelPrecision, validateModelPrecision } from './modelPrecisionManager.js';
|
||||
// Storage configuration
|
||||
export { autoDetectStorage, StorageType, StoragePreset, logStorageConfig } from './storageAutoConfig.js';
|
||||
// Shared configuration for multi-instance
|
||||
export { SharedConfigManager } from './sharedConfigManager.js';
|
||||
// Main zero-config processor
|
||||
export { processZeroConfig, createEmbeddingFunctionWithPrecision } from './zeroConfig.js';
|
||||
// Strongly-typed presets and enums
|
||||
export { PresetName, PresetCategory, ModelPrecision, StorageOption, FeatureSet, DistributedRole, PRESET_CONFIGS, getPreset, isValidPreset, getPresetsByCategory, getAllPresetNames, getPresetDescription, presetToBrainyConfig } from './distributedPresets.js';
|
||||
// Extensible configuration
|
||||
export { registerStorageAugmentation, registerPresetAugmentation, getConfigRegistry } from './extensibleConfig.js';
|
||||
/**
|
||||
* Main zero-config processor
|
||||
* This is what Brainy will call
|
||||
*/
|
||||
export async function applyZeroConfig(input) {
|
||||
// Handle legacy config (full object) by detecting known legacy properties
|
||||
if (input && typeof input === 'object' &&
|
||||
(input.storage?.forceMemoryStorage || input.storage?.forceFileSystemStorage || input.storage?.s3Storage)) {
|
||||
// This is a legacy config object - pass through unchanged
|
||||
console.log('📦 Using legacy configuration format');
|
||||
return input;
|
||||
}
|
||||
// Process as zero-config (includes new object format)
|
||||
const { processZeroConfig } = await import('./zeroConfig.js');
|
||||
return processZeroConfig(input);
|
||||
}
|
||||
//# sourceMappingURL=index.js.map
|
||||
|
|
@ -0,0 +1 @@
|
|||
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,sBAAsB;AACtB,OAAO,EACL,wBAAwB,EAGxB,wBAAwB,EACxB,YAAY,EACZ,cAAc,EACf,MAAM,sBAAsB,CAAA;AAE7B,0BAA0B;AAC1B,OAAO,EACL,qBAAqB,EACrB,iBAAiB,EACjB,iBAAiB,EACjB,kBAAkB,EAClB,sBAAsB,EACvB,MAAM,4BAA4B,CAAA;AAEnC,wBAAwB;AACxB,OAAO,EACL,iBAAiB,EACjB,WAAW,EACX,aAAa,EAEb,gBAAgB,EAIjB,MAAM,wBAAwB,CAAA;AAE/B,0CAA0C;AAC1C,OAAO,EAEL,mBAAmB,EACpB,MAAM,0BAA0B,CAAA;AAEjC,6BAA6B;AAC7B,OAAO,EAEL,iBAAiB,EACjB,oCAAoC,EACrC,MAAM,iBAAiB,CAAA;AAExB,mCAAmC;AACnC,OAAO,EACL,UAAU,EACV,cAAc,EACd,cAAc,EACd,aAAa,EACb,UAAU,EACV,eAAe,EAEf,cAAc,EACd,SAAS,EACT,aAAa,EACb,oBAAoB,EACpB,iBAAiB,EACjB,oBAAoB,EACpB,oBAAoB,EACrB,MAAM,yBAAyB,CAAA;AAEhC,2BAA2B;AAC3B,OAAO,EAEL,2BAA2B,EAC3B,0BAA0B,EAC1B,iBAAiB,EAClB,MAAM,uBAAuB,CAAA;AAE9B;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,KAAoB;IACxD,0EAA0E;IAC1E,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ;QAClC,CAAC,KAAK,CAAC,OAAO,EAAE,kBAAkB,IAAI,KAAK,CAAC,OAAO,EAAE,sBAAsB,IAAI,KAAK,CAAC,OAAO,EAAE,SAAS,CAAC,EAAE,CAAC;QAC7G,0DAA0D;QAC1D,OAAO,CAAC,GAAG,CAAC,sCAAsC,CAAC,CAAA;QACnD,OAAO,KAAK,CAAA;IACd,CAAC;IAED,sDAAsD;IACtD,MAAM,EAAE,iBAAiB,EAAE,GAAG,MAAM,MAAM,CAAC,iBAAiB,CAAC,CAAA;IAC7D,OAAO,iBAAiB,CAAC,KAAK,CAAC,CAAA;AACjC,CAAC"}
|
||||
33
.recovery-workspace/dist-backup-20250910-141917/config/modelAutoConfig.d.ts
vendored
Normal file
33
.recovery-workspace/dist-backup-20250910-141917/config/modelAutoConfig.d.ts
vendored
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
/**
|
||||
* Model Configuration Auto-Selection
|
||||
* Intelligently selects model precision based on environment
|
||||
* while allowing manual override
|
||||
*/
|
||||
export type ModelPrecision = 'fp32' | 'q8';
|
||||
export type ModelPreset = 'fast' | 'small' | 'auto';
|
||||
interface ModelConfigResult {
|
||||
precision: ModelPrecision;
|
||||
reason: string;
|
||||
autoSelected: boolean;
|
||||
}
|
||||
/**
|
||||
* Auto-select model precision based on environment and resources
|
||||
* DEFAULT: Q8 for optimal size/performance balance
|
||||
* @param override - Manual override: 'fp32', 'q8', 'fast' (fp32), 'small' (q8), or 'auto'
|
||||
*/
|
||||
export declare function autoSelectModelPrecision(override?: ModelPrecision | ModelPreset): ModelConfigResult;
|
||||
/**
|
||||
* Convenience function to check if models need to be downloaded
|
||||
* This replaces the need for BRAINY_ALLOW_REMOTE_MODELS
|
||||
*/
|
||||
export declare function shouldAutoDownloadModels(): boolean;
|
||||
/**
|
||||
* Get the model path with intelligent defaults
|
||||
* This replaces the need for BRAINY_MODELS_PATH env var
|
||||
*/
|
||||
export declare function getModelPath(): string;
|
||||
/**
|
||||
* Log model configuration decision (only in verbose mode)
|
||||
*/
|
||||
export declare function logModelConfig(config: ModelConfigResult, verbose?: boolean): void;
|
||||
export {};
|
||||
|
|
@ -0,0 +1,198 @@
|
|||
/**
|
||||
* Model Configuration Auto-Selection
|
||||
* Intelligently selects model precision based on environment
|
||||
* while allowing manual override
|
||||
*/
|
||||
import { isBrowser, isNode } from '../utils/environment.js';
|
||||
import { setModelPrecision } from './modelPrecisionManager.js';
|
||||
/**
|
||||
* Auto-select model precision based on environment and resources
|
||||
* DEFAULT: Q8 for optimal size/performance balance
|
||||
* @param override - Manual override: 'fp32', 'q8', 'fast' (fp32), 'small' (q8), or 'auto'
|
||||
*/
|
||||
export function autoSelectModelPrecision(override) {
|
||||
// Handle direct precision override
|
||||
if (override === 'fp32' || override === 'q8') {
|
||||
setModelPrecision(override); // Update central config
|
||||
return {
|
||||
precision: override,
|
||||
reason: `Manually specified: ${override}`,
|
||||
autoSelected: false
|
||||
};
|
||||
}
|
||||
// Handle preset overrides
|
||||
if (override === 'fast') {
|
||||
setModelPrecision('fp32'); // Update central config
|
||||
return {
|
||||
precision: 'fp32',
|
||||
reason: 'Preset: fast (fp32 for best quality)',
|
||||
autoSelected: false
|
||||
};
|
||||
}
|
||||
if (override === 'small') {
|
||||
setModelPrecision('q8'); // Update central config
|
||||
return {
|
||||
precision: 'q8',
|
||||
reason: 'Preset: small (q8 for reduced size)',
|
||||
autoSelected: false
|
||||
};
|
||||
}
|
||||
// Auto-selection logic
|
||||
return autoDetectBestPrecision();
|
||||
}
|
||||
/**
|
||||
* Automatically detect the best model precision for the environment
|
||||
* NEW DEFAULT: Q8 for optimal size/performance (75% smaller, 99% accuracy)
|
||||
*/
|
||||
function autoDetectBestPrecision() {
|
||||
// Check if user explicitly wants FP32 via environment variable
|
||||
if (process.env.BRAINY_FORCE_FP32 === 'true') {
|
||||
setModelPrecision('fp32');
|
||||
return {
|
||||
precision: 'fp32',
|
||||
reason: 'FP32 forced via BRAINY_FORCE_FP32 environment variable',
|
||||
autoSelected: false
|
||||
};
|
||||
}
|
||||
// Browser environment - use Q8 for smaller download/memory
|
||||
if (isBrowser()) {
|
||||
setModelPrecision('q8');
|
||||
return {
|
||||
precision: 'q8',
|
||||
reason: 'Browser environment - using Q8 (23MB vs 90MB)',
|
||||
autoSelected: true
|
||||
};
|
||||
}
|
||||
// Serverless environments - use Q8 for faster cold starts
|
||||
if (isServerlessEnvironment()) {
|
||||
setModelPrecision('q8');
|
||||
return {
|
||||
precision: 'q8',
|
||||
reason: 'Serverless environment - using Q8 for 75% faster cold starts',
|
||||
autoSelected: true
|
||||
};
|
||||
}
|
||||
// Check available memory
|
||||
const memoryMB = getAvailableMemoryMB();
|
||||
// Only use FP32 if explicitly high memory AND user opts in
|
||||
if (memoryMB >= 4096 && process.env.BRAINY_PREFER_QUALITY === 'true') {
|
||||
setModelPrecision('fp32');
|
||||
return {
|
||||
precision: 'fp32',
|
||||
reason: `High memory (${memoryMB}MB) + quality preference - using FP32`,
|
||||
autoSelected: true
|
||||
};
|
||||
}
|
||||
// DEFAULT TO Q8 - Optimal for 99% of use cases
|
||||
// Q8 provides 99% accuracy at 25% of the size
|
||||
setModelPrecision('q8');
|
||||
return {
|
||||
precision: 'q8',
|
||||
reason: 'Default: Q8 model (23MB, 99% accuracy, 4x faster loads)',
|
||||
autoSelected: true
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Check if running in a serverless environment
|
||||
*/
|
||||
function isServerlessEnvironment() {
|
||||
if (!isNode())
|
||||
return false;
|
||||
return !!(process.env.AWS_LAMBDA_FUNCTION_NAME ||
|
||||
process.env.VERCEL ||
|
||||
process.env.NETLIFY ||
|
||||
process.env.CLOUDFLARE_WORKERS ||
|
||||
process.env.FUNCTIONS_WORKER_RUNTIME ||
|
||||
process.env.K_SERVICE // Google Cloud Run
|
||||
);
|
||||
}
|
||||
/**
|
||||
* Get available memory in MB
|
||||
*/
|
||||
function getAvailableMemoryMB() {
|
||||
if (isBrowser()) {
|
||||
// @ts-ignore - navigator.deviceMemory is experimental
|
||||
if (navigator.deviceMemory) {
|
||||
// @ts-ignore
|
||||
return navigator.deviceMemory * 1024; // Device memory in GB
|
||||
}
|
||||
return 256; // Conservative default for browsers
|
||||
}
|
||||
if (isNode()) {
|
||||
try {
|
||||
// Try to get memory info synchronously for Node.js
|
||||
// This will be available in Node.js environments
|
||||
if (typeof process !== 'undefined' && process.memoryUsage) {
|
||||
// Use RSS (Resident Set Size) as a proxy for available memory
|
||||
const rss = process.memoryUsage().rss;
|
||||
// Assume we can use up to 4GB or 50% more than current usage
|
||||
return Math.min(4096, Math.floor(rss / (1024 * 1024) * 1.5));
|
||||
}
|
||||
}
|
||||
catch {
|
||||
// Fall through to default
|
||||
}
|
||||
return 1024; // Default 1GB for Node.js
|
||||
}
|
||||
return 512; // Conservative default
|
||||
}
|
||||
/**
|
||||
* Convenience function to check if models need to be downloaded
|
||||
* This replaces the need for BRAINY_ALLOW_REMOTE_MODELS
|
||||
*/
|
||||
export function shouldAutoDownloadModels() {
|
||||
// Always allow downloads unless explicitly disabled
|
||||
// This eliminates the need for BRAINY_ALLOW_REMOTE_MODELS
|
||||
const explicitlyDisabled = process.env.BRAINY_ALLOW_REMOTE_MODELS === 'false';
|
||||
if (explicitlyDisabled) {
|
||||
console.warn('Model downloads disabled via BRAINY_ALLOW_REMOTE_MODELS=false');
|
||||
return false;
|
||||
}
|
||||
// In production, always allow downloads for seamless operation
|
||||
if (process.env.NODE_ENV === 'production') {
|
||||
return true;
|
||||
}
|
||||
// In development, allow downloads with a one-time notice
|
||||
if (process.env.NODE_ENV === 'development') {
|
||||
return true;
|
||||
}
|
||||
// Default: allow downloads
|
||||
return true;
|
||||
}
|
||||
/**
|
||||
* Get the model path with intelligent defaults
|
||||
* This replaces the need for BRAINY_MODELS_PATH env var
|
||||
*/
|
||||
export function getModelPath() {
|
||||
// Check if user explicitly set a path (keeping this for advanced users)
|
||||
if (process.env.BRAINY_MODELS_PATH) {
|
||||
return process.env.BRAINY_MODELS_PATH;
|
||||
}
|
||||
// Browser - use cache API or IndexedDB (handled by transformers.js)
|
||||
if (isBrowser()) {
|
||||
return 'browser-cache';
|
||||
}
|
||||
// Serverless - use /tmp for ephemeral storage
|
||||
if (isServerlessEnvironment()) {
|
||||
return '/tmp/.brainy/models';
|
||||
}
|
||||
// Node.js - use home directory for persistent storage
|
||||
if (isNode()) {
|
||||
// Use process.env.HOME as a fallback
|
||||
const homeDir = process.env.HOME || process.env.USERPROFILE || '~';
|
||||
return `${homeDir}/.brainy/models`;
|
||||
}
|
||||
// Fallback
|
||||
return './.brainy/models';
|
||||
}
|
||||
/**
|
||||
* Log model configuration decision (only in verbose mode)
|
||||
*/
|
||||
export function logModelConfig(config, verbose = false) {
|
||||
if (!verbose && process.env.NODE_ENV === 'production') {
|
||||
return; // Silent in production unless verbose
|
||||
}
|
||||
const icon = config.autoSelected ? '🤖' : '👤';
|
||||
console.log(`${icon} Model: ${config.precision.toUpperCase()} - ${config.reason}`);
|
||||
}
|
||||
//# sourceMappingURL=modelAutoConfig.js.map
|
||||
|
|
@ -0,0 +1 @@
|
|||
{"version":3,"file":"modelAutoConfig.js","sourceRoot":"","sources":["../../src/config/modelAutoConfig.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAA;AAC3D,OAAO,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAA;AAW9D;;;;GAIG;AACH,MAAM,UAAU,wBAAwB,CAAC,QAAuC;IAC9E,mCAAmC;IACnC,IAAI,QAAQ,KAAK,MAAM,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;QAC7C,iBAAiB,CAAC,QAAQ,CAAC,CAAA,CAAC,wBAAwB;QACpD,OAAO;YACL,SAAS,EAAE,QAAQ;YACnB,MAAM,EAAE,uBAAuB,QAAQ,EAAE;YACzC,YAAY,EAAE,KAAK;SACpB,CAAA;IACH,CAAC;IAED,0BAA0B;IAC1B,IAAI,QAAQ,KAAK,MAAM,EAAE,CAAC;QACxB,iBAAiB,CAAC,MAAM,CAAC,CAAA,CAAC,wBAAwB;QAClD,OAAO;YACL,SAAS,EAAE,MAAM;YACjB,MAAM,EAAE,sCAAsC;YAC9C,YAAY,EAAE,KAAK;SACpB,CAAA;IACH,CAAC;IAED,IAAI,QAAQ,KAAK,OAAO,EAAE,CAAC;QACzB,iBAAiB,CAAC,IAAI,CAAC,CAAA,CAAC,wBAAwB;QAChD,OAAO;YACL,SAAS,EAAE,IAAI;YACf,MAAM,EAAE,qCAAqC;YAC7C,YAAY,EAAE,KAAK;SACpB,CAAA;IACH,CAAC;IAED,uBAAuB;IACvB,OAAO,uBAAuB,EAAE,CAAA;AAClC,CAAC;AAED;;;GAGG;AACH,SAAS,uBAAuB;IAC9B,+DAA+D;IAC/D,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,KAAK,MAAM,EAAE,CAAC;QAC7C,iBAAiB,CAAC,MAAM,CAAC,CAAA;QACzB,OAAO;YACL,SAAS,EAAE,MAAM;YACjB,MAAM,EAAE,wDAAwD;YAChE,YAAY,EAAE,KAAK;SACpB,CAAA;IACH,CAAC;IAED,2DAA2D;IAC3D,IAAI,SAAS,EAAE,EAAE,CAAC;QAChB,iBAAiB,CAAC,IAAI,CAAC,CAAA;QACvB,OAAO;YACL,SAAS,EAAE,IAAI;YACf,MAAM,EAAE,+CAA+C;YACvD,YAAY,EAAE,IAAI;SACnB,CAAA;IACH,CAAC;IAED,0DAA0D;IAC1D,IAAI,uBAAuB,EAAE,EAAE,CAAC;QAC9B,iBAAiB,CAAC,IAAI,CAAC,CAAA;QACvB,OAAO;YACL,SAAS,EAAE,IAAI;YACf,MAAM,EAAE,8DAA8D;YACtE,YAAY,EAAE,IAAI;SACnB,CAAA;IACH,CAAC;IAED,yBAAyB;IACzB,MAAM,QAAQ,GAAG,oBAAoB,EAAE,CAAA;IAEvC,2DAA2D;IAC3D,IAAI,QAAQ,IAAI,IAAI,IAAI,OAAO,CAAC,GAAG,CAAC,qBAAqB,KAAK,MAAM,EAAE,CAAC;QACrE,iBAAiB,CAAC,MAAM,CAAC,CAAA;QACzB,OAAO;YACL,SAAS,EAAE,MAAM;YACjB,MAAM,EAAE,gBAAgB,QAAQ,uCAAuC;YACvE,YAAY,EAAE,IAAI;SACnB,CAAA;IACH,CAAC;IAED,+CAA+C;IAC/C,8CAA8C;IAC9C,iBAAiB,CAAC,IAAI,CAAC,CAAA;IACvB,OAAO;QACL,SAAS,EAAE,IAAI;QACf,MAAM,EAAE,yDAAyD;QACjE,YAAY,EAAE,IAAI;KACnB,CAAA;AACH,CAAC;AAED;;GAEG;AACH,SAAS,uBAAuB;IAC9B,IAAI,CAAC,MAAM,EAAE;QAAE,OAAO,KAAK,CAAA;IAE3B,OAAO,CAAC,CAAC,CACP,OAAO,CAAC,GAAG,CAAC,wBAAwB;QACpC,OAAO,CAAC,GAAG,CAAC,MAAM;QAClB,OAAO,CAAC,GAAG,CAAC,OAAO;QACnB,OAAO,CAAC,GAAG,CAAC,kBAAkB;QAC9B,OAAO,CAAC,GAAG,CAAC,wBAAwB;QACpC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,mBAAmB;KAC1C,CAAA;AACH,CAAC;AAED;;GAEG;AACH,SAAS,oBAAoB;IAC3B,IAAI,SAAS,EAAE,EAAE,CAAC;QAChB,sDAAsD;QACtD,IAAI,SAAS,CAAC,YAAY,EAAE,CAAC;YAC3B,aAAa;YACb,OAAO,SAAS,CAAC,YAAY,GAAG,IAAI,CAAA,CAAC,sBAAsB;QAC7D,CAAC;QACD,OAAO,GAAG,CAAA,CAAC,oCAAoC;IACjD,CAAC;IAED,IAAI,MAAM,EAAE,EAAE,CAAC;QACb,IAAI,CAAC;YACH,mDAAmD;YACnD,iDAAiD;YACjD,IAAI,OAAO,OAAO,KAAK,WAAW,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;gBAC1D,8DAA8D;gBAC9D,MAAM,GAAG,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,GAAG,CAAA;gBACrC,6DAA6D;gBAC7D,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC,CAAA;YAC9D,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,0BAA0B;QAC5B,CAAC;QACD,OAAO,IAAI,CAAA,CAAC,0BAA0B;IACxC,CAAC;IAED,OAAO,GAAG,CAAA,CAAC,uBAAuB;AACpC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,wBAAwB;IACtC,oDAAoD;IACpD,0DAA0D;IAC1D,MAAM,kBAAkB,GAAG,OAAO,CAAC,GAAG,CAAC,0BAA0B,KAAK,OAAO,CAAA;IAE7E,IAAI,kBAAkB,EAAE,CAAC;QACvB,OAAO,CAAC,IAAI,CAAC,+DAA+D,CAAC,CAAA;QAC7E,OAAO,KAAK,CAAA;IACd,CAAC;IAED,+DAA+D;IAC/D,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,YAAY,EAAE,CAAC;QAC1C,OAAO,IAAI,CAAA;IACb,CAAC;IAED,yDAAyD;IACzD,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,aAAa,EAAE,CAAC;QAC3C,OAAO,IAAI,CAAA;IACb,CAAC;IAED,2BAA2B;IAC3B,OAAO,IAAI,CAAA;AACb,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,YAAY;IAC1B,wEAAwE;IACxE,IAAI,OAAO,CAAC,GAAG,CAAC,kBAAkB,EAAE,CAAC;QACnC,OAAO,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAA;IACvC,CAAC;IAED,oEAAoE;IACpE,IAAI,SAAS,EAAE,EAAE,CAAC;QAChB,OAAO,eAAe,CAAA;IACxB,CAAC;IAED,8CAA8C;IAC9C,IAAI,uBAAuB,EAAE,EAAE,CAAC;QAC9B,OAAO,qBAAqB,CAAA;IAC9B,CAAC;IAED,sDAAsD;IACtD,IAAI,MAAM,EAAE,EAAE,CAAC;QACb,qCAAqC;QACrC,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,OAAO,CAAC,GAAG,CAAC,WAAW,IAAI,GAAG,CAAA;QAClE,OAAO,GAAG,OAAO,iBAAiB,CAAA;IACpC,CAAC;IAED,WAAW;IACX,OAAO,kBAAkB,CAAA;AAC3B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,MAAyB,EAAE,UAAmB,KAAK;IAChF,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,YAAY,EAAE,CAAC;QACtD,OAAM,CAAC,sCAAsC;IAC/C,CAAC;IAED,MAAM,IAAI,GAAG,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAA;IAC9C,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,WAAW,MAAM,CAAC,SAAS,CAAC,WAAW,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;AACpF,CAAC"}
|
||||
42
.recovery-workspace/dist-backup-20250910-141917/config/modelPrecisionManager.d.ts
vendored
Normal file
42
.recovery-workspace/dist-backup-20250910-141917/config/modelPrecisionManager.d.ts
vendored
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
/**
|
||||
* Central Model Precision Manager
|
||||
*
|
||||
* Single source of truth for model precision configuration.
|
||||
* Ensures consistent usage of Q8 or FP32 models throughout the system.
|
||||
*/
|
||||
import { ModelPrecision } from './modelAutoConfig.js';
|
||||
export declare class ModelPrecisionManager {
|
||||
private static instance;
|
||||
private precision;
|
||||
private isLocked;
|
||||
private constructor();
|
||||
static getInstance(): ModelPrecisionManager;
|
||||
/**
|
||||
* Get the current model precision
|
||||
*/
|
||||
getPrecision(): ModelPrecision;
|
||||
/**
|
||||
* Set the model precision (can only be done before first model load)
|
||||
*/
|
||||
setPrecision(precision: ModelPrecision): void;
|
||||
/**
|
||||
* Lock the precision (called after first model load)
|
||||
*/
|
||||
lock(): void;
|
||||
/**
|
||||
* Check if precision is locked
|
||||
*/
|
||||
isConfigLocked(): boolean;
|
||||
/**
|
||||
* Get precision info for logging
|
||||
*/
|
||||
getInfo(): string;
|
||||
/**
|
||||
* Validate that a given precision matches the configured one
|
||||
*/
|
||||
validatePrecision(precision: ModelPrecision): boolean;
|
||||
}
|
||||
export declare const getModelPrecision: () => ModelPrecision;
|
||||
export declare const setModelPrecision: (precision: ModelPrecision) => void;
|
||||
export declare const lockModelPrecision: () => void;
|
||||
export declare const validateModelPrecision: (precision: ModelPrecision) => boolean;
|
||||
|
|
@ -0,0 +1,98 @@
|
|||
/**
|
||||
* Central Model Precision Manager
|
||||
*
|
||||
* Single source of truth for model precision configuration.
|
||||
* Ensures consistent usage of Q8 or FP32 models throughout the system.
|
||||
*/
|
||||
export class ModelPrecisionManager {
|
||||
constructor() {
|
||||
this.precision = 'q8'; // DEFAULT TO Q8
|
||||
this.isLocked = false;
|
||||
// Check environment variable override
|
||||
const envPrecision = process.env.BRAINY_MODEL_PRECISION;
|
||||
if (envPrecision === 'fp32' || envPrecision === 'q8') {
|
||||
this.precision = envPrecision;
|
||||
console.log(`Model precision set from environment: ${envPrecision.toUpperCase()}`);
|
||||
}
|
||||
else {
|
||||
console.log('Using default model precision: Q8 (75% smaller, 99% accuracy)');
|
||||
}
|
||||
}
|
||||
static getInstance() {
|
||||
if (!ModelPrecisionManager.instance) {
|
||||
ModelPrecisionManager.instance = new ModelPrecisionManager();
|
||||
}
|
||||
return ModelPrecisionManager.instance;
|
||||
}
|
||||
/**
|
||||
* Get the current model precision
|
||||
*/
|
||||
getPrecision() {
|
||||
return this.precision;
|
||||
}
|
||||
/**
|
||||
* Set the model precision (can only be done before first model load)
|
||||
*/
|
||||
setPrecision(precision) {
|
||||
if (this.isLocked) {
|
||||
console.warn(`⚠️ Cannot change precision after model initialization. Current: ${this.precision.toUpperCase()}`);
|
||||
return;
|
||||
}
|
||||
if (precision !== this.precision) {
|
||||
console.log(`Model precision changed: ${this.precision.toUpperCase()} → ${precision.toUpperCase()}`);
|
||||
this.precision = precision;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Lock the precision (called after first model load)
|
||||
*/
|
||||
lock() {
|
||||
if (!this.isLocked) {
|
||||
this.isLocked = true;
|
||||
console.log(`Model precision locked: ${this.precision.toUpperCase()}`);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Check if precision is locked
|
||||
*/
|
||||
isConfigLocked() {
|
||||
return this.isLocked;
|
||||
}
|
||||
/**
|
||||
* Get precision info for logging
|
||||
*/
|
||||
getInfo() {
|
||||
const info = this.precision === 'q8'
|
||||
? 'Q8 (quantized, 23MB, 99% accuracy)'
|
||||
: 'FP32 (full precision, 90MB, 100% accuracy)';
|
||||
return `${info}${this.isLocked ? ' [LOCKED]' : ''}`;
|
||||
}
|
||||
/**
|
||||
* Validate that a given precision matches the configured one
|
||||
*/
|
||||
validatePrecision(precision) {
|
||||
if (precision !== this.precision) {
|
||||
console.error(`❌ Precision mismatch! Expected: ${this.precision.toUpperCase()}, Got: ${precision.toUpperCase()}`);
|
||||
console.error('This will cause incompatible embeddings!');
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
// Export singleton instance getter
|
||||
export const getModelPrecision = () => {
|
||||
return ModelPrecisionManager.getInstance().getPrecision();
|
||||
};
|
||||
// Export setter (for configuration phase)
|
||||
export const setModelPrecision = (precision) => {
|
||||
ModelPrecisionManager.getInstance().setPrecision(precision);
|
||||
};
|
||||
// Export lock function (for after model initialization)
|
||||
export const lockModelPrecision = () => {
|
||||
ModelPrecisionManager.getInstance().lock();
|
||||
};
|
||||
// Export validation function
|
||||
export const validateModelPrecision = (precision) => {
|
||||
return ModelPrecisionManager.getInstance().validatePrecision(precision);
|
||||
};
|
||||
//# sourceMappingURL=modelPrecisionManager.js.map
|
||||
|
|
@ -0,0 +1 @@
|
|||
{"version":3,"file":"modelPrecisionManager.js","sourceRoot":"","sources":["../../src/config/modelPrecisionManager.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH,MAAM,OAAO,qBAAqB;IAKhC;QAHQ,cAAS,GAAmB,IAAI,CAAA,CAAC,gBAAgB;QACjD,aAAQ,GAAG,KAAK,CAAA;QAGtB,sCAAsC;QACtC,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAA;QACvD,IAAI,YAAY,KAAK,MAAM,IAAI,YAAY,KAAK,IAAI,EAAE,CAAC;YACrD,IAAI,CAAC,SAAS,GAAG,YAAY,CAAA;YAC7B,OAAO,CAAC,GAAG,CAAC,yCAAyC,YAAY,CAAC,WAAW,EAAE,EAAE,CAAC,CAAA;QACpF,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CAAC,+DAA+D,CAAC,CAAA;QAC9E,CAAC;IACH,CAAC;IAED,MAAM,CAAC,WAAW;QAChB,IAAI,CAAC,qBAAqB,CAAC,QAAQ,EAAE,CAAC;YACpC,qBAAqB,CAAC,QAAQ,GAAG,IAAI,qBAAqB,EAAE,CAAA;QAC9D,CAAC;QACD,OAAO,qBAAqB,CAAC,QAAQ,CAAA;IACvC,CAAC;IAED;;OAEG;IACH,YAAY;QACV,OAAO,IAAI,CAAC,SAAS,CAAA;IACvB,CAAC;IAED;;OAEG;IACH,YAAY,CAAC,SAAyB;QACpC,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,OAAO,CAAC,IAAI,CAAC,mEAAmE,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,EAAE,CAAC,CAAA;YAC/G,OAAM;QACR,CAAC;QAED,IAAI,SAAS,KAAK,IAAI,CAAC,SAAS,EAAE,CAAC;YACjC,OAAO,CAAC,GAAG,CAAC,4BAA4B,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,MAAM,SAAS,CAAC,WAAW,EAAE,EAAE,CAAC,CAAA;YACpG,IAAI,CAAC,SAAS,GAAG,SAAS,CAAA;QAC5B,CAAC;IACH,CAAC;IAED;;OAEG;IACH,IAAI;QACF,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAA;YACpB,OAAO,CAAC,GAAG,CAAC,2BAA2B,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,EAAE,CAAC,CAAA;QACxE,CAAC;IACH,CAAC;IAED;;OAEG;IACH,cAAc;QACZ,OAAO,IAAI,CAAC,QAAQ,CAAA;IACtB,CAAC;IAED;;OAEG;IACH,OAAO;QACL,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,KAAK,IAAI;YAClC,CAAC,CAAC,oCAAoC;YACtC,CAAC,CAAC,4CAA4C,CAAA;QAChD,OAAO,GAAG,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,EAAE,CAAA;IACrD,CAAC;IAED;;OAEG;IACH,iBAAiB,CAAC,SAAyB;QACzC,IAAI,SAAS,KAAK,IAAI,CAAC,SAAS,EAAE,CAAC;YACjC,OAAO,CAAC,KAAK,CAAC,mCAAmC,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,UAAU,SAAS,CAAC,WAAW,EAAE,EAAE,CAAC,CAAA;YACjH,OAAO,CAAC,KAAK,CAAC,0CAA0C,CAAC,CAAA;YACzD,OAAO,KAAK,CAAA;QACd,CAAC;QACD,OAAO,IAAI,CAAA;IACb,CAAC;CACF;AAED,mCAAmC;AACnC,MAAM,CAAC,MAAM,iBAAiB,GAAG,GAAmB,EAAE;IACpD,OAAO,qBAAqB,CAAC,WAAW,EAAE,CAAC,YAAY,EAAE,CAAA;AAC3D,CAAC,CAAA;AAED,0CAA0C;AAC1C,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,SAAyB,EAAQ,EAAE;IACnE,qBAAqB,CAAC,WAAW,EAAE,CAAC,YAAY,CAAC,SAAS,CAAC,CAAA;AAC7D,CAAC,CAAA;AAED,wDAAwD;AACxD,MAAM,CAAC,MAAM,kBAAkB,GAAG,GAAS,EAAE;IAC3C,qBAAqB,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAA;AAC5C,CAAC,CAAA;AAED,6BAA6B;AAC7B,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,SAAyB,EAAW,EAAE;IAC3E,OAAO,qBAAqB,CAAC,WAAW,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAA;AACzE,CAAC,CAAA"}
|
||||
67
.recovery-workspace/dist-backup-20250910-141917/config/sharedConfigManager.d.ts
vendored
Normal file
67
.recovery-workspace/dist-backup-20250910-141917/config/sharedConfigManager.d.ts
vendored
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
/**
|
||||
* Shared Configuration Manager
|
||||
* Ensures configuration consistency across multiple instances using shared storage
|
||||
*/
|
||||
import { ModelPrecision } from './modelAutoConfig.js';
|
||||
export interface SharedConfig {
|
||||
version: string;
|
||||
precision: ModelPrecision;
|
||||
dimensions: number;
|
||||
hnswM: number;
|
||||
hnswEfConstruction: number;
|
||||
distanceFunction: string;
|
||||
createdAt: string;
|
||||
lastUpdated: string;
|
||||
instanceCount?: number;
|
||||
lastAccessedBy?: string;
|
||||
}
|
||||
/**
|
||||
* Manages configuration consistency for shared storage
|
||||
*/
|
||||
export declare class SharedConfigManager {
|
||||
private static CONFIG_FILE;
|
||||
/**
|
||||
* Load or create shared configuration
|
||||
* When connecting to existing data, this OVERRIDES auto-configuration!
|
||||
*/
|
||||
static loadOrCreateSharedConfig(storage: any, localConfig: any): Promise<{
|
||||
config: any;
|
||||
warnings: string[];
|
||||
}>;
|
||||
/**
|
||||
* Check if storage type is shared (multi-instance)
|
||||
*/
|
||||
private static isSharedStorage;
|
||||
/**
|
||||
* Load configuration from shared storage
|
||||
*/
|
||||
private static loadConfigFromStorage;
|
||||
/**
|
||||
* Save configuration to shared storage
|
||||
*/
|
||||
private static saveConfigToStorage;
|
||||
/**
|
||||
* Create shared configuration from local config
|
||||
*/
|
||||
private static createSharedConfig;
|
||||
/**
|
||||
* Check for critical configuration mismatches
|
||||
*/
|
||||
private static checkCriticalMismatches;
|
||||
/**
|
||||
* Merge local config with shared config (shared takes precedence for critical params)
|
||||
*/
|
||||
private static mergeWithSharedConfig;
|
||||
/**
|
||||
* Update access information in shared config
|
||||
*/
|
||||
private static updateAccessInfo;
|
||||
/**
|
||||
* Get unique identifier for this instance
|
||||
*/
|
||||
private static getInstanceIdentifier;
|
||||
/**
|
||||
* Validate that a configuration is compatible with shared data
|
||||
*/
|
||||
static validateCompatibility(config1: SharedConfig, config2: SharedConfig): boolean;
|
||||
}
|
||||
|
|
@ -0,0 +1,215 @@
|
|||
/**
|
||||
* Shared Configuration Manager
|
||||
* Ensures configuration consistency across multiple instances using shared storage
|
||||
*/
|
||||
/**
|
||||
* Manages configuration consistency for shared storage
|
||||
*/
|
||||
export class SharedConfigManager {
|
||||
/**
|
||||
* Load or create shared configuration
|
||||
* When connecting to existing data, this OVERRIDES auto-configuration!
|
||||
*/
|
||||
static async loadOrCreateSharedConfig(storage, localConfig) {
|
||||
const warnings = [];
|
||||
try {
|
||||
// Check if we're using shared storage
|
||||
if (!this.isSharedStorage(localConfig.storageType)) {
|
||||
// Local storage - use local config
|
||||
return { config: localConfig, warnings: [] };
|
||||
}
|
||||
// Try to load existing configuration from shared storage
|
||||
const existingConfig = await this.loadConfigFromStorage(storage);
|
||||
if (existingConfig) {
|
||||
// EXISTING SHARED DATA - Must use its configuration!
|
||||
console.log('📁 Found existing shared data configuration');
|
||||
// Check for critical mismatches
|
||||
const mismatches = this.checkCriticalMismatches(localConfig, existingConfig);
|
||||
if (mismatches.length > 0) {
|
||||
console.warn('⚠️ Configuration override required for shared storage:');
|
||||
mismatches.forEach(m => {
|
||||
console.warn(` - ${m.param}: ${m.local} → ${m.shared} (using shared)`);
|
||||
warnings.push(`${m.param} overridden: ${m.local} → ${m.shared}`);
|
||||
});
|
||||
}
|
||||
// Override critical parameters with shared values
|
||||
const mergedConfig = this.mergeWithSharedConfig(localConfig, existingConfig);
|
||||
// Update last accessed
|
||||
await this.updateAccessInfo(storage, existingConfig);
|
||||
return { config: mergedConfig, warnings };
|
||||
}
|
||||
else {
|
||||
// NEW SHARED STORAGE - Save our configuration
|
||||
console.log('📝 Initializing new shared storage with configuration');
|
||||
const sharedConfig = this.createSharedConfig(localConfig);
|
||||
await this.saveConfigToStorage(storage, sharedConfig);
|
||||
return { config: localConfig, warnings: [] };
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
console.error('Failed to manage shared configuration:', error);
|
||||
warnings.push('Could not verify shared configuration - proceeding with caution');
|
||||
return { config: localConfig, warnings };
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Check if storage type is shared (multi-instance)
|
||||
*/
|
||||
static isSharedStorage(storageType) {
|
||||
return ['s3', 'gcs', 'r2'].includes(storageType);
|
||||
}
|
||||
/**
|
||||
* Load configuration from shared storage
|
||||
*/
|
||||
static async loadConfigFromStorage(storage) {
|
||||
try {
|
||||
const configData = await storage.get(this.CONFIG_FILE);
|
||||
if (!configData)
|
||||
return null;
|
||||
const config = JSON.parse(configData);
|
||||
return config;
|
||||
}
|
||||
catch (error) {
|
||||
// Config doesn't exist yet
|
||||
return null;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Save configuration to shared storage
|
||||
*/
|
||||
static async saveConfigToStorage(storage, config) {
|
||||
try {
|
||||
await storage.set(this.CONFIG_FILE, JSON.stringify(config, null, 2));
|
||||
}
|
||||
catch (error) {
|
||||
console.error('Failed to save shared configuration:', error);
|
||||
throw new Error('Cannot initialize shared storage without saving configuration');
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Create shared configuration from local config
|
||||
*/
|
||||
static createSharedConfig(localConfig) {
|
||||
return {
|
||||
version: '2.10.0', // Brainy version
|
||||
precision: localConfig.embeddingOptions?.precision || 'fp32',
|
||||
dimensions: 384, // Fixed for all-MiniLM-L6-v2
|
||||
hnswM: localConfig.hnsw?.M || 16,
|
||||
hnswEfConstruction: localConfig.hnsw?.efConstruction || 200,
|
||||
distanceFunction: localConfig.distanceFunction || 'cosine',
|
||||
createdAt: new Date().toISOString(),
|
||||
lastUpdated: new Date().toISOString(),
|
||||
instanceCount: 1,
|
||||
lastAccessedBy: this.getInstanceIdentifier()
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Check for critical configuration mismatches
|
||||
*/
|
||||
static checkCriticalMismatches(localConfig, sharedConfig) {
|
||||
const mismatches = [];
|
||||
// Model precision - CRITICAL!
|
||||
const localPrecision = localConfig.embeddingOptions?.precision || 'fp32';
|
||||
if (localPrecision !== sharedConfig.precision) {
|
||||
mismatches.push({
|
||||
param: 'Model Precision',
|
||||
local: localPrecision,
|
||||
shared: sharedConfig.precision
|
||||
});
|
||||
}
|
||||
// HNSW parameters - Important for index consistency
|
||||
const localM = localConfig.hnsw?.M || 16;
|
||||
if (localM !== sharedConfig.hnswM) {
|
||||
mismatches.push({
|
||||
param: 'HNSW M',
|
||||
local: localM,
|
||||
shared: sharedConfig.hnswM
|
||||
});
|
||||
}
|
||||
const localEf = localConfig.hnsw?.efConstruction || 200;
|
||||
if (localEf !== sharedConfig.hnswEfConstruction) {
|
||||
mismatches.push({
|
||||
param: 'HNSW efConstruction',
|
||||
local: localEf,
|
||||
shared: sharedConfig.hnswEfConstruction
|
||||
});
|
||||
}
|
||||
// Distance function
|
||||
const localDistance = localConfig.distanceFunction || 'cosine';
|
||||
if (localDistance !== sharedConfig.distanceFunction) {
|
||||
mismatches.push({
|
||||
param: 'Distance Function',
|
||||
local: localDistance,
|
||||
shared: sharedConfig.distanceFunction
|
||||
});
|
||||
}
|
||||
return mismatches;
|
||||
}
|
||||
/**
|
||||
* Merge local config with shared config (shared takes precedence for critical params)
|
||||
*/
|
||||
static mergeWithSharedConfig(localConfig, sharedConfig) {
|
||||
return {
|
||||
...localConfig,
|
||||
// Override critical parameters with shared values
|
||||
embeddingOptions: {
|
||||
...localConfig.embeddingOptions,
|
||||
precision: sharedConfig.precision // MUST use shared precision!
|
||||
},
|
||||
hnsw: {
|
||||
...localConfig.hnsw,
|
||||
M: sharedConfig.hnswM,
|
||||
efConstruction: sharedConfig.hnswEfConstruction
|
||||
},
|
||||
distanceFunction: sharedConfig.distanceFunction,
|
||||
// Add metadata about shared configuration
|
||||
_sharedConfig: {
|
||||
loaded: true,
|
||||
version: sharedConfig.version,
|
||||
createdAt: sharedConfig.createdAt,
|
||||
precision: sharedConfig.precision
|
||||
}
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Update access information in shared config
|
||||
*/
|
||||
static async updateAccessInfo(storage, config) {
|
||||
try {
|
||||
config.lastUpdated = new Date().toISOString();
|
||||
config.instanceCount = (config.instanceCount || 0) + 1;
|
||||
config.lastAccessedBy = this.getInstanceIdentifier();
|
||||
await this.saveConfigToStorage(storage, config);
|
||||
}
|
||||
catch {
|
||||
// Non-critical - don't fail if we can't update access info
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Get unique identifier for this instance
|
||||
*/
|
||||
static getInstanceIdentifier() {
|
||||
if (process.env.HOSTNAME)
|
||||
return process.env.HOSTNAME;
|
||||
if (process.env.CONTAINER_ID)
|
||||
return process.env.CONTAINER_ID;
|
||||
if (process.env.K_SERVICE)
|
||||
return process.env.K_SERVICE;
|
||||
if (process.env.AWS_LAMBDA_FUNCTION_NAME)
|
||||
return process.env.AWS_LAMBDA_FUNCTION_NAME;
|
||||
// Generate a random identifier
|
||||
return `instance-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
|
||||
}
|
||||
/**
|
||||
* Validate that a configuration is compatible with shared data
|
||||
*/
|
||||
static validateCompatibility(config1, config2) {
|
||||
return (config1.precision === config2.precision &&
|
||||
config1.dimensions === config2.dimensions &&
|
||||
config1.hnswM === config2.hnswM &&
|
||||
config1.hnswEfConstruction === config2.hnswEfConstruction &&
|
||||
config1.distanceFunction === config2.distanceFunction);
|
||||
}
|
||||
}
|
||||
SharedConfigManager.CONFIG_FILE = '.brainy/config.json';
|
||||
//# sourceMappingURL=sharedConfigManager.js.map
|
||||
File diff suppressed because one or more lines are too long
41
.recovery-workspace/dist-backup-20250910-141917/config/storageAutoConfig.d.ts
vendored
Normal file
41
.recovery-workspace/dist-backup-20250910-141917/config/storageAutoConfig.d.ts
vendored
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
/**
|
||||
* Storage Configuration Auto-Detection
|
||||
* Intelligently selects storage based on environment and available services
|
||||
*/
|
||||
/**
|
||||
* Low-level storage implementation types
|
||||
*/
|
||||
export declare enum StorageType {
|
||||
MEMORY = "memory",
|
||||
FILESYSTEM = "filesystem",
|
||||
OPFS = "opfs",
|
||||
S3 = "s3",
|
||||
GCS = "gcs",
|
||||
R2 = "r2"
|
||||
}
|
||||
/**
|
||||
* High-level storage presets (maps to StorageType)
|
||||
*/
|
||||
export declare enum StoragePreset {
|
||||
AUTO = "auto",
|
||||
MEMORY = "memory",
|
||||
DISK = "disk",
|
||||
CLOUD = "cloud"
|
||||
}
|
||||
export type StorageTypeString = 'memory' | 'filesystem' | 'opfs' | 's3' | 'gcs' | 'r2';
|
||||
export type StoragePresetString = 'auto' | 'memory' | 'disk' | 'cloud';
|
||||
export interface StorageConfigResult {
|
||||
type: StorageType | StorageTypeString;
|
||||
config: any;
|
||||
reason: string;
|
||||
autoSelected: boolean;
|
||||
}
|
||||
/**
|
||||
* Auto-detect the best storage configuration
|
||||
* @param override - Manual override: specific type or preset
|
||||
*/
|
||||
export declare function autoDetectStorage(override?: StorageType | StoragePreset | StorageTypeString | StoragePresetString | any): Promise<StorageConfigResult>;
|
||||
/**
|
||||
* Log storage configuration decision
|
||||
*/
|
||||
export declare function logStorageConfig(config: StorageConfigResult, verbose?: boolean): void;
|
||||
|
|
@ -0,0 +1,328 @@
|
|||
/**
|
||||
* Storage Configuration Auto-Detection
|
||||
* Intelligently selects storage based on environment and available services
|
||||
*/
|
||||
import { isBrowser, isNode } from '../utils/environment.js';
|
||||
/**
|
||||
* Low-level storage implementation types
|
||||
*/
|
||||
export var StorageType;
|
||||
(function (StorageType) {
|
||||
StorageType["MEMORY"] = "memory";
|
||||
StorageType["FILESYSTEM"] = "filesystem";
|
||||
StorageType["OPFS"] = "opfs";
|
||||
StorageType["S3"] = "s3";
|
||||
StorageType["GCS"] = "gcs";
|
||||
StorageType["R2"] = "r2";
|
||||
})(StorageType || (StorageType = {}));
|
||||
/**
|
||||
* High-level storage presets (maps to StorageType)
|
||||
*/
|
||||
export var StoragePreset;
|
||||
(function (StoragePreset) {
|
||||
StoragePreset["AUTO"] = "auto";
|
||||
StoragePreset["MEMORY"] = "memory";
|
||||
StoragePreset["DISK"] = "disk";
|
||||
StoragePreset["CLOUD"] = "cloud";
|
||||
})(StoragePreset || (StoragePreset = {}));
|
||||
/**
|
||||
* Auto-detect the best storage configuration
|
||||
* @param override - Manual override: specific type or preset
|
||||
*/
|
||||
export async function autoDetectStorage(override) {
|
||||
// Handle direct storage config object
|
||||
if (override && typeof override === 'object') {
|
||||
return {
|
||||
type: override.type || 'memory',
|
||||
config: override,
|
||||
reason: 'Manually configured storage',
|
||||
autoSelected: false
|
||||
};
|
||||
}
|
||||
// Handle storage type override (enum values or strings)
|
||||
if (override && Object.values(StorageType).includes(override)) {
|
||||
return {
|
||||
type: override,
|
||||
config: override,
|
||||
reason: `Manually specified: ${override}`,
|
||||
autoSelected: false
|
||||
};
|
||||
}
|
||||
// Handle presets (both enum and string values)
|
||||
if (override === StoragePreset.MEMORY || override === 'memory') {
|
||||
return {
|
||||
type: StorageType.MEMORY,
|
||||
config: StorageType.MEMORY,
|
||||
reason: 'Preset: memory storage',
|
||||
autoSelected: false
|
||||
};
|
||||
}
|
||||
if (override === StoragePreset.DISK || override === 'disk') {
|
||||
const diskType = isBrowser() ? StorageType.OPFS : StorageType.FILESYSTEM;
|
||||
return {
|
||||
type: diskType,
|
||||
config: diskType,
|
||||
reason: `Preset: disk storage (${diskType})`,
|
||||
autoSelected: false
|
||||
};
|
||||
}
|
||||
if (override === StoragePreset.CLOUD || override === 'cloud') {
|
||||
const cloudStorage = await detectCloudStorage();
|
||||
if (cloudStorage) {
|
||||
return {
|
||||
...cloudStorage,
|
||||
reason: `Preset: cloud storage (${cloudStorage.type})`,
|
||||
autoSelected: false
|
||||
};
|
||||
}
|
||||
// Fallback to disk if no cloud storage detected
|
||||
const diskType = isBrowser() ? StorageType.OPFS : StorageType.FILESYSTEM;
|
||||
return {
|
||||
type: diskType,
|
||||
config: diskType,
|
||||
reason: 'Preset: cloud (none detected, using disk)',
|
||||
autoSelected: false
|
||||
};
|
||||
}
|
||||
// Auto-detection logic
|
||||
return await autoDetectBestStorage();
|
||||
}
|
||||
/**
|
||||
* Automatically detect the best storage option
|
||||
*/
|
||||
async function autoDetectBestStorage() {
|
||||
// Check for cloud storage first (highest priority in production)
|
||||
const cloudStorage = await detectCloudStorage();
|
||||
if (cloudStorage && process.env.NODE_ENV === 'production') {
|
||||
return {
|
||||
...cloudStorage,
|
||||
reason: `Auto-detected ${cloudStorage.type} in production`,
|
||||
autoSelected: true
|
||||
};
|
||||
}
|
||||
// Browser environment
|
||||
if (isBrowser()) {
|
||||
// Check for OPFS support
|
||||
if (await hasOPFSSupport()) {
|
||||
return {
|
||||
type: StorageType.OPFS,
|
||||
config: { requestPersistentStorage: true },
|
||||
reason: 'Browser with OPFS support detected',
|
||||
autoSelected: true
|
||||
};
|
||||
}
|
||||
// Fallback to memory for browsers without OPFS
|
||||
return {
|
||||
type: StorageType.MEMORY,
|
||||
config: StorageType.MEMORY,
|
||||
reason: 'Browser without OPFS - using memory',
|
||||
autoSelected: true
|
||||
};
|
||||
}
|
||||
// Serverless environment - prefer memory or cloud
|
||||
if (isServerlessEnvironment()) {
|
||||
if (cloudStorage) {
|
||||
return {
|
||||
...cloudStorage,
|
||||
reason: `Serverless with ${cloudStorage.type} detected`,
|
||||
autoSelected: true
|
||||
};
|
||||
}
|
||||
return {
|
||||
type: StorageType.MEMORY,
|
||||
config: StorageType.MEMORY,
|
||||
reason: 'Serverless environment - using memory',
|
||||
autoSelected: true
|
||||
};
|
||||
}
|
||||
// Node.js environment - use filesystem
|
||||
if (isNode()) {
|
||||
const dataPath = await findBestDataPath();
|
||||
return {
|
||||
type: StorageType.FILESYSTEM,
|
||||
config: StorageType.FILESYSTEM,
|
||||
reason: `Node.js environment - using filesystem at ${dataPath}`,
|
||||
autoSelected: true
|
||||
};
|
||||
}
|
||||
// Fallback to memory
|
||||
return {
|
||||
type: StorageType.MEMORY,
|
||||
config: StorageType.MEMORY,
|
||||
reason: 'Default fallback - using memory',
|
||||
autoSelected: true
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Detect cloud storage from environment variables
|
||||
*/
|
||||
async function detectCloudStorage() {
|
||||
// AWS S3 Detection
|
||||
if (hasAWSConfig()) {
|
||||
return {
|
||||
type: StorageType.S3,
|
||||
config: {
|
||||
s3Storage: {
|
||||
bucketName: process.env.AWS_BUCKET || process.env.S3_BUCKET_NAME || 'brainy-data',
|
||||
region: process.env.AWS_REGION || process.env.AWS_DEFAULT_REGION || 'us-east-1',
|
||||
// Credentials will be picked up by AWS SDK automatically
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
// Google Cloud Storage Detection
|
||||
if (hasGCPConfig()) {
|
||||
return {
|
||||
type: StorageType.GCS,
|
||||
config: {
|
||||
gcsStorage: {
|
||||
bucketName: process.env.GCS_BUCKET || process.env.GOOGLE_STORAGE_BUCKET || 'brainy-data',
|
||||
// Credentials will be picked up by GCP SDK automatically
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
// Cloudflare R2 Detection
|
||||
if (hasR2Config()) {
|
||||
return {
|
||||
type: StorageType.R2,
|
||||
config: {
|
||||
r2Storage: {
|
||||
bucketName: process.env.R2_BUCKET || 'brainy-data',
|
||||
accountId: process.env.CLOUDFLARE_ACCOUNT_ID,
|
||||
accessKeyId: process.env.R2_ACCESS_KEY_ID,
|
||||
secretAccessKey: process.env.R2_SECRET_ACCESS_KEY
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
/**
|
||||
* Check if AWS S3 is configured
|
||||
*/
|
||||
function hasAWSConfig() {
|
||||
return !!(
|
||||
// Explicit S3 bucket
|
||||
process.env.AWS_BUCKET ||
|
||||
process.env.S3_BUCKET_NAME ||
|
||||
// AWS credentials (SDK will find them)
|
||||
process.env.AWS_ACCESS_KEY_ID ||
|
||||
process.env.AWS_PROFILE ||
|
||||
// AWS environment indicators
|
||||
process.env.AWS_EXECUTION_ENV ||
|
||||
process.env.AWS_LAMBDA_FUNCTION_NAME ||
|
||||
process.env.ECS_CONTAINER_METADATA_URI);
|
||||
}
|
||||
/**
|
||||
* Check if Google Cloud Storage is configured
|
||||
*/
|
||||
function hasGCPConfig() {
|
||||
return !!(
|
||||
// Explicit GCS bucket
|
||||
process.env.GCS_BUCKET ||
|
||||
process.env.GOOGLE_STORAGE_BUCKET ||
|
||||
// GCP credentials
|
||||
process.env.GOOGLE_APPLICATION_CREDENTIALS ||
|
||||
// GCP environment indicators
|
||||
process.env.GOOGLE_CLOUD_PROJECT ||
|
||||
process.env.K_SERVICE ||
|
||||
process.env.GAE_SERVICE);
|
||||
}
|
||||
/**
|
||||
* Check if Cloudflare R2 is configured
|
||||
*/
|
||||
function hasR2Config() {
|
||||
return !!(process.env.R2_BUCKET ||
|
||||
(process.env.CLOUDFLARE_ACCOUNT_ID && process.env.R2_ACCESS_KEY_ID));
|
||||
}
|
||||
/**
|
||||
* Check if running in serverless environment
|
||||
*/
|
||||
function isServerlessEnvironment() {
|
||||
if (!isNode())
|
||||
return false;
|
||||
return !!(process.env.AWS_LAMBDA_FUNCTION_NAME ||
|
||||
process.env.VERCEL ||
|
||||
process.env.NETLIFY ||
|
||||
process.env.CLOUDFLARE_WORKERS ||
|
||||
process.env.FUNCTIONS_WORKER_RUNTIME ||
|
||||
process.env.K_SERVICE ||
|
||||
process.env.RAILWAY_ENVIRONMENT ||
|
||||
process.env.FLY_APP_NAME);
|
||||
}
|
||||
/**
|
||||
* Check for OPFS support in browser
|
||||
*/
|
||||
async function hasOPFSSupport() {
|
||||
if (!isBrowser())
|
||||
return false;
|
||||
try {
|
||||
return 'storage' in navigator &&
|
||||
'getDirectory' in navigator.storage;
|
||||
}
|
||||
catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Find the best path for filesystem storage
|
||||
*/
|
||||
async function findBestDataPath() {
|
||||
if (!isNode())
|
||||
return './brainy-data';
|
||||
const homeDir = process.env.HOME || process.env.USERPROFILE || '~';
|
||||
const tempDir = process.env.TMPDIR || process.env.TEMP || '/tmp';
|
||||
const candidates = [
|
||||
// User-specified path
|
||||
process.env.BRAINY_DATA_PATH,
|
||||
// Current directory
|
||||
'./brainy-data',
|
||||
// Home directory
|
||||
`${homeDir}/.brainy/data`,
|
||||
// Temp directory (last resort)
|
||||
`${tempDir}/brainy-data`
|
||||
].filter(Boolean);
|
||||
// Find first writable directory
|
||||
for (const candidate of candidates) {
|
||||
if (await isWritable(candidate)) {
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
// Default fallback
|
||||
return candidates[1]; // ./brainy-data
|
||||
}
|
||||
/**
|
||||
* Check if a directory is writable
|
||||
*/
|
||||
async function isWritable(dirPath) {
|
||||
if (!isNode())
|
||||
return false;
|
||||
try {
|
||||
// Dynamic import fs for Node.js
|
||||
const { promises: fs } = await import('fs');
|
||||
const path = await import('path');
|
||||
// Try to create directory if it doesn't exist
|
||||
await fs.mkdir(dirPath, { recursive: true });
|
||||
// Try to write a test file
|
||||
const testFile = path.join(dirPath, '.write-test');
|
||||
await fs.writeFile(testFile, 'test');
|
||||
await fs.unlink(testFile);
|
||||
return true;
|
||||
}
|
||||
catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
// Legacy getStorageConfig function removed - now using simple string types
|
||||
/**
|
||||
* Log storage configuration decision
|
||||
*/
|
||||
export function logStorageConfig(config, verbose = false) {
|
||||
if (!verbose && process.env.NODE_ENV === 'production') {
|
||||
return; // Silent in production unless verbose
|
||||
}
|
||||
const icon = config.autoSelected ? '🤖' : '👤';
|
||||
console.log(`${icon} Storage: ${config.type.toUpperCase()} - ${config.reason}`);
|
||||
}
|
||||
//# sourceMappingURL=storageAutoConfig.js.map
|
||||
File diff suppressed because one or more lines are too long
68
.recovery-workspace/dist-backup-20250910-141917/config/zeroConfig.d.ts
vendored
Normal file
68
.recovery-workspace/dist-backup-20250910-141917/config/zeroConfig.d.ts
vendored
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
/**
|
||||
* Zero-Configuration System for Brainy
|
||||
* Provides intelligent defaults while preserving full control
|
||||
*/
|
||||
import { ModelPrecision, ModelPreset } from './modelAutoConfig.js';
|
||||
import { StorageType, StoragePreset } from './storageAutoConfig.js';
|
||||
/**
|
||||
* Simplified configuration interface
|
||||
* Everything is optional - zero config by default!
|
||||
*/
|
||||
export interface BrainyZeroConfig {
|
||||
/**
|
||||
* Configuration preset for common scenarios
|
||||
* - 'production': Optimized for production (disk storage, auto model, default features)
|
||||
* - 'development': Optimized for development (memory storage, fp32, verbose logging)
|
||||
* - 'minimal': Minimal footprint (memory storage, q8, minimal features)
|
||||
* - 'zero': True zero config (all auto-detected)
|
||||
* - 'writer': Write-only instance for distributed setups (no index loading)
|
||||
* - 'reader': Read-only instance for distributed setups (no write operations)
|
||||
*/
|
||||
mode?: 'production' | 'development' | 'minimal' | 'zero' | 'writer' | 'reader';
|
||||
/**
|
||||
* Model precision configuration
|
||||
* - 'fp32': Full precision (best quality, larger size)
|
||||
* - 'q8': Quantized 8-bit (smaller size, slightly lower quality)
|
||||
* - 'fast': Alias for fp32
|
||||
* - 'small': Alias for q8
|
||||
* - 'auto': Auto-detect based on environment (default)
|
||||
*/
|
||||
model?: ModelPrecision | ModelPreset;
|
||||
/**
|
||||
* Storage configuration
|
||||
* - 'memory': In-memory only (no persistence)
|
||||
* - 'disk': Local disk (filesystem or OPFS)
|
||||
* - 'cloud': Cloud storage (S3/GCS/R2 if configured)
|
||||
* - 'auto': Auto-detect best option (default)
|
||||
* - Object: Custom storage configuration
|
||||
*/
|
||||
storage?: StorageType | StoragePreset | any;
|
||||
/**
|
||||
* Feature set configuration
|
||||
* - 'minimal': Core features only (fastest startup)
|
||||
* - 'default': Standard features (balanced)
|
||||
* - 'full': All features enabled (most capable)
|
||||
* - Array: Specific features to enable
|
||||
*/
|
||||
features?: 'minimal' | 'default' | 'full' | string[];
|
||||
/**
|
||||
* Logging verbosity
|
||||
* - true: Show configuration decisions and progress
|
||||
* - false: Silent operation (default in production)
|
||||
*/
|
||||
verbose?: boolean;
|
||||
/**
|
||||
* Advanced configuration (escape hatch for power users)
|
||||
* Any additional configuration can be passed here
|
||||
*/
|
||||
advanced?: any;
|
||||
}
|
||||
/**
|
||||
* Process zero-config input into full configuration
|
||||
*/
|
||||
export declare function processZeroConfig(input?: string | BrainyZeroConfig): Promise<any>;
|
||||
/**
|
||||
* Create embedding function with specified precision
|
||||
* This ensures the model precision is respected
|
||||
*/
|
||||
export declare function createEmbeddingFunctionWithPrecision(precision: ModelPrecision): Promise<any>;
|
||||
|
|
@ -0,0 +1,301 @@
|
|||
/**
|
||||
* Zero-Configuration System for Brainy
|
||||
* Provides intelligent defaults while preserving full control
|
||||
*/
|
||||
import { autoSelectModelPrecision, getModelPath, shouldAutoDownloadModels } from './modelAutoConfig.js';
|
||||
import { autoDetectStorage } from './storageAutoConfig.js';
|
||||
import { AutoConfiguration } from '../utils/autoConfiguration.js';
|
||||
/**
|
||||
* Configuration presets for common scenarios
|
||||
*/
|
||||
const PRESETS = {
|
||||
production: {
|
||||
storage: 'disk',
|
||||
model: 'auto',
|
||||
features: 'default',
|
||||
verbose: false
|
||||
},
|
||||
development: {
|
||||
storage: 'memory',
|
||||
model: 'q8', // Q8 is now the default for all presets
|
||||
features: 'full',
|
||||
verbose: true
|
||||
},
|
||||
minimal: {
|
||||
storage: 'memory',
|
||||
model: 'q8',
|
||||
features: 'minimal',
|
||||
verbose: false
|
||||
},
|
||||
zero: {
|
||||
storage: 'auto',
|
||||
model: 'auto',
|
||||
features: 'default',
|
||||
verbose: false
|
||||
},
|
||||
writer: {
|
||||
storage: 'auto',
|
||||
model: 'auto',
|
||||
features: 'minimal',
|
||||
verbose: false,
|
||||
// Writer-specific settings
|
||||
distributed: true,
|
||||
role: 'writer',
|
||||
writeOnly: true,
|
||||
allowDirectReads: true // Allow deduplication checks
|
||||
},
|
||||
reader: {
|
||||
storage: 'auto',
|
||||
model: 'auto',
|
||||
features: 'default',
|
||||
verbose: false,
|
||||
// Reader-specific settings
|
||||
distributed: true,
|
||||
role: 'reader',
|
||||
readOnly: true,
|
||||
lazyLoadInReadOnlyMode: true // Optimize for search
|
||||
}
|
||||
};
|
||||
/**
|
||||
* Feature sets configuration
|
||||
*/
|
||||
const FEATURE_SETS = {
|
||||
minimal: [
|
||||
'core',
|
||||
'search',
|
||||
'storage'
|
||||
],
|
||||
default: [
|
||||
'core',
|
||||
'search',
|
||||
'storage',
|
||||
'cache',
|
||||
'metadata-index',
|
||||
'batch-processing',
|
||||
'entity-registry',
|
||||
'request-deduplicator'
|
||||
],
|
||||
full: [
|
||||
'core',
|
||||
'search',
|
||||
'storage',
|
||||
'cache',
|
||||
'metadata-index',
|
||||
'batch-processing',
|
||||
'entity-registry',
|
||||
'request-deduplicator',
|
||||
'connection-pool',
|
||||
'wal',
|
||||
'monitoring',
|
||||
'metrics',
|
||||
'intelligent-verb-scoring',
|
||||
'triple-intelligence',
|
||||
'neural-api'
|
||||
]
|
||||
};
|
||||
/**
|
||||
* Process zero-config input into full configuration
|
||||
*/
|
||||
export async function processZeroConfig(input) {
|
||||
let config = {};
|
||||
// Handle string shorthand (preset name)
|
||||
if (typeof input === 'string') {
|
||||
if (input in PRESETS) {
|
||||
config = { mode: input };
|
||||
}
|
||||
else {
|
||||
throw new Error(`Unknown preset: ${input}. Valid presets: ${Object.keys(PRESETS).join(', ')}`);
|
||||
}
|
||||
}
|
||||
else if (input) {
|
||||
config = input;
|
||||
}
|
||||
// Apply preset if specified
|
||||
if (config.mode && config.mode in PRESETS) {
|
||||
const preset = PRESETS[config.mode];
|
||||
config = {
|
||||
...preset,
|
||||
...config,
|
||||
// Preserve explicit overrides
|
||||
model: config.model ?? preset.model,
|
||||
storage: config.storage ?? preset.storage,
|
||||
features: config.features ?? preset.features,
|
||||
verbose: config.verbose ?? preset.verbose
|
||||
};
|
||||
}
|
||||
// Auto-detect environment if not in preset mode
|
||||
const environment = detectEnvironmentMode();
|
||||
// Process model configuration
|
||||
const modelConfig = autoSelectModelPrecision(config.model);
|
||||
// Process storage configuration
|
||||
const storageConfig = await autoDetectStorage(config.storage);
|
||||
// Process features configuration
|
||||
const features = processFeatures(config.features);
|
||||
// Get auto-configuration recommendations
|
||||
const autoConfig = await AutoConfiguration.getInstance().detectAndConfigure({
|
||||
expectedDataSize: estimateDataSize(environment),
|
||||
s3Available: storageConfig.type === 's3',
|
||||
memoryBudget: undefined // Let it auto-detect
|
||||
});
|
||||
// Determine verbosity
|
||||
const verbose = config.verbose ?? (process.env.NODE_ENV === 'development');
|
||||
// Log configuration decisions if verbose
|
||||
if (verbose) {
|
||||
logConfigurationSummary({
|
||||
mode: config.mode || 'auto',
|
||||
model: modelConfig,
|
||||
storage: storageConfig,
|
||||
features: features,
|
||||
environment: environment,
|
||||
autoConfig: autoConfig
|
||||
});
|
||||
}
|
||||
// Build final configuration
|
||||
const finalConfig = {
|
||||
// Model configuration
|
||||
embeddingFunction: undefined, // Will be created with correct precision
|
||||
embeddingOptions: {
|
||||
precision: modelConfig.precision,
|
||||
modelPath: getModelPath(),
|
||||
allowRemoteDownload: shouldAutoDownloadModels()
|
||||
},
|
||||
// Storage configuration
|
||||
storage: storageConfig.config,
|
||||
storageType: storageConfig.type,
|
||||
// HNSW configuration from auto-config
|
||||
hnsw: {
|
||||
M: autoConfig.recommendedConfig.enablePartitioning ? 32 : 16,
|
||||
efConstruction: autoConfig.recommendedConfig.enablePartitioning ? 400 : 200,
|
||||
maxDatasetSize: autoConfig.recommendedConfig.expectedDatasetSize,
|
||||
partitioning: autoConfig.recommendedConfig.enablePartitioning,
|
||||
maxNodesPerPartition: autoConfig.recommendedConfig.maxNodesPerPartition
|
||||
},
|
||||
// Cache configuration from auto-config
|
||||
cache: {
|
||||
autoTune: true,
|
||||
hotCacheMaxSize: Math.floor(autoConfig.recommendedConfig.maxMemoryUsage / (1024 * 1024 * 10)), // 10% of memory budget
|
||||
batchSize: autoConfig.recommendedConfig.enablePartitioning ? 100 : 50
|
||||
},
|
||||
// Features configuration
|
||||
enabledFeatures: features,
|
||||
// Metadata index configuration
|
||||
metadataIndex: features.includes('metadata-index') ? {
|
||||
enabled: true,
|
||||
autoRebuild: true
|
||||
} : undefined,
|
||||
// Intelligent verb scoring
|
||||
intelligentVerbScoring: features.includes('intelligent-verb-scoring') ? {
|
||||
enabled: true
|
||||
} : undefined,
|
||||
// Logging configuration
|
||||
logging: {
|
||||
verbose: verbose
|
||||
},
|
||||
// Performance flags from auto-config
|
||||
optimizations: autoConfig.optimizationFlags,
|
||||
// Advanced overrides (if any)
|
||||
...config.advanced
|
||||
};
|
||||
// Apply distributed preset settings if applicable
|
||||
if (config.mode === 'writer' || config.mode === 'reader') {
|
||||
const presetSettings = PRESETS[config.mode]; // Cast to any since we know these presets have additional properties
|
||||
// Apply distributed-specific settings
|
||||
finalConfig.distributed = presetSettings.distributed;
|
||||
finalConfig.readOnly = presetSettings.readOnly || false;
|
||||
finalConfig.writeOnly = presetSettings.writeOnly || false;
|
||||
finalConfig.allowDirectReads = presetSettings.allowDirectReads || false;
|
||||
finalConfig.lazyLoadInReadOnlyMode = presetSettings.lazyLoadInReadOnlyMode || false;
|
||||
// Set distributed role in distributed config
|
||||
if (finalConfig.distributed) {
|
||||
finalConfig.distributed = {
|
||||
enabled: true,
|
||||
role: presetSettings.role
|
||||
};
|
||||
}
|
||||
// Log distributed mode if verbose
|
||||
if (verbose) {
|
||||
console.log(`📡 Distributed mode: ${config.mode.toUpperCase()}`);
|
||||
console.log(` Role: ${presetSettings.role}`);
|
||||
console.log(` Read-only: ${finalConfig.readOnly}`);
|
||||
console.log(` Write-only: ${finalConfig.writeOnly}`);
|
||||
}
|
||||
}
|
||||
return finalConfig;
|
||||
}
|
||||
/**
|
||||
* Detect environment mode if not specified
|
||||
*/
|
||||
function detectEnvironmentMode() {
|
||||
if (process.env.NODE_ENV === 'production')
|
||||
return 'production';
|
||||
if (process.env.NODE_ENV === 'development')
|
||||
return 'development';
|
||||
if (process.env.NODE_ENV === 'test')
|
||||
return 'development';
|
||||
// Check for CI environments
|
||||
if (process.env.CI || process.env.GITHUB_ACTIONS)
|
||||
return 'production';
|
||||
// Check for production indicators
|
||||
if (process.env.VERCEL_ENV === 'production' ||
|
||||
process.env.NETLIFY_ENV === 'production' ||
|
||||
process.env.RAILWAY_ENVIRONMENT === 'production') {
|
||||
return 'production';
|
||||
}
|
||||
return 'unknown';
|
||||
}
|
||||
/**
|
||||
* Process features configuration
|
||||
*/
|
||||
function processFeatures(features) {
|
||||
if (Array.isArray(features)) {
|
||||
return features;
|
||||
}
|
||||
if (features && features in FEATURE_SETS) {
|
||||
return FEATURE_SETS[features];
|
||||
}
|
||||
// Default based on environment
|
||||
const env = detectEnvironmentMode();
|
||||
if (env === 'production')
|
||||
return FEATURE_SETS.default;
|
||||
if (env === 'development')
|
||||
return FEATURE_SETS.full;
|
||||
return FEATURE_SETS.default;
|
||||
}
|
||||
/**
|
||||
* Estimate dataset size based on environment
|
||||
*/
|
||||
function estimateDataSize(environment) {
|
||||
switch (environment) {
|
||||
case 'production': return 100000;
|
||||
case 'development': return 10000;
|
||||
default: return 50000;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Log configuration summary
|
||||
*/
|
||||
function logConfigurationSummary(config) {
|
||||
console.log('\n🧠 Brainy Zero-Config Summary');
|
||||
console.log('================================');
|
||||
console.log(`Mode: ${config.mode}`);
|
||||
console.log(`Environment: ${config.environment}`);
|
||||
console.log(`Model: ${config.model.precision.toUpperCase()} (${config.model.reason})`);
|
||||
console.log(`Storage: ${config.storage.type.toUpperCase()} (${config.storage.reason})`);
|
||||
console.log(`Features: ${config.features.length} enabled`);
|
||||
console.log(`Memory Budget: ${Math.floor(config.autoConfig.recommendedConfig.maxMemoryUsage / (1024 * 1024))}MB`);
|
||||
console.log(`Expected Dataset: ${config.autoConfig.recommendedConfig.expectedDatasetSize.toLocaleString()} items`);
|
||||
console.log('================================\n');
|
||||
}
|
||||
/**
|
||||
* Create embedding function with specified precision
|
||||
* This ensures the model precision is respected
|
||||
*/
|
||||
export async function createEmbeddingFunctionWithPrecision(precision) {
|
||||
const { createEmbeddingFunction } = await import('../utils/embedding.js');
|
||||
// Create embedding function with specified precision
|
||||
return createEmbeddingFunction({
|
||||
precision: precision,
|
||||
verbose: false // Silent by default in zero-config
|
||||
});
|
||||
}
|
||||
//# sourceMappingURL=zeroConfig.js.map
|
||||
File diff suppressed because one or more lines are too long
Loading…
Add table
Add a link
Reference in a new issue