**feat: enhance BrainyData storage configuration with support for custom S3 endpoints and extended types**

### Changes:
- Introduced `ExtendedBrainyDataConfig` interface to include `rootDirectory` in the storage configuration.
- Enhanced S3 storage setup:
  - Added support for `customS3Storage` when `S3_ENDPOINT` is provided.
  - Retained standard `s3Storage` configuration for AWS S3 without custom endpoints.
- Updated fallback logic for default storage types (`filesystem`, `memory`) with consistent handling for `rootDirectory`.
- Refined `initializeBrainy` function to improve readability and configuration extensibility.
- Updated TypeScript typings for improved clarity and maintainability.

### Purpose:
Improved the `initializeBrainy` function by extending storage configuration capabilities, allowing seamless integration of custom S3 endpoints alongside standard AWS S3 settings. Enhanced overall code readability and adaptability for diverse storage configurations.
This commit is contained in:
David Snelling 2025-06-19 15:12:51 -07:00
parent 25f7ab73b4
commit 487bfa473b

View file

@ -1,8 +1,17 @@
import { BrainyData } from '@soulcraft/brainy'; import { BrainyData, BrainyDataConfig } from '@soulcraft/brainy'
import dotenv from 'dotenv'; import dotenv from 'dotenv'
// Load environment variables // Load environment variables
dotenv.config(); dotenv.config()
/**
* Extended configuration interface that includes rootDirectory
*/
interface ExtendedBrainyDataConfig extends BrainyDataConfig {
storage: BrainyDataConfig['storage'] & {
rootDirectory?: string
}
}
/** /**
* Initialize the Brainy instance with appropriate configuration * Initialize the Brainy instance with appropriate configuration
@ -10,39 +19,50 @@ dotenv.config();
*/ */
export async function initializeBrainy(): Promise<BrainyData> { export async function initializeBrainy(): Promise<BrainyData> {
// Get storage configuration from environment variables // Get storage configuration from environment variables
const storageType = process.env.STORAGE_TYPE || 'filesystem'; const storageType = process.env.STORAGE_TYPE || 'filesystem'
// Create configuration object // Create configuration object
const config: any = { const config: ExtendedBrainyDataConfig = {
storage: {} storage: {}
}; }
// Configure storage based on type // Configure storage based on type
switch (storageType) { switch (storageType) {
case 's3': case 's3':
// Configure S3 storage // Configure S3 storage
config.storage.s3Storage = { if (process.env.S3_ENDPOINT) {
// Use customS3Storage for S3-compatible services with custom endpoints
config.storage.customS3Storage = {
bucketName: process.env.S3_BUCKET_NAME, bucketName: process.env.S3_BUCKET_NAME,
accessKeyId: process.env.S3_ACCESS_KEY_ID, accessKeyId: process.env.S3_ACCESS_KEY_ID,
secretAccessKey: process.env.S3_SECRET_ACCESS_KEY, secretAccessKey: process.env.S3_SECRET_ACCESS_KEY,
region: process.env.S3_REGION || 'us-east-1', region: process.env.S3_REGION || 'us-east-1',
endpoint: process.env.S3_ENDPOINT // Optional for custom S3-compatible services endpoint: process.env.S3_ENDPOINT
}; }
break; } else {
// Use standard S3 storage for AWS S3
config.storage.s3Storage = {
bucketName: process.env.S3_BUCKET_NAME,
accessKeyId: process.env.S3_ACCESS_KEY_ID,
secretAccessKey: process.env.S3_SECRET_ACCESS_KEY,
region: process.env.S3_REGION || 'us-east-1'
}
}
break
case 'filesystem': case 'filesystem':
// Configure filesystem storage // Configure filesystem storage
config.storage.rootDirectory = process.env.STORAGE_ROOT_DIR || './data'; config.storage.rootDirectory = process.env.STORAGE_ROOT_DIR || './data'
break; break
case 'memory': case 'memory':
// No additional configuration needed for memory storage // No additional configuration needed for memory storage
break; break
default: default:
// Default to filesystem storage // Default to filesystem storage
config.storage.rootDirectory = process.env.STORAGE_ROOT_DIR || './data'; config.storage.rootDirectory = process.env.STORAGE_ROOT_DIR || './data'
break; break
} }
// Note: Universal Sentence Encoder is now the only embedding option // Note: Universal Sentence Encoder is now the only embedding option
@ -58,20 +78,20 @@ export async function initializeBrainy(): Promise<BrainyData> {
efSearch: process.env.HNSW_EF_SEARCH efSearch: process.env.HNSW_EF_SEARCH
? parseInt(process.env.HNSW_EF_SEARCH, 10) ? parseInt(process.env.HNSW_EF_SEARCH, 10)
: 50 : 50
}; }
} }
// Create and initialize the Brainy instance // Create and initialize the Brainy instance
const brainy = new BrainyData(config); const brainy = new BrainyData(config)
await brainy.init(); await brainy.init()
return brainy; return brainy
} }
/** /**
* Get the current storage type being used by Brainy * Get the current storage type being used by Brainy
*/ */
export async function getStorageType(brainy: BrainyData): Promise<string> { export async function getStorageType(brainy: BrainyData): Promise<string> {
const status = await brainy.status(); const status = await brainy.status()
return status.storageType || 'unknown'; return status.type || 'unknown'
} }