diff --git a/src/brainyData.ts b/src/brainyData.ts index 3ee3a281..eb6b9741 100644 --- a/src/brainyData.ts +++ b/src/brainyData.ts @@ -135,6 +135,60 @@ export interface BrainyDataConfig { verbose?: boolean } + /** + * Timeout configuration for async operations + * Controls how long operations wait before timing out + */ + timeouts?: { + /** + * Timeout for get operations in milliseconds + * Default: 30000 (30 seconds) + */ + get?: number + + /** + * Timeout for add operations in milliseconds + * Default: 60000 (60 seconds) + */ + add?: number + + /** + * Timeout for delete operations in milliseconds + * Default: 30000 (30 seconds) + */ + delete?: number + } + + /** + * Retry policy configuration for failed operations + * Controls how operations are retried on failure + */ + retryPolicy?: { + /** + * Maximum number of retry attempts + * Default: 3 + */ + maxRetries?: number + + /** + * Initial delay between retries in milliseconds + * Default: 1000 (1 second) + */ + initialDelay?: number + + /** + * Maximum delay between retries in milliseconds + * Default: 10000 (10 seconds) + */ + maxDelay?: number + + /** + * Multiplier for exponential backoff + * Default: 2 + */ + backoffMultiplier?: number + } + /** * Real-time update configuration * Controls how the database handles updates when data is added by external processes @@ -181,6 +235,10 @@ export class BrainyData implements BrainyDataInterface { private _dimensions: number private loggingConfig: BrainyDataConfig['logging'] = {verbose: true} + // Timeout and retry configuration + private timeoutConfig: BrainyDataConfig['timeouts'] = {} + private retryConfig: BrainyDataConfig['retryPolicy'] = {} + // Real-time update properties private realtimeUpdateConfig: Required> = { enabled: false, @@ -268,6 +326,10 @@ export class BrainyData implements BrainyDataInterface { // Store storage configuration for later use in init() this.storageConfig = config.storage || {} + // Store timeout and retry configuration + this.timeoutConfig = config.timeouts || {} + this.retryConfig = config.retryPolicy || {} + // Store remote server configuration if provided if (config.remoteServer) { this.remoteServerConfig = config.remoteServer diff --git a/src/errors/brainyError.ts b/src/errors/brainyError.ts new file mode 100644 index 00000000..81cfd56f --- /dev/null +++ b/src/errors/brainyError.ts @@ -0,0 +1,179 @@ +/** + * Custom error types for Brainy operations + * Provides better error classification and handling + */ + +export type BrainyErrorType = 'TIMEOUT' | 'NETWORK' | 'STORAGE' | 'NOT_FOUND' | 'RETRY_EXHAUSTED' + +/** + * Custom error class for Brainy operations + * Provides error type classification and retry information + */ +export class BrainyError extends Error { + public readonly type: BrainyErrorType + public readonly retryable: boolean + public readonly originalError?: Error + public readonly attemptNumber?: number + public readonly maxRetries?: number + + constructor( + message: string, + type: BrainyErrorType, + retryable: boolean = false, + originalError?: Error, + attemptNumber?: number, + maxRetries?: number + ) { + super(message) + this.name = 'BrainyError' + this.type = type + this.retryable = retryable + this.originalError = originalError + this.attemptNumber = attemptNumber + this.maxRetries = maxRetries + + // Maintain proper stack trace for where our error was thrown (only available on V8) + if (Error.captureStackTrace) { + Error.captureStackTrace(this, BrainyError) + } + } + + /** + * Create a timeout error + */ + static timeout(operation: string, timeoutMs: number, originalError?: Error): BrainyError { + return new BrainyError( + `Operation '${operation}' timed out after ${timeoutMs}ms`, + 'TIMEOUT', + true, + originalError + ) + } + + /** + * Create a network error + */ + static network(message: string, originalError?: Error): BrainyError { + return new BrainyError( + `Network error: ${message}`, + 'NETWORK', + true, + originalError + ) + } + + /** + * Create a storage error + */ + static storage(message: string, originalError?: Error): BrainyError { + return new BrainyError( + `Storage error: ${message}`, + 'STORAGE', + true, + originalError + ) + } + + /** + * Create a not found error + */ + static notFound(resource: string): BrainyError { + return new BrainyError( + `Resource not found: ${resource}`, + 'NOT_FOUND', + false + ) + } + + /** + * Create a retry exhausted error + */ + static retryExhausted(operation: string, maxRetries: number, lastError?: Error): BrainyError { + return new BrainyError( + `Operation '${operation}' failed after ${maxRetries} retry attempts`, + 'RETRY_EXHAUSTED', + false, + lastError, + maxRetries, + maxRetries + ) + } + + /** + * Check if an error is retryable + */ + static isRetryable(error: Error): boolean { + if (error instanceof BrainyError) { + return error.retryable + } + + // Check for common retryable error patterns + const message = error.message.toLowerCase() + const name = error.name.toLowerCase() + + // Network-related errors that are typically retryable + if ( + message.includes('timeout') || + message.includes('network') || + message.includes('connection') || + message.includes('econnreset') || + message.includes('enotfound') || + message.includes('etimedout') || + name.includes('timeout') + ) { + return true + } + + // AWS SDK specific retryable errors + if ( + message.includes('throttling') || + message.includes('rate limit') || + message.includes('service unavailable') || + message.includes('internal server error') || + message.includes('bad gateway') || + message.includes('gateway timeout') + ) { + return true + } + + return false + } + + /** + * Convert a generic error to a BrainyError with appropriate classification + */ + static fromError(error: Error, operation?: string): BrainyError { + if (error instanceof BrainyError) { + return error + } + + const message = error.message.toLowerCase() + const name = error.name.toLowerCase() + + // Classify the error based on common patterns + if (message.includes('timeout') || name.includes('timeout')) { + return BrainyError.timeout(operation || 'unknown', 0, error) + } + + if ( + message.includes('network') || + message.includes('connection') || + message.includes('econnreset') || + message.includes('enotfound') || + message.includes('etimedout') + ) { + return BrainyError.network(error.message, error) + } + + if ( + message.includes('nosuchkey') || + message.includes('not found') || + message.includes('does not exist') + ) { + return BrainyError.notFound(operation || 'resource') + } + + // Default to storage error for unclassified errors + return BrainyError.storage(error.message, error) + } +} diff --git a/src/storage/adapters/s3CompatibleStorage.ts b/src/storage/adapters/s3CompatibleStorage.ts index 72596608..e7bc9e9f 100644 --- a/src/storage/adapters/s3CompatibleStorage.ts +++ b/src/storage/adapters/s3CompatibleStorage.ts @@ -6,6 +6,8 @@ import {GraphVerb, HNSWNoun, StatisticsData} from '../../coreTypes.js' import {BaseStorage, NOUNS_DIR, VERBS_DIR, METADATA_DIR, INDEX_DIR, STATISTICS_KEY} from '../baseStorage.js' +import {StorageOperationExecutors, OperationConfig} from '../../utils/operationUtils.js' +import {BrainyError} from '../../errors/brainyError.js' // Type aliases for better readability type HNSWNode = HNSWNoun @@ -76,6 +78,9 @@ export class S3CompatibleStorage extends BaseStorage { // Change log for efficient synchronization private changeLogPrefix: string = 'change-log/' + + // Operation executors for timeout and retry handling + private operationExecutors: StorageOperationExecutors /** * Initialize the storage adapter @@ -90,6 +95,7 @@ export class S3CompatibleStorage extends BaseStorage { secretAccessKey: string sessionToken?: string serviceType?: string + operationConfig?: OperationConfig }) { super() this.bucketName = options.bucketName @@ -101,6 +107,9 @@ export class S3CompatibleStorage extends BaseStorage { this.sessionToken = options.sessionToken this.serviceType = options.serviceType || 's3' + // Initialize operation executors with timeout and retry configuration + this.operationExecutors = new StorageOperationExecutors(options.operationConfig) + // Set up prefixes for different types of data this.nounPrefix = `${NOUNS_DIR}/` this.verbPrefix = `${VERBS_DIR}/` @@ -938,61 +947,62 @@ export class S3CompatibleStorage extends BaseStorage { public async getMetadata(id: string): Promise { await this.ensureInitialized() - try { - // Import the GetObjectCommand only when needed - const {GetObjectCommand} = await import('@aws-sdk/client-s3') - - console.log(`Getting metadata for ${id} from bucket ${this.bucketName}`) - const key = `${this.metadataPrefix}${id}.json` - console.log(`Looking for metadata at key: ${key}`) - - // Try to get the metadata from the metadata directory - const response = await this.s3Client!.send( - new GetObjectCommand({ - Bucket: this.bucketName, - Key: key - }) - ) - - // Check if response is null or undefined (can happen in mock implementations) - if (!response || !response.Body) { - console.log(`No metadata found for ${id}`) - return null - } - - // Convert the response body to a string - const bodyContents = await response.Body.transformToString() - console.log(`Retrieved metadata body: ${bodyContents}`) - - // Parse the JSON string + return this.operationExecutors.executeGet(async () => { try { - const parsedMetadata = JSON.parse(bodyContents) - console.log(`Successfully retrieved metadata for ${id}:`, parsedMetadata) - return parsedMetadata - } catch (parseError) { - console.error(`Failed to parse metadata for ${id}:`, parseError) - return null - } - } catch (error: any) { - // Check if this is a "NoSuchKey" error (object doesn't exist) - // In AWS SDK, this would be error.name === 'NoSuchKey' - // In our mock, we might get different error types - if ( - error.name === 'NoSuchKey' || - (error.message && ( - error.message.includes('NoSuchKey') || - error.message.includes('not found') || - error.message.includes('does not exist') - )) - ) { - console.log(`Metadata not found for ${id}`) - return null - } + // Import the GetObjectCommand only when needed + const {GetObjectCommand} = await import('@aws-sdk/client-s3') - // For other types of errors, log and re-throw - console.error(`Error getting metadata for ${id}:`, error) - throw error - } + console.log(`Getting metadata for ${id} from bucket ${this.bucketName}`) + const key = `${this.metadataPrefix}${id}.json` + console.log(`Looking for metadata at key: ${key}`) + + // Try to get the metadata from the metadata directory + const response = await this.s3Client!.send( + new GetObjectCommand({ + Bucket: this.bucketName, + Key: key + }) + ) + + // Check if response is null or undefined (can happen in mock implementations) + if (!response || !response.Body) { + console.log(`No metadata found for ${id}`) + return null + } + + // Convert the response body to a string + const bodyContents = await response.Body.transformToString() + console.log(`Retrieved metadata body: ${bodyContents}`) + + // Parse the JSON string + try { + const parsedMetadata = JSON.parse(bodyContents) + console.log(`Successfully retrieved metadata for ${id}:`, parsedMetadata) + return parsedMetadata + } catch (parseError) { + console.error(`Failed to parse metadata for ${id}:`, parseError) + return null + } + } catch (error: any) { + // Check if this is a "NoSuchKey" error (object doesn't exist) + // In AWS SDK, this would be error.name === 'NoSuchKey' + // In our mock, we might get different error types + if ( + error.name === 'NoSuchKey' || + (error.message && ( + error.message.includes('NoSuchKey') || + error.message.includes('not found') || + error.message.includes('does not exist') + )) + ) { + console.log(`Metadata not found for ${id}`) + return null + } + + // For other types of errors, convert to BrainyError for better classification + throw BrainyError.fromError(error, `getMetadata(${id})`) + } + }, `getMetadata(${id})`) } /** diff --git a/src/storage/storageFactory.ts b/src/storage/storageFactory.ts index 81450b40..bb9f4831 100644 --- a/src/storage/storageFactory.ts +++ b/src/storage/storageFactory.ts @@ -9,6 +9,7 @@ import {OPFSStorage} from './adapters/opfsStorage.js' import {S3CompatibleStorage, R2Storage} from './adapters/s3CompatibleStorage.js' import {FileSystemStorage} from './adapters/fileSystemStorage.js' import {isBrowser} from '../utils/environment.js' +import {OperationConfig} from '../utils/operationUtils.js' /** * Options for creating a storage adapter @@ -165,6 +166,11 @@ export interface StorageOptions { */ serviceType?: string } + + /** + * Operation configuration for timeout and retry behavior + */ + operationConfig?: OperationConfig } /** @@ -238,7 +244,8 @@ export async function createStorage( accessKeyId: options.s3Storage.accessKeyId, secretAccessKey: options.s3Storage.secretAccessKey, sessionToken: options.s3Storage.sessionToken, - serviceType: 's3' + serviceType: 's3', + operationConfig: options.operationConfig }) } else { console.warn('S3 storage configuration is missing, falling back to memory storage') diff --git a/src/utils/operationUtils.ts b/src/utils/operationUtils.ts new file mode 100644 index 00000000..91a905be --- /dev/null +++ b/src/utils/operationUtils.ts @@ -0,0 +1,204 @@ +/** + * Utility functions for timeout and retry logic + * Used by storage adapters to handle network operations reliably + */ + +import { BrainyError } from '../errors/brainyError.js' + +export interface TimeoutConfig { + get?: number + add?: number + delete?: number +} + +export interface RetryConfig { + maxRetries?: number + initialDelay?: number + maxDelay?: number + backoffMultiplier?: number +} + +export interface OperationConfig { + timeouts?: TimeoutConfig + retryPolicy?: RetryConfig +} + +// Default configuration values +export const DEFAULT_TIMEOUTS: Required = { + get: 30000, // 30 seconds + add: 60000, // 1 minute + delete: 30000 // 30 seconds +} + +export const DEFAULT_RETRY_POLICY: Required = { + maxRetries: 3, + initialDelay: 1000, + maxDelay: 10000, + backoffMultiplier: 2 +} + +/** + * Wraps a promise with a timeout + */ +export function withTimeout( + promise: Promise, + timeoutMs: number, + operation: string +): Promise { + return new Promise((resolve, reject) => { + const timeoutId = setTimeout(() => { + reject(BrainyError.timeout(operation, timeoutMs)) + }, timeoutMs) + + promise + .then((result) => { + clearTimeout(timeoutId) + resolve(result) + }) + .catch((error) => { + clearTimeout(timeoutId) + reject(error) + }) + }) +} + +/** + * Calculates the delay for exponential backoff + */ +function calculateBackoffDelay( + attemptNumber: number, + initialDelay: number, + maxDelay: number, + backoffMultiplier: number +): number { + const delay = initialDelay * Math.pow(backoffMultiplier, attemptNumber - 1) + return Math.min(delay, maxDelay) +} + +/** + * Sleeps for the specified number of milliseconds + */ +function sleep(ms: number): Promise { + return new Promise(resolve => setTimeout(resolve, ms)) +} + +/** + * Executes an operation with retry logic and exponential backoff + */ +export async function withRetry( + operation: () => Promise, + operationName: string, + config: RetryConfig = {} +): Promise { + const { + maxRetries = DEFAULT_RETRY_POLICY.maxRetries, + initialDelay = DEFAULT_RETRY_POLICY.initialDelay, + maxDelay = DEFAULT_RETRY_POLICY.maxDelay, + backoffMultiplier = DEFAULT_RETRY_POLICY.backoffMultiplier + } = config + + let lastError: Error | undefined + + for (let attempt = 1; attempt <= maxRetries + 1; attempt++) { + try { + return await operation() + } catch (error) { + lastError = error instanceof Error ? error : new Error(String(error)) + + // If this is the last attempt, don't retry + if (attempt > maxRetries) { + break + } + + // Check if the error is retryable + if (!BrainyError.isRetryable(lastError)) { + throw BrainyError.fromError(lastError, operationName) + } + + // Calculate delay for exponential backoff + const delay = calculateBackoffDelay(attempt, initialDelay, maxDelay, backoffMultiplier) + + console.warn( + `Operation '${operationName}' failed on attempt ${attempt}/${maxRetries + 1}. ` + + `Retrying in ${delay}ms. Error: ${lastError.message}` + ) + + // Wait before retrying + await sleep(delay) + } + } + + // All retries exhausted + throw BrainyError.retryExhausted(operationName, maxRetries, lastError) +} + +/** + * Executes an operation with both timeout and retry logic + */ +export async function withTimeoutAndRetry( + operation: () => Promise, + operationName: string, + timeoutMs: number, + retryConfig: RetryConfig = {} +): Promise { + return withRetry( + () => withTimeout(operation(), timeoutMs, operationName), + operationName, + retryConfig + ) +} + +/** + * Creates a configured operation executor for a specific operation type + */ +export function createOperationExecutor( + operationType: keyof TimeoutConfig, + config: OperationConfig = {} +) { + const timeouts = { ...DEFAULT_TIMEOUTS, ...config.timeouts } + const retryPolicy = { ...DEFAULT_RETRY_POLICY, ...config.retryPolicy } + const timeoutMs = timeouts[operationType] + + return async function executeOperation( + operation: () => Promise, + operationName: string + ): Promise { + return withTimeoutAndRetry(operation, operationName, timeoutMs, retryPolicy) + } +} + +/** + * Storage operation executors for different operation types + */ +export class StorageOperationExecutors { + private getExecutor: ReturnType + private addExecutor: ReturnType + private deleteExecutor: ReturnType + + constructor(config: OperationConfig = {}) { + this.getExecutor = createOperationExecutor('get', config) + this.addExecutor = createOperationExecutor('add', config) + this.deleteExecutor = createOperationExecutor('delete', config) + } + + /** + * Execute a get operation with timeout and retry + */ + async executeGet(operation: () => Promise, operationName: string): Promise { + return this.getExecutor(operation, operationName) + } + + /** + * Execute an add operation with timeout and retry + */ + async executeAdd(operation: () => Promise, operationName: string): Promise { + return this.addExecutor(operation, operationName) + } + + /** + * Execute a delete operation with timeout and retry + */ + async executeDelete(operation: () => Promise, operationName: string): Promise { + return this.deleteExecutor(operation, operationName) + } +}