/** * Storage Factory * Creates the appropriate storage adapter based on the environment and configuration */ import { StorageAdapter } from '../coreTypes.js' import { MemoryStorage } from './adapters/memoryStorage.js' import { OPFSStorage } from './adapters/opfsStorage.js' import { S3CompatibleStorage } from './adapters/s3CompatibleStorage.js' import { R2Storage } from './adapters/r2Storage.js' import { GcsStorage } from './adapters/gcsStorage.js' import { AzureBlobStorage } from './adapters/azureBlobStorage.js' import { TypeAwareStorageAdapter } from './adapters/typeAwareStorageAdapter.js' // FileSystemStorage is dynamically imported to avoid issues in browser environments import { isBrowser } from '../utils/environment.js' import { OperationConfig } from '../utils/operationUtils.js' /** * Options for creating a storage adapter */ export interface StorageOptions { /** * The type of storage to use * - 'auto': Automatically select the best storage adapter based on the environment * - 'memory': Use in-memory storage * - 'opfs': Use Origin Private File System storage (browser only) * - 'filesystem': Use file system storage (Node.js only) * - 's3': Use Amazon S3 storage * - 'r2': Use Cloudflare R2 storage * - 'gcs': Use Google Cloud Storage (native SDK with ADC) * - 'gcs-native': DEPRECATED - Use 'gcs' instead * - 'azure': Use Azure Blob Storage (native SDK with Managed Identity) * - 'type-aware': Use type-first storage adapter (wraps another adapter) */ type?: 'auto' | 'memory' | 'opfs' | 'filesystem' | 's3' | 'r2' | 'gcs' | 'gcs-native' | 'azure' | 'type-aware' /** * Force the use of memory storage even if other storage types are available */ forceMemoryStorage?: boolean /** * Force the use of file system storage even if other storage types are available */ forceFileSystemStorage?: boolean /** * Request persistent storage permission from the user (browser only) */ requestPersistentStorage?: boolean /** * Root directory for file system storage (Node.js only) */ rootDirectory?: string /** * Nested options object for backward compatibility with BrainyConfig.storage.options * Supports flexible API patterns */ options?: { rootDirectory?: string path?: string [key: string]: any } /** * Configuration for Amazon S3 storage */ s3Storage?: { /** * S3 bucket name */ bucketName: string /** * AWS region (e.g., 'us-east-1') */ region?: string /** * AWS access key ID */ accessKeyId: string /** * AWS secret access key */ secretAccessKey: string /** * AWS session token (optional) */ sessionToken?: string } /** * Configuration for Cloudflare R2 storage */ r2Storage?: { /** * R2 bucket name */ bucketName: string /** * Cloudflare account ID */ accountId: string /** * R2 access key ID */ accessKeyId: string /** * R2 secret access key */ secretAccessKey: string } /** * Configuration for Google Cloud Storage (Legacy S3-compatible with HMAC keys) * @deprecated Use gcsNativeStorage instead for better performance with ADC * This is only needed if you must use HMAC keys for backward compatibility */ gcsStorage?: { /** * GCS bucket name */ bucketName: string /** * GCS region (e.g., 'us-central1') */ region?: string /** * GCS access key ID */ accessKeyId: string /** * GCS secret access key */ secretAccessKey: string /** * GCS endpoint (e.g., 'https://storage.googleapis.com') */ endpoint?: string } /** * Configuration for Google Cloud Storage (native SDK with ADC) * This is the recommended way to use GCS with Brainy * Supports Application Default Credentials for zero-config cloud deployments */ gcsNativeStorage?: { /** * GCS bucket name */ bucketName: string /** * Service account key file path (optional, uses ADC if not provided) */ keyFilename?: string /** * Service account credentials object (optional, uses ADC if not provided) */ credentials?: object /** * HMAC access key ID (backward compatibility, not recommended) * @deprecated Use ADC, keyFilename, or credentials instead */ accessKeyId?: string /** * HMAC secret access key (backward compatibility, not recommended) * @deprecated Use ADC, keyFilename, or credentials instead */ secretAccessKey?: string /** * Skip initial bucket scan for counting entities * Useful for large buckets where the scan would timeout * If true, counts start at 0 and are updated incrementally * @default false */ skipInitialScan?: boolean /** * Skip loading and saving the counts file entirely * Useful for very large datasets where counts aren't critical * @default false */ skipCountsFile?: boolean } /** * Configuration for Azure Blob Storage (native SDK with Managed Identity) */ azureStorage?: { /** * Azure container name */ containerName: string /** * Azure Storage account name (for Managed Identity or SAS) */ accountName?: string /** * Azure Storage account key (optional, uses Managed Identity if not provided) */ accountKey?: string /** * Azure connection string (highest priority if provided) */ connectionString?: string /** * SAS token (optional, alternative to account key) */ sasToken?: string } /** * Configuration for Type-Aware Storage (type-first architecture) * Wraps another storage adapter and adds type-first routing */ typeAwareStorage?: { /** * Underlying storage adapter to use * Can be any of: 'memory', 'filesystem', 's3', 'r2', 'gcs', 'gcs-native' */ underlyingType?: 'memory' | 'filesystem' | 's3' | 'r2' | 'gcs' | 'gcs-native' /** * Options for the underlying storage adapter */ underlyingOptions?: StorageOptions /** * Enable verbose logging for debugging */ verbose?: boolean } /** * Configuration for custom S3-compatible storage */ customS3Storage?: { /** * S3-compatible bucket name */ bucketName: string /** * S3-compatible region */ region?: string /** * S3-compatible endpoint URL */ endpoint: string /** * S3-compatible access key ID */ accessKeyId: string /** * S3-compatible secret access key */ secretAccessKey: string /** * S3-compatible service type (for logging and error messages) */ serviceType?: string } /** * Operation configuration for timeout and retry behavior */ operationConfig?: OperationConfig /** * Cache configuration for optimizing data access * Particularly important for S3 and other remote storage */ cacheConfig?: { /** * Maximum size of the hot cache (most frequently accessed items) * For large datasets, consider values between 5000-50000 depending on available memory */ hotCacheMaxSize?: number /** * Threshold at which to start evicting items from the hot cache * Expressed as a fraction of hotCacheMaxSize (0.0 to 1.0) * Default: 0.8 (start evicting when cache is 80% full) */ hotCacheEvictionThreshold?: number /** * Time-to-live for items in the warm cache in milliseconds * Default: 3600000 (1 hour) */ warmCacheTTL?: number /** * Batch size for operations like prefetching * Larger values improve throughput but use more memory */ batchSize?: number /** * Whether to enable auto-tuning of cache parameters * When true, the system will automatically adjust cache sizes based on usage patterns * Default: true */ autoTune?: boolean /** * The interval (in milliseconds) at which to auto-tune cache parameters * Only applies when autoTune is true * Default: 60000 (1 minute) */ autoTuneInterval?: number /** * Whether the storage is in read-only mode * This affects cache sizing and prefetching strategies */ readOnly?: boolean } /** * COW (Copy-on-Write) configuration for instant fork() capability * v5.0.0+ */ branch?: string // Current branch name (default: 'main') enableCOW?: boolean // Enable Copy-on-Write support (default: false for v5.0.0) enableCompression?: boolean // Enable zstd compression for COW blobs (default: true) } /** * Extract filesystem root directory from options * Single source of truth for path resolution - supports all API variants * Zero-config philosophy: flexible input, predictable output */ function getFileSystemPath(options: StorageOptions): string { return ( options.rootDirectory || // Official storageFactory API (options as any).path || // User-friendly API options.options?.rootDirectory || // Nested options API options.options?.path || // Nested path API './brainy-data' // Zero-config fallback ) } /** * Wrap any storage adapter with TypeAwareStorageAdapter * v5.0.0: TypeAware is now the standard interface for ALL storage adapters * This provides type-first organization, fixed-size type counts, and efficient type queries * * @param underlying - The base storage adapter (memory, filesystem, S3, etc.) * @param options - Storage options (for COW configuration) * @param verbose - Optional verbose logging * @returns TypeAwareStorageAdapter wrapping the underlying storage */ async function wrapWithTypeAware( underlying: StorageAdapter, options?: StorageOptions, verbose = false ): Promise { const wrapped = new TypeAwareStorageAdapter({ underlyingStorage: underlying as any, verbose }) as any // Initialize COW if enabled if (options?.enableCOW && typeof wrapped.initializeCOW === 'function') { await wrapped.initializeCOW({ branch: options.branch || 'main', enableCompression: options.enableCompression !== false }) } return wrapped } /** * Create a storage adapter based on the environment and configuration * @param options Options for creating the storage adapter * @returns Promise that resolves to a storage adapter */ export async function createStorage( options: StorageOptions = {} ): Promise { // If memory storage is forced, use it regardless of other options if (options.forceMemoryStorage) { console.log('Using memory storage (forced) + TypeAware wrapper') return await wrapWithTypeAware(new MemoryStorage(), options) } // If file system storage is forced, use it regardless of other options if (options.forceFileSystemStorage) { if (isBrowser()) { console.warn( 'FileSystemStorage is not available in browser environments, falling back to memory storage' ) return await wrapWithTypeAware(new MemoryStorage(), options) } const fsPath = getFileSystemPath(options) console.log(`Using file system storage (forced): ${fsPath} + TypeAware wrapper`) try { const { FileSystemStorage } = await import( './adapters/fileSystemStorage.js' ) return await wrapWithTypeAware(new FileSystemStorage(fsPath), options) } catch (error) { console.warn( 'Failed to load FileSystemStorage, falling back to memory storage:', error ) return await wrapWithTypeAware(new MemoryStorage(), options) } } // If a specific storage type is specified, use it if (options.type && options.type !== 'auto') { switch (options.type) { case 'memory': console.log('Using memory storage + TypeAware wrapper') return await wrapWithTypeAware(new MemoryStorage(), options) case 'opfs': { // Check if OPFS is available const opfsStorage = new OPFSStorage() if (opfsStorage.isOPFSAvailable()) { console.log('Using OPFS storage + TypeAware wrapper') await opfsStorage.init() // Request persistent storage if specified if (options.requestPersistentStorage) { const isPersistent = await opfsStorage.requestPersistentStorage() console.log( `Persistent storage ${isPersistent ? 'granted' : 'denied'}` ) } return await wrapWithTypeAware(opfsStorage, options) } else { console.warn( 'OPFS storage is not available, falling back to memory storage' ) return await wrapWithTypeAware(new MemoryStorage(), options) } } case 'filesystem': { if (isBrowser()) { console.warn( 'FileSystemStorage is not available in browser environments, falling back to memory storage' ) return await wrapWithTypeAware(new MemoryStorage(), options) } const fsPath = getFileSystemPath(options) console.log(`Using file system storage: ${fsPath} + TypeAware wrapper`) try { const { FileSystemStorage } = await import( './adapters/fileSystemStorage.js' ) return await wrapWithTypeAware(new FileSystemStorage(fsPath)) } catch (error) { console.warn( 'Failed to load FileSystemStorage, falling back to memory storage:', error ) return await wrapWithTypeAware(new MemoryStorage(), options) } } case 's3': if (options.s3Storage) { console.log('Using Amazon S3 storage + TypeAware wrapper') return await wrapWithTypeAware(new S3CompatibleStorage({ bucketName: options.s3Storage.bucketName, region: options.s3Storage.region, accessKeyId: options.s3Storage.accessKeyId, secretAccessKey: options.s3Storage.secretAccessKey, sessionToken: options.s3Storage.sessionToken, serviceType: 's3', operationConfig: options.operationConfig, cacheConfig: options.cacheConfig })) } else { console.warn( 'S3 storage configuration is missing, falling back to memory storage' ) return await wrapWithTypeAware(new MemoryStorage(), options) } case 'r2': if (options.r2Storage) { console.log('Using Cloudflare R2 storage (dedicated adapter) + TypeAware wrapper') return await wrapWithTypeAware(new R2Storage({ bucketName: options.r2Storage.bucketName, accountId: options.r2Storage.accountId, accessKeyId: options.r2Storage.accessKeyId, secretAccessKey: options.r2Storage.secretAccessKey, cacheConfig: options.cacheConfig })) } else { console.warn( 'R2 storage configuration is missing, falling back to memory storage' ) return await wrapWithTypeAware(new MemoryStorage(), options) } case 'gcs-native': // DEPRECATED: gcs-native is deprecated in favor of just 'gcs' console.warn( '⚠️ DEPRECATED: type "gcs-native" is deprecated. Use type "gcs" instead.' ) console.warn( ' This will continue to work but may be removed in a future version.' ) // Fall through to 'gcs' case case 'gcs': { // Prefer gcsNativeStorage, but also accept gcsStorage for backward compatibility const gcsNative = options.gcsNativeStorage const gcsLegacy = options.gcsStorage if (!gcsNative && !gcsLegacy) { console.warn( 'GCS storage configuration is missing, falling back to memory storage' ) return await wrapWithTypeAware(new MemoryStorage(), options) } // If using legacy gcsStorage with HMAC keys, use S3-compatible adapter if (gcsLegacy && gcsLegacy.accessKeyId && gcsLegacy.secretAccessKey) { console.warn( '⚠️ GCS with HMAC keys detected. Consider using gcsNativeStorage with ADC instead.' ) console.warn( ' Native GCS with Application Default Credentials is recommended for better performance and security.' ) // Use S3-compatible storage for HMAC keys return await wrapWithTypeAware(new S3CompatibleStorage({ bucketName: gcsLegacy.bucketName, region: gcsLegacy.region, endpoint: gcsLegacy.endpoint || 'https://storage.googleapis.com', accessKeyId: gcsLegacy.accessKeyId, secretAccessKey: gcsLegacy.secretAccessKey, serviceType: 'gcs', cacheConfig: options.cacheConfig })) } // Use native GCS SDK (the correct default!) const config = gcsNative || gcsLegacy! console.log('Using Google Cloud Storage (native SDK) + TypeAware wrapper') return await wrapWithTypeAware(new GcsStorage({ bucketName: config.bucketName, keyFilename: gcsNative?.keyFilename, credentials: gcsNative?.credentials, accessKeyId: config.accessKeyId, secretAccessKey: config.secretAccessKey, skipInitialScan: gcsNative?.skipInitialScan, skipCountsFile: gcsNative?.skipCountsFile, cacheConfig: options.cacheConfig })) } case 'azure': if (options.azureStorage) { console.log('Using Azure Blob Storage (native SDK) + TypeAware wrapper') return await wrapWithTypeAware(new AzureBlobStorage({ containerName: options.azureStorage.containerName, accountName: options.azureStorage.accountName, accountKey: options.azureStorage.accountKey, connectionString: options.azureStorage.connectionString, sasToken: options.azureStorage.sasToken, cacheConfig: options.cacheConfig })) } else { console.warn( 'Azure storage configuration is missing, falling back to memory storage' ) return await wrapWithTypeAware(new MemoryStorage(), options) } case 'type-aware': // v5.0.0: TypeAware is now the default for ALL adapters // Redirect to the underlying type instead console.warn( '⚠️ type-aware is deprecated in v5.0.0 - TypeAware is now always enabled.' ) console.warn( ' Just use the underlying storage type (e.g., "filesystem", "s3", etc.)' ) // Recursively create storage with underlying type return await createStorage({ ...options, type: options.typeAwareStorage?.underlyingType || 'auto' }) default: console.warn( `Unknown storage type: ${options.type}, falling back to memory storage` ) return await wrapWithTypeAware(new MemoryStorage(), options) } } // If custom S3-compatible storage is specified, use it if (options.customS3Storage) { console.log( `Using custom S3-compatible storage: ${options.customS3Storage.serviceType || 'custom'} + TypeAware wrapper` ) return await wrapWithTypeAware(new S3CompatibleStorage({ bucketName: options.customS3Storage.bucketName, region: options.customS3Storage.region, endpoint: options.customS3Storage.endpoint, accessKeyId: options.customS3Storage.accessKeyId, secretAccessKey: options.customS3Storage.secretAccessKey, serviceType: options.customS3Storage.serviceType || 'custom', cacheConfig: options.cacheConfig })) } // If R2 storage is specified, use it if (options.r2Storage) { console.log('Using Cloudflare R2 storage (dedicated adapter) + TypeAware wrapper') return await wrapWithTypeAware(new R2Storage({ bucketName: options.r2Storage.bucketName, accountId: options.r2Storage.accountId, accessKeyId: options.r2Storage.accessKeyId, secretAccessKey: options.r2Storage.secretAccessKey, cacheConfig: options.cacheConfig })) } // If S3 storage is specified, use it if (options.s3Storage) { console.log('Using Amazon S3 storage + TypeAware wrapper') return await wrapWithTypeAware(new S3CompatibleStorage({ bucketName: options.s3Storage.bucketName, region: options.s3Storage.region, accessKeyId: options.s3Storage.accessKeyId, secretAccessKey: options.s3Storage.secretAccessKey, sessionToken: options.s3Storage.sessionToken, serviceType: 's3', cacheConfig: options.cacheConfig })) } // If GCS storage is specified (native or legacy S3-compatible) // Prefer gcsNativeStorage, but also accept gcsStorage for backward compatibility const gcsNative = options.gcsNativeStorage const gcsLegacy = options.gcsStorage if (gcsNative || gcsLegacy) { // If using legacy gcsStorage with HMAC keys, use S3-compatible adapter if (gcsLegacy && gcsLegacy.accessKeyId && gcsLegacy.secretAccessKey) { console.warn( '⚠️ GCS with HMAC keys detected. Consider using gcsNativeStorage with ADC instead.' ) console.warn( ' Native GCS with Application Default Credentials is recommended for better performance and security.' ) // Use S3-compatible storage for HMAC keys console.log('Using Google Cloud Storage (S3-compatible with HMAC - auto-detected) + TypeAware wrapper') return await wrapWithTypeAware(new S3CompatibleStorage({ bucketName: gcsLegacy.bucketName, region: gcsLegacy.region, endpoint: gcsLegacy.endpoint || 'https://storage.googleapis.com', accessKeyId: gcsLegacy.accessKeyId, secretAccessKey: gcsLegacy.secretAccessKey, serviceType: 'gcs', cacheConfig: options.cacheConfig })) } // Use native GCS SDK (the correct default!) const config = gcsNative || gcsLegacy! console.log('Using Google Cloud Storage (native SDK - auto-detected) + TypeAware wrapper') return await wrapWithTypeAware(new GcsStorage({ bucketName: config.bucketName, keyFilename: gcsNative?.keyFilename, credentials: gcsNative?.credentials, accessKeyId: config.accessKeyId, secretAccessKey: config.secretAccessKey, skipInitialScan: gcsNative?.skipInitialScan, skipCountsFile: gcsNative?.skipCountsFile, cacheConfig: options.cacheConfig })) } // If Azure storage is specified, use it if (options.azureStorage) { console.log('Using Azure Blob Storage (native SDK) + TypeAware wrapper') return await wrapWithTypeAware(new AzureBlobStorage({ containerName: options.azureStorage.containerName, accountName: options.azureStorage.accountName, accountKey: options.azureStorage.accountKey, connectionString: options.azureStorage.connectionString, sasToken: options.azureStorage.sasToken, cacheConfig: options.cacheConfig })) } // Auto-detect the best storage adapter based on the environment // First, check if we're in Node.js (prioritize for test environments) if (!isBrowser()) { try { // Check if we're in a Node.js environment if ( typeof process !== 'undefined' && process.versions && process.versions.node ) { const fsPath = getFileSystemPath(options) console.log(`Using file system storage (auto-detected): ${fsPath} + TypeAware wrapper`) try { const { FileSystemStorage } = await import( './adapters/fileSystemStorage.js' ) return await wrapWithTypeAware(new FileSystemStorage(fsPath)) } catch (fsError) { console.warn( 'Failed to load FileSystemStorage, falling back to memory storage:', fsError ) } } } catch (error) { // Not in a Node.js environment or file system is not available console.warn('Not in a Node.js environment:', error) } } // Next, try OPFS (browser only) if (isBrowser()) { const opfsStorage = new OPFSStorage() if (opfsStorage.isOPFSAvailable()) { console.log('Using OPFS storage (auto-detected) + TypeAware wrapper') await opfsStorage.init() // Request persistent storage if specified if (options.requestPersistentStorage) { const isPersistent = await opfsStorage.requestPersistentStorage() console.log(`Persistent storage ${isPersistent ? 'granted' : 'denied'}`) } return await wrapWithTypeAware(opfsStorage, options) } } // Finally, fall back to memory storage console.log('Using memory storage (auto-detected) + TypeAware wrapper') return await wrapWithTypeAware(new MemoryStorage(), options) } /** * Export storage adapters */ export { MemoryStorage, OPFSStorage, S3CompatibleStorage, R2Storage, GcsStorage, AzureBlobStorage, TypeAwareStorageAdapter } // Export FileSystemStorage conditionally // NOTE: FileSystemStorage is now only imported dynamically to avoid fs imports in browser builds // export { FileSystemStorage } from './adapters/fileSystemStorage.js'