feat: add native Google Cloud Storage adapter with ADC support
Implement native @google-cloud/storage adapter for better performance
and easier authentication in Cloud Run/GCE environments.
Features:
- Application Default Credentials (ADC) for zero-config auth
- Service account authentication (keyFilename, credentials)
- HMAC fallback for backward compatibility
- Full UUID-based sharding preservation
- Write buffers for high-volume mode
- Multi-level caching and adaptive backpressure
- Complete feature parity with S3-compatible adapter
Benefits over S3-compatible GCS:
- No HMAC key management required
- Native SDK performance optimizations
- Automatic authentication in Cloud Run/GCE
- Simpler configuration
Configuration:
- type: 'gcs-native'
- gcsNativeStorage: { bucketName, keyFilename?, credentials? }
- Zero data migration required (same path structure)
🤖 Generated with [Claude Code](https://claude.com/claude-code)
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
parent
8a9cf1bd51
commit
e2aa8e3253
6 changed files with 2824 additions and 35 deletions
1533
src/storage/adapters/gcsStorage.ts
Normal file
1533
src/storage/adapters/gcsStorage.ts
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -10,6 +10,7 @@ import {
|
|||
S3CompatibleStorage,
|
||||
R2Storage
|
||||
} from './adapters/s3CompatibleStorage.js'
|
||||
import { GcsStorage } from './adapters/gcsStorage.js'
|
||||
// FileSystemStorage is dynamically imported to avoid issues in browser environments
|
||||
import { isBrowser } from '../utils/environment.js'
|
||||
import { OperationConfig } from '../utils/operationUtils.js'
|
||||
|
|
@ -26,9 +27,10 @@ export interface StorageOptions {
|
|||
* - 'filesystem': Use file system storage (Node.js only)
|
||||
* - 's3': Use Amazon S3 storage
|
||||
* - 'r2': Use Cloudflare R2 storage
|
||||
* - 'gcs': Use Google Cloud Storage
|
||||
* - 'gcs': Use Google Cloud Storage (S3-compatible with HMAC keys)
|
||||
* - 'gcs-native': Use Google Cloud Storage (native SDK with ADC)
|
||||
*/
|
||||
type?: 'auto' | 'memory' | 'opfs' | 'filesystem' | 's3' | 'r2' | 'gcs'
|
||||
type?: 'auto' | 'memory' | 'opfs' | 'filesystem' | 's3' | 'r2' | 'gcs' | 'gcs-native'
|
||||
|
||||
/**
|
||||
* Force the use of memory storage even if other storage types are available
|
||||
|
|
@ -106,7 +108,7 @@ export interface StorageOptions {
|
|||
}
|
||||
|
||||
/**
|
||||
* Configuration for Google Cloud Storage
|
||||
* Configuration for Google Cloud Storage (S3-compatible with HMAC keys)
|
||||
*/
|
||||
gcsStorage?: {
|
||||
/**
|
||||
|
|
@ -135,6 +137,36 @@ export interface StorageOptions {
|
|||
endpoint?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Configuration for Google Cloud Storage (native SDK with ADC)
|
||||
*/
|
||||
gcsNativeStorage?: {
|
||||
/**
|
||||
* GCS bucket name
|
||||
*/
|
||||
bucketName: string
|
||||
|
||||
/**
|
||||
* Service account key file path (optional, uses ADC if not provided)
|
||||
*/
|
||||
keyFilename?: string
|
||||
|
||||
/**
|
||||
* Service account credentials object (optional, uses ADC if not provided)
|
||||
*/
|
||||
credentials?: object
|
||||
|
||||
/**
|
||||
* HMAC access key ID (backward compatibility, not recommended)
|
||||
*/
|
||||
accessKeyId?: string
|
||||
|
||||
/**
|
||||
* HMAC secret access key (backward compatibility, not recommended)
|
||||
*/
|
||||
secretAccessKey?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Configuration for custom S3-compatible storage
|
||||
*/
|
||||
|
|
@ -357,7 +389,7 @@ export async function createStorage(
|
|||
|
||||
case 'gcs':
|
||||
if (options.gcsStorage) {
|
||||
console.log('Using Google Cloud Storage')
|
||||
console.log('Using Google Cloud Storage (S3-compatible)')
|
||||
return new S3CompatibleStorage({
|
||||
bucketName: options.gcsStorage.bucketName,
|
||||
region: options.gcsStorage.region,
|
||||
|
|
@ -375,6 +407,24 @@ export async function createStorage(
|
|||
return new MemoryStorage()
|
||||
}
|
||||
|
||||
case 'gcs-native':
|
||||
if (options.gcsNativeStorage) {
|
||||
console.log('Using Google Cloud Storage (native SDK)')
|
||||
return new GcsStorage({
|
||||
bucketName: options.gcsNativeStorage.bucketName,
|
||||
keyFilename: options.gcsNativeStorage.keyFilename,
|
||||
credentials: options.gcsNativeStorage.credentials,
|
||||
accessKeyId: options.gcsNativeStorage.accessKeyId,
|
||||
secretAccessKey: options.gcsNativeStorage.secretAccessKey,
|
||||
cacheConfig: options.cacheConfig
|
||||
})
|
||||
} else {
|
||||
console.warn(
|
||||
'GCS native storage configuration is missing, falling back to memory storage'
|
||||
)
|
||||
return new MemoryStorage()
|
||||
}
|
||||
|
||||
default:
|
||||
console.warn(
|
||||
`Unknown storage type: ${options.type}, falling back to memory storage`
|
||||
|
|
@ -426,9 +476,22 @@ export async function createStorage(
|
|||
})
|
||||
}
|
||||
|
||||
// If GCS storage is specified, use it
|
||||
// If GCS native storage is specified, use it (prioritize native over S3-compatible)
|
||||
if (options.gcsNativeStorage) {
|
||||
console.log('Using Google Cloud Storage (native SDK)')
|
||||
return new GcsStorage({
|
||||
bucketName: options.gcsNativeStorage.bucketName,
|
||||
keyFilename: options.gcsNativeStorage.keyFilename,
|
||||
credentials: options.gcsNativeStorage.credentials,
|
||||
accessKeyId: options.gcsNativeStorage.accessKeyId,
|
||||
secretAccessKey: options.gcsNativeStorage.secretAccessKey,
|
||||
cacheConfig: options.cacheConfig
|
||||
})
|
||||
}
|
||||
|
||||
// If GCS storage is specified, use it (S3-compatible)
|
||||
if (options.gcsStorage) {
|
||||
console.log('Using Google Cloud Storage')
|
||||
console.log('Using Google Cloud Storage (S3-compatible)')
|
||||
return new S3CompatibleStorage({
|
||||
bucketName: options.gcsStorage.bucketName,
|
||||
region: options.gcsStorage.region,
|
||||
|
|
@ -498,7 +561,8 @@ export {
|
|||
MemoryStorage,
|
||||
OPFSStorage,
|
||||
S3CompatibleStorage,
|
||||
R2Storage
|
||||
R2Storage,
|
||||
GcsStorage
|
||||
}
|
||||
|
||||
// Export FileSystemStorage conditionally
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue