feat: add native Google Cloud Storage adapter with ADC support

Implement native @google-cloud/storage adapter for better performance
and easier authentication in Cloud Run/GCE environments.

Features:
- Application Default Credentials (ADC) for zero-config auth
- Service account authentication (keyFilename, credentials)
- HMAC fallback for backward compatibility
- Full UUID-based sharding preservation
- Write buffers for high-volume mode
- Multi-level caching and adaptive backpressure
- Complete feature parity with S3-compatible adapter

Benefits over S3-compatible GCS:
- No HMAC key management required
- Native SDK performance optimizations
- Automatic authentication in Cloud Run/GCE
- Simpler configuration

Configuration:
- type: 'gcs-native'
- gcsNativeStorage: { bucketName, keyFilename?, credentials? }
- Zero data migration required (same path structure)

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
David Snelling 2025-10-08 14:08:43 -07:00
parent 8a9cf1bd51
commit e2aa8e3253
6 changed files with 2824 additions and 35 deletions

File diff suppressed because it is too large Load diff

View file

@ -10,6 +10,7 @@ import {
S3CompatibleStorage,
R2Storage
} from './adapters/s3CompatibleStorage.js'
import { GcsStorage } from './adapters/gcsStorage.js'
// FileSystemStorage is dynamically imported to avoid issues in browser environments
import { isBrowser } from '../utils/environment.js'
import { OperationConfig } from '../utils/operationUtils.js'
@ -26,9 +27,10 @@ export interface StorageOptions {
* - 'filesystem': Use file system storage (Node.js only)
* - 's3': Use Amazon S3 storage
* - 'r2': Use Cloudflare R2 storage
* - 'gcs': Use Google Cloud Storage
* - 'gcs': Use Google Cloud Storage (S3-compatible with HMAC keys)
* - 'gcs-native': Use Google Cloud Storage (native SDK with ADC)
*/
type?: 'auto' | 'memory' | 'opfs' | 'filesystem' | 's3' | 'r2' | 'gcs'
type?: 'auto' | 'memory' | 'opfs' | 'filesystem' | 's3' | 'r2' | 'gcs' | 'gcs-native'
/**
* Force the use of memory storage even if other storage types are available
@ -106,7 +108,7 @@ export interface StorageOptions {
}
/**
* Configuration for Google Cloud Storage
* Configuration for Google Cloud Storage (S3-compatible with HMAC keys)
*/
gcsStorage?: {
/**
@ -135,6 +137,36 @@ export interface StorageOptions {
endpoint?: string
}
/**
* Configuration for Google Cloud Storage (native SDK with ADC)
*/
gcsNativeStorage?: {
/**
* GCS bucket name
*/
bucketName: string
/**
* Service account key file path (optional, uses ADC if not provided)
*/
keyFilename?: string
/**
* Service account credentials object (optional, uses ADC if not provided)
*/
credentials?: object
/**
* HMAC access key ID (backward compatibility, not recommended)
*/
accessKeyId?: string
/**
* HMAC secret access key (backward compatibility, not recommended)
*/
secretAccessKey?: string
}
/**
* Configuration for custom S3-compatible storage
*/
@ -357,7 +389,7 @@ export async function createStorage(
case 'gcs':
if (options.gcsStorage) {
console.log('Using Google Cloud Storage')
console.log('Using Google Cloud Storage (S3-compatible)')
return new S3CompatibleStorage({
bucketName: options.gcsStorage.bucketName,
region: options.gcsStorage.region,
@ -375,6 +407,24 @@ export async function createStorage(
return new MemoryStorage()
}
case 'gcs-native':
if (options.gcsNativeStorage) {
console.log('Using Google Cloud Storage (native SDK)')
return new GcsStorage({
bucketName: options.gcsNativeStorage.bucketName,
keyFilename: options.gcsNativeStorage.keyFilename,
credentials: options.gcsNativeStorage.credentials,
accessKeyId: options.gcsNativeStorage.accessKeyId,
secretAccessKey: options.gcsNativeStorage.secretAccessKey,
cacheConfig: options.cacheConfig
})
} else {
console.warn(
'GCS native storage configuration is missing, falling back to memory storage'
)
return new MemoryStorage()
}
default:
console.warn(
`Unknown storage type: ${options.type}, falling back to memory storage`
@ -426,9 +476,22 @@ export async function createStorage(
})
}
// If GCS storage is specified, use it
// If GCS native storage is specified, use it (prioritize native over S3-compatible)
if (options.gcsNativeStorage) {
console.log('Using Google Cloud Storage (native SDK)')
return new GcsStorage({
bucketName: options.gcsNativeStorage.bucketName,
keyFilename: options.gcsNativeStorage.keyFilename,
credentials: options.gcsNativeStorage.credentials,
accessKeyId: options.gcsNativeStorage.accessKeyId,
secretAccessKey: options.gcsNativeStorage.secretAccessKey,
cacheConfig: options.cacheConfig
})
}
// If GCS storage is specified, use it (S3-compatible)
if (options.gcsStorage) {
console.log('Using Google Cloud Storage')
console.log('Using Google Cloud Storage (S3-compatible)')
return new S3CompatibleStorage({
bucketName: options.gcsStorage.bucketName,
region: options.gcsStorage.region,
@ -498,7 +561,8 @@ export {
MemoryStorage,
OPFSStorage,
S3CompatibleStorage,
R2Storage
R2Storage,
GcsStorage
}
// Export FileSystemStorage conditionally