feat: simplify GCS storage naming and add Cloud Run deployment options
Changes: - **NEW**: type: 'gcs' now uses native @google-cloud/storage SDK (more intuitive!) - **DEPRECATED**: type: 'gcs-native' is deprecated (use 'gcs' instead) - **NEW**: Add skipInitialScan option to skip bucket scan on init (fixes Cloud Run timeouts) - **NEW**: Add skipCountsFile option to disable counts persistence - **NEW**: Add 2-minute timeout to bucket scans with helpful error messages - **IMPROVED**: Better error handling and recovery for bucket scan failures - **IMPROVED**: Automatic detection of HMAC keys routes to S3CompatibleStorage - **IMPROVED**: Backward compatibility maintained - all existing configs still work Migration Guide: - If using type: 'gcs-native' → Change to type: 'gcs' (or remove type, it auto-detects) - If using HMAC keys with gcsStorage → Consider migrating to ADC for better performance - For Cloud Run timeouts → Add skipInitialScan: true to gcsNativeStorage config Why This Fixes the Waitlist Bug: - Cloud Run containers were timing out during bucket scans - skipInitialScan option allows bypassing expensive bucket scans - Timeout handling prevents silent failures - Better error messages guide users to solutions Resolves issue where GCS native adapter was confusingly named 'gcs-native' while legacy S3-compatible mode used 'gcs'. Now 'gcs' correctly uses the native SDK by default, as users expect. Previous configs continue to work. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
parent
368dd90348
commit
38343c0128
4 changed files with 218 additions and 87 deletions
|
|
@ -2912,31 +2912,27 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
*/
|
||||
private normalizeConfig(config?: BrainyConfig): Required<BrainyConfig> {
|
||||
// Validate storage configuration
|
||||
if (config?.storage?.type && !['auto', 'memory', 'filesystem', 'opfs', 'remote', 's3', 'r2', 'gcs', 'gcs-native'].includes(config.storage.type)) {
|
||||
throw new Error(`Invalid storage type: ${config.storage.type}. Must be one of: auto, memory, filesystem, opfs, remote, s3, r2, gcs, gcs-native`)
|
||||
if (config?.storage?.type && !['auto', 'memory', 'filesystem', 'opfs', 'remote', 's3', 'r2', 'gcs', 'gcs-native', 'azure'].includes(config.storage.type)) {
|
||||
throw new Error(`Invalid storage type: ${config.storage.type}. Must be one of: auto, memory, filesystem, opfs, remote, s3, r2, gcs, gcs-native, azure`)
|
||||
}
|
||||
|
||||
// Validate storage type/config pairing (catch common mismatches)
|
||||
// Warn about deprecated gcs-native
|
||||
if (config?.storage?.type === ('gcs-native' as any)) {
|
||||
console.warn('⚠️ DEPRECATED: type "gcs-native" is deprecated. Use type "gcs" instead.')
|
||||
console.warn(' This will continue to work but may be removed in a future version.')
|
||||
}
|
||||
|
||||
// Validate storage type/config pairing (now more lenient)
|
||||
if (config?.storage) {
|
||||
const storage = config.storage as any
|
||||
|
||||
// Check for gcs/gcsNativeStorage mismatch
|
||||
if (storage.type === 'gcs' && storage.gcsNativeStorage) {
|
||||
throw new Error(
|
||||
`Storage type/config mismatch: type 'gcs' requires 'gcsStorage' config object (S3-compatible). ` +
|
||||
`You provided 'gcsNativeStorage' which requires type 'gcs-native'. ` +
|
||||
`Either change type to 'gcs-native' or use 'gcsStorage' instead of 'gcsNativeStorage'.`
|
||||
)
|
||||
// Warn about legacy gcsStorage config with HMAC keys
|
||||
if (storage.gcsStorage && storage.gcsStorage.accessKeyId && storage.gcsStorage.secretAccessKey) {
|
||||
console.warn('⚠️ GCS with HMAC keys (gcsStorage) is legacy. Consider migrating to native GCS (gcsNativeStorage) with ADC.')
|
||||
}
|
||||
|
||||
// Check for gcs-native/gcsStorage mismatch
|
||||
if (storage.type === 'gcs-native' && storage.gcsStorage) {
|
||||
throw new Error(
|
||||
`Storage type/config mismatch: type 'gcs-native' requires 'gcsNativeStorage' config object. ` +
|
||||
`You provided 'gcsStorage' which requires type 'gcs' (S3-compatible). ` +
|
||||
`Either change type to 'gcs' or use 'gcsNativeStorage' instead of 'gcsStorage'.`
|
||||
)
|
||||
}
|
||||
// No longer throw errors for mismatches - storageFactory now handles this intelligently
|
||||
// Both 'gcs' and 'gcs-native' can now use either gcsStorage or gcsNativeStorage
|
||||
}
|
||||
|
||||
// Validate model configuration
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue