fix: prevent circuit breaker activation and data loss during bulk imports

Storage-aware batching system prevents rate limiting issues on cloud storage (GCS, S3, R2, Azure). Replaces entity-by-entity creation with addMany()/relateMany() batch operations in ImportCoordinator. Separate read/write circuit breakers prevent read lockouts during write throttling. Each storage adapter auto-configures optimal batch sizes and delays. Fixes silent data loss and 30+ second lockouts on 1000+ row imports.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

src/storage/adapters/azureBlobStorage.ts

@@ -25,6 +25,7 @@ import {
 } from '../../coreTypes.js'
 import {
   BaseStorage,
+  StorageBatchConfig,
   NOUNS_DIR,
   VERBS_DIR,
   METADATA_DIR,
@@ -162,6 +163,32 @@ export class AzureBlobStorage extends BaseStorage {
     }
   }
 
+  /**
+   * Get Azure Blob-optimized batch configuration
+   *
+   * Azure Blob Storage has moderate rate limits between GCS and S3:
+   * - Medium batch sizes (75 items)
+   * - Parallel processing supported
+   * - Moderate delays (75ms)
+   *
+   * Azure can handle ~2000 operations/second with good performance
+   *
+   * @returns Azure Blob-optimized batch configuration
+   * @since v4.11.0
+   */
+  public getBatchConfig(): StorageBatchConfig {
+    return {
+      maxBatchSize: 75,
+      batchDelayMs: 75,
+      maxConcurrent: 75,
+      supportsParallelWrites: true,  // Azure handles parallel reasonably
+      rateLimit: {
+        operationsPerSecond: 2000,  // Moderate limits
+        burstCapacity: 500
+      }
+    }
+  }
+
   /**
    * Initialize the storage adapter
    */

src/storage/adapters/baseStorageAdapter.ts

@@ -14,6 +14,7 @@ import {
   NounMetadata,
   VerbMetadata
 } from '../../coreTypes.js'
+import { StorageBatchConfig } from '../baseStorage.js'
 import { extractFieldNamesFromJson, mapToStandardField } from '../../utils/fieldNameTracking.js'
 import { getGlobalMutex, cleanupMutexes } from '../../utils/mutex.js'
 
@@ -90,6 +91,33 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
     details?: Record<string, any>
   }>
 
+  /**
+   * Get optimal batch configuration for this storage adapter
+   * Override in subclasses to provide storage-specific optimization
+   *
+   * This method allows each storage adapter to declare its optimal batch behavior
+   * for rate limiting and performance. The configuration is used by addMany(),
+   * relateMany(), and import operations to automatically adapt to storage capabilities.
+   *
+   * @returns Batch configuration optimized for this storage type
+   * @since v4.11.0
+   */
+  public getBatchConfig(): StorageBatchConfig {
+    // Conservative defaults that work safely across all storage types
+    // Cloud storage adapters should override with higher throughput values
+    // Local storage adapters should override with no delays
+    return {
+      maxBatchSize: 50,
+      batchDelayMs: 100,
+      maxConcurrent: 50,
+      supportsParallelWrites: false,
+      rateLimit: {
+        operationsPerSecond: 100,
+        burstCapacity: 200
+      }
+    }
+  }
+
   // NOTE: getAllNouns and getAllVerbs have been removed to prevent expensive full scans.
   // Use getNouns() and getVerbs() with pagination instead.
 

src/storage/adapters/fileSystemStorage.ts

@@ -16,6 +16,7 @@ import {
 } from '../../coreTypes.js'
 import {
   BaseStorage,
+  StorageBatchConfig,
   SYSTEM_DIR,
   STATISTICS_KEY
 } from '../baseStorage.js'
@@ -119,6 +120,31 @@ export class FileSystemStorage extends BaseStorage {
     // Defer path operations until init() when path module is guaranteed to be loaded
   }
 
+  /**
+   * Get FileSystem-optimized batch configuration
+   *
+   * File system storage is I/O bound but not rate limited:
+   * - Large batch sizes (500 items)
+   * - No delays needed (0ms)
+   * - Moderate concurrency (100 operations) - limited by I/O threads
+   * - Parallel processing supported
+   *
+   * @returns FileSystem-optimized batch configuration
+   * @since v4.11.0
+   */
+  public getBatchConfig(): StorageBatchConfig {
+    return {
+      maxBatchSize: 500,
+      batchDelayMs: 0,
+      maxConcurrent: 100,
+      supportsParallelWrites: true,  // Filesystem handles parallel I/O
+      rateLimit: {
+        operationsPerSecond: 5000,  // Depends on disk speed
+        burstCapacity: 2000
+      }
+    }
+  }
+
   /**
    * Initialize the storage adapter
    */

src/storage/adapters/gcsStorage.ts

@@ -22,6 +22,7 @@ import {
 } from '../../coreTypes.js'
 import {
   BaseStorage,
+  StorageBatchConfig,
   NOUNS_DIR,
   VERBS_DIR,
   METADATA_DIR,
@@ -175,6 +176,33 @@ export class GcsStorage extends BaseStorage {
     }
   }
 
+  /**
+   * Get GCS-optimized batch configuration
+   *
+   * GCS has strict rate limits (~5000 writes/second per bucket) and benefits from:
+   * - Moderate batch sizes (50 items)
+   * - Sequential processing (not parallel)
+   * - Delays between batches (100ms)
+   *
+   * Note: Each entity write involves 2 operations (vector + metadata),
+   * so 800 ops/sec = ~400 entities/sec = ~2500 actual GCS writes/sec
+   *
+   * @returns GCS-optimized batch configuration
+   * @since v4.11.0
+   */
+  public getBatchConfig(): StorageBatchConfig {
+    return {
+      maxBatchSize: 50,
+      batchDelayMs: 100,
+      maxConcurrent: 50,
+      supportsParallelWrites: false,  // Sequential is safer for GCS rate limits
+      rateLimit: {
+        operationsPerSecond: 800,  // Conservative estimate for entity operations
+        burstCapacity: 200
+      }
+    }
+  }
+
   /**
    * Initialize the storage adapter
    */

src/storage/adapters/memoryStorage.ts

@@ -14,7 +14,7 @@ import {
   StatisticsData,
   NounType
 } from '../../coreTypes.js'
-import { BaseStorage, STATISTICS_KEY } from '../baseStorage.js'
+import { BaseStorage, StorageBatchConfig, STATISTICS_KEY } from '../baseStorage.js'
 import { PaginatedResult } from '../../types/paginationTypes.js'
 
 // No type aliases needed - using the original types directly
@@ -47,6 +47,31 @@ export class MemoryStorage extends BaseStorage {
     super()
   }
 
+  /**
+   * Get Memory-optimized batch configuration
+   *
+   * Memory storage has no rate limits and can handle very high throughput:
+   * - Large batch sizes (1000 items)
+   * - No delays needed (0ms)
+   * - High concurrency (1000 operations)
+   * - Parallel processing maximizes throughput
+   *
+   * @returns Memory-optimized batch configuration
+   * @since v4.11.0
+   */
+  public getBatchConfig(): StorageBatchConfig {
+    return {
+      maxBatchSize: 1000,
+      batchDelayMs: 0,
+      maxConcurrent: 1000,
+      supportsParallelWrites: true,  // Memory loves parallel operations
+      rateLimit: {
+        operationsPerSecond: 100000,  // Virtually unlimited
+        burstCapacity: 100000
+      }
+    }
+  }
+
   /**
    * Initialize the storage adapter
    * Nothing to initialize for in-memory storage

src/storage/adapters/opfsStorage.ts

@@ -16,6 +16,7 @@ import {
 } from '../../coreTypes.js'
 import {
   BaseStorage,
+  StorageBatchConfig,
   NOUNS_DIR,
   VERBS_DIR,
   METADATA_DIR,
@@ -80,6 +81,31 @@ export class OPFSStorage extends BaseStorage {
       'getDirectory' in navigator.storage
   }
 
+  /**
+   * Get OPFS-optimized batch configuration
+   *
+   * OPFS (Origin Private File System) is browser-based storage with moderate performance:
+   * - Moderate batch sizes (100 items)
+   * - Small delays (10ms) for browser event loop
+   * - Limited concurrency (50 operations) - browser constraints
+   * - Sequential processing preferred for stability
+   *
+   * @returns OPFS-optimized batch configuration
+   * @since v4.11.0
+   */
+  public getBatchConfig(): StorageBatchConfig {
+    return {
+      maxBatchSize: 100,
+      batchDelayMs: 10,
+      maxConcurrent: 50,
+      supportsParallelWrites: false,  // Sequential safer in browser
+      rateLimit: {
+        operationsPerSecond: 1000,
+        burstCapacity: 500
+      }
+    }
+  }
+
   /**
    * Initialize the storage adapter
    */

src/storage/adapters/r2Storage.ts

@@ -25,6 +25,7 @@ import {
 } from '../../coreTypes.js'
 import {
   BaseStorage,
+  StorageBatchConfig,
   NOUNS_DIR,
   VERBS_DIR,
   METADATA_DIR,
@@ -169,6 +170,35 @@ export class R2Storage extends BaseStorage {
     }
   }
 
+  /**
+   * Get R2-optimized batch configuration
+   *
+   * Cloudflare R2 has S3-compatible characteristics with some advantages:
+   * - Zero egress fees (can cache more aggressively)
+   * - Global edge network
+   * - Similar throughput to S3
+   *
+   * R2 benefits from the same configuration as S3:
+   * - Larger batch sizes (100 items)
+   * - Parallel processing
+   * - Short delays (50ms)
+   *
+   * @returns R2-optimized batch configuration
+   * @since v4.11.0
+   */
+  public getBatchConfig(): StorageBatchConfig {
+    return {
+      maxBatchSize: 100,
+      batchDelayMs: 50,
+      maxConcurrent: 100,
+      supportsParallelWrites: true,  // R2 handles parallel writes like S3
+      rateLimit: {
+        operationsPerSecond: 3500,  // Similar to S3 throughput
+        burstCapacity: 1000
+      }
+    }
+  }
+
   /**
    * Initialize the storage adapter
    */

src/storage/adapters/s3CompatibleStorage.ts

@@ -18,6 +18,7 @@ import {
 } from '../../coreTypes.js'
 import {
   BaseStorage,
+  StorageBatchConfig,
   NOUNS_DIR,
   VERBS_DIR,
   METADATA_DIR,
@@ -210,6 +211,32 @@ export class S3CompatibleStorage extends BaseStorage {
     this.verbCacheManager = new CacheManager<Edge>(options.cacheConfig)
   }
 
+  /**
+   * Get S3-optimized batch configuration
+   *
+   * S3 has higher throughput than GCS and handles parallel writes efficiently:
+   * - Larger batch sizes (100 items)
+   * - Parallel processing supported
+   * - Shorter delays between batches (50ms)
+   *
+   * S3 can handle ~3500 operations/second per bucket with good performance
+   *
+   * @returns S3-optimized batch configuration
+   * @since v4.11.0
+   */
+  public getBatchConfig(): StorageBatchConfig {
+    return {
+      maxBatchSize: 100,
+      batchDelayMs: 50,
+      maxConcurrent: 100,
+      supportsParallelWrites: true,  // S3 handles parallel writes efficiently
+      rateLimit: {
+        operationsPerSecond: 3500,  // S3 is more permissive than GCS
+        burstCapacity: 1000
+      }
+    }
+  }
+
   /**
    * Initialize the storage adapter
    */

src/storage/baseStorage.ts

@@ -32,6 +32,36 @@ interface StorageKeyInfo {
   fullPath: string
 }
 
+/**
+ * Storage adapter batch configuration profile
+ * Each storage adapter declares its optimal batch behavior for rate limiting
+ * and performance optimization
+ *
+ * @since v4.11.0
+ */
+export interface StorageBatchConfig {
+  /** Maximum items per batch */
+  maxBatchSize: number
+
+  /** Delay between batches in milliseconds (for rate limiting) */
+  batchDelayMs: number
+
+  /** Maximum concurrent operations this storage can handle */
+  maxConcurrent: number
+
+  /** Whether storage can handle parallel writes efficiently */
+  supportsParallelWrites: boolean
+
+  /** Rate limit characteristics of this storage adapter */
+  rateLimit: {
+    /** Approximate operations per second this storage can handle */
+    operationsPerSecond: number
+
+    /** Maximum burst capacity before throttling occurs */
+    burstCapacity: number
+  }
+}
+
 // Clean directory structure (v4.7.2+)
 // All storage adapters use this consistent structure
 export const NOUNS_METADATA_DIR = 'entities/nouns/metadata'