feat: progressive init and readiness API for cloud storage
- Add `initMode` option to all cloud storage adapters (GCS, S3, Azure) - 'auto' (default): progressive in cloud, strict locally - 'progressive': <200ms cold starts, lazy bucket validation - 'strict': blocking validation (current behavior) - Add cloud environment auto-detection for: - Cloud Run (K_SERVICE, K_REVISION) - AWS Lambda (AWS_LAMBDA_FUNCTION_NAME) - Cloud Functions (FUNCTIONS_TARGET) - Azure Functions (AZURE_FUNCTIONS_ENVIRONMENT) - Add readiness API to Brainy class: - `brain.ready`: Promise that resolves when init() completes - `brain.isFullyInitialized()`: checks if background tasks done - `brain.awaitBackgroundInit()`: waits for background tasks - Lazy bucket validation on first write (not during init) - Background count synchronization - Update AWS/GCP deployment docs with readiness patterns 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
parent
19c5e67035
commit
d938a6bd4f
10 changed files with 1470 additions and 62 deletions
|
|
@ -41,6 +41,7 @@ import { getGlobalBackpressure } from '../../utils/adaptiveBackpressure.js'
|
|||
import { getWriteBuffer, WriteBuffer } from '../../utils/writeBuffer.js'
|
||||
import { getCoalescer, RequestCoalescer } from '../../utils/requestCoalescer.js'
|
||||
import { getShardIdFromUuid, getAllShardIds, getShardIdByIndex, TOTAL_SHARDS } from '../sharding.js'
|
||||
import { InitMode } from './baseStorageAdapter.js'
|
||||
|
||||
// Type aliases for better readability
|
||||
type HNSWNode = HNSWNoun
|
||||
|
|
@ -119,7 +120,26 @@ export class AzureBlobStorage extends BaseStorage {
|
|||
|
||||
/**
|
||||
* Initialize the storage adapter
|
||||
*
|
||||
* @param options Configuration options for Azure Blob Storage
|
||||
*
|
||||
* @example Zero-config (recommended) - auto-detects Azure Functions for fast init
|
||||
* ```typescript
|
||||
* const storage = new AzureBlobStorage({
|
||||
* containerName: 'my-container',
|
||||
* accountName: 'myaccount'
|
||||
* })
|
||||
* await storage.init() // <200ms in Azure Functions, blocking locally
|
||||
* ```
|
||||
*
|
||||
* @example Force progressive mode for all environments
|
||||
* ```typescript
|
||||
* const storage = new AzureBlobStorage({
|
||||
* containerName: 'my-container',
|
||||
* accountName: 'myaccount',
|
||||
* initMode: 'progressive' // Always <200ms init
|
||||
* })
|
||||
* ```
|
||||
*/
|
||||
constructor(options: {
|
||||
containerName: string
|
||||
|
|
@ -140,6 +160,21 @@ export class AzureBlobStorage extends BaseStorage {
|
|||
hotCacheEvictionThreshold?: number
|
||||
warmCacheTTL?: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialization mode for fast cold starts (v7.3.0+)
|
||||
*
|
||||
* - `'auto'` (default): Progressive in cloud environments (Azure Functions),
|
||||
* strict locally. Zero-config optimization.
|
||||
* - `'progressive'`: Always use fast init (<200ms). Container validation and
|
||||
* count loading happen in background. First write validates container.
|
||||
* - `'strict'`: Traditional blocking init. Validates container and loads counts
|
||||
* before init() returns.
|
||||
*
|
||||
* @since v7.3.0
|
||||
*/
|
||||
initMode?: InitMode
|
||||
|
||||
readOnly?: boolean
|
||||
}) {
|
||||
super()
|
||||
|
|
@ -150,6 +185,11 @@ export class AzureBlobStorage extends BaseStorage {
|
|||
this.sasToken = options.sasToken
|
||||
this.readOnly = options.readOnly || false
|
||||
|
||||
// v7.3.0: Handle initMode
|
||||
if (options.initMode) {
|
||||
this.initMode = options.initMode
|
||||
}
|
||||
|
||||
// Set up prefixes for different types of data using entity-based structure
|
||||
this.nounPrefix = `${getDirectoryPath('noun', 'vector')}/`
|
||||
this.verbPrefix = `${getDirectoryPath('verb', 'vector')}/`
|
||||
|
|
@ -256,6 +296,22 @@ export class AzureBlobStorage extends BaseStorage {
|
|||
|
||||
/**
|
||||
* Initialize the storage adapter
|
||||
*
|
||||
* v7.3.0: Supports progressive initialization for fast cold starts
|
||||
*
|
||||
* | Mode | Init Time | When |
|
||||
* |------|-----------|------|
|
||||
* | `progressive` | <200ms | Azure Functions |
|
||||
* | `strict` | 100-500ms+ | Local development, tests |
|
||||
* | `auto` | Detected | Default - best of both |
|
||||
*
|
||||
* In progressive mode:
|
||||
* - SDK import and client creation: ~50ms (unavoidable)
|
||||
* - Write buffers and caches: ~10ms
|
||||
* - Mark as initialized: READY
|
||||
* - Background: validate container, load counts
|
||||
*
|
||||
* First write operation validates container existence (lazy validation).
|
||||
*/
|
||||
public async init(): Promise<void> {
|
||||
if (this.isInitialized) {
|
||||
|
|
@ -263,7 +319,7 @@ export class AzureBlobStorage extends BaseStorage {
|
|||
}
|
||||
|
||||
try {
|
||||
// Import Azure Storage SDK only when needed
|
||||
// Import Azure Storage SDK only when needed (~50ms)
|
||||
const { BlobServiceClient } = await import('@azure/storage-blob')
|
||||
|
||||
// Configure the Azure Blob Storage client based on available credentials
|
||||
|
|
@ -306,17 +362,14 @@ export class AzureBlobStorage extends BaseStorage {
|
|||
throw new Error('Azure Blob Storage requires either connectionString, accountName+accountKey, accountName+sasToken, or accountName (for Managed Identity)')
|
||||
}
|
||||
|
||||
// Get reference to the container
|
||||
// Get reference to the container (no network calls)
|
||||
this.containerClient = this.blobServiceClient.getContainerClient(this.containerName)
|
||||
|
||||
// Create container if it doesn't exist
|
||||
const exists = await this.containerClient.exists()
|
||||
if (!exists) {
|
||||
await this.containerClient.create()
|
||||
prodLog.info(`✅ Created Azure container: ${this.containerName}`)
|
||||
} else {
|
||||
prodLog.info(`✅ Connected to Azure container: ${this.containerName}`)
|
||||
}
|
||||
// Determine initialization mode
|
||||
const effectiveMode = this.resolveInitMode()
|
||||
const isCloud = this.detectCloudEnvironment()
|
||||
|
||||
prodLog.info(`🚀 Azure init mode: ${effectiveMode} (detected cloud: ${isCloud})`)
|
||||
|
||||
// Initialize write buffers for high-volume mode
|
||||
const storageId = `azure-${this.containerName}`
|
||||
|
|
@ -345,23 +398,183 @@ export class AzureBlobStorage extends BaseStorage {
|
|||
}
|
||||
)
|
||||
|
||||
// Initialize counts from storage
|
||||
await this.initializeCounts()
|
||||
|
||||
// Clear any stale cache entries from previous runs
|
||||
prodLog.info('🧹 Clearing cache from previous run to prevent cache poisoning')
|
||||
this.nounCacheManager.clear()
|
||||
this.verbCacheManager.clear()
|
||||
prodLog.info('✅ Cache cleared - starting fresh')
|
||||
|
||||
// v6.0.0: Initialize GraphAdjacencyIndex and type statistics
|
||||
await super.init()
|
||||
// v7.3.0: Progressive vs Strict initialization
|
||||
if (effectiveMode === 'progressive') {
|
||||
// PROGRESSIVE MODE: Fast init, background validation
|
||||
// Mark as initialized immediately - ready to accept operations
|
||||
// Container validation happens lazily on first write
|
||||
// Count loading happens in background
|
||||
|
||||
prodLog.info(`✅ Azure progressive init complete: ${this.containerName} (validation deferred)`)
|
||||
|
||||
// v6.0.0: Initialize GraphAdjacencyIndex and type statistics
|
||||
await super.init()
|
||||
|
||||
// Schedule background tasks (non-blocking)
|
||||
this.scheduleBackgroundInit()
|
||||
} else {
|
||||
// STRICT MODE: Traditional blocking initialization
|
||||
// Verify container exists or create it (blocking)
|
||||
const exists = await this.containerClient.exists()
|
||||
if (!exists) {
|
||||
await this.containerClient.create()
|
||||
prodLog.info(`✅ Created Azure container: ${this.containerName}`)
|
||||
} else {
|
||||
prodLog.info(`✅ Connected to Azure container: ${this.containerName}`)
|
||||
}
|
||||
this.bucketValidated = true
|
||||
|
||||
// Initialize counts from storage (blocking)
|
||||
await this.initializeCounts()
|
||||
this.countsLoaded = true
|
||||
|
||||
// v6.0.0: Initialize GraphAdjacencyIndex and type statistics
|
||||
await super.init()
|
||||
|
||||
// Mark background tasks as complete (nothing to do in background)
|
||||
this.backgroundTasksComplete = true
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.error('Failed to initialize Azure Blob Storage:', error)
|
||||
throw new Error(`Failed to initialize Azure Blob Storage: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
// =============================================
|
||||
// Progressive Initialization (v7.3.0+)
|
||||
// =============================================
|
||||
|
||||
/**
|
||||
* Run background initialization tasks for Azure.
|
||||
*
|
||||
* Called in progressive mode after init() returns. Performs:
|
||||
* 1. Container validation (in background)
|
||||
* 2. Count loading from storage (in background)
|
||||
*
|
||||
* These tasks don't block the main thread, allowing fast cold starts.
|
||||
*
|
||||
* @protected
|
||||
* @override
|
||||
* @since v7.3.0
|
||||
*/
|
||||
protected async runBackgroundInit(): Promise<void> {
|
||||
const startTime = Date.now()
|
||||
prodLog.info('[Azure Background] Starting background initialization...')
|
||||
|
||||
// Run validation and count loading in parallel
|
||||
const validationPromise = this.validateContainerInBackground()
|
||||
const countsPromise = this.loadCountsInBackground()
|
||||
|
||||
// Wait for both to complete (but we're already initialized)
|
||||
await Promise.all([validationPromise, countsPromise])
|
||||
|
||||
const elapsed = Date.now() - startTime
|
||||
prodLog.info(`[Azure Background] Background init complete in ${elapsed}ms`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate container existence in background.
|
||||
*
|
||||
* Creates container if it doesn't exist. Stores result in
|
||||
* bucketValidated/bucketValidationError for lazy use.
|
||||
*
|
||||
* @private
|
||||
* @since v7.3.0
|
||||
*/
|
||||
private async validateContainerInBackground(): Promise<void> {
|
||||
try {
|
||||
const exists = await this.containerClient!.exists()
|
||||
if (!exists) {
|
||||
// Try to create container
|
||||
try {
|
||||
await this.containerClient!.create()
|
||||
prodLog.info(`[Azure Background] Created container: ${this.containerName}`)
|
||||
} catch (createError: any) {
|
||||
// Another process might have created it - check again
|
||||
const existsNow = await this.containerClient!.exists()
|
||||
if (!existsNow) {
|
||||
throw createError
|
||||
}
|
||||
}
|
||||
}
|
||||
this.bucketValidated = true
|
||||
prodLog.info(`[Azure Background] Container validated: ${this.containerName}`)
|
||||
} catch (error: any) {
|
||||
this.bucketValidationError = new Error(
|
||||
`Container ${this.containerName} validation failed: ${error.message || error}`
|
||||
)
|
||||
prodLog.warn(`[Azure Background] Container validation failed: ${this.containerName}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Load counts from storage in background.
|
||||
*
|
||||
* @private
|
||||
* @since v7.3.0
|
||||
*/
|
||||
private async loadCountsInBackground(): Promise<void> {
|
||||
try {
|
||||
await this.initializeCounts()
|
||||
this.countsLoaded = true
|
||||
prodLog.info(`[Azure Background] Counts loaded: ${this.totalNounCount} nouns, ${this.totalVerbCount} verbs`)
|
||||
} catch (error: any) {
|
||||
// Non-fatal in progressive mode - counts start at 0
|
||||
prodLog.warn(`[Azure Background] Failed to load counts (starting at 0): ${error.message}`)
|
||||
this.countsLoaded = true // Mark as loaded even on error (0 is valid)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure container is validated before write operations.
|
||||
*
|
||||
* In progressive mode, container validation is deferred until the first
|
||||
* write operation. This method validates the container and caches the
|
||||
* result (or error) for subsequent calls.
|
||||
*
|
||||
* @throws Error if container validation fails
|
||||
* @protected
|
||||
* @override
|
||||
* @since v7.3.0
|
||||
*/
|
||||
protected async ensureValidatedForWrite(): Promise<void> {
|
||||
// If already validated, nothing to do
|
||||
if (this.bucketValidated) {
|
||||
return
|
||||
}
|
||||
|
||||
// If we have a cached validation error from background init, throw it
|
||||
if (this.bucketValidationError) {
|
||||
throw this.bucketValidationError
|
||||
}
|
||||
|
||||
// Perform synchronous validation (first write in progressive mode)
|
||||
try {
|
||||
prodLog.info(`[Azure] Lazy container validation on first write: ${this.containerName}`)
|
||||
const exists = await this.containerClient!.exists()
|
||||
if (!exists) {
|
||||
// Try to create container
|
||||
await this.containerClient!.create()
|
||||
prodLog.info(`[Azure] Created container: ${this.containerName}`)
|
||||
}
|
||||
this.bucketValidated = true
|
||||
prodLog.info(`[Azure] Container validated successfully: ${this.containerName}`)
|
||||
} catch (error: any) {
|
||||
// Cache the error for fast-fail on subsequent writes
|
||||
const wrappedError = new Error(
|
||||
`Container ${this.containerName} validation failed: ${error.message || error}`
|
||||
)
|
||||
this.bucketValidationError = wrappedError
|
||||
throw wrappedError
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the Azure blob name for a noun using UUID-based sharding
|
||||
*
|
||||
|
|
@ -681,10 +894,14 @@ export class AzureBlobStorage extends BaseStorage {
|
|||
/**
|
||||
* Write an object to a specific path in Azure
|
||||
* Primitive operation required by base class
|
||||
*
|
||||
* v7.3.0: Performs lazy container validation on first write in progressive mode.
|
||||
* @protected
|
||||
*/
|
||||
protected async writeObjectToPath(path: string, data: any): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
// v7.3.0: Lazy container validation for progressive init
|
||||
await this.ensureValidatedForWrite()
|
||||
|
||||
try {
|
||||
this.logger.trace(`Writing object to path: ${path}`)
|
||||
|
|
@ -736,10 +953,14 @@ export class AzureBlobStorage extends BaseStorage {
|
|||
/**
|
||||
* Delete an object from a specific path in Azure
|
||||
* Primitive operation required by base class
|
||||
*
|
||||
* v7.3.0: Performs lazy container validation on first delete in progressive mode.
|
||||
* @protected
|
||||
*/
|
||||
protected async deleteObjectFromPath(path: string): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
// v7.3.0: Lazy container validation for progressive init
|
||||
await this.ensureValidatedForWrite()
|
||||
|
||||
try {
|
||||
this.logger.trace(`Deleting object at path: ${path}`)
|
||||
|
|
|
|||
|
|
@ -18,6 +18,45 @@ import { StorageBatchConfig } from '../baseStorage.js'
|
|||
import { extractFieldNamesFromJson, mapToStandardField } from '../../utils/fieldNameTracking.js'
|
||||
import { getGlobalMutex, cleanupMutexes } from '../../utils/mutex.js'
|
||||
|
||||
// =============================================================================
|
||||
// Progressive Initialization Types (v7.3.0+)
|
||||
// =============================================================================
|
||||
|
||||
/**
|
||||
* Initialization mode for cloud storage adapters.
|
||||
*
|
||||
* Controls how storage adapters initialize during startup, enabling
|
||||
* fast cold starts in serverless environments while maintaining strict
|
||||
* validation for local development.
|
||||
*
|
||||
* @since v7.3.0
|
||||
*
|
||||
* | Mode | Description | Use Case |
|
||||
* |------|-------------|----------|
|
||||
* | `'auto'` | Auto-detect environment (progressive in cloud, strict locally) | Default, zero-config |
|
||||
* | `'progressive'` | Fast init (<200ms), validate lazily on first write | Serverless, Cloud Run |
|
||||
* | `'strict'` | Blocking validation during init (traditional behavior) | Local dev, testing |
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* // Zero-config: auto-detects Cloud Run, Lambda, etc.
|
||||
* const storage = new GCSStorageAdapter({ bucket: 'my-bucket' })
|
||||
*
|
||||
* // Force progressive mode for all environments
|
||||
* const storage = new GCSStorageAdapter({
|
||||
* bucket: 'my-bucket',
|
||||
* initMode: 'progressive'
|
||||
* })
|
||||
*
|
||||
* // Force strict validation (useful for testing)
|
||||
* const storage = new GCSStorageAdapter({
|
||||
* bucket: 'my-bucket',
|
||||
* initMode: 'strict'
|
||||
* })
|
||||
* ```
|
||||
*/
|
||||
export type InitMode = 'progressive' | 'strict' | 'auto'
|
||||
|
||||
/**
|
||||
* Base class for storage adapters that implements statistics tracking
|
||||
*/
|
||||
|
|
@ -255,6 +294,77 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
|||
status: 'normal' | 'throttled' | 'recovering'
|
||||
}> = new Map()
|
||||
|
||||
// =============================================
|
||||
// Progressive Initialization State (v7.3.0+)
|
||||
// =============================================
|
||||
// These properties enable fast cold starts in cloud environments
|
||||
// by deferring validation and count loading to background tasks.
|
||||
|
||||
/**
|
||||
* Initialization mode for this storage adapter.
|
||||
*
|
||||
* - `'auto'` (default): Progressive in cloud environments, strict locally
|
||||
* - `'progressive'`: Always use fast init with lazy validation
|
||||
* - `'strict'`: Always validate during init (traditional behavior)
|
||||
*
|
||||
* @protected
|
||||
* @since v7.3.0
|
||||
*/
|
||||
protected initMode: InitMode = 'auto'
|
||||
|
||||
/**
|
||||
* Whether the bucket/container has been validated to exist.
|
||||
*
|
||||
* In progressive mode, this starts as `false` and is set to `true`
|
||||
* after the first successful write operation or background validation.
|
||||
*
|
||||
* @protected
|
||||
* @since v7.3.0
|
||||
*/
|
||||
protected bucketValidated = false
|
||||
|
||||
/**
|
||||
* Error from bucket validation, if any.
|
||||
*
|
||||
* Stored here so subsequent operations can fail fast with the same error.
|
||||
*
|
||||
* @protected
|
||||
* @since v7.3.0
|
||||
*/
|
||||
protected bucketValidationError: Error | null = null
|
||||
|
||||
/**
|
||||
* Whether counts have been loaded from storage.
|
||||
*
|
||||
* In progressive mode, counts start at 0 and are loaded in the background.
|
||||
* Operations work immediately; counts become accurate after background load.
|
||||
*
|
||||
* @protected
|
||||
* @since v7.3.0
|
||||
*/
|
||||
protected countsLoaded = false
|
||||
|
||||
/**
|
||||
* Whether all background initialization tasks have completed.
|
||||
*
|
||||
* Useful for tests and diagnostics to ensure full initialization.
|
||||
*
|
||||
* @protected
|
||||
* @since v7.3.0
|
||||
*/
|
||||
protected backgroundTasksComplete = false
|
||||
|
||||
/**
|
||||
* Promise that resolves when background initialization completes.
|
||||
*
|
||||
* Can be awaited by callers who need to ensure full initialization
|
||||
* before proceeding (e.g., in tests).
|
||||
*
|
||||
* @protected
|
||||
* @since v7.3.0
|
||||
*/
|
||||
protected backgroundInitPromise: Promise<void> | null = null
|
||||
|
||||
// Statistics-specific methods that must be implemented by subclasses
|
||||
protected abstract saveStatisticsData(
|
||||
statistics: StatisticsData
|
||||
|
|
@ -1124,6 +1234,169 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
|||
return false
|
||||
}
|
||||
|
||||
// =============================================
|
||||
// Progressive Initialization Methods (v7.3.0+)
|
||||
// =============================================
|
||||
|
||||
/**
|
||||
* Detect if running in a cloud/serverless environment.
|
||||
*
|
||||
* Cloud environments benefit from progressive initialization to minimize
|
||||
* cold start times. This method checks for common environment variables
|
||||
* set by cloud providers.
|
||||
*
|
||||
* | Provider | Environment Variable |
|
||||
* |----------|---------------------|
|
||||
* | Cloud Run | `K_SERVICE`, `K_REVISION` |
|
||||
* | Lambda | `AWS_LAMBDA_FUNCTION_NAME` |
|
||||
* | Cloud Functions | `FUNCTIONS_TARGET` |
|
||||
* | Azure Functions | `AZURE_FUNCTIONS_ENVIRONMENT` |
|
||||
*
|
||||
* @returns `true` if running in a detected cloud environment
|
||||
* @protected
|
||||
* @since v7.3.0
|
||||
*/
|
||||
protected detectCloudEnvironment(): boolean {
|
||||
return !!(
|
||||
process.env.K_SERVICE || // Google Cloud Run
|
||||
process.env.K_REVISION || // Google Cloud Run (revision)
|
||||
process.env.AWS_LAMBDA_FUNCTION_NAME || // AWS Lambda
|
||||
process.env.FUNCTIONS_TARGET || // Google Cloud Functions
|
||||
process.env.AZURE_FUNCTIONS_ENVIRONMENT // Azure Functions
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine the effective initialization mode.
|
||||
*
|
||||
* Resolves `'auto'` to either `'progressive'` or `'strict'` based on
|
||||
* the detected environment.
|
||||
*
|
||||
* @returns The resolved init mode (`'progressive'` or `'strict'`)
|
||||
* @protected
|
||||
* @since v7.3.0
|
||||
*/
|
||||
protected resolveInitMode(): 'progressive' | 'strict' {
|
||||
if (this.initMode === 'auto') {
|
||||
return this.detectCloudEnvironment() ? 'progressive' : 'strict'
|
||||
}
|
||||
return this.initMode
|
||||
}
|
||||
|
||||
/**
|
||||
* Schedule background initialization tasks.
|
||||
*
|
||||
* This method is called in progressive mode after the adapter is marked
|
||||
* as initialized. It schedules non-blocking tasks to:
|
||||
* 1. Validate bucket/container existence
|
||||
* 2. Load counts from storage
|
||||
*
|
||||
* Override in subclasses to add storage-specific background tasks.
|
||||
* Always call `super.scheduleBackgroundInit()` first.
|
||||
*
|
||||
* @protected
|
||||
* @since v7.3.0
|
||||
*/
|
||||
protected scheduleBackgroundInit(): void {
|
||||
// Create a promise that tracks all background work
|
||||
this.backgroundInitPromise = this.runBackgroundInit()
|
||||
|
||||
// Don't await - let it run in background
|
||||
this.backgroundInitPromise
|
||||
.then(() => {
|
||||
this.backgroundTasksComplete = true
|
||||
})
|
||||
.catch((error) => {
|
||||
// Log but don't throw - progressive mode is best-effort
|
||||
console.error('[Progressive Init] Background initialization failed:', error)
|
||||
this.backgroundTasksComplete = true // Mark complete even on error
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Run background initialization tasks.
|
||||
*
|
||||
* Override in subclasses to add storage-specific tasks.
|
||||
* The default implementation does nothing (for local storage adapters).
|
||||
*
|
||||
* @protected
|
||||
* @since v7.3.0
|
||||
*/
|
||||
protected async runBackgroundInit(): Promise<void> {
|
||||
// Default implementation: nothing to do for local storage
|
||||
// Cloud storage adapters override this to:
|
||||
// 1. Validate bucket/container
|
||||
// 2. Load counts from storage
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for background initialization to complete.
|
||||
*
|
||||
* Useful in tests or when you need to ensure all background tasks
|
||||
* have finished before proceeding.
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const storage = new GCSStorageAdapter({ bucket: 'my-bucket' })
|
||||
* await storage.init()
|
||||
*
|
||||
* // Wait for background validation and count loading
|
||||
* await storage.awaitBackgroundInit()
|
||||
*
|
||||
* // Now counts are accurate and bucket is validated
|
||||
* const count = await storage.getNounCount()
|
||||
* ```
|
||||
*
|
||||
* @public
|
||||
* @since v7.3.0
|
||||
*/
|
||||
public async awaitBackgroundInit(): Promise<void> {
|
||||
if (this.backgroundInitPromise) {
|
||||
await this.backgroundInitPromise
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if background initialization has completed.
|
||||
*
|
||||
* @returns `true` if all background tasks are complete
|
||||
* @public
|
||||
* @since v7.3.0
|
||||
*/
|
||||
public isBackgroundInitComplete(): boolean {
|
||||
return this.backgroundTasksComplete
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure bucket/container is validated before write operations.
|
||||
*
|
||||
* In progressive mode, bucket validation is deferred until the first
|
||||
* write operation. This method performs lazy validation and caches
|
||||
* the result (or error) for subsequent calls.
|
||||
*
|
||||
* Override in subclasses to implement storage-specific validation.
|
||||
* The base implementation does nothing (for local storage adapters).
|
||||
*
|
||||
* @throws Error if bucket validation fails
|
||||
* @protected
|
||||
* @since v7.3.0
|
||||
*/
|
||||
protected async ensureValidatedForWrite(): Promise<void> {
|
||||
// If already validated, nothing to do
|
||||
if (this.bucketValidated) {
|
||||
return
|
||||
}
|
||||
|
||||
// If we have a cached validation error, throw it
|
||||
if (this.bucketValidationError) {
|
||||
throw this.bucketValidationError
|
||||
}
|
||||
|
||||
// Default implementation: no validation needed (local storage)
|
||||
// Cloud storage adapters override this to validate bucket/container
|
||||
this.bucketValidated = true
|
||||
}
|
||||
|
||||
/**
|
||||
* Schedule a smart batched persist operation.
|
||||
*
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@ import { getGlobalBackpressure } from '../../utils/adaptiveBackpressure.js'
|
|||
import { getWriteBuffer, WriteBuffer } from '../../utils/writeBuffer.js'
|
||||
import { getCoalescer, RequestCoalescer } from '../../utils/requestCoalescer.js'
|
||||
import { getShardIdFromUuid, getAllShardIds, getShardIdByIndex, TOTAL_SHARDS } from '../sharding.js'
|
||||
import { InitMode } from './baseStorageAdapter.js'
|
||||
|
||||
// Type aliases for better readability
|
||||
type HNSWNode = HNSWNoun
|
||||
|
|
@ -128,7 +129,30 @@ export class GcsStorage extends BaseStorage {
|
|||
|
||||
/**
|
||||
* Initialize the storage adapter
|
||||
*
|
||||
* @param options Configuration options for Google Cloud Storage
|
||||
*
|
||||
* @example Zero-config (recommended) - auto-detects Cloud Run/Lambda for fast init
|
||||
* ```typescript
|
||||
* const storage = new GcsStorage({ bucketName: 'my-bucket' })
|
||||
* await storage.init() // <200ms in Cloud Run, blocking locally
|
||||
* ```
|
||||
*
|
||||
* @example Force progressive mode for all environments
|
||||
* ```typescript
|
||||
* const storage = new GcsStorage({
|
||||
* bucketName: 'my-bucket',
|
||||
* initMode: 'progressive' // Always <200ms init
|
||||
* })
|
||||
* ```
|
||||
*
|
||||
* @example Force strict validation (useful for tests)
|
||||
* ```typescript
|
||||
* const storage = new GcsStorage({
|
||||
* bucketName: 'my-bucket',
|
||||
* initMode: 'strict' // Always validate during init
|
||||
* })
|
||||
* ```
|
||||
*/
|
||||
constructor(options: {
|
||||
bucketName: string
|
||||
|
|
@ -141,8 +165,30 @@ export class GcsStorage extends BaseStorage {
|
|||
accessKeyId?: string
|
||||
secretAccessKey?: string
|
||||
|
||||
// Initialization configuration
|
||||
/**
|
||||
* Initialization mode for fast cold starts (v7.3.0+)
|
||||
*
|
||||
* - `'auto'` (default): Progressive in cloud environments (Cloud Run, Lambda),
|
||||
* strict locally. Zero-config optimization.
|
||||
* - `'progressive'`: Always use fast init (<200ms). Bucket validation and
|
||||
* count loading happen in background. First write validates bucket.
|
||||
* - `'strict'`: Traditional blocking init. Validates bucket and loads counts
|
||||
* before init() returns.
|
||||
*
|
||||
* @since v7.3.0
|
||||
*/
|
||||
initMode?: InitMode
|
||||
|
||||
/**
|
||||
* @deprecated Use `initMode: 'progressive'` instead.
|
||||
* Will be removed in v8.0.0.
|
||||
*/
|
||||
skipInitialScan?: boolean
|
||||
|
||||
/**
|
||||
* @deprecated Use `initMode: 'progressive'` instead.
|
||||
* Will be removed in v8.0.0.
|
||||
*/
|
||||
skipCountsFile?: boolean
|
||||
|
||||
// Cache and operation configuration
|
||||
|
|
@ -159,8 +205,28 @@ export class GcsStorage extends BaseStorage {
|
|||
this.credentials = options.credentials
|
||||
this.accessKeyId = options.accessKeyId
|
||||
this.secretAccessKey = options.secretAccessKey
|
||||
this.skipInitialScan = options.skipInitialScan || false
|
||||
this.skipCountsFile = options.skipCountsFile || false
|
||||
|
||||
// v7.3.0: Handle initMode and deprecated skip* flags
|
||||
if (options.initMode) {
|
||||
this.initMode = options.initMode
|
||||
}
|
||||
|
||||
// Deprecation warnings for skip* flags
|
||||
if (options.skipInitialScan) {
|
||||
console.warn(
|
||||
'[GcsStorage] DEPRECATION WARNING: skipInitialScan is deprecated. ' +
|
||||
'Use initMode: "progressive" instead. Will be removed in v8.0.0.'
|
||||
)
|
||||
this.skipInitialScan = true
|
||||
}
|
||||
if (options.skipCountsFile) {
|
||||
console.warn(
|
||||
'[GcsStorage] DEPRECATION WARNING: skipCountsFile is deprecated. ' +
|
||||
'Use initMode: "progressive" instead. Will be removed in v8.0.0.'
|
||||
)
|
||||
this.skipCountsFile = true
|
||||
}
|
||||
|
||||
this.readOnly = options.readOnly || false
|
||||
|
||||
// Set up prefixes for different types of data using entity-based structure
|
||||
|
|
@ -179,6 +245,22 @@ export class GcsStorage extends BaseStorage {
|
|||
|
||||
/**
|
||||
* Initialize the storage adapter
|
||||
*
|
||||
* v7.3.0: Supports progressive initialization for fast cold starts
|
||||
*
|
||||
* | Mode | Init Time | When |
|
||||
* |------|-----------|------|
|
||||
* | `progressive` | <200ms | Cloud Run, Lambda, Azure Functions |
|
||||
* | `strict` | 100-500ms+ | Local development, tests |
|
||||
* | `auto` | Detected | Default - best of both |
|
||||
*
|
||||
* In progressive mode:
|
||||
* - SDK import and bucket reference: ~50ms (unavoidable)
|
||||
* - Write buffers and caches: ~10ms
|
||||
* - Mark as initialized: READY
|
||||
* - Background: validate bucket, load counts
|
||||
*
|
||||
* First write operation validates bucket existence (lazy validation).
|
||||
*/
|
||||
public async init(): Promise<void> {
|
||||
if (this.isInitialized) {
|
||||
|
|
@ -186,7 +268,7 @@ export class GcsStorage extends BaseStorage {
|
|||
}
|
||||
|
||||
try {
|
||||
// Import Google Cloud Storage SDK only when needed
|
||||
// Import Google Cloud Storage SDK only when needed (~50ms)
|
||||
const { Storage } = await import('@google-cloud/storage')
|
||||
|
||||
// Configure the GCS client based on available credentials
|
||||
|
|
@ -216,19 +298,17 @@ export class GcsStorage extends BaseStorage {
|
|||
prodLog.info('🔐 GCS: Using Application Default Credentials (ADC)')
|
||||
}
|
||||
|
||||
// Create the GCS client
|
||||
// Create the GCS client (no network calls)
|
||||
this.storage = new Storage(clientConfig)
|
||||
|
||||
// Get reference to the bucket
|
||||
// Get reference to the bucket (no network calls)
|
||||
this.bucket = this.storage.bucket(this.bucketName)
|
||||
|
||||
// Verify bucket exists and is accessible
|
||||
const [exists] = await this.bucket.exists()
|
||||
if (!exists) {
|
||||
throw new Error(`Bucket ${this.bucketName} does not exist or is not accessible`)
|
||||
}
|
||||
// Determine initialization mode
|
||||
const effectiveMode = this.resolveInitMode()
|
||||
const isCloud = this.detectCloudEnvironment()
|
||||
|
||||
prodLog.info(`✅ Connected to GCS bucket: ${this.bucketName}`)
|
||||
prodLog.info(`🚀 GCS init mode: ${effectiveMode} (detected cloud: ${isCloud})`)
|
||||
|
||||
// Initialize write buffers for high-volume mode
|
||||
const storageId = `gcs-${this.bucketName}`
|
||||
|
|
@ -257,9 +337,6 @@ export class GcsStorage extends BaseStorage {
|
|||
}
|
||||
)
|
||||
|
||||
// Initialize counts from storage
|
||||
await this.initializeCounts()
|
||||
|
||||
// CRITICAL FIX (v3.37.7): Clear any stale cache entries from previous runs
|
||||
// This prevents cache poisoning from causing silent failures on container restart
|
||||
prodLog.info('🧹 Clearing cache from previous run to prevent cache poisoning')
|
||||
|
|
@ -267,14 +344,170 @@ export class GcsStorage extends BaseStorage {
|
|||
this.verbCacheManager.clear()
|
||||
prodLog.info('✅ Cache cleared - starting fresh')
|
||||
|
||||
// v6.0.0: Initialize GraphAdjacencyIndex and type statistics
|
||||
await super.init()
|
||||
// v7.3.0: Progressive vs Strict initialization
|
||||
if (effectiveMode === 'progressive') {
|
||||
// PROGRESSIVE MODE: Fast init, background validation
|
||||
// Mark as initialized immediately - ready to accept operations
|
||||
// Bucket validation happens lazily on first write
|
||||
// Count loading happens in background
|
||||
|
||||
prodLog.info(`✅ GCS progressive init complete: ${this.bucketName} (validation deferred)`)
|
||||
|
||||
// v6.0.0: Initialize GraphAdjacencyIndex and type statistics
|
||||
await super.init()
|
||||
|
||||
// Schedule background tasks (non-blocking)
|
||||
this.scheduleBackgroundInit()
|
||||
} else {
|
||||
// STRICT MODE: Traditional blocking initialization
|
||||
// Verify bucket exists and is accessible (blocking)
|
||||
const [exists] = await this.bucket.exists()
|
||||
if (!exists) {
|
||||
throw new Error(`Bucket ${this.bucketName} does not exist or is not accessible`)
|
||||
}
|
||||
this.bucketValidated = true
|
||||
|
||||
prodLog.info(`✅ Connected to GCS bucket: ${this.bucketName}`)
|
||||
|
||||
// Initialize counts from storage (blocking)
|
||||
await this.initializeCounts()
|
||||
this.countsLoaded = true
|
||||
|
||||
// v6.0.0: Initialize GraphAdjacencyIndex and type statistics
|
||||
await super.init()
|
||||
|
||||
// Mark background tasks as complete (nothing to do in background)
|
||||
this.backgroundTasksComplete = true
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.error('Failed to initialize GCS storage:', error)
|
||||
throw new Error(`Failed to initialize GCS storage: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
// =============================================
|
||||
// Progressive Initialization (v7.3.0+)
|
||||
// =============================================
|
||||
|
||||
/**
|
||||
* Run background initialization tasks for GCS.
|
||||
*
|
||||
* Called in progressive mode after init() returns. Performs:
|
||||
* 1. Bucket validation (in background)
|
||||
* 2. Count loading from storage (in background)
|
||||
*
|
||||
* These tasks don't block the main thread, allowing fast cold starts.
|
||||
*
|
||||
* @protected
|
||||
* @override
|
||||
* @since v7.3.0
|
||||
*/
|
||||
protected async runBackgroundInit(): Promise<void> {
|
||||
const startTime = Date.now()
|
||||
prodLog.info('[GCS Background] Starting background initialization...')
|
||||
|
||||
// Run validation and count loading in parallel
|
||||
const validationPromise = this.validateBucketInBackground()
|
||||
const countsPromise = this.loadCountsInBackground()
|
||||
|
||||
// Wait for both to complete (but we're already initialized)
|
||||
await Promise.all([validationPromise, countsPromise])
|
||||
|
||||
const elapsed = Date.now() - startTime
|
||||
prodLog.info(`[GCS Background] Background init complete in ${elapsed}ms`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate bucket existence in background.
|
||||
*
|
||||
* Stores result in bucketValidated/bucketValidationError for lazy use.
|
||||
*
|
||||
* @private
|
||||
* @since v7.3.0
|
||||
*/
|
||||
private async validateBucketInBackground(): Promise<void> {
|
||||
try {
|
||||
const [exists] = await this.bucket!.exists()
|
||||
if (exists) {
|
||||
this.bucketValidated = true
|
||||
prodLog.info(`[GCS Background] Bucket validated: ${this.bucketName}`)
|
||||
} else {
|
||||
this.bucketValidationError = new Error(
|
||||
`Bucket ${this.bucketName} does not exist or is not accessible`
|
||||
)
|
||||
prodLog.warn(`[GCS Background] Bucket validation failed: ${this.bucketName}`)
|
||||
}
|
||||
} catch (error: any) {
|
||||
this.bucketValidationError = new Error(
|
||||
`Bucket validation failed: ${error.message || error}`
|
||||
)
|
||||
prodLog.error(`[GCS Background] Bucket validation error:`, error)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Load counts from storage in background.
|
||||
*
|
||||
* Uses the existing initializeCounts() logic but in background.
|
||||
*
|
||||
* @private
|
||||
* @since v7.3.0
|
||||
*/
|
||||
private async loadCountsInBackground(): Promise<void> {
|
||||
try {
|
||||
await this.initializeCounts()
|
||||
this.countsLoaded = true
|
||||
prodLog.info(`[GCS Background] Counts loaded: ${this.totalNounCount} nouns, ${this.totalVerbCount} verbs`)
|
||||
} catch (error: any) {
|
||||
// Non-fatal in progressive mode - counts start at 0
|
||||
prodLog.warn(`[GCS Background] Failed to load counts (starting at 0): ${error.message}`)
|
||||
this.countsLoaded = true // Mark as loaded even on error (0 is valid)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure bucket is validated before write operations.
|
||||
*
|
||||
* In progressive mode, bucket validation is deferred until the first
|
||||
* write operation. This method validates the bucket and caches the
|
||||
* result (or error) for subsequent calls.
|
||||
*
|
||||
* @throws Error if bucket does not exist or is not accessible
|
||||
* @protected
|
||||
* @override
|
||||
* @since v7.3.0
|
||||
*/
|
||||
protected async ensureValidatedForWrite(): Promise<void> {
|
||||
// If already validated, nothing to do
|
||||
if (this.bucketValidated) {
|
||||
return
|
||||
}
|
||||
|
||||
// If we have a cached validation error from background init, throw it
|
||||
if (this.bucketValidationError) {
|
||||
throw this.bucketValidationError
|
||||
}
|
||||
|
||||
// Perform synchronous validation (first write in progressive mode)
|
||||
try {
|
||||
prodLog.info(`[GCS] Lazy bucket validation on first write: ${this.bucketName}`)
|
||||
const [exists] = await this.bucket!.exists()
|
||||
if (!exists) {
|
||||
const error = new Error(`Bucket ${this.bucketName} does not exist or is not accessible`)
|
||||
this.bucketValidationError = error
|
||||
throw error
|
||||
}
|
||||
this.bucketValidated = true
|
||||
prodLog.info(`[GCS] Bucket validated successfully: ${this.bucketName}`)
|
||||
} catch (error: any) {
|
||||
// Cache the error for fast-fail on subsequent writes
|
||||
if (!this.bucketValidationError) {
|
||||
this.bucketValidationError = error
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the GCS object key for a noun using UUID-based sharding
|
||||
*
|
||||
|
|
@ -423,8 +656,13 @@ export class GcsStorage extends BaseStorage {
|
|||
|
||||
/**
|
||||
* Save a node directly to GCS (bypass buffer)
|
||||
*
|
||||
* v7.3.0: Performs lazy bucket validation on first write in progressive mode.
|
||||
*/
|
||||
private async saveNodeDirect(node: HNSWNode): Promise<void> {
|
||||
// v7.3.0: Lazy bucket validation for progressive init
|
||||
await this.ensureValidatedForWrite()
|
||||
|
||||
// Apply backpressure before starting operation
|
||||
const requestId = await this.applyBackpressure()
|
||||
|
||||
|
|
@ -598,10 +836,14 @@ export class GcsStorage extends BaseStorage {
|
|||
/**
|
||||
* Write an object to a specific path in GCS
|
||||
* Primitive operation required by base class
|
||||
*
|
||||
* v7.3.0: Performs lazy bucket validation on first write in progressive mode.
|
||||
* @protected
|
||||
*/
|
||||
protected async writeObjectToPath(path: string, data: any): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
// v7.3.0: Lazy bucket validation for progressive init
|
||||
await this.ensureValidatedForWrite()
|
||||
|
||||
try {
|
||||
this.logger.trace(`Writing object to path: ${path}`)
|
||||
|
|
@ -741,10 +983,14 @@ export class GcsStorage extends BaseStorage {
|
|||
/**
|
||||
* Delete an object from a specific path in GCS
|
||||
* Primitive operation required by base class
|
||||
*
|
||||
* v7.3.0: Performs lazy bucket validation on first delete in progressive mode.
|
||||
* @protected
|
||||
*/
|
||||
protected async deleteObjectFromPath(path: string): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
// v7.3.0: Lazy bucket validation for progressive init
|
||||
await this.ensureValidatedForWrite()
|
||||
|
||||
try {
|
||||
this.logger.trace(`Deleting object at path: ${path}`)
|
||||
|
|
@ -814,8 +1060,13 @@ export class GcsStorage extends BaseStorage {
|
|||
|
||||
/**
|
||||
* Save an edge directly to GCS (bypass buffer)
|
||||
*
|
||||
* v7.3.0: Performs lazy bucket validation on first write in progressive mode.
|
||||
*/
|
||||
private async saveEdgeDirect(edge: Edge): Promise<void> {
|
||||
// v7.3.0: Lazy bucket validation for progressive init
|
||||
await this.ensureValidatedForWrite()
|
||||
|
||||
const requestId = await this.applyBackpressure()
|
||||
|
||||
try {
|
||||
|
|
|
|||
|
|
@ -40,6 +40,7 @@ import { getGlobalBackpressure } from '../../utils/adaptiveBackpressure.js'
|
|||
import { getWriteBuffer, WriteBuffer } from '../../utils/writeBuffer.js'
|
||||
import { getCoalescer, RequestCoalescer } from '../../utils/requestCoalescer.js'
|
||||
import { getShardIdFromUuid, getAllShardIds, getShardIdByIndex, TOTAL_SHARDS } from '../sharding.js'
|
||||
import { InitMode } from './baseStorageAdapter.js'
|
||||
|
||||
// Type aliases for better readability
|
||||
type HNSWNode = HNSWNoun
|
||||
|
|
@ -169,7 +170,28 @@ export class S3CompatibleStorage extends BaseStorage {
|
|||
|
||||
/**
|
||||
* Initialize the storage adapter
|
||||
*
|
||||
* @param options Configuration options for the S3-compatible storage
|
||||
*
|
||||
* @example Zero-config (recommended) - auto-detects Lambda/Cloud Run for fast init
|
||||
* ```typescript
|
||||
* const storage = new S3CompatibleStorage({
|
||||
* bucketName: 'my-bucket',
|
||||
* accessKeyId: 'key',
|
||||
* secretAccessKey: 'secret'
|
||||
* })
|
||||
* await storage.init() // <200ms in Lambda, blocking locally
|
||||
* ```
|
||||
*
|
||||
* @example Force progressive mode for all environments
|
||||
* ```typescript
|
||||
* const storage = new S3CompatibleStorage({
|
||||
* bucketName: 'my-bucket',
|
||||
* accessKeyId: 'key',
|
||||
* secretAccessKey: 'secret',
|
||||
* initMode: 'progressive' // Always <200ms init
|
||||
* })
|
||||
* ```
|
||||
*/
|
||||
constructor(options: {
|
||||
bucketName: string
|
||||
|
|
@ -186,6 +208,21 @@ export class S3CompatibleStorage extends BaseStorage {
|
|||
hotCacheEvictionThreshold?: number
|
||||
warmCacheTTL?: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialization mode for fast cold starts (v7.3.0+)
|
||||
*
|
||||
* - `'auto'` (default): Progressive in cloud environments (Lambda, Cloud Run),
|
||||
* strict locally. Zero-config optimization.
|
||||
* - `'progressive'`: Always use fast init (<200ms). Bucket validation and
|
||||
* count loading happen in background. First write validates bucket.
|
||||
* - `'strict'`: Traditional blocking init. Validates bucket and loads counts
|
||||
* before init() returns.
|
||||
*
|
||||
* @since v7.3.0
|
||||
*/
|
||||
initMode?: InitMode
|
||||
|
||||
readOnly?: boolean
|
||||
}) {
|
||||
super()
|
||||
|
|
@ -199,6 +236,11 @@ export class S3CompatibleStorage extends BaseStorage {
|
|||
this.serviceType = options.serviceType || 's3'
|
||||
this.readOnly = options.readOnly || false
|
||||
|
||||
// v7.3.0: Handle initMode
|
||||
if (options.initMode) {
|
||||
this.initMode = options.initMode
|
||||
}
|
||||
|
||||
// Initialize operation executors with timeout and retry configuration
|
||||
this.operationExecutors = new StorageOperationExecutors(
|
||||
options.operationConfig
|
||||
|
|
@ -315,6 +357,22 @@ export class S3CompatibleStorage extends BaseStorage {
|
|||
|
||||
/**
|
||||
* Initialize the storage adapter
|
||||
*
|
||||
* v7.3.0: Supports progressive initialization for fast cold starts
|
||||
*
|
||||
* | Mode | Init Time | When |
|
||||
* |------|-----------|------|
|
||||
* | `progressive` | <200ms | Lambda, Cloud Run, Azure Functions |
|
||||
* | `strict` | 100-500ms+ | Local development, tests |
|
||||
* | `auto` | Detected | Default - best of both |
|
||||
*
|
||||
* In progressive mode:
|
||||
* - SDK import and client creation: ~50ms (unavoidable)
|
||||
* - Write buffers and caches: ~10ms
|
||||
* - Mark as initialized: READY
|
||||
* - Background: validate bucket, load counts
|
||||
*
|
||||
* First write operation validates bucket existence (lazy validation).
|
||||
*/
|
||||
public async init(): Promise<void> {
|
||||
if (this.isInitialized) {
|
||||
|
|
@ -322,7 +380,7 @@ export class S3CompatibleStorage extends BaseStorage {
|
|||
}
|
||||
|
||||
try {
|
||||
// Import AWS SDK modules only when needed
|
||||
// Import AWS SDK modules only when needed (~50ms)
|
||||
const { S3Client } = await import('@aws-sdk/client-s3')
|
||||
|
||||
// Configure the S3 client based on the service type
|
||||
|
|
@ -354,17 +412,15 @@ export class S3CompatibleStorage extends BaseStorage {
|
|||
clientConfig.endpoint = `https://${this.accountId}.r2.cloudflarestorage.com`
|
||||
}
|
||||
|
||||
// Create the S3 client
|
||||
// Create the S3 client (no network calls)
|
||||
this.s3Client = new S3Client(clientConfig)
|
||||
|
||||
// Ensure the bucket exists and is accessible
|
||||
const { HeadBucketCommand } = await import('@aws-sdk/client-s3')
|
||||
await this.s3Client.send(
|
||||
new HeadBucketCommand({
|
||||
Bucket: this.bucketName
|
||||
})
|
||||
)
|
||||
|
||||
// Determine initialization mode
|
||||
const effectiveMode = this.resolveInitMode()
|
||||
const isCloud = this.detectCloudEnvironment()
|
||||
|
||||
prodLog.info(`🚀 S3 init mode: ${effectiveMode} (detected cloud: ${isCloud})`)
|
||||
|
||||
// Create storage adapter proxies for the cache managers
|
||||
const nounStorageAdapter = {
|
||||
get: async (id: string) => this.getNoun_internal(id),
|
||||
|
|
@ -375,13 +431,13 @@ export class S3CompatibleStorage extends BaseStorage {
|
|||
// Process in batches to avoid overwhelming the S3 API
|
||||
const batchSize = this.getBatchSize()
|
||||
const batches: string[][] = []
|
||||
|
||||
|
||||
// Split into batches
|
||||
for (let i = 0; i < ids.length; i += batchSize) {
|
||||
const batch = ids.slice(i, i + batchSize)
|
||||
batches.push(batch)
|
||||
}
|
||||
|
||||
|
||||
// Process each batch
|
||||
for (const batch of batches) {
|
||||
const batchResults = await Promise.all(
|
||||
|
|
@ -390,7 +446,7 @@ export class S3CompatibleStorage extends BaseStorage {
|
|||
return { id, node }
|
||||
})
|
||||
)
|
||||
|
||||
|
||||
// Add results to map
|
||||
for (const { id, node } of batchResults) {
|
||||
if (node) {
|
||||
|
|
@ -398,7 +454,7 @@ export class S3CompatibleStorage extends BaseStorage {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
return result
|
||||
},
|
||||
clear: async () => {
|
||||
|
|
@ -406,7 +462,7 @@ export class S3CompatibleStorage extends BaseStorage {
|
|||
// This would be implemented if needed
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
const verbStorageAdapter = {
|
||||
get: async (id: string) => this.getVerb_internal(id),
|
||||
set: async (id: string, edge: Edge) => this.saveVerb_internal(edge),
|
||||
|
|
@ -416,13 +472,13 @@ export class S3CompatibleStorage extends BaseStorage {
|
|||
// Process in batches to avoid overwhelming the S3 API
|
||||
const batchSize = this.getBatchSize()
|
||||
const batches: string[][] = []
|
||||
|
||||
|
||||
// Split into batches
|
||||
for (let i = 0; i < ids.length; i += batchSize) {
|
||||
const batch = ids.slice(i, i + batchSize)
|
||||
batches.push(batch)
|
||||
}
|
||||
|
||||
|
||||
// Process each batch
|
||||
for (const batch of batches) {
|
||||
const batchResults = await Promise.all(
|
||||
|
|
@ -431,7 +487,7 @@ export class S3CompatibleStorage extends BaseStorage {
|
|||
return { id, edge }
|
||||
})
|
||||
)
|
||||
|
||||
|
||||
// Add results to map
|
||||
for (const { id, edge } of batchResults) {
|
||||
if (edge) {
|
||||
|
|
@ -439,7 +495,7 @@ export class S3CompatibleStorage extends BaseStorage {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
return result
|
||||
},
|
||||
clear: async () => {
|
||||
|
|
@ -447,22 +503,16 @@ export class S3CompatibleStorage extends BaseStorage {
|
|||
// This would be implemented if needed
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Set storage adapters for cache managers
|
||||
this.nounCacheManager.setStorageAdapters(nounStorageAdapter, nounStorageAdapter)
|
||||
this.verbCacheManager.setStorageAdapters(verbStorageAdapter, verbStorageAdapter)
|
||||
|
||||
// Initialize write buffers for high-volume scenarios
|
||||
this.initializeBuffers()
|
||||
|
||||
|
||||
// Initialize request coalescer
|
||||
this.initializeCoalescer()
|
||||
|
||||
// Auto-cleanup legacy /index folder on initialization
|
||||
await this.cleanupLegacyIndexFolder()
|
||||
|
||||
// Initialize counts from storage
|
||||
await this.initializeCounts()
|
||||
|
||||
// CRITICAL FIX (v3.37.7): Clear any stale cache entries from previous runs
|
||||
// This prevents cache poisoning from causing silent failures on container restart
|
||||
|
|
@ -474,8 +524,45 @@ export class S3CompatibleStorage extends BaseStorage {
|
|||
prodLog.info('🧹 Node cache is empty - starting fresh')
|
||||
}
|
||||
|
||||
// v6.0.0: Initialize GraphAdjacencyIndex and type statistics
|
||||
await super.init()
|
||||
// v7.3.0: Progressive vs Strict initialization
|
||||
if (effectiveMode === 'progressive') {
|
||||
// PROGRESSIVE MODE: Fast init, background validation
|
||||
// Mark as initialized immediately - ready to accept operations
|
||||
// Bucket validation happens lazily on first write
|
||||
// Count loading happens in background
|
||||
|
||||
prodLog.info(`✅ S3 progressive init complete: ${this.bucketName} (validation deferred)`)
|
||||
|
||||
// v6.0.0: Initialize GraphAdjacencyIndex and type statistics
|
||||
await super.init()
|
||||
|
||||
// Schedule background tasks (non-blocking)
|
||||
this.scheduleBackgroundInit()
|
||||
} else {
|
||||
// STRICT MODE: Traditional blocking initialization
|
||||
// Ensure the bucket exists and is accessible (blocking)
|
||||
const { HeadBucketCommand } = await import('@aws-sdk/client-s3')
|
||||
await this.s3Client.send(
|
||||
new HeadBucketCommand({
|
||||
Bucket: this.bucketName
|
||||
})
|
||||
)
|
||||
this.bucketValidated = true
|
||||
|
||||
// Auto-cleanup legacy /index folder on initialization
|
||||
await this.cleanupLegacyIndexFolder()
|
||||
|
||||
// Initialize counts from storage (blocking)
|
||||
await this.initializeCounts()
|
||||
this.countsLoaded = true
|
||||
|
||||
// v6.0.0: Initialize GraphAdjacencyIndex and type statistics
|
||||
await super.init()
|
||||
|
||||
// Mark background tasks as complete (nothing to do in background)
|
||||
this.backgroundTasksComplete = true
|
||||
}
|
||||
|
||||
this.logger.info(`Initialized ${this.serviceType} storage with bucket ${this.bucketName}`)
|
||||
} catch (error) {
|
||||
this.logger.error(`Failed to initialize ${this.serviceType} storage:`, error)
|
||||
|
|
@ -485,6 +572,130 @@ export class S3CompatibleStorage extends BaseStorage {
|
|||
}
|
||||
}
|
||||
|
||||
// =============================================
|
||||
// Progressive Initialization (v7.3.0+)
|
||||
// =============================================
|
||||
|
||||
/**
|
||||
* Run background initialization tasks for S3.
|
||||
*
|
||||
* Called in progressive mode after init() returns. Performs:
|
||||
* 1. Bucket validation (in background)
|
||||
* 2. Legacy folder cleanup (in background)
|
||||
* 3. Count loading from storage (in background)
|
||||
*
|
||||
* These tasks don't block the main thread, allowing fast cold starts.
|
||||
*
|
||||
* @protected
|
||||
* @override
|
||||
* @since v7.3.0
|
||||
*/
|
||||
protected async runBackgroundInit(): Promise<void> {
|
||||
const startTime = Date.now()
|
||||
prodLog.info('[S3 Background] Starting background initialization...')
|
||||
|
||||
// Run validation, cleanup, and count loading in parallel
|
||||
const validationPromise = this.validateBucketInBackground()
|
||||
const cleanupPromise = this.cleanupLegacyIndexFolder().catch((err) => {
|
||||
prodLog.warn(`[S3 Background] Legacy cleanup failed (non-fatal): ${err.message}`)
|
||||
})
|
||||
const countsPromise = this.loadCountsInBackground()
|
||||
|
||||
// Wait for all to complete (but we're already initialized)
|
||||
await Promise.all([validationPromise, cleanupPromise, countsPromise])
|
||||
|
||||
const elapsed = Date.now() - startTime
|
||||
prodLog.info(`[S3 Background] Background init complete in ${elapsed}ms`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate bucket existence in background.
|
||||
*
|
||||
* Stores result in bucketValidated/bucketValidationError for lazy use.
|
||||
*
|
||||
* @private
|
||||
* @since v7.3.0
|
||||
*/
|
||||
private async validateBucketInBackground(): Promise<void> {
|
||||
try {
|
||||
const { HeadBucketCommand } = await import('@aws-sdk/client-s3')
|
||||
await this.s3Client!.send(
|
||||
new HeadBucketCommand({
|
||||
Bucket: this.bucketName
|
||||
})
|
||||
)
|
||||
this.bucketValidated = true
|
||||
prodLog.info(`[S3 Background] Bucket validated: ${this.bucketName}`)
|
||||
} catch (error: any) {
|
||||
this.bucketValidationError = new Error(
|
||||
`Bucket ${this.bucketName} does not exist or is not accessible: ${error.message || error}`
|
||||
)
|
||||
prodLog.warn(`[S3 Background] Bucket validation failed: ${this.bucketName}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Load counts from storage in background.
|
||||
*
|
||||
* @private
|
||||
* @since v7.3.0
|
||||
*/
|
||||
private async loadCountsInBackground(): Promise<void> {
|
||||
try {
|
||||
await this.initializeCounts()
|
||||
this.countsLoaded = true
|
||||
prodLog.info(`[S3 Background] Counts loaded: ${this.totalNounCount} nouns, ${this.totalVerbCount} verbs`)
|
||||
} catch (error: any) {
|
||||
// Non-fatal in progressive mode - counts start at 0
|
||||
prodLog.warn(`[S3 Background] Failed to load counts (starting at 0): ${error.message}`)
|
||||
this.countsLoaded = true // Mark as loaded even on error (0 is valid)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure bucket is validated before write operations.
|
||||
*
|
||||
* In progressive mode, bucket validation is deferred until the first
|
||||
* write operation. This method validates the bucket and caches the
|
||||
* result (or error) for subsequent calls.
|
||||
*
|
||||
* @throws Error if bucket does not exist or is not accessible
|
||||
* @protected
|
||||
* @override
|
||||
* @since v7.3.0
|
||||
*/
|
||||
protected async ensureValidatedForWrite(): Promise<void> {
|
||||
// If already validated, nothing to do
|
||||
if (this.bucketValidated) {
|
||||
return
|
||||
}
|
||||
|
||||
// If we have a cached validation error from background init, throw it
|
||||
if (this.bucketValidationError) {
|
||||
throw this.bucketValidationError
|
||||
}
|
||||
|
||||
// Perform synchronous validation (first write in progressive mode)
|
||||
try {
|
||||
prodLog.info(`[S3] Lazy bucket validation on first write: ${this.bucketName}`)
|
||||
const { HeadBucketCommand } = await import('@aws-sdk/client-s3')
|
||||
await this.s3Client!.send(
|
||||
new HeadBucketCommand({
|
||||
Bucket: this.bucketName
|
||||
})
|
||||
)
|
||||
this.bucketValidated = true
|
||||
prodLog.info(`[S3] Bucket validated successfully: ${this.bucketName}`)
|
||||
} catch (error: any) {
|
||||
// Cache the error for fast-fail on subsequent writes
|
||||
const wrappedError = new Error(
|
||||
`Bucket ${this.bucketName} does not exist or is not accessible: ${error.message || error}`
|
||||
)
|
||||
this.bucketValidationError = wrappedError
|
||||
throw wrappedError
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Set distributed components for multi-node coordination
|
||||
*
|
||||
|
|
@ -1588,9 +1799,13 @@ export class S3CompatibleStorage extends BaseStorage {
|
|||
/**
|
||||
* Primitive operation: Write object to path
|
||||
* All metadata operations use this internally via base class routing
|
||||
*
|
||||
* v7.3.0: Performs lazy bucket validation on first write in progressive mode.
|
||||
*/
|
||||
protected async writeObjectToPath(path: string, data: any): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
// v7.3.0: Lazy bucket validation for progressive init
|
||||
await this.ensureValidatedForWrite()
|
||||
|
||||
// Apply backpressure before starting operation
|
||||
const requestId = await this.applyBackpressure()
|
||||
|
|
@ -1681,9 +1896,13 @@ export class S3CompatibleStorage extends BaseStorage {
|
|||
/**
|
||||
* Primitive operation: Delete object from path
|
||||
* All metadata operations use this internally via base class routing
|
||||
*
|
||||
* v7.3.0: Performs lazy bucket validation on first delete in progressive mode.
|
||||
*/
|
||||
protected async deleteObjectFromPath(path: string): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
// v7.3.0: Lazy bucket validation for progressive init
|
||||
await this.ensureValidatedForWrite()
|
||||
|
||||
try {
|
||||
const { DeleteObjectCommand } = await import('@aws-sdk/client-s3')
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue