feat: COW always-on architecture + cloud storage clear() fix (v5.11.0)
Major architectural improvements and critical bug fixes: ## COW Always-On Architecture - Removed cowEnabled flag from BaseStorage (COW cannot be disabled) - Eliminated marker file system (checkClearMarker, createClearMarker) - Simplified all code paths to assume COW is always enabled - COW automatically re-initializes after clear() operations ## Critical Bug Fix: Cloud Storage clear() - Fixed GCS clear() using correct paths (branches/ instead of entities/nouns/) - Fixed S3Compatible clear() path structure - Fixed R2 clear() implementation - Fixed Azure, FileSystem, OPFS, Memory clear() COW flag handling - clear() now deletes: branches/, _cow/, _system/ - Result: Cloud buckets can now be fully cleared (previously impossible) ## Container Memory Detection - Auto-detect Docker/K8s/Cloud Run memory limits (cgroup v1/v2) - Smart memory allocation (75% graph data, 25% query operations) - Environment variable support (CLOUD_RUN_MEMORY, MEMORY_LIMIT) - Production-grade containerized deployment support ## CommitLog streamHistory Feature - Added streamable commit history with pagination - Efficient memory usage for large commit histories - Support for branch filtering and time ranges ## Comprehensive Storage Documentation - Complete v5.11.0 file structure reference - Detailed path construction algorithms - 8 common storage scenarios with examples - Type-first storage, sharding, COW architecture explained - Public docs: docs/architecture/data-storage-architecture.md (1063 lines) ## Files Modified (14 files) - All 8 storage adapters (GCS, S3, R2, Azure, FS, OPFS, Memory, Historical) - BaseStorage core architecture - CommitLog with streaming - Brainy memory configuration - Parameter validation with container detection - Storage architecture documentation ## Breaking Changes NONE - COW was already enabled by default. This removes the ability to disable it. ## Migration No action required. Upgrade and clear() will work correctly on cloud storage. ## Impact - Users can now clear cloud storage buckets completely - No more corrupted buckets after clear() operations - Container deployments automatically optimize memory allocation - COW is mandatory and always enabled (safer, simpler) v5.11.0 - Production ready
This commit is contained in:
parent
28160a3052
commit
3e8b9aacc8
17 changed files with 2925 additions and 1207 deletions
|
|
@ -1145,19 +1145,11 @@ export class AzureBlobStorage extends BaseStorage {
|
|||
}
|
||||
}
|
||||
|
||||
// CRITICAL: Reset COW state to prevent automatic reinitialization
|
||||
// When COW data is cleared, we must also clear the COW managers
|
||||
// Otherwise initializeCOW() will auto-recreate initial commit on next operation
|
||||
// v5.11.0: Reset COW managers (but don't disable COW - it's always enabled)
|
||||
// COW will re-initialize automatically on next use
|
||||
this.refManager = undefined
|
||||
this.blobStorage = undefined
|
||||
this.commitLog = undefined
|
||||
this.cowEnabled = false
|
||||
|
||||
// v5.10.4: Create persistent marker blob (CRITICAL FIX)
|
||||
// Bug: cowEnabled = false only affects current instance, not future instances
|
||||
// Fix: Create marker blob that persists across instance restarts
|
||||
// When new instance calls initializeCOW(), it checks for this marker
|
||||
await this.createClearMarker()
|
||||
|
||||
// Clear caches
|
||||
this.nounCacheManager.clear()
|
||||
|
|
@ -1216,40 +1208,10 @@ export class AzureBlobStorage extends BaseStorage {
|
|||
* @returns true if marker blob exists, false otherwise
|
||||
* @protected
|
||||
*/
|
||||
protected async checkClearMarker(): Promise<boolean> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
const markerPath = `${this.systemPrefix}cow-disabled`
|
||||
const blockBlobClient = this.containerClient!.getBlockBlobClient(markerPath)
|
||||
const exists = await blockBlobClient.exists()
|
||||
return exists
|
||||
} catch (error) {
|
||||
this.logger.warn('AzureBlobStorage.checkClearMarker: Error checking marker', error)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create marker indicating COW has been explicitly disabled
|
||||
* v5.10.4: Called by clear() to prevent COW reinitialization on new instances
|
||||
* @protected
|
||||
* v5.11.0: Removed checkClearMarker() and createClearMarker() methods
|
||||
* COW is now always enabled - marker files are no longer used
|
||||
*/
|
||||
protected async createClearMarker(): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
const markerPath = `${this.systemPrefix}cow-disabled`
|
||||
const blockBlobClient = this.containerClient!.getBlockBlobClient(markerPath)
|
||||
// Create empty marker blob
|
||||
await blockBlobClient.upload(Buffer.from(''), 0, {
|
||||
blobHTTPHeaders: { blobContentType: 'text/plain' }
|
||||
})
|
||||
} catch (error) {
|
||||
this.logger.error('AzureBlobStorage.createClearMarker: Failed to create marker blob', error)
|
||||
// Don't throw - marker creation failure shouldn't break clear()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save statistics data to storage
|
||||
|
|
|
|||
|
|
@ -1023,19 +1023,11 @@ export class FileSystemStorage extends BaseStorage {
|
|||
// Delete the entire _cow/ directory (not just contents)
|
||||
await fs.promises.rm(cowDir, { recursive: true, force: true })
|
||||
|
||||
// CRITICAL: Reset COW state to prevent automatic reinitialization
|
||||
// When COW data is cleared, we must also clear the COW managers
|
||||
// Otherwise initializeCOW() will auto-recreate initial commit on next operation
|
||||
// v5.11.0: Reset COW managers (but don't disable COW - it's always enabled)
|
||||
// COW will re-initialize automatically on next use
|
||||
this.refManager = undefined
|
||||
this.blobStorage = undefined
|
||||
this.commitLog = undefined
|
||||
this.cowEnabled = false
|
||||
|
||||
// v5.10.4: Create persistent marker file (CRITICAL FIX)
|
||||
// Bug: cowEnabled = false only affects current instance, not future instances
|
||||
// Fix: Create marker file that persists across instance restarts
|
||||
// When new instance calls initializeCOW(), it checks for this marker
|
||||
await this.createClearMarker()
|
||||
}
|
||||
|
||||
// Clear the statistics cache
|
||||
|
|
@ -1080,42 +1072,10 @@ export class FileSystemStorage extends BaseStorage {
|
|||
* @returns true if marker file exists, false otherwise
|
||||
* @protected
|
||||
*/
|
||||
protected async checkClearMarker(): Promise<boolean> {
|
||||
// Check if fs module is available
|
||||
if (!fs || !fs.promises) {
|
||||
return false
|
||||
}
|
||||
|
||||
try {
|
||||
const markerPath = path.join(this.systemDir, 'cow-disabled')
|
||||
await fs.promises.access(markerPath, fs.constants.F_OK)
|
||||
return true // Marker exists
|
||||
} catch (error) {
|
||||
return false // Marker doesn't exist (ENOENT) or can't be accessed
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create marker indicating COW has been explicitly disabled
|
||||
* v5.10.4: Called by clear() to prevent COW reinitialization on new instances
|
||||
* @protected
|
||||
* v5.11.0: Removed checkClearMarker() and createClearMarker() methods
|
||||
* COW is now always enabled - marker files are no longer used
|
||||
*/
|
||||
protected async createClearMarker(): Promise<void> {
|
||||
// Check if fs module is available
|
||||
if (!fs || !fs.promises) {
|
||||
console.warn('FileSystemStorage.createClearMarker: fs module not available, skipping marker creation')
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
const markerPath = path.join(this.systemDir, 'cow-disabled')
|
||||
// Create empty marker file
|
||||
await fs.promises.writeFile(markerPath, '', 'utf8')
|
||||
} catch (error) {
|
||||
console.error('FileSystemStorage.createClearMarker: Failed to create marker file', error)
|
||||
// Don't throw - marker creation failure shouldn't break clear()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get information about storage usage and capacity
|
||||
|
|
|
|||
|
|
@ -995,31 +995,21 @@ export class GcsStorage extends BaseStorage {
|
|||
}
|
||||
}
|
||||
|
||||
// Clear all data directories
|
||||
await deleteObjectsWithPrefix(this.nounPrefix)
|
||||
await deleteObjectsWithPrefix(this.verbPrefix)
|
||||
await deleteObjectsWithPrefix(this.metadataPrefix)
|
||||
await deleteObjectsWithPrefix(this.verbMetadataPrefix)
|
||||
await deleteObjectsWithPrefix(this.systemPrefix)
|
||||
// v5.11.0: Clear ALL data using correct paths
|
||||
// Delete entire branches/ directory (includes ALL entities, ALL types, ALL VFS data, ALL forks)
|
||||
await deleteObjectsWithPrefix('branches/')
|
||||
|
||||
// v5.6.1: Clear COW (copy-on-write) version control data
|
||||
// This includes all git-like versioning data (commits, trees, blobs, refs)
|
||||
// Must be deleted to fully clear all data including version history
|
||||
// Delete COW version control data
|
||||
await deleteObjectsWithPrefix('_cow/')
|
||||
|
||||
// CRITICAL: Reset COW state to prevent automatic reinitialization
|
||||
// When COW data is cleared, we must also clear the COW managers
|
||||
// Otherwise initializeCOW() will auto-recreate initial commit on next operation
|
||||
// Delete system metadata
|
||||
await deleteObjectsWithPrefix('_system/')
|
||||
|
||||
// v5.11.0: Reset COW managers (but don't disable COW - it's always enabled)
|
||||
// COW will re-initialize automatically on next use
|
||||
this.refManager = undefined
|
||||
this.blobStorage = undefined
|
||||
this.commitLog = undefined
|
||||
this.cowEnabled = false
|
||||
|
||||
// v5.10.4: Create persistent marker object (CRITICAL FIX)
|
||||
// Bug: cowEnabled = false only affects current instance, not future instances
|
||||
// Fix: Create marker object that persists across instance restarts
|
||||
// When new instance calls initializeCOW(), it checks for this marker
|
||||
await this.createClearMarker()
|
||||
|
||||
// Clear caches
|
||||
this.nounCacheManager.clear()
|
||||
|
|
@ -1081,38 +1071,10 @@ export class GcsStorage extends BaseStorage {
|
|||
* @returns true if marker object exists, false otherwise
|
||||
* @protected
|
||||
*/
|
||||
protected async checkClearMarker(): Promise<boolean> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
const markerPath = `${this.systemPrefix}cow-disabled`
|
||||
const file = this.bucket!.file(markerPath)
|
||||
const [exists] = await file.exists()
|
||||
return exists
|
||||
} catch (error) {
|
||||
this.logger.warn('GCSStorage.checkClearMarker: Error checking marker', error)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create marker indicating COW has been explicitly disabled
|
||||
* v5.10.4: Called by clear() to prevent COW reinitialization on new instances
|
||||
* @protected
|
||||
* v5.11.0: Removed checkClearMarker() and createClearMarker() methods
|
||||
* COW is now always enabled - marker files are no longer used
|
||||
*/
|
||||
protected async createClearMarker(): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
const markerPath = `${this.systemPrefix}cow-disabled`
|
||||
const file = this.bucket!.file(markerPath)
|
||||
// Create empty marker object
|
||||
await file.save('', { contentType: 'text/plain' })
|
||||
} catch (error) {
|
||||
this.logger.error('GCSStorage.createClearMarker: Failed to create marker object', error)
|
||||
// Don't throw - marker creation failure shouldn't break clear()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save statistics data to storage
|
||||
|
|
|
|||
|
|
@ -316,18 +316,10 @@ export class HistoricalStorageAdapter extends BaseStorage {
|
|||
* @returns Always false (read-only adapter doesn't manage COW state)
|
||||
* @protected
|
||||
*/
|
||||
protected async checkClearMarker(): Promise<boolean> {
|
||||
return false // Read-only adapter - COW state managed by underlying storage
|
||||
}
|
||||
|
||||
/**
|
||||
* Create marker indicating COW has been explicitly disabled
|
||||
* v5.10.4: No-op for HistoricalStorageAdapter (read-only)
|
||||
* @protected
|
||||
* v5.11.0: Removed checkClearMarker() and createClearMarker() methods
|
||||
* COW is now always enabled - marker files are no longer used
|
||||
*/
|
||||
protected async createClearMarker(): Promise<void> {
|
||||
// No-op: HistoricalStorageAdapter is read-only, doesn't create markers
|
||||
}
|
||||
|
||||
// ============= Override Write Methods (Read-Only) =============
|
||||
|
||||
|
|
|
|||
|
|
@ -202,19 +202,10 @@ export class MemoryStorage extends BaseStorage {
|
|||
* @returns Always false (marker doesn't persist in memory)
|
||||
* @protected
|
||||
*/
|
||||
protected async checkClearMarker(): Promise<boolean> {
|
||||
return false // MemoryStorage doesn't persist - marker doesn't survive restart
|
||||
}
|
||||
|
||||
/**
|
||||
* Create marker indicating COW has been explicitly disabled
|
||||
* v5.10.4: No-op for MemoryStorage (doesn't persist)
|
||||
* @protected
|
||||
* v5.11.0: Removed checkClearMarker() and createClearMarker() methods
|
||||
* COW is now always enabled - marker files are no longer used
|
||||
*/
|
||||
protected async createClearMarker(): Promise<void> {
|
||||
// No-op: MemoryStorage doesn't persist, so marker is not needed
|
||||
// clear() in memory already resets all state, no marker survives restart
|
||||
}
|
||||
|
||||
/**
|
||||
* Save statistics data to storage
|
||||
|
|
|
|||
|
|
@ -488,19 +488,11 @@ export class OPFSStorage extends BaseStorage {
|
|||
// Delete the entire _cow/ directory (not just contents)
|
||||
await this.rootDir!.removeEntry('_cow', { recursive: true })
|
||||
|
||||
// CRITICAL: Reset COW state to prevent automatic reinitialization
|
||||
// When COW data is cleared, we must also clear the COW managers
|
||||
// Otherwise initializeCOW() will auto-recreate initial commit on next operation
|
||||
// v5.11.0: Reset COW managers (but don't disable COW - it's always enabled)
|
||||
// COW will re-initialize automatically on next use
|
||||
this.refManager = undefined
|
||||
this.blobStorage = undefined
|
||||
this.commitLog = undefined
|
||||
this.cowEnabled = false
|
||||
|
||||
// v5.10.4: Create persistent marker file (CRITICAL FIX)
|
||||
// Bug: cowEnabled = false only affects current instance, not future instances
|
||||
// Fix: Create marker file that persists across instance restarts
|
||||
// When new instance calls initializeCOW(), it checks for this marker
|
||||
await this.createClearMarker()
|
||||
} catch (error: any) {
|
||||
// Ignore if _cow directory doesn't exist (not all instances use COW)
|
||||
if (error.name !== 'NotFoundError') {
|
||||
|
|
@ -528,46 +520,10 @@ export class OPFSStorage extends BaseStorage {
|
|||
* @returns true if marker file exists, false otherwise
|
||||
* @protected
|
||||
*/
|
||||
protected async checkClearMarker(): Promise<boolean> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Get system directory (may not exist yet)
|
||||
const systemDir = await this.rootDir!.getDirectoryHandle('system', { create: false })
|
||||
// Try to get the marker file
|
||||
await systemDir.getFileHandle('cow-disabled', { create: false })
|
||||
return true // Marker exists
|
||||
} catch (error: any) {
|
||||
if (error.name === 'NotFoundError') {
|
||||
return false // Marker doesn't exist (or system dir doesn't exist)
|
||||
}
|
||||
// Other errors (permissions, etc.) - treat as marker not existing
|
||||
console.warn('OPFSStorage.checkClearMarker: Error checking marker', error)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create marker indicating COW has been explicitly disabled
|
||||
* v5.10.4: Called by clear() to prevent COW reinitialization on new instances
|
||||
* @protected
|
||||
* v5.11.0: Removed checkClearMarker() and createClearMarker() methods
|
||||
* COW is now always enabled - marker files are no longer used
|
||||
*/
|
||||
protected async createClearMarker(): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Get or create system directory
|
||||
const systemDir = await this.rootDir!.getDirectoryHandle('system', { create: true })
|
||||
// Create empty marker file
|
||||
const fileHandle = await systemDir.getFileHandle('cow-disabled', { create: true })
|
||||
const writable = await fileHandle.createWritable()
|
||||
await writable.write(new Uint8Array(0)) // Empty file
|
||||
await writable.close()
|
||||
} catch (error) {
|
||||
console.error('OPFSStorage.createClearMarker: Failed to create marker file', error)
|
||||
// Don't throw - marker creation failure shouldn't break clear()
|
||||
}
|
||||
}
|
||||
|
||||
// Quota monitoring configuration (v4.0.0)
|
||||
private quotaWarningThreshold = 0.8 // Warn at 80% usage
|
||||
|
|
|
|||
|
|
@ -1032,30 +1032,30 @@ export class R2Storage extends BaseStorage {
|
|||
|
||||
prodLog.info('🧹 R2: Clearing all data from bucket...')
|
||||
|
||||
// Clear all prefixes (v5.6.1: includes _cow/ for version control data)
|
||||
// _cow/ stores all git-like versioning data (commits, trees, blobs, refs)
|
||||
// Must be deleted to fully clear all data including version history
|
||||
for (const prefix of [this.nounPrefix, this.verbPrefix, this.metadataPrefix, this.verbMetadataPrefix, this.systemPrefix, '_cow/']) {
|
||||
const objects = await this.listObjectsUnderPath(prefix)
|
||||
|
||||
for (const key of objects) {
|
||||
await this.deleteObjectFromPath(key)
|
||||
}
|
||||
// v5.11.0: Clear ALL data using correct paths
|
||||
// Delete entire branches/ directory (includes ALL entities, ALL types, ALL VFS data, ALL forks)
|
||||
const branchObjects = await this.listObjectsUnderPath('branches/')
|
||||
for (const key of branchObjects) {
|
||||
await this.deleteObjectFromPath(key)
|
||||
}
|
||||
|
||||
// CRITICAL: Reset COW state to prevent automatic reinitialization
|
||||
// When COW data is cleared, we must also clear the COW managers
|
||||
// Otherwise initializeCOW() will auto-recreate initial commit on next operation
|
||||
// Delete COW version control data
|
||||
const cowObjects = await this.listObjectsUnderPath('_cow/')
|
||||
for (const key of cowObjects) {
|
||||
await this.deleteObjectFromPath(key)
|
||||
}
|
||||
|
||||
// Delete system metadata
|
||||
const systemObjects = await this.listObjectsUnderPath('_system/')
|
||||
for (const key of systemObjects) {
|
||||
await this.deleteObjectFromPath(key)
|
||||
}
|
||||
|
||||
// v5.11.0: Reset COW managers (but don't disable COW - it's always enabled)
|
||||
// COW will re-initialize automatically on next use
|
||||
this.refManager = undefined
|
||||
this.blobStorage = undefined
|
||||
this.commitLog = undefined
|
||||
this.cowEnabled = false
|
||||
|
||||
// v5.10.4: Create persistent marker object (CRITICAL FIX)
|
||||
// Bug: cowEnabled = false only affects current instance, not future instances
|
||||
// Fix: Create marker object that persists across instance restarts
|
||||
// When new instance calls initializeCOW(), it checks for this marker
|
||||
await this.createClearMarker()
|
||||
|
||||
this.nounCacheManager.clear()
|
||||
this.verbCacheManager.clear()
|
||||
|
|
@ -1098,36 +1098,10 @@ export class R2Storage extends BaseStorage {
|
|||
* @returns true if marker object exists, false otherwise
|
||||
* @protected
|
||||
*/
|
||||
protected async checkClearMarker(): Promise<boolean> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
const markerPath = `${this.systemPrefix}cow-disabled`
|
||||
const data = await this.readObjectFromPath(markerPath)
|
||||
return data !== null // Marker exists if we got any data
|
||||
} catch (error) {
|
||||
prodLog.warn('R2Storage.checkClearMarker: Error checking marker', error)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create marker indicating COW has been explicitly disabled
|
||||
* v5.10.4: Called by clear() to prevent COW reinitialization on new instances
|
||||
* @protected
|
||||
* v5.11.0: Removed checkClearMarker() and createClearMarker() methods
|
||||
* COW is now always enabled - marker files are no longer used
|
||||
*/
|
||||
protected async createClearMarker(): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
const markerPath = `${this.systemPrefix}cow-disabled`
|
||||
// Create empty marker object
|
||||
await this.writeObjectToPath(markerPath, '')
|
||||
} catch (error) {
|
||||
prodLog.error('R2Storage.createClearMarker: Failed to create marker object', error)
|
||||
// Don't throw - marker creation failure shouldn't break clear()
|
||||
}
|
||||
}
|
||||
|
||||
// v5.4.0: Removed getNounsWithPagination override - use BaseStorage's type-first implementation
|
||||
|
||||
|
|
|
|||
|
|
@ -2109,39 +2109,21 @@ export class S3CompatibleStorage extends BaseStorage {
|
|||
}
|
||||
}
|
||||
|
||||
// Delete all objects in the nouns directory
|
||||
await deleteObjectsWithPrefix(this.nounPrefix)
|
||||
// v5.11.0: Clear ALL data using correct paths
|
||||
// Delete entire branches/ directory (includes ALL entities, ALL types, ALL VFS data, ALL forks)
|
||||
await deleteObjectsWithPrefix('branches/')
|
||||
|
||||
// Delete all objects in the verbs directory
|
||||
await deleteObjectsWithPrefix(this.verbPrefix)
|
||||
|
||||
// Delete all objects in the noun metadata directory
|
||||
await deleteObjectsWithPrefix(this.metadataPrefix)
|
||||
|
||||
// Delete all objects in the verb metadata directory
|
||||
await deleteObjectsWithPrefix(this.verbMetadataPrefix)
|
||||
|
||||
// Delete all objects in the index directory
|
||||
await deleteObjectsWithPrefix(this.indexPrefix)
|
||||
|
||||
// v5.6.1: Delete COW (copy-on-write) version control data
|
||||
// This includes all git-like versioning data (commits, trees, blobs, refs)
|
||||
// Must be deleted to fully clear all data including version history
|
||||
// Delete COW version control data
|
||||
await deleteObjectsWithPrefix('_cow/')
|
||||
|
||||
// CRITICAL: Reset COW state to prevent automatic reinitialization
|
||||
// When COW data is cleared, we must also clear the COW managers
|
||||
// Otherwise initializeCOW() will auto-recreate initial commit on next operation
|
||||
// Delete system metadata
|
||||
await deleteObjectsWithPrefix('_system/')
|
||||
|
||||
// v5.11.0: Reset COW managers (but don't disable COW - it's always enabled)
|
||||
// COW will re-initialize automatically on next use
|
||||
this.refManager = undefined
|
||||
this.blobStorage = undefined
|
||||
this.commitLog = undefined
|
||||
this.cowEnabled = false
|
||||
|
||||
// v5.10.4: Create persistent marker object (CRITICAL FIX)
|
||||
// Bug: cowEnabled = false only affects current instance, not future instances
|
||||
// Fix: Create marker object that persists across instance restarts
|
||||
// When new instance calls initializeCOW(), it checks for this marker
|
||||
await this.createClearMarker()
|
||||
|
||||
// Clear the statistics cache
|
||||
this.statisticsCache = null
|
||||
|
|
@ -2273,55 +2255,10 @@ export class S3CompatibleStorage extends BaseStorage {
|
|||
* @returns true if marker object exists, false otherwise
|
||||
* @protected
|
||||
*/
|
||||
protected async checkClearMarker(): Promise<boolean> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
const { HeadObjectCommand } = await import('@aws-sdk/client-s3')
|
||||
const markerKey = `${this.systemPrefix}cow-disabled`
|
||||
|
||||
await this.s3Client!.send(
|
||||
new HeadObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: markerKey
|
||||
})
|
||||
)
|
||||
return true // Marker exists
|
||||
} catch (error: any) {
|
||||
if (error.name === 'NotFound' || error.$metadata?.httpStatusCode === 404) {
|
||||
return false // Marker doesn't exist
|
||||
}
|
||||
prodLog.warn('S3CompatibleStorage.checkClearMarker: Error checking marker', error)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create marker indicating COW has been explicitly disabled
|
||||
* v5.10.4: Called by clear() to prevent COW reinitialization on new instances
|
||||
* @protected
|
||||
* v5.11.0: Removed checkClearMarker() and createClearMarker() methods
|
||||
* COW is now always enabled - marker files are no longer used
|
||||
*/
|
||||
protected async createClearMarker(): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
const { PutObjectCommand } = await import('@aws-sdk/client-s3')
|
||||
const markerKey = `${this.systemPrefix}cow-disabled`
|
||||
|
||||
// Create empty marker object
|
||||
await this.s3Client!.send(
|
||||
new PutObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: markerKey,
|
||||
Body: Buffer.from(''),
|
||||
ContentType: 'text/plain'
|
||||
})
|
||||
)
|
||||
} catch (error) {
|
||||
prodLog.error('S3CompatibleStorage.createClearMarker: Failed to create marker object', error)
|
||||
// Don't throw - marker creation failure shouldn't break clear()
|
||||
}
|
||||
}
|
||||
|
||||
// Batch update timer ID
|
||||
protected statisticsBatchUpdateTimerId: NodeJS.Timeout | null = null
|
||||
|
|
|
|||
|
|
@ -156,7 +156,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
public blobStorage?: BlobStorage
|
||||
public commitLog?: CommitLog
|
||||
public currentBranch: string = 'main'
|
||||
protected cowEnabled: boolean = false
|
||||
// v5.11.0: Removed cowEnabled flag - COW is ALWAYS enabled (mandatory, cannot be disabled)
|
||||
|
||||
// Type-first indexing support (v5.4.0)
|
||||
// Built into all storage adapters for billion-scale efficiency
|
||||
|
|
@ -275,13 +275,11 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
* Called during init() to ensure all data is stored with branch prefixes from the start
|
||||
* RefManager/BlobStorage/CommitLog are lazy-initialized on first fork()
|
||||
* @param branch - Branch name to use (default: 'main')
|
||||
*
|
||||
* v5.11.0: COW is always enabled - this method now just sets the branch name (idempotent)
|
||||
*/
|
||||
public enableCOWLightweight(branch: string = 'main'): void {
|
||||
if (this.cowEnabled) {
|
||||
return
|
||||
}
|
||||
this.currentBranch = branch
|
||||
this.cowEnabled = true
|
||||
// RefManager/BlobStorage/CommitLog remain undefined until first fork()
|
||||
}
|
||||
|
||||
|
|
@ -300,30 +298,17 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
branch?: string
|
||||
enableCompression?: boolean
|
||||
}): Promise<void> {
|
||||
// v5.10.4: Check for persistent marker file (CRITICAL FIX)
|
||||
// Bug: Setting cowEnabled = false on OLD instance doesn't affect NEW instances
|
||||
// Fix: Check storage for persistent marker created by clear()
|
||||
// If marker exists, COW was explicitly disabled and should NOT reinitialize
|
||||
const markerExists = await this.checkClearMarker()
|
||||
if (markerExists) {
|
||||
return // COW was disabled by clear() - don't recreate _cow/ directory
|
||||
}
|
||||
// v5.11.0: COW is ALWAYS enabled - idempotent initialization only
|
||||
// Removed marker file check (cowEnabled flag removed, COW is mandatory)
|
||||
|
||||
// v5.6.1: If COW was explicitly disabled (e.g., via clear()), don't reinitialize
|
||||
// This prevents automatic recreation of COW data after clear() operations
|
||||
if (this.cowEnabled === false) {
|
||||
// Check if RefManager already initialized (idempotent)
|
||||
if (this.refManager && this.blobStorage && this.commitLog) {
|
||||
return
|
||||
}
|
||||
|
||||
// Check if RefManager already initialized (full COW setup complete)
|
||||
if (this.refManager) {
|
||||
return
|
||||
}
|
||||
|
||||
// Enable lightweight COW if not already enabled
|
||||
if (!this.cowEnabled) {
|
||||
this.currentBranch = options?.branch || 'main'
|
||||
this.cowEnabled = true
|
||||
// Set current branch if provided
|
||||
if (options?.branch) {
|
||||
this.currentBranch = options.branch
|
||||
}
|
||||
|
||||
// Create COWStorageAdapter bridge
|
||||
|
|
@ -436,7 +421,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
}
|
||||
}
|
||||
|
||||
this.cowEnabled = true
|
||||
// v5.11.0: COW is always enabled - no flag to set
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -452,10 +437,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
return basePath // COW metadata is global across all branches
|
||||
}
|
||||
|
||||
if (!this.cowEnabled) {
|
||||
return basePath // COW disabled, use direct path
|
||||
}
|
||||
|
||||
// v5.11.0: COW is always enabled - always use branch-scoped paths
|
||||
const targetBranch = branch || this.currentBranch || 'main'
|
||||
|
||||
// Branch-scoped path: branches/<branch>/<basePath>
|
||||
|
|
@ -485,18 +467,10 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
* Read object with inheritance from parent branches (COW layer)
|
||||
* Tries current branch first, then walks commit history
|
||||
* @protected - Available to subclasses for COW implementation
|
||||
*
|
||||
* v5.11.0: COW is always enabled - always use branch-scoped paths with inheritance
|
||||
*/
|
||||
protected async readWithInheritance(path: string, branch?: string): Promise<any | null> {
|
||||
if (!this.cowEnabled) {
|
||||
// COW disabled: check write cache, then direct read
|
||||
// v5.7.2: Check cache first for read-after-write consistency
|
||||
const cachedData = this.writeCache.get(path)
|
||||
if (cachedData !== undefined) {
|
||||
return cachedData
|
||||
}
|
||||
return this.readObjectFromPath(path)
|
||||
}
|
||||
|
||||
const targetBranch = branch || this.currentBranch || 'main'
|
||||
const branchPath = this.resolveBranchPath(path, targetBranch)
|
||||
|
||||
|
|
@ -585,12 +559,10 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
* This enables fork to see parent's data in pagination operations
|
||||
*
|
||||
* Simplified approach: All branches inherit from main
|
||||
*
|
||||
* v5.11.0: COW is always enabled - always use inheritance
|
||||
*/
|
||||
protected async listObjectsWithInheritance(prefix: string, branch?: string): Promise<string[]> {
|
||||
if (!this.cowEnabled) {
|
||||
return this.listObjectsInBranch(prefix, branch)
|
||||
}
|
||||
|
||||
const targetBranch = branch || this.currentBranch || 'main'
|
||||
|
||||
// Collect paths from current branch
|
||||
|
|
@ -1714,21 +1686,9 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
public abstract clear(): Promise<void>
|
||||
|
||||
/**
|
||||
* Check if COW has been explicitly disabled via clear()
|
||||
* v5.10.4: Fixes bug where clear() doesn't persist across instance restarts
|
||||
* Each adapter checks for a marker file/object (e.g., "_system/cow-disabled")
|
||||
* @returns true if COW was disabled by clear(), false otherwise
|
||||
* @protected
|
||||
* v5.11.0: Removed checkClearMarker() and createClearMarker() abstract methods
|
||||
* COW is now always enabled - marker files are no longer used
|
||||
*/
|
||||
protected abstract checkClearMarker(): Promise<boolean>
|
||||
|
||||
/**
|
||||
* Create marker indicating COW has been explicitly disabled
|
||||
* v5.10.4: Called by clear() to prevent COW reinitialization on new instances
|
||||
* Each adapter creates a marker file/object (e.g., "_system/cow-disabled")
|
||||
* @protected
|
||||
*/
|
||||
protected abstract createClearMarker(): Promise<void>
|
||||
|
||||
/**
|
||||
* Get information about storage usage and capacity
|
||||
|
|
|
|||
|
|
@ -267,6 +267,54 @@ export class CommitLog {
|
|||
return commits
|
||||
}
|
||||
|
||||
/**
|
||||
* Stream commit history (memory-efficient for large histories)
|
||||
*
|
||||
* Yields commits one at a time without accumulating in memory.
|
||||
* Use this for large commit histories (1000s of commits) where
|
||||
* memory efficiency is important.
|
||||
*
|
||||
* @param ref - Starting ref
|
||||
* @param options - Walk options
|
||||
* @yields Commits in reverse chronological order (newest first)
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* // Stream all commits without memory accumulation
|
||||
* for await (const commit of commitLog.streamHistory('main', { maxCount: 10000 })) {
|
||||
* console.log(commit.message)
|
||||
* }
|
||||
* ```
|
||||
*/
|
||||
async *streamHistory(
|
||||
ref: string,
|
||||
options?: {
|
||||
maxCount?: number
|
||||
since?: number
|
||||
until?: number
|
||||
}
|
||||
): AsyncIterableIterator<CommitObject> {
|
||||
let count = 0
|
||||
|
||||
for await (const commit of this.walk(ref, {
|
||||
maxDepth: options?.maxCount,
|
||||
until: options?.until
|
||||
})) {
|
||||
// Filter by since timestamp if provided
|
||||
if (options?.since && commit.timestamp < options.since) {
|
||||
continue
|
||||
}
|
||||
|
||||
yield commit
|
||||
count++
|
||||
|
||||
// Stop after maxCount commits
|
||||
if (options?.maxCount && count >= options.maxCount) {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Count commits between two commits
|
||||
*
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue