fix: critical clear() data persistence regression (v5.10.4)
Workshop team reported that brain.clear() doesn't fully delete persistent storage. After calling clear() and creating a new Brainy instance, all data was restored from storage. This is a CRITICAL data integrity bug. Root causes (3 bugs fixed): 1. **FileSystemStorage deleting wrong directory**: Data stored in branches/main/entities/ but clear() was only deleting old pre-v5.4.0 structure (nouns/, verbs/, metadata/) 2. **COW reinitialization after clear()**: Setting cowEnabled=false on old instance doesn't affect new instances. Fixed with persistent marker file. 3. **Metadata index cache not cleared**: find() with type filters returned stale data after clear(). Fixed by recreating MetadataIndexManager. Changes: - FileSystemStorage: Clear branches/ directory (where data actually lives) - All storage adapters: Add checkClearMarker()/createClearMarker() methods - BaseStorage: Check for cow-disabled marker before initializing COW - Brainy: Recreate metadataIndex after clear() to flush cached data - Tests: Comprehensive regression suite (8 tests) to prevent recurrence Fixes Workshop bug report: /media/dpsifr/storage/home/Projects/workshop/BRAINY_V5_10_2_CLEAR_BUG.md 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
parent
53420f6c9a
commit
aba15638dc
11 changed files with 613 additions and 17 deletions
|
|
@ -2205,6 +2205,11 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
this.index = this.setupIndex()
|
this.index = this.setupIndex()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// v5.10.4: Recreate metadata index to clear cached data
|
||||||
|
// Bug: Metadata index cache was not being cleared, causing find() with type filters to return stale data
|
||||||
|
this.metadataIndex = new MetadataIndexManager(this.storage)
|
||||||
|
await this.metadataIndex.init()
|
||||||
|
|
||||||
// Reset dimensions
|
// Reset dimensions
|
||||||
this.dimensions = undefined
|
this.dimensions = undefined
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1153,6 +1153,12 @@ export class AzureBlobStorage extends BaseStorage {
|
||||||
this.commitLog = undefined
|
this.commitLog = undefined
|
||||||
this.cowEnabled = false
|
this.cowEnabled = false
|
||||||
|
|
||||||
|
// v5.10.4: Create persistent marker blob (CRITICAL FIX)
|
||||||
|
// Bug: cowEnabled = false only affects current instance, not future instances
|
||||||
|
// Fix: Create marker blob that persists across instance restarts
|
||||||
|
// When new instance calls initializeCOW(), it checks for this marker
|
||||||
|
await this.createClearMarker()
|
||||||
|
|
||||||
// Clear caches
|
// Clear caches
|
||||||
this.nounCacheManager.clear()
|
this.nounCacheManager.clear()
|
||||||
this.verbCacheManager.clear()
|
this.verbCacheManager.clear()
|
||||||
|
|
@ -1204,6 +1210,47 @@ export class AzureBlobStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if COW has been explicitly disabled via clear()
|
||||||
|
* v5.10.4: Fixes bug where clear() doesn't persist across instance restarts
|
||||||
|
* @returns true if marker blob exists, false otherwise
|
||||||
|
* @protected
|
||||||
|
*/
|
||||||
|
protected async checkClearMarker(): Promise<boolean> {
|
||||||
|
await this.ensureInitialized()
|
||||||
|
|
||||||
|
try {
|
||||||
|
const markerPath = `${this.systemPrefix}cow-disabled`
|
||||||
|
const blockBlobClient = this.containerClient!.getBlockBlobClient(markerPath)
|
||||||
|
const exists = await blockBlobClient.exists()
|
||||||
|
return exists
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.warn('AzureBlobStorage.checkClearMarker: Error checking marker', error)
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create marker indicating COW has been explicitly disabled
|
||||||
|
* v5.10.4: Called by clear() to prevent COW reinitialization on new instances
|
||||||
|
* @protected
|
||||||
|
*/
|
||||||
|
protected async createClearMarker(): Promise<void> {
|
||||||
|
await this.ensureInitialized()
|
||||||
|
|
||||||
|
try {
|
||||||
|
const markerPath = `${this.systemPrefix}cow-disabled`
|
||||||
|
const blockBlobClient = this.containerClient!.getBlockBlobClient(markerPath)
|
||||||
|
// Create empty marker blob
|
||||||
|
await blockBlobClient.upload(Buffer.from(''), 0, {
|
||||||
|
blobHTTPHeaders: { blobContentType: 'text/plain' }
|
||||||
|
})
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.error('AzureBlobStorage.createClearMarker: Failed to create marker blob', error)
|
||||||
|
// Don't throw - marker creation failure shouldn't break clear()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Save statistics data to storage
|
* Save statistics data to storage
|
||||||
*/
|
*/
|
||||||
|
|
|
||||||
|
|
@ -1001,20 +1001,13 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Remove all files in the nouns directory
|
// v5.10.4: Clear the entire branches/ directory (branch-based storage)
|
||||||
await removeDirectoryContents(this.nounsDir)
|
// Bug fix: Data is stored in branches/main/entities/, not just entities/
|
||||||
|
// The branch-based structure was introduced for COW support
|
||||||
// Remove all files in the verbs directory
|
const branchesDir = path.join(this.rootDir, 'branches')
|
||||||
await removeDirectoryContents(this.verbsDir)
|
if (await this.directoryExists(branchesDir)) {
|
||||||
|
await removeDirectoryContents(branchesDir)
|
||||||
// Remove all files in the metadata directory
|
}
|
||||||
await removeDirectoryContents(this.metadataDir)
|
|
||||||
|
|
||||||
// Remove all files in the noun metadata directory
|
|
||||||
await removeDirectoryContents(this.nounMetadataDir)
|
|
||||||
|
|
||||||
// Remove all files in the verb metadata directory
|
|
||||||
await removeDirectoryContents(this.verbMetadataDir)
|
|
||||||
|
|
||||||
// Remove all files in both system directories
|
// Remove all files in both system directories
|
||||||
await removeDirectoryContents(this.systemDir)
|
await removeDirectoryContents(this.systemDir)
|
||||||
|
|
@ -1037,6 +1030,12 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
this.blobStorage = undefined
|
this.blobStorage = undefined
|
||||||
this.commitLog = undefined
|
this.commitLog = undefined
|
||||||
this.cowEnabled = false
|
this.cowEnabled = false
|
||||||
|
|
||||||
|
// v5.10.4: Create persistent marker file (CRITICAL FIX)
|
||||||
|
// Bug: cowEnabled = false only affects current instance, not future instances
|
||||||
|
// Fix: Create marker file that persists across instance restarts
|
||||||
|
// When new instance calls initializeCOW(), it checks for this marker
|
||||||
|
await this.createClearMarker()
|
||||||
}
|
}
|
||||||
|
|
||||||
// Clear the statistics cache
|
// Clear the statistics cache
|
||||||
|
|
@ -1075,6 +1074,49 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if COW has been explicitly disabled via clear()
|
||||||
|
* v5.10.4: Fixes bug where clear() doesn't persist across instance restarts
|
||||||
|
* @returns true if marker file exists, false otherwise
|
||||||
|
* @protected
|
||||||
|
*/
|
||||||
|
protected async checkClearMarker(): Promise<boolean> {
|
||||||
|
// Check if fs module is available
|
||||||
|
if (!fs || !fs.promises) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const markerPath = path.join(this.systemDir, 'cow-disabled')
|
||||||
|
await fs.promises.access(markerPath, fs.constants.F_OK)
|
||||||
|
return true // Marker exists
|
||||||
|
} catch (error) {
|
||||||
|
return false // Marker doesn't exist (ENOENT) or can't be accessed
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create marker indicating COW has been explicitly disabled
|
||||||
|
* v5.10.4: Called by clear() to prevent COW reinitialization on new instances
|
||||||
|
* @protected
|
||||||
|
*/
|
||||||
|
protected async createClearMarker(): Promise<void> {
|
||||||
|
// Check if fs module is available
|
||||||
|
if (!fs || !fs.promises) {
|
||||||
|
console.warn('FileSystemStorage.createClearMarker: fs module not available, skipping marker creation')
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const markerPath = path.join(this.systemDir, 'cow-disabled')
|
||||||
|
// Create empty marker file
|
||||||
|
await fs.promises.writeFile(markerPath, '', 'utf8')
|
||||||
|
} catch (error) {
|
||||||
|
console.error('FileSystemStorage.createClearMarker: Failed to create marker file', error)
|
||||||
|
// Don't throw - marker creation failure shouldn't break clear()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get information about storage usage and capacity
|
* Get information about storage usage and capacity
|
||||||
*/
|
*/
|
||||||
|
|
|
||||||
|
|
@ -1015,6 +1015,12 @@ export class GcsStorage extends BaseStorage {
|
||||||
this.commitLog = undefined
|
this.commitLog = undefined
|
||||||
this.cowEnabled = false
|
this.cowEnabled = false
|
||||||
|
|
||||||
|
// v5.10.4: Create persistent marker object (CRITICAL FIX)
|
||||||
|
// Bug: cowEnabled = false only affects current instance, not future instances
|
||||||
|
// Fix: Create marker object that persists across instance restarts
|
||||||
|
// When new instance calls initializeCOW(), it checks for this marker
|
||||||
|
await this.createClearMarker()
|
||||||
|
|
||||||
// Clear caches
|
// Clear caches
|
||||||
this.nounCacheManager.clear()
|
this.nounCacheManager.clear()
|
||||||
this.verbCacheManager.clear()
|
this.verbCacheManager.clear()
|
||||||
|
|
@ -1069,6 +1075,45 @@ export class GcsStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if COW has been explicitly disabled via clear()
|
||||||
|
* v5.10.4: Fixes bug where clear() doesn't persist across instance restarts
|
||||||
|
* @returns true if marker object exists, false otherwise
|
||||||
|
* @protected
|
||||||
|
*/
|
||||||
|
protected async checkClearMarker(): Promise<boolean> {
|
||||||
|
await this.ensureInitialized()
|
||||||
|
|
||||||
|
try {
|
||||||
|
const markerPath = `${this.systemPrefix}cow-disabled`
|
||||||
|
const file = this.bucket!.file(markerPath)
|
||||||
|
const [exists] = await file.exists()
|
||||||
|
return exists
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.warn('GCSStorage.checkClearMarker: Error checking marker', error)
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create marker indicating COW has been explicitly disabled
|
||||||
|
* v5.10.4: Called by clear() to prevent COW reinitialization on new instances
|
||||||
|
* @protected
|
||||||
|
*/
|
||||||
|
protected async createClearMarker(): Promise<void> {
|
||||||
|
await this.ensureInitialized()
|
||||||
|
|
||||||
|
try {
|
||||||
|
const markerPath = `${this.systemPrefix}cow-disabled`
|
||||||
|
const file = this.bucket!.file(markerPath)
|
||||||
|
// Create empty marker object
|
||||||
|
await file.save('', { contentType: 'text/plain' })
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.error('GCSStorage.createClearMarker: Failed to create marker object', error)
|
||||||
|
// Don't throw - marker creation failure shouldn't break clear()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Save statistics data to storage
|
* Save statistics data to storage
|
||||||
*/
|
*/
|
||||||
|
|
|
||||||
|
|
@ -310,6 +310,25 @@ export class HistoricalStorageAdapter extends BaseStorage {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if COW has been explicitly disabled via clear()
|
||||||
|
* v5.10.4: No-op for HistoricalStorageAdapter (read-only, doesn't manage COW)
|
||||||
|
* @returns Always false (read-only adapter doesn't manage COW state)
|
||||||
|
* @protected
|
||||||
|
*/
|
||||||
|
protected async checkClearMarker(): Promise<boolean> {
|
||||||
|
return false // Read-only adapter - COW state managed by underlying storage
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create marker indicating COW has been explicitly disabled
|
||||||
|
* v5.10.4: No-op for HistoricalStorageAdapter (read-only)
|
||||||
|
* @protected
|
||||||
|
*/
|
||||||
|
protected async createClearMarker(): Promise<void> {
|
||||||
|
// No-op: HistoricalStorageAdapter is read-only, doesn't create markers
|
||||||
|
}
|
||||||
|
|
||||||
// ============= Override Write Methods (Read-Only) =============
|
// ============= Override Write Methods (Read-Only) =============
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
||||||
|
|
@ -196,6 +196,26 @@ export class MemoryStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if COW has been explicitly disabled via clear()
|
||||||
|
* v5.10.4: No-op for MemoryStorage (doesn't persist)
|
||||||
|
* @returns Always false (marker doesn't persist in memory)
|
||||||
|
* @protected
|
||||||
|
*/
|
||||||
|
protected async checkClearMarker(): Promise<boolean> {
|
||||||
|
return false // MemoryStorage doesn't persist - marker doesn't survive restart
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create marker indicating COW has been explicitly disabled
|
||||||
|
* v5.10.4: No-op for MemoryStorage (doesn't persist)
|
||||||
|
* @protected
|
||||||
|
*/
|
||||||
|
protected async createClearMarker(): Promise<void> {
|
||||||
|
// No-op: MemoryStorage doesn't persist, so marker is not needed
|
||||||
|
// clear() in memory already resets all state, no marker survives restart
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Save statistics data to storage
|
* Save statistics data to storage
|
||||||
* @param statistics The statistics data to save
|
* @param statistics The statistics data to save
|
||||||
|
|
|
||||||
|
|
@ -495,6 +495,12 @@ export class OPFSStorage extends BaseStorage {
|
||||||
this.blobStorage = undefined
|
this.blobStorage = undefined
|
||||||
this.commitLog = undefined
|
this.commitLog = undefined
|
||||||
this.cowEnabled = false
|
this.cowEnabled = false
|
||||||
|
|
||||||
|
// v5.10.4: Create persistent marker file (CRITICAL FIX)
|
||||||
|
// Bug: cowEnabled = false only affects current instance, not future instances
|
||||||
|
// Fix: Create marker file that persists across instance restarts
|
||||||
|
// When new instance calls initializeCOW(), it checks for this marker
|
||||||
|
await this.createClearMarker()
|
||||||
} catch (error: any) {
|
} catch (error: any) {
|
||||||
// Ignore if _cow directory doesn't exist (not all instances use COW)
|
// Ignore if _cow directory doesn't exist (not all instances use COW)
|
||||||
if (error.name !== 'NotFoundError') {
|
if (error.name !== 'NotFoundError') {
|
||||||
|
|
@ -516,6 +522,53 @@ export class OPFSStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if COW has been explicitly disabled via clear()
|
||||||
|
* v5.10.4: Fixes bug where clear() doesn't persist across instance restarts
|
||||||
|
* @returns true if marker file exists, false otherwise
|
||||||
|
* @protected
|
||||||
|
*/
|
||||||
|
protected async checkClearMarker(): Promise<boolean> {
|
||||||
|
await this.ensureInitialized()
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Get system directory (may not exist yet)
|
||||||
|
const systemDir = await this.rootDir!.getDirectoryHandle('system', { create: false })
|
||||||
|
// Try to get the marker file
|
||||||
|
await systemDir.getFileHandle('cow-disabled', { create: false })
|
||||||
|
return true // Marker exists
|
||||||
|
} catch (error: any) {
|
||||||
|
if (error.name === 'NotFoundError') {
|
||||||
|
return false // Marker doesn't exist (or system dir doesn't exist)
|
||||||
|
}
|
||||||
|
// Other errors (permissions, etc.) - treat as marker not existing
|
||||||
|
console.warn('OPFSStorage.checkClearMarker: Error checking marker', error)
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create marker indicating COW has been explicitly disabled
|
||||||
|
* v5.10.4: Called by clear() to prevent COW reinitialization on new instances
|
||||||
|
* @protected
|
||||||
|
*/
|
||||||
|
protected async createClearMarker(): Promise<void> {
|
||||||
|
await this.ensureInitialized()
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Get or create system directory
|
||||||
|
const systemDir = await this.rootDir!.getDirectoryHandle('system', { create: true })
|
||||||
|
// Create empty marker file
|
||||||
|
const fileHandle = await systemDir.getFileHandle('cow-disabled', { create: true })
|
||||||
|
const writable = await fileHandle.createWritable()
|
||||||
|
await writable.write(new Uint8Array(0)) // Empty file
|
||||||
|
await writable.close()
|
||||||
|
} catch (error) {
|
||||||
|
console.error('OPFSStorage.createClearMarker: Failed to create marker file', error)
|
||||||
|
// Don't throw - marker creation failure shouldn't break clear()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Quota monitoring configuration (v4.0.0)
|
// Quota monitoring configuration (v4.0.0)
|
||||||
private quotaWarningThreshold = 0.8 // Warn at 80% usage
|
private quotaWarningThreshold = 0.8 // Warn at 80% usage
|
||||||
private quotaCriticalThreshold = 0.95 // Critical at 95% usage
|
private quotaCriticalThreshold = 0.95 // Critical at 95% usage
|
||||||
|
|
|
||||||
|
|
@ -1051,6 +1051,12 @@ export class R2Storage extends BaseStorage {
|
||||||
this.commitLog = undefined
|
this.commitLog = undefined
|
||||||
this.cowEnabled = false
|
this.cowEnabled = false
|
||||||
|
|
||||||
|
// v5.10.4: Create persistent marker object (CRITICAL FIX)
|
||||||
|
// Bug: cowEnabled = false only affects current instance, not future instances
|
||||||
|
// Fix: Create marker object that persists across instance restarts
|
||||||
|
// When new instance calls initializeCOW(), it checks for this marker
|
||||||
|
await this.createClearMarker()
|
||||||
|
|
||||||
this.nounCacheManager.clear()
|
this.nounCacheManager.clear()
|
||||||
this.verbCacheManager.clear()
|
this.verbCacheManager.clear()
|
||||||
|
|
||||||
|
|
@ -1086,6 +1092,43 @@ export class R2Storage extends BaseStorage {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if COW has been explicitly disabled via clear()
|
||||||
|
* v5.10.4: Fixes bug where clear() doesn't persist across instance restarts
|
||||||
|
* @returns true if marker object exists, false otherwise
|
||||||
|
* @protected
|
||||||
|
*/
|
||||||
|
protected async checkClearMarker(): Promise<boolean> {
|
||||||
|
await this.ensureInitialized()
|
||||||
|
|
||||||
|
try {
|
||||||
|
const markerPath = `${this.systemPrefix}cow-disabled`
|
||||||
|
const data = await this.readObjectFromPath(markerPath)
|
||||||
|
return data !== null // Marker exists if we got any data
|
||||||
|
} catch (error) {
|
||||||
|
prodLog.warn('R2Storage.checkClearMarker: Error checking marker', error)
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create marker indicating COW has been explicitly disabled
|
||||||
|
* v5.10.4: Called by clear() to prevent COW reinitialization on new instances
|
||||||
|
* @protected
|
||||||
|
*/
|
||||||
|
protected async createClearMarker(): Promise<void> {
|
||||||
|
await this.ensureInitialized()
|
||||||
|
|
||||||
|
try {
|
||||||
|
const markerPath = `${this.systemPrefix}cow-disabled`
|
||||||
|
// Create empty marker object
|
||||||
|
await this.writeObjectToPath(markerPath, '')
|
||||||
|
} catch (error) {
|
||||||
|
prodLog.error('R2Storage.createClearMarker: Failed to create marker object', error)
|
||||||
|
// Don't throw - marker creation failure shouldn't break clear()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// v5.4.0: Removed getNounsWithPagination override - use BaseStorage's type-first implementation
|
// v5.4.0: Removed getNounsWithPagination override - use BaseStorage's type-first implementation
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -2137,6 +2137,12 @@ export class S3CompatibleStorage extends BaseStorage {
|
||||||
this.commitLog = undefined
|
this.commitLog = undefined
|
||||||
this.cowEnabled = false
|
this.cowEnabled = false
|
||||||
|
|
||||||
|
// v5.10.4: Create persistent marker object (CRITICAL FIX)
|
||||||
|
// Bug: cowEnabled = false only affects current instance, not future instances
|
||||||
|
// Fix: Create marker object that persists across instance restarts
|
||||||
|
// When new instance calls initializeCOW(), it checks for this marker
|
||||||
|
await this.createClearMarker()
|
||||||
|
|
||||||
// Clear the statistics cache
|
// Clear the statistics cache
|
||||||
this.statisticsCache = null
|
this.statisticsCache = null
|
||||||
this.statisticsModified = false
|
this.statisticsModified = false
|
||||||
|
|
@ -2261,6 +2267,62 @@ export class S3CompatibleStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if COW has been explicitly disabled via clear()
|
||||||
|
* v5.10.4: Fixes bug where clear() doesn't persist across instance restarts
|
||||||
|
* @returns true if marker object exists, false otherwise
|
||||||
|
* @protected
|
||||||
|
*/
|
||||||
|
protected async checkClearMarker(): Promise<boolean> {
|
||||||
|
await this.ensureInitialized()
|
||||||
|
|
||||||
|
try {
|
||||||
|
const { HeadObjectCommand } = await import('@aws-sdk/client-s3')
|
||||||
|
const markerKey = `${this.systemPrefix}cow-disabled`
|
||||||
|
|
||||||
|
await this.s3Client!.send(
|
||||||
|
new HeadObjectCommand({
|
||||||
|
Bucket: this.bucketName,
|
||||||
|
Key: markerKey
|
||||||
|
})
|
||||||
|
)
|
||||||
|
return true // Marker exists
|
||||||
|
} catch (error: any) {
|
||||||
|
if (error.name === 'NotFound' || error.$metadata?.httpStatusCode === 404) {
|
||||||
|
return false // Marker doesn't exist
|
||||||
|
}
|
||||||
|
prodLog.warn('S3CompatibleStorage.checkClearMarker: Error checking marker', error)
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create marker indicating COW has been explicitly disabled
|
||||||
|
* v5.10.4: Called by clear() to prevent COW reinitialization on new instances
|
||||||
|
* @protected
|
||||||
|
*/
|
||||||
|
protected async createClearMarker(): Promise<void> {
|
||||||
|
await this.ensureInitialized()
|
||||||
|
|
||||||
|
try {
|
||||||
|
const { PutObjectCommand } = await import('@aws-sdk/client-s3')
|
||||||
|
const markerKey = `${this.systemPrefix}cow-disabled`
|
||||||
|
|
||||||
|
// Create empty marker object
|
||||||
|
await this.s3Client!.send(
|
||||||
|
new PutObjectCommand({
|
||||||
|
Bucket: this.bucketName,
|
||||||
|
Key: markerKey,
|
||||||
|
Body: Buffer.from(''),
|
||||||
|
ContentType: 'text/plain'
|
||||||
|
})
|
||||||
|
)
|
||||||
|
} catch (error) {
|
||||||
|
prodLog.error('S3CompatibleStorage.createClearMarker: Failed to create marker object', error)
|
||||||
|
// Don't throw - marker creation failure shouldn't break clear()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Batch update timer ID
|
// Batch update timer ID
|
||||||
protected statisticsBatchUpdateTimerId: NodeJS.Timeout | null = null
|
protected statisticsBatchUpdateTimerId: NodeJS.Timeout | null = null
|
||||||
// Flag to indicate if statistics have been modified since last save
|
// Flag to indicate if statistics have been modified since last save
|
||||||
|
|
|
||||||
|
|
@ -300,6 +300,15 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
branch?: string
|
branch?: string
|
||||||
enableCompression?: boolean
|
enableCompression?: boolean
|
||||||
}): Promise<void> {
|
}): Promise<void> {
|
||||||
|
// v5.10.4: Check for persistent marker file (CRITICAL FIX)
|
||||||
|
// Bug: Setting cowEnabled = false on OLD instance doesn't affect NEW instances
|
||||||
|
// Fix: Check storage for persistent marker created by clear()
|
||||||
|
// If marker exists, COW was explicitly disabled and should NOT reinitialize
|
||||||
|
const markerExists = await this.checkClearMarker()
|
||||||
|
if (markerExists) {
|
||||||
|
return // COW was disabled by clear() - don't recreate _cow/ directory
|
||||||
|
}
|
||||||
|
|
||||||
// v5.6.1: If COW was explicitly disabled (e.g., via clear()), don't reinitialize
|
// v5.6.1: If COW was explicitly disabled (e.g., via clear()), don't reinitialize
|
||||||
// This prevents automatic recreation of COW data after clear() operations
|
// This prevents automatic recreation of COW data after clear() operations
|
||||||
if (this.cowEnabled === false) {
|
if (this.cowEnabled === false) {
|
||||||
|
|
@ -1704,6 +1713,23 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
*/
|
*/
|
||||||
public abstract clear(): Promise<void>
|
public abstract clear(): Promise<void>
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if COW has been explicitly disabled via clear()
|
||||||
|
* v5.10.4: Fixes bug where clear() doesn't persist across instance restarts
|
||||||
|
* Each adapter checks for a marker file/object (e.g., "_system/cow-disabled")
|
||||||
|
* @returns true if COW was disabled by clear(), false otherwise
|
||||||
|
* @protected
|
||||||
|
*/
|
||||||
|
protected abstract checkClearMarker(): Promise<boolean>
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create marker indicating COW has been explicitly disabled
|
||||||
|
* v5.10.4: Called by clear() to prevent COW reinitialization on new instances
|
||||||
|
* Each adapter creates a marker file/object (e.g., "_system/cow-disabled")
|
||||||
|
* @protected
|
||||||
|
*/
|
||||||
|
protected abstract createClearMarker(): Promise<void>
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get information about storage usage and capacity
|
* Get information about storage usage and capacity
|
||||||
* This method should be implemented by each specific adapter
|
* This method should be implemented by each specific adapter
|
||||||
|
|
|
||||||
234
tests/integration/clear-persistence.test.ts
Normal file
234
tests/integration/clear-persistence.test.ts
Normal file
|
|
@ -0,0 +1,234 @@
|
||||||
|
/**
|
||||||
|
* Integration tests for clear() bug fix (v5.10.4)
|
||||||
|
*
|
||||||
|
* Bug report: Workshop team reported that brain.clear() doesn't fully delete persistent storage.
|
||||||
|
* After calling clear() and creating a new Brainy instance, all data was restored from _cow/ directory.
|
||||||
|
*
|
||||||
|
* Root cause: Setting cowEnabled = false on old instance doesn't affect new instances.
|
||||||
|
* Fix: Create persistent marker file that survives instance restarts.
|
||||||
|
*
|
||||||
|
* These tests verify:
|
||||||
|
* 1. clear() actually deletes all data
|
||||||
|
* 2. New instances don't restore data from _cow/
|
||||||
|
* 3. Marker file persists across restarts
|
||||||
|
* 4. Works for all storage adapters
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||||
|
import { Brainy } from '../../src/brainy.js'
|
||||||
|
import * as fs from 'fs'
|
||||||
|
import * as path from 'path'
|
||||||
|
|
||||||
|
describe('Clear Persistence Bug Fix (v5.10.4)', () => {
|
||||||
|
const testStoragePath = './test-clear-persistence-' + Date.now()
|
||||||
|
|
||||||
|
afterEach(async () => {
|
||||||
|
// Cleanup test storage
|
||||||
|
try {
|
||||||
|
if (fs.existsSync(testStoragePath)) {
|
||||||
|
await fs.promises.rm(testStoragePath, { recursive: true, force: true })
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
console.warn('Failed to cleanup test storage:', error)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should fully clear persistent storage (Workshop scenario)', async () => {
|
||||||
|
// Step 1: Create and populate instance
|
||||||
|
const brain1 = new Brainy({
|
||||||
|
storage: {
|
||||||
|
type: 'filesystem',
|
||||||
|
path: testStoragePath,
|
||||||
|
enableCompression: true
|
||||||
|
}
|
||||||
|
})
|
||||||
|
await brain1.init()
|
||||||
|
|
||||||
|
// Add data (using only entities, not VFS)
|
||||||
|
const entityId = await brain1.add({ data: 'Test Entity', type: 'concept' })
|
||||||
|
|
||||||
|
// Verify data exists (filter by type to exclude VFS directories)
|
||||||
|
const entityCount1 = (await brain1.find({ type: 'concept' })).length
|
||||||
|
expect(entityCount1).toBe(1)
|
||||||
|
|
||||||
|
// Step 2: Clear all data
|
||||||
|
await brain1.clear()
|
||||||
|
|
||||||
|
// Verify cleared in same instance (filter by type to exclude VFS directories)
|
||||||
|
const entityCount2 = (await brain1.find({ type: 'concept' })).length
|
||||||
|
expect(entityCount2).toBe(0)
|
||||||
|
|
||||||
|
// Step 3: Create NEW instance (simulate server restart)
|
||||||
|
const brain2 = new Brainy({
|
||||||
|
storage: {
|
||||||
|
type: 'filesystem',
|
||||||
|
path: testStoragePath,
|
||||||
|
enableCompression: true
|
||||||
|
}
|
||||||
|
})
|
||||||
|
await brain2.init()
|
||||||
|
|
||||||
|
// CRITICAL: Verify data is NOT restored (THE BUG)
|
||||||
|
const entityCount3 = (await brain2.find({ type: 'concept' })).length
|
||||||
|
|
||||||
|
expect(entityCount3).toBe(0) // Should be 0, not 1
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should create cow-disabled marker file', async () => {
|
||||||
|
const brain = new Brainy({
|
||||||
|
storage: {
|
||||||
|
type: 'filesystem',
|
||||||
|
path: testStoragePath
|
||||||
|
}
|
||||||
|
})
|
||||||
|
await brain.init()
|
||||||
|
|
||||||
|
// Add some data to ensure COW is initialized
|
||||||
|
await brain.add({ data: 'Test', type: 'concept' })
|
||||||
|
|
||||||
|
// Clear
|
||||||
|
await brain.clear()
|
||||||
|
|
||||||
|
// Check for marker file
|
||||||
|
const markerPath = path.join(testStoragePath, '_system', 'cow-disabled')
|
||||||
|
const markerExists = fs.existsSync(markerPath)
|
||||||
|
expect(markerExists).toBe(true)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should delete _cow/ directory', async () => {
|
||||||
|
const brain = new Brainy({
|
||||||
|
storage: {
|
||||||
|
type: 'filesystem',
|
||||||
|
path: testStoragePath
|
||||||
|
}
|
||||||
|
})
|
||||||
|
await brain.init()
|
||||||
|
|
||||||
|
// Add data to create COW commits
|
||||||
|
await brain.add({ data:'Test', type: 'concept' })
|
||||||
|
|
||||||
|
// Verify _cow/ exists
|
||||||
|
const cowPath = path.join(testStoragePath, '_cow')
|
||||||
|
const cowExistsBefore = fs.existsSync(cowPath)
|
||||||
|
expect(cowExistsBefore).toBe(true)
|
||||||
|
|
||||||
|
// Clear
|
||||||
|
await brain.clear()
|
||||||
|
|
||||||
|
// Verify _cow/ is deleted
|
||||||
|
const cowExistsAfter = fs.existsSync(cowPath)
|
||||||
|
expect(cowExistsAfter).toBe(false)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should prevent COW reinitialization after clear()', async () => {
|
||||||
|
// Create and clear
|
||||||
|
const brain1 = new Brainy({
|
||||||
|
storage: { type: 'filesystem', path: testStoragePath }
|
||||||
|
})
|
||||||
|
await brain1.init()
|
||||||
|
await brain1.add({ data:'Test', type: 'concept' })
|
||||||
|
await brain1.clear()
|
||||||
|
|
||||||
|
// Create new instance
|
||||||
|
const brain2 = new Brainy({
|
||||||
|
storage: { type: 'filesystem', path: testStoragePath }
|
||||||
|
})
|
||||||
|
await brain2.init()
|
||||||
|
|
||||||
|
// Add new data - should NOT recreate _cow/
|
||||||
|
// (COW stays disabled because marker exists)
|
||||||
|
await brain2.add({ data:'New Data', type: 'concept' })
|
||||||
|
|
||||||
|
// Verify _cow/ still doesn't exist
|
||||||
|
const cowPath = path.join(testStoragePath, '_cow')
|
||||||
|
const cowExists = fs.existsSync(cowPath)
|
||||||
|
expect(cowExists).toBe(false)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should work across multiple clear() calls', async () => {
|
||||||
|
const storagePath = testStoragePath + '-multi'
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Iteration 1
|
||||||
|
const brain1 = new Brainy({ storage: { type: 'filesystem', path: storagePath }})
|
||||||
|
await brain1.init()
|
||||||
|
await brain1.add({ data:'Entity 1', type: 'concept' })
|
||||||
|
expect((await brain1.find({ type: 'concept' })).length).toBe(1)
|
||||||
|
await brain1.clear()
|
||||||
|
|
||||||
|
// Iteration 2
|
||||||
|
const brain2 = new Brainy({ storage: { type: 'filesystem', path: storagePath }})
|
||||||
|
await brain2.init()
|
||||||
|
expect((await brain2.find({ type: 'concept' })).length).toBe(0)
|
||||||
|
await brain2.add({ data:'Entity 2', type: 'concept' })
|
||||||
|
expect((await brain2.find({ type: 'concept' })).length).toBe(1)
|
||||||
|
await brain2.clear()
|
||||||
|
|
||||||
|
// Iteration 3
|
||||||
|
const brain3 = new Brainy({ storage: { type: 'filesystem', path: storagePath }})
|
||||||
|
await brain3.init()
|
||||||
|
expect((await brain3.find({ type: 'concept' })).length).toBe(0)
|
||||||
|
} finally {
|
||||||
|
// Cleanup
|
||||||
|
if (fs.existsSync(storagePath)) {
|
||||||
|
await fs.promises.rm(storagePath, { recursive: true, force: true })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should clear both entities and relations', async () => {
|
||||||
|
const brain1 = new Brainy({ storage: { type: 'filesystem', path: testStoragePath }})
|
||||||
|
await brain1.init()
|
||||||
|
|
||||||
|
// Add graph data
|
||||||
|
const entity1Id = await brain1.add({ data:'Entity 1', type: 'person' })
|
||||||
|
const entity2Id = await brain1.add({ data:'Entity 2', type: 'concept' })
|
||||||
|
await brain1.relate({ from: entity1Id, to: entity2Id, type: 'relatedTo' })
|
||||||
|
|
||||||
|
// Verify data exists
|
||||||
|
expect((await brain1.find({ type: 'person' })).length).toBe(1)
|
||||||
|
expect((await brain1.find({ type: 'concept' })).length).toBe(1)
|
||||||
|
expect((await brain1.getRelations({})).length).toBe(1)
|
||||||
|
|
||||||
|
// Clear
|
||||||
|
await brain1.clear()
|
||||||
|
|
||||||
|
// Create new instance
|
||||||
|
const brain2 = new Brainy({ storage: { type: 'filesystem', path: testStoragePath }})
|
||||||
|
await brain2.init()
|
||||||
|
|
||||||
|
// Verify everything is cleared
|
||||||
|
expect((await brain2.find({ type: 'person' })).length).toBe(0)
|
||||||
|
expect((await brain2.find({ type: 'concept' })).length).toBe(0)
|
||||||
|
expect((await brain2.getRelations({})).length).toBe(0)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should handle clear() on empty storage', async () => {
|
||||||
|
const brain = new Brainy({ storage: { type: 'filesystem', path: testStoragePath }})
|
||||||
|
await brain.init()
|
||||||
|
|
||||||
|
// Clear without adding any data
|
||||||
|
await expect(brain.clear()).resolves.not.toThrow()
|
||||||
|
|
||||||
|
// Should create marker even if no data was present
|
||||||
|
const markerPath = path.join(testStoragePath, '_system', 'cow-disabled')
|
||||||
|
const markerExists = fs.existsSync(markerPath)
|
||||||
|
expect(markerExists).toBe(true)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('Clear() works for MemoryStorage', () => {
|
||||||
|
it('should clear memory storage completely', async () => {
|
||||||
|
const brain1 = new Brainy({ storage: { type: 'memory' }})
|
||||||
|
await brain1.init()
|
||||||
|
|
||||||
|
await brain1.add({ data:'Test', type: 'concept' })
|
||||||
|
expect((await brain1.find({ type: 'concept' })).length).toBe(1)
|
||||||
|
|
||||||
|
await brain1.clear()
|
||||||
|
expect((await brain1.find({ type: 'concept' })).length).toBe(0)
|
||||||
|
|
||||||
|
// Note: MemoryStorage doesn't persist, so we can't test instance restart
|
||||||
|
// The marker methods are no-ops for MemoryStorage
|
||||||
|
})
|
||||||
|
})
|
||||||
Loading…
Add table
Add a link
Reference in a new issue