fix: critical clear() data persistence regression (v5.10.4)

Workshop team reported that brain.clear() doesn't fully delete persistent storage.
After calling clear() and creating a new Brainy instance, all data was restored
from storage. This is a CRITICAL data integrity bug.

Root causes (3 bugs fixed):
1. **FileSystemStorage deleting wrong directory**: Data stored in branches/main/entities/
   but clear() was only deleting old pre-v5.4.0 structure (nouns/, verbs/, metadata/)
2. **COW reinitialization after clear()**: Setting cowEnabled=false on old instance
   doesn't affect new instances. Fixed with persistent marker file.
3. **Metadata index cache not cleared**: find() with type filters returned stale data
   after clear(). Fixed by recreating MetadataIndexManager.

Changes:
- FileSystemStorage: Clear branches/ directory (where data actually lives)
- All storage adapters: Add checkClearMarker()/createClearMarker() methods
- BaseStorage: Check for cow-disabled marker before initializing COW
- Brainy: Recreate metadataIndex after clear() to flush cached data
- Tests: Comprehensive regression suite (8 tests) to prevent recurrence

Fixes Workshop bug report: /media/dpsifr/storage/home/Projects/workshop/BRAINY_V5_10_2_CLEAR_BUG.md

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

src/brainy.ts

@@ -2196,7 +2196,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
     return this.augmentationRegistry.execute('clear', {}, async () => {
       // Clear storage
       await this.storage.clear()
-      
+
       // Reset index
       if ('clear' in this.index && typeof this.index.clear === 'function') {
         await this.index.clear()
@@ -2204,10 +2204,15 @@ export class Brainy<T = any> implements BrainyInterface<T> {
         // Recreate index if no clear method
         this.index = this.setupIndex()
       }
-      
+
+      // v5.10.4: Recreate metadata index to clear cached data
+      // Bug: Metadata index cache was not being cleared, causing find() with type filters to return stale data
+      this.metadataIndex = new MetadataIndexManager(this.storage)
+      await this.metadataIndex.init()
+
       // Reset dimensions
       this.dimensions = undefined
-      
+
       // Clear any cached sub-APIs
       this._neural = undefined
       this._nlp = undefined

src/storage/adapters/azureBlobStorage.ts

@@ -1153,6 +1153,12 @@ export class AzureBlobStorage extends BaseStorage {
       this.commitLog = undefined
       this.cowEnabled = false
 
+      // v5.10.4: Create persistent marker blob (CRITICAL FIX)
+      // Bug: cowEnabled = false only affects current instance, not future instances
+      // Fix: Create marker blob that persists across instance restarts
+      // When new instance calls initializeCOW(), it checks for this marker
+      await this.createClearMarker()
+
       // Clear caches
       this.nounCacheManager.clear()
       this.verbCacheManager.clear()
@@ -1204,6 +1210,47 @@ export class AzureBlobStorage extends BaseStorage {
     }
   }
 
+  /**
+   * Check if COW has been explicitly disabled via clear()
+   * v5.10.4: Fixes bug where clear() doesn't persist across instance restarts
+   * @returns true if marker blob exists, false otherwise
+   * @protected
+   */
+  protected async checkClearMarker(): Promise<boolean> {
+    await this.ensureInitialized()
+
+    try {
+      const markerPath = `${this.systemPrefix}cow-disabled`
+      const blockBlobClient = this.containerClient!.getBlockBlobClient(markerPath)
+      const exists = await blockBlobClient.exists()
+      return exists
+    } catch (error) {
+      this.logger.warn('AzureBlobStorage.checkClearMarker: Error checking marker', error)
+      return false
+    }
+  }
+
+  /**
+   * Create marker indicating COW has been explicitly disabled
+   * v5.10.4: Called by clear() to prevent COW reinitialization on new instances
+   * @protected
+   */
+  protected async createClearMarker(): Promise<void> {
+    await this.ensureInitialized()
+
+    try {
+      const markerPath = `${this.systemPrefix}cow-disabled`
+      const blockBlobClient = this.containerClient!.getBlockBlobClient(markerPath)
+      // Create empty marker blob
+      await blockBlobClient.upload(Buffer.from(''), 0, {
+        blobHTTPHeaders: { blobContentType: 'text/plain' }
+      })
+    } catch (error) {
+      this.logger.error('AzureBlobStorage.createClearMarker: Failed to create marker blob', error)
+      // Don't throw - marker creation failure shouldn't break clear()
+    }
+  }
+
   /**
    * Save statistics data to storage
    */

src/storage/adapters/fileSystemStorage.ts

@@ -1001,20 +1001,13 @@ export class FileSystemStorage extends BaseStorage {
       }
     }
 
-    // Remove all files in the nouns directory
-    await removeDirectoryContents(this.nounsDir)
-
-    // Remove all files in the verbs directory
-    await removeDirectoryContents(this.verbsDir)
-
-    // Remove all files in the metadata directory
-    await removeDirectoryContents(this.metadataDir)
-
-    // Remove all files in the noun metadata directory
-    await removeDirectoryContents(this.nounMetadataDir)
-
-    // Remove all files in the verb metadata directory
-    await removeDirectoryContents(this.verbMetadataDir)
+    // v5.10.4: Clear the entire branches/ directory (branch-based storage)
+    // Bug fix: Data is stored in branches/main/entities/, not just entities/
+    // The branch-based structure was introduced for COW support
+    const branchesDir = path.join(this.rootDir, 'branches')
+    if (await this.directoryExists(branchesDir)) {
+      await removeDirectoryContents(branchesDir)
+    }
 
     // Remove all files in both system directories
     await removeDirectoryContents(this.systemDir)
@@ -1037,6 +1030,12 @@ export class FileSystemStorage extends BaseStorage {
       this.blobStorage = undefined
       this.commitLog = undefined
       this.cowEnabled = false
+
+      // v5.10.4: Create persistent marker file (CRITICAL FIX)
+      // Bug: cowEnabled = false only affects current instance, not future instances
+      // Fix: Create marker file that persists across instance restarts
+      // When new instance calls initializeCOW(), it checks for this marker
+      await this.createClearMarker()
     }
 
     // Clear the statistics cache
@@ -1075,6 +1074,49 @@ export class FileSystemStorage extends BaseStorage {
     return result
   }
 
+  /**
+   * Check if COW has been explicitly disabled via clear()
+   * v5.10.4: Fixes bug where clear() doesn't persist across instance restarts
+   * @returns true if marker file exists, false otherwise
+   * @protected
+   */
+  protected async checkClearMarker(): Promise<boolean> {
+    // Check if fs module is available
+    if (!fs || !fs.promises) {
+      return false
+    }
+
+    try {
+      const markerPath = path.join(this.systemDir, 'cow-disabled')
+      await fs.promises.access(markerPath, fs.constants.F_OK)
+      return true // Marker exists
+    } catch (error) {
+      return false // Marker doesn't exist (ENOENT) or can't be accessed
+    }
+  }
+
+  /**
+   * Create marker indicating COW has been explicitly disabled
+   * v5.10.4: Called by clear() to prevent COW reinitialization on new instances
+   * @protected
+   */
+  protected async createClearMarker(): Promise<void> {
+    // Check if fs module is available
+    if (!fs || !fs.promises) {
+      console.warn('FileSystemStorage.createClearMarker: fs module not available, skipping marker creation')
+      return
+    }
+
+    try {
+      const markerPath = path.join(this.systemDir, 'cow-disabled')
+      // Create empty marker file
+      await fs.promises.writeFile(markerPath, '', 'utf8')
+    } catch (error) {
+      console.error('FileSystemStorage.createClearMarker: Failed to create marker file', error)
+      // Don't throw - marker creation failure shouldn't break clear()
+    }
+  }
+
   /**
    * Get information about storage usage and capacity
    */

src/storage/adapters/gcsStorage.ts

@@ -1015,6 +1015,12 @@ export class GcsStorage extends BaseStorage {
       this.commitLog = undefined
       this.cowEnabled = false
 
+      // v5.10.4: Create persistent marker object (CRITICAL FIX)
+      // Bug: cowEnabled = false only affects current instance, not future instances
+      // Fix: Create marker object that persists across instance restarts
+      // When new instance calls initializeCOW(), it checks for this marker
+      await this.createClearMarker()
+
       // Clear caches
       this.nounCacheManager.clear()
       this.verbCacheManager.clear()
@@ -1069,6 +1075,45 @@ export class GcsStorage extends BaseStorage {
     }
   }
 
+  /**
+   * Check if COW has been explicitly disabled via clear()
+   * v5.10.4: Fixes bug where clear() doesn't persist across instance restarts
+   * @returns true if marker object exists, false otherwise
+   * @protected
+   */
+  protected async checkClearMarker(): Promise<boolean> {
+    await this.ensureInitialized()
+
+    try {
+      const markerPath = `${this.systemPrefix}cow-disabled`
+      const file = this.bucket!.file(markerPath)
+      const [exists] = await file.exists()
+      return exists
+    } catch (error) {
+      this.logger.warn('GCSStorage.checkClearMarker: Error checking marker', error)
+      return false
+    }
+  }
+
+  /**
+   * Create marker indicating COW has been explicitly disabled
+   * v5.10.4: Called by clear() to prevent COW reinitialization on new instances
+   * @protected
+   */
+  protected async createClearMarker(): Promise<void> {
+    await this.ensureInitialized()
+
+    try {
+      const markerPath = `${this.systemPrefix}cow-disabled`
+      const file = this.bucket!.file(markerPath)
+      // Create empty marker object
+      await file.save('', { contentType: 'text/plain' })
+    } catch (error) {
+      this.logger.error('GCSStorage.createClearMarker: Failed to create marker object', error)
+      // Don't throw - marker creation failure shouldn't break clear()
+    }
+  }
+
   /**
    * Save statistics data to storage
    */

src/storage/adapters/historicalStorageAdapter.ts

@@ -310,6 +310,25 @@ export class HistoricalStorageAdapter extends BaseStorage {
     }
   }
 
+  /**
+   * Check if COW has been explicitly disabled via clear()
+   * v5.10.4: No-op for HistoricalStorageAdapter (read-only, doesn't manage COW)
+   * @returns Always false (read-only adapter doesn't manage COW state)
+   * @protected
+   */
+  protected async checkClearMarker(): Promise<boolean> {
+    return false // Read-only adapter - COW state managed by underlying storage
+  }
+
+  /**
+   * Create marker indicating COW has been explicitly disabled
+   * v5.10.4: No-op for HistoricalStorageAdapter (read-only)
+   * @protected
+   */
+  protected async createClearMarker(): Promise<void> {
+    // No-op: HistoricalStorageAdapter is read-only, doesn't create markers
+  }
+
   // ============= Override Write Methods (Read-Only) =============
 
   /**

src/storage/adapters/memoryStorage.ts

@@ -196,6 +196,26 @@ export class MemoryStorage extends BaseStorage {
     }
   }
 
+  /**
+   * Check if COW has been explicitly disabled via clear()
+   * v5.10.4: No-op for MemoryStorage (doesn't persist)
+   * @returns Always false (marker doesn't persist in memory)
+   * @protected
+   */
+  protected async checkClearMarker(): Promise<boolean> {
+    return false // MemoryStorage doesn't persist - marker doesn't survive restart
+  }
+
+  /**
+   * Create marker indicating COW has been explicitly disabled
+   * v5.10.4: No-op for MemoryStorage (doesn't persist)
+   * @protected
+   */
+  protected async createClearMarker(): Promise<void> {
+    // No-op: MemoryStorage doesn't persist, so marker is not needed
+    // clear() in memory already resets all state, no marker survives restart
+  }
+
   /**
    * Save statistics data to storage
    * @param statistics The statistics data to save

src/storage/adapters/opfsStorage.ts

@@ -495,6 +495,12 @@ export class OPFSStorage extends BaseStorage {
         this.blobStorage = undefined
         this.commitLog = undefined
         this.cowEnabled = false
+
+        // v5.10.4: Create persistent marker file (CRITICAL FIX)
+        // Bug: cowEnabled = false only affects current instance, not future instances
+        // Fix: Create marker file that persists across instance restarts
+        // When new instance calls initializeCOW(), it checks for this marker
+        await this.createClearMarker()
       } catch (error: any) {
         // Ignore if _cow directory doesn't exist (not all instances use COW)
         if (error.name !== 'NotFoundError') {
@@ -516,6 +522,53 @@ export class OPFSStorage extends BaseStorage {
     }
   }
 
+  /**
+   * Check if COW has been explicitly disabled via clear()
+   * v5.10.4: Fixes bug where clear() doesn't persist across instance restarts
+   * @returns true if marker file exists, false otherwise
+   * @protected
+   */
+  protected async checkClearMarker(): Promise<boolean> {
+    await this.ensureInitialized()
+
+    try {
+      // Get system directory (may not exist yet)
+      const systemDir = await this.rootDir!.getDirectoryHandle('system', { create: false })
+      // Try to get the marker file
+      await systemDir.getFileHandle('cow-disabled', { create: false })
+      return true // Marker exists
+    } catch (error: any) {
+      if (error.name === 'NotFoundError') {
+        return false // Marker doesn't exist (or system dir doesn't exist)
+      }
+      // Other errors (permissions, etc.) - treat as marker not existing
+      console.warn('OPFSStorage.checkClearMarker: Error checking marker', error)
+      return false
+    }
+  }
+
+  /**
+   * Create marker indicating COW has been explicitly disabled
+   * v5.10.4: Called by clear() to prevent COW reinitialization on new instances
+   * @protected
+   */
+  protected async createClearMarker(): Promise<void> {
+    await this.ensureInitialized()
+
+    try {
+      // Get or create system directory
+      const systemDir = await this.rootDir!.getDirectoryHandle('system', { create: true })
+      // Create empty marker file
+      const fileHandle = await systemDir.getFileHandle('cow-disabled', { create: true })
+      const writable = await fileHandle.createWritable()
+      await writable.write(new Uint8Array(0)) // Empty file
+      await writable.close()
+    } catch (error) {
+      console.error('OPFSStorage.createClearMarker: Failed to create marker file', error)
+      // Don't throw - marker creation failure shouldn't break clear()
+    }
+  }
+
   // Quota monitoring configuration (v4.0.0)
   private quotaWarningThreshold = 0.8  // Warn at 80% usage
   private quotaCriticalThreshold = 0.95  // Critical at 95% usage

src/storage/adapters/r2Storage.ts

@@ -1051,6 +1051,12 @@ export class R2Storage extends BaseStorage {
     this.commitLog = undefined
     this.cowEnabled = false
 
+    // v5.10.4: Create persistent marker object (CRITICAL FIX)
+    // Bug: cowEnabled = false only affects current instance, not future instances
+    // Fix: Create marker object that persists across instance restarts
+    // When new instance calls initializeCOW(), it checks for this marker
+    await this.createClearMarker()
+
     this.nounCacheManager.clear()
     this.verbCacheManager.clear()
 
@@ -1086,6 +1092,43 @@ export class R2Storage extends BaseStorage {
     }
   }
 
+  /**
+   * Check if COW has been explicitly disabled via clear()
+   * v5.10.4: Fixes bug where clear() doesn't persist across instance restarts
+   * @returns true if marker object exists, false otherwise
+   * @protected
+   */
+  protected async checkClearMarker(): Promise<boolean> {
+    await this.ensureInitialized()
+
+    try {
+      const markerPath = `${this.systemPrefix}cow-disabled`
+      const data = await this.readObjectFromPath(markerPath)
+      return data !== null // Marker exists if we got any data
+    } catch (error) {
+      prodLog.warn('R2Storage.checkClearMarker: Error checking marker', error)
+      return false
+    }
+  }
+
+  /**
+   * Create marker indicating COW has been explicitly disabled
+   * v5.10.4: Called by clear() to prevent COW reinitialization on new instances
+   * @protected
+   */
+  protected async createClearMarker(): Promise<void> {
+    await this.ensureInitialized()
+
+    try {
+      const markerPath = `${this.systemPrefix}cow-disabled`
+      // Create empty marker object
+      await this.writeObjectToPath(markerPath, '')
+    } catch (error) {
+      prodLog.error('R2Storage.createClearMarker: Failed to create marker object', error)
+      // Don't throw - marker creation failure shouldn't break clear()
+    }
+  }
+
   // v5.4.0: Removed getNounsWithPagination override - use BaseStorage's type-first implementation
 
 

src/storage/adapters/s3CompatibleStorage.ts

@@ -2137,6 +2137,12 @@ export class S3CompatibleStorage extends BaseStorage {
       this.commitLog = undefined
       this.cowEnabled = false
 
+      // v5.10.4: Create persistent marker object (CRITICAL FIX)
+      // Bug: cowEnabled = false only affects current instance, not future instances
+      // Fix: Create marker object that persists across instance restarts
+      // When new instance calls initializeCOW(), it checks for this marker
+      await this.createClearMarker()
+
       // Clear the statistics cache
       this.statisticsCache = null
       this.statisticsModified = false
@@ -2261,6 +2267,62 @@ export class S3CompatibleStorage extends BaseStorage {
     }
   }
 
+  /**
+   * Check if COW has been explicitly disabled via clear()
+   * v5.10.4: Fixes bug where clear() doesn't persist across instance restarts
+   * @returns true if marker object exists, false otherwise
+   * @protected
+   */
+  protected async checkClearMarker(): Promise<boolean> {
+    await this.ensureInitialized()
+
+    try {
+      const { HeadObjectCommand } = await import('@aws-sdk/client-s3')
+      const markerKey = `${this.systemPrefix}cow-disabled`
+
+      await this.s3Client!.send(
+        new HeadObjectCommand({
+          Bucket: this.bucketName,
+          Key: markerKey
+        })
+      )
+      return true // Marker exists
+    } catch (error: any) {
+      if (error.name === 'NotFound' || error.$metadata?.httpStatusCode === 404) {
+        return false // Marker doesn't exist
+      }
+      prodLog.warn('S3CompatibleStorage.checkClearMarker: Error checking marker', error)
+      return false
+    }
+  }
+
+  /**
+   * Create marker indicating COW has been explicitly disabled
+   * v5.10.4: Called by clear() to prevent COW reinitialization on new instances
+   * @protected
+   */
+  protected async createClearMarker(): Promise<void> {
+    await this.ensureInitialized()
+
+    try {
+      const { PutObjectCommand } = await import('@aws-sdk/client-s3')
+      const markerKey = `${this.systemPrefix}cow-disabled`
+
+      // Create empty marker object
+      await this.s3Client!.send(
+        new PutObjectCommand({
+          Bucket: this.bucketName,
+          Key: markerKey,
+          Body: Buffer.from(''),
+          ContentType: 'text/plain'
+        })
+      )
+    } catch (error) {
+      prodLog.error('S3CompatibleStorage.createClearMarker: Failed to create marker object', error)
+      // Don't throw - marker creation failure shouldn't break clear()
+    }
+  }
+
   // Batch update timer ID
   protected statisticsBatchUpdateTimerId: NodeJS.Timeout | null = null
   // Flag to indicate if statistics have been modified since last save

src/storage/baseStorage.ts

@@ -300,6 +300,15 @@ export abstract class BaseStorage extends BaseStorageAdapter {
     branch?: string
     enableCompression?: boolean
   }): Promise<void> {
+    // v5.10.4: Check for persistent marker file (CRITICAL FIX)
+    // Bug: Setting cowEnabled = false on OLD instance doesn't affect NEW instances
+    // Fix: Check storage for persistent marker created by clear()
+    // If marker exists, COW was explicitly disabled and should NOT reinitialize
+    const markerExists = await this.checkClearMarker()
+    if (markerExists) {
+      return // COW was disabled by clear() - don't recreate _cow/ directory
+    }
+
     // v5.6.1: If COW was explicitly disabled (e.g., via clear()), don't reinitialize
     // This prevents automatic recreation of COW data after clear() operations
     if (this.cowEnabled === false) {
@@ -1704,6 +1713,23 @@ export abstract class BaseStorage extends BaseStorageAdapter {
    */
   public abstract clear(): Promise<void>
 
+  /**
+   * Check if COW has been explicitly disabled via clear()
+   * v5.10.4: Fixes bug where clear() doesn't persist across instance restarts
+   * Each adapter checks for a marker file/object (e.g., "_system/cow-disabled")
+   * @returns true if COW was disabled by clear(), false otherwise
+   * @protected
+   */
+  protected abstract checkClearMarker(): Promise<boolean>
+
+  /**
+   * Create marker indicating COW has been explicitly disabled
+   * v5.10.4: Called by clear() to prevent COW reinitialization on new instances
+   * Each adapter creates a marker file/object (e.g., "_system/cow-disabled")
+   * @protected
+   */
+  protected abstract createClearMarker(): Promise<void>
+
   /**
    * Get information about storage usage and capacity
    * This method should be implemented by each specific adapter