feat: comprehensive metadata namespace architecture and cleanup system

BREAKING CHANGE: Remove hard delete option from deleteVerb() for consistent API

- Add complete metadata namespace architecture with O(1) soft delete performance
- Implement periodic cleanup system for old soft-deleted items
- Add restore methods for both nouns and verbs
- Require metadata contracts for all augmentations
- Eliminate namespace collisions with clean separation (_brainy, _augmentations, _audit)
- Optimize index performance using flattened dot-notation for O(1) lookups
- Add comprehensive augmentation safety system with type-safe access control
- Maintain full backward compatibility for existing data
- Add enterprise-grade cleanup with configurable age thresholds and batch processing

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>

src/utils/periodicCleanup.ts

@@ -0,0 +1,292 @@
+/**
+ * Periodic Cleanup for Soft-Deleted Items
+ * 
+ * SAFETY-FIRST APPROACH:
+ * - Maintains durability guarantees (storage-first)
+ * - Coordinates HNSW and metadata index consistency
+ * - Isolated from live operations
+ * - Graceful failure handling
+ */
+
+import { prodLog } from './logger.js'
+import { DELETED_FIELD, isDeleted } from './metadataNamespace.js'
+import type { StorageAdapter } from '../coreTypes.js'
+import type { HNSWIndex } from '../hnsw/hnswIndex.js'
+import type { MetadataIndexManager } from './metadataIndex.js'
+
+export interface CleanupConfig {
+  /** Age in milliseconds after which soft-deleted items are eligible for cleanup */
+  maxAge: number
+  /** Maximum number of items to clean up in one batch */
+  batchSize: number
+  /** Interval between cleanup runs (milliseconds) */
+  cleanupInterval: number
+  /** Whether to run cleanup automatically */
+  enabled: boolean
+}
+
+export interface CleanupStats {
+  itemsProcessed: number
+  itemsDeleted: number
+  errors: number
+  lastRun: number
+  nextRun: number
+}
+
+/**
+ * Coordinates safe cleanup of old soft-deleted items across all indexes
+ * 
+ * CRITICAL SAFETY FEATURES:
+ * 1. Storage-first deletion (durability)
+ * 2. Index consistency coordination
+ * 3. Batch processing with limits
+ * 4. Error isolation and recovery
+ */
+export class PeriodicCleanup {
+  private storage: StorageAdapter
+  private hnswIndex: HNSWIndex
+  private metadataIndex: MetadataIndexManager | null
+  private config: CleanupConfig
+  private stats: CleanupStats
+  private cleanupTimer: NodeJS.Timeout | null = null
+  private running = false
+
+  constructor(
+    storage: StorageAdapter,
+    hnswIndex: HNSWIndex,
+    metadataIndex: MetadataIndexManager | null,
+    config: Partial<CleanupConfig> = {}
+  ) {
+    this.storage = storage
+    this.hnswIndex = hnswIndex
+    this.metadataIndex = metadataIndex
+    
+    // Default: clean up items deleted more than 1 hour ago
+    this.config = {
+      maxAge: config.maxAge ?? 60 * 60 * 1000, // 1 hour
+      batchSize: config.batchSize ?? 100,       // 100 items max per batch
+      cleanupInterval: config.cleanupInterval ?? 15 * 60 * 1000, // Every 15 minutes
+      enabled: config.enabled ?? true
+    }
+    
+    this.stats = {
+      itemsProcessed: 0,
+      itemsDeleted: 0,
+      errors: 0,
+      lastRun: 0,
+      nextRun: 0
+    }
+  }
+
+  /**
+   * Start periodic cleanup
+   */
+  start(): void {
+    if (!this.config.enabled || this.cleanupTimer) {
+      return
+    }
+
+    prodLog.info(`Starting periodic cleanup: maxAge=${this.config.maxAge}, batchSize=${this.config.batchSize}, interval=${this.config.cleanupInterval}`)
+
+    this.scheduleNext()
+  }
+
+  /**
+   * Stop periodic cleanup
+   */
+  stop(): void {
+    if (this.cleanupTimer) {
+      clearTimeout(this.cleanupTimer)
+      this.cleanupTimer = null
+    }
+
+    prodLog.info('Stopped periodic cleanup')
+  }
+
+  /**
+   * Run cleanup manually
+   */
+  async runNow(): Promise<CleanupStats> {
+    if (this.running) {
+      throw new Error('Cleanup already running')
+    }
+
+    return this.performCleanup()
+  }
+
+  /**
+   * Get current cleanup statistics
+   */
+  getStats(): CleanupStats {
+    return { ...this.stats }
+  }
+
+  private scheduleNext(): void {
+    const nextRun = Date.now() + this.config.cleanupInterval
+    this.stats.nextRun = nextRun
+
+    this.cleanupTimer = setTimeout(async () => {
+      await this.performCleanup()
+      this.scheduleNext()
+    }, this.config.cleanupInterval)
+  }
+
+  /**
+   * CRITICAL: Coordinated cleanup across all indexes
+   * 
+   * SAFETY PROTOCOL:
+   * 1. Find eligible items (old + soft-deleted)
+   * 2. Remove from storage FIRST (durability)
+   * 3. Remove from HNSW (graph consistency) 
+   * 4. Remove from metadata index (search consistency)
+   * 5. Track stats and errors
+   */
+  private async performCleanup(): Promise<CleanupStats> {
+    if (this.running) {
+      prodLog.warn('Cleanup already running, skipping')
+      return this.stats
+    }
+
+    this.running = true
+    const startTime = Date.now()
+    this.stats.lastRun = startTime
+
+    try {
+      prodLog.debug(`Starting cleanup run: maxAge=${this.config.maxAge}, cutoffTime=${startTime - this.config.maxAge}`)
+
+      // Step 1: Find eligible items for cleanup
+      const eligibleItems = await this.findEligibleItems(startTime)
+      
+      if (eligibleItems.length === 0) {
+        prodLog.debug('No items eligible for cleanup')
+        return this.stats
+      }
+
+      prodLog.info(`Found ${eligibleItems.length} items eligible for cleanup`)
+
+      // Step 2: Process in batches for safety
+      let processed = 0
+      let deleted = 0
+      let errors = 0
+
+      for (let i = 0; i < eligibleItems.length; i += this.config.batchSize) {
+        const batch = eligibleItems.slice(i, i + this.config.batchSize)
+        
+        const batchResult = await this.processBatch(batch)
+        processed += batchResult.processed
+        deleted += batchResult.deleted
+        errors += batchResult.errors
+
+        // Small delay between batches to avoid overwhelming the system
+        if (i + this.config.batchSize < eligibleItems.length) {
+          await new Promise(resolve => setTimeout(resolve, 10))
+        }
+      }
+
+      // Update stats
+      this.stats.itemsProcessed += processed
+      this.stats.itemsDeleted += deleted
+      this.stats.errors += errors
+
+      prodLog.info(`Cleanup run completed: processed=${processed}, deleted=${deleted}, errors=${errors}, duration=${Date.now() - startTime}ms`)
+
+    } catch (error) {
+      prodLog.error(`Cleanup run failed: ${error}`)
+      this.stats.errors++
+    } finally {
+      this.running = false
+    }
+
+    return this.stats
+  }
+
+  /**
+   * Find items eligible for cleanup (old + soft-deleted)
+   */
+  private async findEligibleItems(currentTime: number): Promise<string[]> {
+    const cutoffTime = currentTime - this.config.maxAge
+    const eligibleItems: string[] = []
+
+    try {
+      // Get all nouns from storage (using pagination to avoid memory issues)
+      const nounsResult = await this.storage.getNouns({
+        pagination: { limit: 1000 } // Process in chunks
+      })
+      
+      for (const noun of nounsResult.items) {
+        try {
+          if (!noun.metadata || !isDeleted(noun.metadata)) {
+            continue // Not deleted, skip
+          }
+
+          // Check if old enough for cleanup
+          const deletedTime = noun.metadata._brainy?.updated || 0
+          if (deletedTime && (currentTime - deletedTime) > this.config.maxAge) {
+            eligibleItems.push(noun.id)
+          }
+
+        } catch (error) {
+          prodLog.warn(`Failed to check item ${noun.id} for cleanup eligibility: ${error}`)
+        }
+      }
+
+    } catch (error) {
+      prodLog.error(`Failed to find eligible items: ${error}`)
+      throw error
+    }
+
+    return eligibleItems
+  }
+
+  /**
+   * Process a batch of items for cleanup
+   * 
+   * CRITICAL: This maintains the durability-first approach:
+   * Storage → HNSW → Metadata Index
+   */
+  private async processBatch(itemIds: string[]): Promise<{
+    processed: number
+    deleted: number  
+    errors: number
+  }> {
+    let processed = 0
+    let deleted = 0
+    let errors = 0
+
+    for (const id of itemIds) {
+      processed++
+
+      try {
+        // STEP 1: Remove from storage FIRST (durability guarantee)
+        try {
+          await this.storage.deleteNoun(id)
+        } catch (storageError) {
+          prodLog.warn(`Failed to delete ${id} from storage: ${storageError}`)
+          errors++
+          continue
+        }
+
+        // STEP 2: Remove from HNSW index (vector search consistency)
+        const hnswResult = this.hnswIndex.removeItem(id)
+        if (!hnswResult) {
+          prodLog.warn(`Failed to remove ${id} from HNSW index (may not have been indexed)`)
+          // Not a critical error - item might not have been in vector index
+        }
+
+        // STEP 3: Remove from metadata index (faceted search consistency)  
+        if (this.metadataIndex) {
+          await this.metadataIndex.removeFromIndex(id)
+        }
+
+        deleted++
+        prodLog.debug(`Successfully cleaned up item ${id}`)
+
+      } catch (error) {
+        errors++
+        prodLog.error(`Failed to cleanup item ${id}: ${error}`)
+      }
+    }
+
+    return { processed, deleted, errors }
+  }
+}