feat: comprehensive metadata namespace architecture and cleanup system

BREAKING CHANGE: Remove hard delete option from deleteVerb() for consistent API

- Add complete metadata namespace architecture with O(1) soft delete performance
- Implement periodic cleanup system for old soft-deleted items
- Add restore methods for both nouns and verbs
- Require metadata contracts for all augmentations
- Eliminate namespace collisions with clean separation (_brainy, _augmentations, _audit)
- Optimize index performance using flattened dot-notation for O(1) lookups
- Add comprehensive augmentation safety system with type-safe access control
- Maintain full backward compatibility for existing data
- Add enterprise-grade cleanup with configurable age thresholds and batch processing

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>

src/utils/deletedItemsIndex.ts

@@ -0,0 +1,106 @@
+/**
+ * Dedicated index for tracking soft-deleted items
+ * This is MUCH more efficient than checking every item in the database
+ * 
+ * Performance characteristics:
+ * - Add deleted item: O(1)
+ * - Remove deleted item: O(1)  
+ * - Check if deleted: O(1)
+ * - Get all deleted: O(d) where d = number of deleted items << total items
+ */
+
+export class DeletedItemsIndex {
+  private deletedIds: Set<string> = new Set()
+  private deletedCount: number = 0
+  
+  /**
+   * Mark an item as deleted
+   */
+  markDeleted(id: string): void {
+    if (!this.deletedIds.has(id)) {
+      this.deletedIds.add(id)
+      this.deletedCount++
+    }
+  }
+  
+  /**
+   * Mark an item as not deleted (restored)
+   */
+  markRestored(id: string): void {
+    if (this.deletedIds.delete(id)) {
+      this.deletedCount--
+    }
+  }
+  
+  /**
+   * Check if an item is deleted - O(1)
+   */
+  isDeleted(id: string): boolean {
+    return this.deletedIds.has(id)
+  }
+  
+  /**
+   * Get all deleted item IDs - O(d)
+   */
+  getAllDeleted(): string[] {
+    return Array.from(this.deletedIds)
+  }
+  
+  /**
+   * Filter out deleted items from results - O(k) where k = result count
+   */
+  filterDeleted<T extends { id?: string }>(items: T[]): T[] {
+    if (this.deletedCount === 0) {
+      // Fast path - no deleted items
+      return items
+    }
+    
+    return items.filter(item => {
+      const id = item.id
+      return id ? !this.deletedIds.has(id) : true
+    })
+  }
+  
+  /**
+   * Get statistics
+   */
+  getStats() {
+    return {
+      deletedCount: this.deletedCount,
+      memoryUsage: this.deletedCount * 100 // Rough estimate: 100 bytes per ID
+    }
+  }
+  
+  /**
+   * Clear all deleted items (for testing)
+   */
+  clear(): void {
+    this.deletedIds.clear()
+    this.deletedCount = 0
+  }
+  
+  /**
+   * Serialize for persistence
+   */
+  serialize(): string {
+    return JSON.stringify(Array.from(this.deletedIds))
+  }
+  
+  /**
+   * Deserialize from persistence
+   */
+  deserialize(data: string): void {
+    try {
+      const ids = JSON.parse(data)
+      this.deletedIds = new Set(ids)
+      this.deletedCount = this.deletedIds.size
+    } catch (e) {
+      console.warn('Failed to deserialize deleted items index')
+    }
+  }
+}
+
+/**
+ * Global singleton for deleted items tracking
+ */
+export const deletedItemsIndex = new DeletedItemsIndex()