brainy/.recovery-workspace/dist-backup-20250910-141917/utils/periodicCleanup.js

/**
 * Periodic Cleanup for Soft-Deleted Items
 *
 * SAFETY-FIRST APPROACH:
 * - Maintains durability guarantees (storage-first)
 * - Coordinates HNSW and metadata index consistency
 * - Isolated from live operations
 * - Graceful failure handling
 */
import { prodLog } from './logger.js';
import { isDeleted } from './metadataNamespace.js';
/**
 * Coordinates safe cleanup of old soft-deleted items across all indexes
 *
 * CRITICAL SAFETY FEATURES:
 * 1. Storage-first deletion (durability)
 * 2. Index consistency coordination
 * 3. Batch processing with limits
 * 4. Error isolation and recovery
 */
export class PeriodicCleanup {
    constructor(storage, hnswIndex, metadataIndex, config = {}) {
        this.cleanupTimer = null;
        this.running = false;
        this.storage = storage;
        this.hnswIndex = hnswIndex;
        this.metadataIndex = metadataIndex;
        // Default: clean up items deleted more than 1 hour ago
        this.config = {
            maxAge: config.maxAge ?? 60 * 60 * 1000, // 1 hour
            batchSize: config.batchSize ?? 100, // 100 items max per batch
            cleanupInterval: config.cleanupInterval ?? 15 * 60 * 1000, // Every 15 minutes
            enabled: config.enabled ?? true
        };
        this.stats = {
            itemsProcessed: 0,
            itemsDeleted: 0,
            errors: 0,
            lastRun: 0,
            nextRun: 0
        };
    }
    /**
     * Start periodic cleanup
     */
    start() {
        if (!this.config.enabled || this.cleanupTimer) {
            return;
        }
        prodLog.info(`Starting periodic cleanup: maxAge=${this.config.maxAge}, batchSize=${this.config.batchSize}, interval=${this.config.cleanupInterval}`);
        this.scheduleNext();
    }
    /**
     * Stop periodic cleanup
     */
    stop() {
        if (this.cleanupTimer) {
            clearTimeout(this.cleanupTimer);
            this.cleanupTimer = null;
        }
        prodLog.info('Stopped periodic cleanup');
    }
    /**
     * Run cleanup manually
     */
    async runNow() {
        if (this.running) {
            throw new Error('Cleanup already running');
        }
        return this.performCleanup();
    }
    /**
     * Get current cleanup statistics
     */
    getStats() {
        return { ...this.stats };
    }
    scheduleNext() {
        const nextRun = Date.now() + this.config.cleanupInterval;
        this.stats.nextRun = nextRun;
        this.cleanupTimer = setTimeout(async () => {
            await this.performCleanup();
            this.scheduleNext();
        }, this.config.cleanupInterval);
    }
    /**
     * CRITICAL: Coordinated cleanup across all indexes
     *
     * SAFETY PROTOCOL:
     * 1. Find eligible items (old + soft-deleted)
     * 2. Remove from storage FIRST (durability)
     * 3. Remove from HNSW (graph consistency)
     * 4. Remove from metadata index (search consistency)
     * 5. Track stats and errors
     */
    async performCleanup() {
        if (this.running) {
            prodLog.warn('Cleanup already running, skipping');
            return this.stats;
        }
        this.running = true;
        const startTime = Date.now();
        this.stats.lastRun = startTime;
        try {
            prodLog.debug(`Starting cleanup run: maxAge=${this.config.maxAge}, cutoffTime=${startTime - this.config.maxAge}`);
            // Step 1: Find eligible items for cleanup
            const eligibleItems = await this.findEligibleItems(startTime);
            if (eligibleItems.length === 0) {
                prodLog.debug('No items eligible for cleanup');
                return this.stats;
            }
            prodLog.info(`Found ${eligibleItems.length} items eligible for cleanup`);
            // Step 2: Process in batches for safety
            let processed = 0;
            let deleted = 0;
            let errors = 0;
            for (let i = 0; i < eligibleItems.length; i += this.config.batchSize) {
                const batch = eligibleItems.slice(i, i + this.config.batchSize);
                const batchResult = await this.processBatch(batch);
                processed += batchResult.processed;
                deleted += batchResult.deleted;
                errors += batchResult.errors;
                // Small delay between batches to avoid overwhelming the system
                if (i + this.config.batchSize < eligibleItems.length) {
                    await new Promise(resolve => setTimeout(resolve, 10));
                }
            }
            // Update stats
            this.stats.itemsProcessed += processed;
            this.stats.itemsDeleted += deleted;
            this.stats.errors += errors;
            prodLog.info(`Cleanup run completed: processed=${processed}, deleted=${deleted}, errors=${errors}, duration=${Date.now() - startTime}ms`);
        }
        catch (error) {
            prodLog.error(`Cleanup run failed: ${error}`);
            this.stats.errors++;
        }
        finally {
            this.running = false;
        }
        return this.stats;
    }
    /**
     * Find items eligible for cleanup (old + soft-deleted)
     */
    async findEligibleItems(currentTime) {
        const cutoffTime = currentTime - this.config.maxAge;
        const eligibleItems = [];
        try {
            // Get all nouns from storage (using pagination to avoid memory issues)
            const nounsResult = await this.storage.getNouns({
                pagination: { limit: 1000 } // Process in chunks
            });
            for (const noun of nounsResult.items) {
                try {
                    if (!noun.metadata || !isDeleted(noun.metadata)) {
                        continue; // Not deleted, skip
                    }
                    // Check if old enough for cleanup
                    const deletedTime = noun.metadata._brainy?.updated || 0;
                    if (deletedTime && (currentTime - deletedTime) > this.config.maxAge) {
                        eligibleItems.push(noun.id);
                    }
                }
                catch (error) {
                    prodLog.warn(`Failed to check item ${noun.id} for cleanup eligibility: ${error}`);
                }
            }
        }
        catch (error) {
            prodLog.error(`Failed to find eligible items: ${error}`);
            throw error;
        }
        return eligibleItems;
    }
    /**
     * Process a batch of items for cleanup
     *
     * CRITICAL: This maintains the durability-first approach:
     * Storage → HNSW → Metadata Index
     */
    async processBatch(itemIds) {
        let processed = 0;
        let deleted = 0;
        let errors = 0;
        for (const id of itemIds) {
            processed++;
            try {
                // STEP 1: Remove from storage FIRST (durability guarantee)
                try {
                    await this.storage.deleteNoun(id);
                }
                catch (storageError) {
                    prodLog.warn(`Failed to delete ${id} from storage: ${storageError}`);
                    errors++;
                    continue;
                }
                // STEP 2: Remove from HNSW index (vector search consistency)
                const hnswResult = this.hnswIndex.removeItem(id);
                if (!hnswResult) {
                    prodLog.warn(`Failed to remove ${id} from HNSW index (may not have been indexed)`);
                    // Not a critical error - item might not have been in vector index
                }
                // STEP 3: Remove from metadata index (faceted search consistency)  
                if (this.metadataIndex) {
                    await this.metadataIndex.removeFromIndex(id);
                }
                deleted++;
                prodLog.debug(`Successfully cleaned up item ${id}`);
            }
            catch (error) {
                errors++;
                prodLog.error(`Failed to cleanup item ${id}: ${error}`);
            }
        }
        return { processed, deleted, errors };
    }
}
//# sourceMappingURL=periodicCleanup.js.map