chore: recovery checkpoint - v3.0 API successfully recovered
CRITICAL CHECKPOINT - DO NOT PUSH TO GITHUB Recovery Status: - Successfully recovered brainy.ts from compiled JavaScript - All core v3.0 API methods functional (add, get, update, delete, relate, find, etc.) - Neural subsystem intact (562KB embedded patterns, NLP working) - Augmentation pipeline operational (20+ augmentations) - HNSW clustering system complete - Triple Intelligence compiled (needs constructor fix) - Test suite validates functionality Changes preserved: - 898 files with changes from last 3 days - 144,475 insertions - All augmentation improvements - All test coverage enhancements - Complete v3.0 feature set This is a LOCAL checkpoint only - contains recovered work after corruption incident. Created backup in .backups/brainy-full-20250910-151314.tar.gz Branch: recovery-checkpoint-20250910-151433 Date: Wed Sep 10 03:18:04 PM PDT 2025
This commit is contained in:
parent
f65455fb22
commit
8ff382ca3b
895 changed files with 143654 additions and 28268 deletions
|
|
@ -0,0 +1,219 @@
|
|||
/**
|
||||
* Periodic Cleanup for Soft-Deleted Items
|
||||
*
|
||||
* SAFETY-FIRST APPROACH:
|
||||
* - Maintains durability guarantees (storage-first)
|
||||
* - Coordinates HNSW and metadata index consistency
|
||||
* - Isolated from live operations
|
||||
* - Graceful failure handling
|
||||
*/
|
||||
import { prodLog } from './logger.js';
|
||||
import { isDeleted } from './metadataNamespace.js';
|
||||
/**
|
||||
* Coordinates safe cleanup of old soft-deleted items across all indexes
|
||||
*
|
||||
* CRITICAL SAFETY FEATURES:
|
||||
* 1. Storage-first deletion (durability)
|
||||
* 2. Index consistency coordination
|
||||
* 3. Batch processing with limits
|
||||
* 4. Error isolation and recovery
|
||||
*/
|
||||
export class PeriodicCleanup {
|
||||
constructor(storage, hnswIndex, metadataIndex, config = {}) {
|
||||
this.cleanupTimer = null;
|
||||
this.running = false;
|
||||
this.storage = storage;
|
||||
this.hnswIndex = hnswIndex;
|
||||
this.metadataIndex = metadataIndex;
|
||||
// Default: clean up items deleted more than 1 hour ago
|
||||
this.config = {
|
||||
maxAge: config.maxAge ?? 60 * 60 * 1000, // 1 hour
|
||||
batchSize: config.batchSize ?? 100, // 100 items max per batch
|
||||
cleanupInterval: config.cleanupInterval ?? 15 * 60 * 1000, // Every 15 minutes
|
||||
enabled: config.enabled ?? true
|
||||
};
|
||||
this.stats = {
|
||||
itemsProcessed: 0,
|
||||
itemsDeleted: 0,
|
||||
errors: 0,
|
||||
lastRun: 0,
|
||||
nextRun: 0
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Start periodic cleanup
|
||||
*/
|
||||
start() {
|
||||
if (!this.config.enabled || this.cleanupTimer) {
|
||||
return;
|
||||
}
|
||||
prodLog.info(`Starting periodic cleanup: maxAge=${this.config.maxAge}, batchSize=${this.config.batchSize}, interval=${this.config.cleanupInterval}`);
|
||||
this.scheduleNext();
|
||||
}
|
||||
/**
|
||||
* Stop periodic cleanup
|
||||
*/
|
||||
stop() {
|
||||
if (this.cleanupTimer) {
|
||||
clearTimeout(this.cleanupTimer);
|
||||
this.cleanupTimer = null;
|
||||
}
|
||||
prodLog.info('Stopped periodic cleanup');
|
||||
}
|
||||
/**
|
||||
* Run cleanup manually
|
||||
*/
|
||||
async runNow() {
|
||||
if (this.running) {
|
||||
throw new Error('Cleanup already running');
|
||||
}
|
||||
return this.performCleanup();
|
||||
}
|
||||
/**
|
||||
* Get current cleanup statistics
|
||||
*/
|
||||
getStats() {
|
||||
return { ...this.stats };
|
||||
}
|
||||
scheduleNext() {
|
||||
const nextRun = Date.now() + this.config.cleanupInterval;
|
||||
this.stats.nextRun = nextRun;
|
||||
this.cleanupTimer = setTimeout(async () => {
|
||||
await this.performCleanup();
|
||||
this.scheduleNext();
|
||||
}, this.config.cleanupInterval);
|
||||
}
|
||||
/**
|
||||
* CRITICAL: Coordinated cleanup across all indexes
|
||||
*
|
||||
* SAFETY PROTOCOL:
|
||||
* 1. Find eligible items (old + soft-deleted)
|
||||
* 2. Remove from storage FIRST (durability)
|
||||
* 3. Remove from HNSW (graph consistency)
|
||||
* 4. Remove from metadata index (search consistency)
|
||||
* 5. Track stats and errors
|
||||
*/
|
||||
async performCleanup() {
|
||||
if (this.running) {
|
||||
prodLog.warn('Cleanup already running, skipping');
|
||||
return this.stats;
|
||||
}
|
||||
this.running = true;
|
||||
const startTime = Date.now();
|
||||
this.stats.lastRun = startTime;
|
||||
try {
|
||||
prodLog.debug(`Starting cleanup run: maxAge=${this.config.maxAge}, cutoffTime=${startTime - this.config.maxAge}`);
|
||||
// Step 1: Find eligible items for cleanup
|
||||
const eligibleItems = await this.findEligibleItems(startTime);
|
||||
if (eligibleItems.length === 0) {
|
||||
prodLog.debug('No items eligible for cleanup');
|
||||
return this.stats;
|
||||
}
|
||||
prodLog.info(`Found ${eligibleItems.length} items eligible for cleanup`);
|
||||
// Step 2: Process in batches for safety
|
||||
let processed = 0;
|
||||
let deleted = 0;
|
||||
let errors = 0;
|
||||
for (let i = 0; i < eligibleItems.length; i += this.config.batchSize) {
|
||||
const batch = eligibleItems.slice(i, i + this.config.batchSize);
|
||||
const batchResult = await this.processBatch(batch);
|
||||
processed += batchResult.processed;
|
||||
deleted += batchResult.deleted;
|
||||
errors += batchResult.errors;
|
||||
// Small delay between batches to avoid overwhelming the system
|
||||
if (i + this.config.batchSize < eligibleItems.length) {
|
||||
await new Promise(resolve => setTimeout(resolve, 10));
|
||||
}
|
||||
}
|
||||
// Update stats
|
||||
this.stats.itemsProcessed += processed;
|
||||
this.stats.itemsDeleted += deleted;
|
||||
this.stats.errors += errors;
|
||||
prodLog.info(`Cleanup run completed: processed=${processed}, deleted=${deleted}, errors=${errors}, duration=${Date.now() - startTime}ms`);
|
||||
}
|
||||
catch (error) {
|
||||
prodLog.error(`Cleanup run failed: ${error}`);
|
||||
this.stats.errors++;
|
||||
}
|
||||
finally {
|
||||
this.running = false;
|
||||
}
|
||||
return this.stats;
|
||||
}
|
||||
/**
|
||||
* Find items eligible for cleanup (old + soft-deleted)
|
||||
*/
|
||||
async findEligibleItems(currentTime) {
|
||||
const cutoffTime = currentTime - this.config.maxAge;
|
||||
const eligibleItems = [];
|
||||
try {
|
||||
// Get all nouns from storage (using pagination to avoid memory issues)
|
||||
const nounsResult = await this.storage.getNouns({
|
||||
pagination: { limit: 1000 } // Process in chunks
|
||||
});
|
||||
for (const noun of nounsResult.items) {
|
||||
try {
|
||||
if (!noun.metadata || !isDeleted(noun.metadata)) {
|
||||
continue; // Not deleted, skip
|
||||
}
|
||||
// Check if old enough for cleanup
|
||||
const deletedTime = noun.metadata._brainy?.updated || 0;
|
||||
if (deletedTime && (currentTime - deletedTime) > this.config.maxAge) {
|
||||
eligibleItems.push(noun.id);
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
prodLog.warn(`Failed to check item ${noun.id} for cleanup eligibility: ${error}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
prodLog.error(`Failed to find eligible items: ${error}`);
|
||||
throw error;
|
||||
}
|
||||
return eligibleItems;
|
||||
}
|
||||
/**
|
||||
* Process a batch of items for cleanup
|
||||
*
|
||||
* CRITICAL: This maintains the durability-first approach:
|
||||
* Storage → HNSW → Metadata Index
|
||||
*/
|
||||
async processBatch(itemIds) {
|
||||
let processed = 0;
|
||||
let deleted = 0;
|
||||
let errors = 0;
|
||||
for (const id of itemIds) {
|
||||
processed++;
|
||||
try {
|
||||
// STEP 1: Remove from storage FIRST (durability guarantee)
|
||||
try {
|
||||
await this.storage.deleteNoun(id);
|
||||
}
|
||||
catch (storageError) {
|
||||
prodLog.warn(`Failed to delete ${id} from storage: ${storageError}`);
|
||||
errors++;
|
||||
continue;
|
||||
}
|
||||
// STEP 2: Remove from HNSW index (vector search consistency)
|
||||
const hnswResult = this.hnswIndex.removeItem(id);
|
||||
if (!hnswResult) {
|
||||
prodLog.warn(`Failed to remove ${id} from HNSW index (may not have been indexed)`);
|
||||
// Not a critical error - item might not have been in vector index
|
||||
}
|
||||
// STEP 3: Remove from metadata index (faceted search consistency)
|
||||
if (this.metadataIndex) {
|
||||
await this.metadataIndex.removeFromIndex(id);
|
||||
}
|
||||
deleted++;
|
||||
prodLog.debug(`Successfully cleaned up item ${id}`);
|
||||
}
|
||||
catch (error) {
|
||||
errors++;
|
||||
prodLog.error(`Failed to cleanup item ${id}: ${error}`);
|
||||
}
|
||||
}
|
||||
return { processed, deleted, errors };
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=periodicCleanup.js.map
|
||||
Loading…
Add table
Add a link
Reference in a new issue