674 lines
26 KiB
JavaScript
674 lines
26 KiB
JavaScript
|
/**
|
||
|
|
* Write-Ahead Log (WAL) Augmentation
|
||
|
|
*
|
||
|
|
* Provides file-based durability and atomicity for storage operations
|
||
|
|
* Automatically enabled for all critical storage operations
|
||
|
|
*
|
||
|
|
* Features:
|
||
|
|
* - True file-based persistence for crash recovery
|
||
|
|
* - Operation replay after startup
|
||
|
|
* - Automatic log rotation and cleanup
|
||
|
|
* - Cross-platform compatibility (filesystem, OPFS, cloud)
|
||
|
|
*/
|
||
|
|
import { BaseAugmentation } from './brainyAugmentation.js';
|
||
|
|
import { v4 as uuidv4 } from '../universal/uuid.js';
|
||
|
|
export class WALAugmentation extends BaseAugmentation {
|
||
|
|
constructor(config = {}) {
|
||
|
|
super(config);
|
||
|
|
this.name = 'wal';
|
||
|
|
this.timing = 'around';
|
||
|
|
this.metadata = 'readonly'; // Reads metadata for logging/recovery
|
||
|
|
this.operations = ['addNoun', 'addVerb', 'saveNoun', 'saveVerb', 'updateMetadata', 'delete', 'deleteVerb', 'clear'];
|
||
|
|
this.priority = 100; // Critical system operation - highest priority
|
||
|
|
// Augmentation metadata
|
||
|
|
this.category = 'internal';
|
||
|
|
this.description = 'Write-ahead logging for durability and crash recovery';
|
||
|
|
this.operationCounter = 0;
|
||
|
|
this.isRecovering = false;
|
||
|
|
// Create unique log ID for this session
|
||
|
|
this.currentLogId = `${this.config.walPrefix}_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Get the augmentation manifest for discovery
|
||
|
|
*/
|
||
|
|
getManifest() {
|
||
|
|
return {
|
||
|
|
id: 'wal',
|
||
|
|
name: 'Write-Ahead Log',
|
||
|
|
version: '2.0.0',
|
||
|
|
description: 'Provides durability and crash recovery through write-ahead logging',
|
||
|
|
longDescription: 'The WAL augmentation ensures data durability by logging all write operations before they are applied. It supports crash recovery, operation replay, and automatic log rotation. Essential for production deployments requiring high reliability.',
|
||
|
|
category: 'internal',
|
||
|
|
configSchema: {
|
||
|
|
type: 'object',
|
||
|
|
properties: {
|
||
|
|
enabled: {
|
||
|
|
type: 'boolean',
|
||
|
|
default: true,
|
||
|
|
description: 'Enable or disable write-ahead logging'
|
||
|
|
},
|
||
|
|
immediateWrites: {
|
||
|
|
type: 'boolean',
|
||
|
|
default: true,
|
||
|
|
description: 'Execute operations immediately then log asynchronously for better performance'
|
||
|
|
},
|
||
|
|
adaptivePersistence: {
|
||
|
|
type: 'boolean',
|
||
|
|
default: true,
|
||
|
|
description: 'Automatically adapt persistence strategy based on operation patterns'
|
||
|
|
},
|
||
|
|
walPrefix: {
|
||
|
|
type: 'string',
|
||
|
|
default: 'wal',
|
||
|
|
description: 'Prefix for WAL file names'
|
||
|
|
},
|
||
|
|
maxSize: {
|
||
|
|
type: 'number',
|
||
|
|
default: 10485760, // 10MB
|
||
|
|
minimum: 1048576, // 1MB
|
||
|
|
maximum: 1073741824, // 1GB
|
||
|
|
description: 'Maximum WAL file size in bytes before rotation'
|
||
|
|
},
|
||
|
|
checkpointInterval: {
|
||
|
|
type: 'number',
|
||
|
|
default: 60000, // 1 minute
|
||
|
|
minimum: 1000, // 1 second
|
||
|
|
maximum: 3600000, // 1 hour
|
||
|
|
description: 'Interval between checkpoints in milliseconds'
|
||
|
|
},
|
||
|
|
autoRecover: {
|
||
|
|
type: 'boolean',
|
||
|
|
default: true,
|
||
|
|
description: 'Automatically recover pending operations on startup'
|
||
|
|
},
|
||
|
|
maxRetries: {
|
||
|
|
type: 'number',
|
||
|
|
default: 3,
|
||
|
|
minimum: 0,
|
||
|
|
maximum: 10,
|
||
|
|
description: 'Maximum number of retries for failed operations'
|
||
|
|
}
|
||
|
|
},
|
||
|
|
required: [],
|
||
|
|
additionalProperties: false
|
||
|
|
},
|
||
|
|
configDefaults: {
|
||
|
|
enabled: true,
|
||
|
|
immediateWrites: true,
|
||
|
|
adaptivePersistence: true,
|
||
|
|
walPrefix: 'wal',
|
||
|
|
maxSize: 10485760,
|
||
|
|
checkpointInterval: 60000,
|
||
|
|
autoRecover: true,
|
||
|
|
maxRetries: 3
|
||
|
|
},
|
||
|
|
configExamples: [
|
||
|
|
{
|
||
|
|
name: 'High Performance',
|
||
|
|
description: 'Optimized for speed with asynchronous logging',
|
||
|
|
config: {
|
||
|
|
enabled: true,
|
||
|
|
immediateWrites: true,
|
||
|
|
adaptivePersistence: true,
|
||
|
|
checkpointInterval: 300000 // 5 minutes
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
name: 'High Durability',
|
||
|
|
description: 'Maximum durability with synchronous writes',
|
||
|
|
config: {
|
||
|
|
enabled: true,
|
||
|
|
immediateWrites: false,
|
||
|
|
adaptivePersistence: false,
|
||
|
|
checkpointInterval: 30000, // 30 seconds
|
||
|
|
maxRetries: 5
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
name: 'Development',
|
||
|
|
description: 'Lightweight configuration for development',
|
||
|
|
config: {
|
||
|
|
enabled: true,
|
||
|
|
immediateWrites: true,
|
||
|
|
maxSize: 5242880, // 5MB
|
||
|
|
checkpointInterval: 120000, // 2 minutes
|
||
|
|
autoRecover: false
|
||
|
|
}
|
||
|
|
}
|
||
|
|
],
|
||
|
|
minBrainyVersion: '2.0.0',
|
||
|
|
keywords: ['durability', 'recovery', 'wal', 'reliability', 'crash-recovery'],
|
||
|
|
documentation: 'https://docs.brainy.dev/augmentations/wal',
|
||
|
|
status: 'stable',
|
||
|
|
performance: {
|
||
|
|
memoryUsage: 'low',
|
||
|
|
cpuUsage: 'low',
|
||
|
|
networkUsage: 'none'
|
||
|
|
},
|
||
|
|
features: ['durability', 'crash-recovery', 'operation-replay', 'log-rotation'],
|
||
|
|
enhancedOperations: ['saveNoun', 'saveVerb', 'addNoun', 'addVerb', 'updateMetadata', 'delete'],
|
||
|
|
metrics: [
|
||
|
|
{
|
||
|
|
name: 'wal_operations_total',
|
||
|
|
type: 'counter',
|
||
|
|
description: 'Total number of operations logged to WAL'
|
||
|
|
},
|
||
|
|
{
|
||
|
|
name: 'wal_recovery_operations',
|
||
|
|
type: 'counter',
|
||
|
|
description: 'Number of operations recovered from WAL'
|
||
|
|
},
|
||
|
|
{
|
||
|
|
name: 'wal_file_size',
|
||
|
|
type: 'gauge',
|
||
|
|
description: 'Current WAL file size in bytes'
|
||
|
|
}
|
||
|
|
],
|
||
|
|
ui: {
|
||
|
|
icon: '📝',
|
||
|
|
color: '#FF5722'
|
||
|
|
}
|
||
|
|
};
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Handle runtime configuration changes
|
||
|
|
*/
|
||
|
|
async onConfigChange(newConfig, oldConfig) {
|
||
|
|
// Handle checkpoint interval changes
|
||
|
|
if (newConfig.checkpointInterval !== oldConfig.checkpointInterval) {
|
||
|
|
if (this.checkpointTimer) {
|
||
|
|
clearInterval(this.checkpointTimer);
|
||
|
|
this.checkpointTimer = undefined;
|
||
|
|
}
|
||
|
|
if (newConfig.checkpointInterval && newConfig.checkpointInterval > 0 && newConfig.enabled) {
|
||
|
|
this.checkpointTimer = setInterval(() => this.createCheckpoint(), newConfig.checkpointInterval);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
// Handle enabled/disabled changes
|
||
|
|
if (newConfig.enabled !== oldConfig.enabled) {
|
||
|
|
if (!newConfig.enabled && this.checkpointTimer) {
|
||
|
|
clearInterval(this.checkpointTimer);
|
||
|
|
this.checkpointTimer = undefined;
|
||
|
|
}
|
||
|
|
this.log(`Write-Ahead Log ${newConfig.enabled ? 'enabled' : 'disabled'}`);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
async onInitialize() {
|
||
|
|
if (!this.config.enabled) {
|
||
|
|
this.log('Write-Ahead Log disabled');
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
this.log('Write-Ahead Log initializing with file-based persistence');
|
||
|
|
// Recover any pending operations from previous sessions
|
||
|
|
if (this.config.autoRecover) {
|
||
|
|
await this.recoverPendingOperations();
|
||
|
|
}
|
||
|
|
// Start checkpoint timer
|
||
|
|
if (this.config.checkpointInterval > 0) {
|
||
|
|
this.checkpointTimer = setInterval(() => this.createCheckpoint(), this.config.checkpointInterval);
|
||
|
|
}
|
||
|
|
this.log('Write-Ahead Log initialized with file-based durability');
|
||
|
|
}
|
||
|
|
shouldExecute(operation, params) {
|
||
|
|
// Only execute if enabled and for write operations that modify data
|
||
|
|
return this.config.enabled && !this.isRecovering && (operation === 'saveNoun' ||
|
||
|
|
operation === 'saveVerb' ||
|
||
|
|
operation === 'addNoun' ||
|
||
|
|
operation === 'addVerb' ||
|
||
|
|
operation === 'updateMetadata' ||
|
||
|
|
operation === 'delete');
|
||
|
|
}
|
||
|
|
async execute(operation, params, next) {
|
||
|
|
if (!this.shouldExecute(operation, params)) {
|
||
|
|
return next();
|
||
|
|
}
|
||
|
|
const entry = {
|
||
|
|
id: uuidv4(),
|
||
|
|
operation,
|
||
|
|
params: this.sanitizeParams(params),
|
||
|
|
timestamp: Date.now(),
|
||
|
|
status: 'pending'
|
||
|
|
};
|
||
|
|
// ZERO-CONFIG INTELLIGENT ADAPTATION:
|
||
|
|
// If immediate writes are enabled, execute first then log asynchronously
|
||
|
|
if (this.config.immediateWrites) {
|
||
|
|
try {
|
||
|
|
// Step 1: Execute operation immediately for user responsiveness
|
||
|
|
const result = await next();
|
||
|
|
// Step 2: Log completion asynchronously (non-blocking)
|
||
|
|
entry.status = 'completed';
|
||
|
|
this.logAsyncWALEntry(entry); // Fire-and-forget logging
|
||
|
|
this.operationCounter++;
|
||
|
|
// Step 3: Background log maintenance (non-blocking)
|
||
|
|
setImmediate(() => this.checkLogRotation());
|
||
|
|
return result;
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
// Log failure asynchronously (non-blocking)
|
||
|
|
entry.status = 'failed';
|
||
|
|
entry.error = error.message;
|
||
|
|
this.logAsyncWALEntry(entry); // Fire-and-forget logging
|
||
|
|
this.log(`Operation ${operation} failed: ${entry.error}`, 'error');
|
||
|
|
throw error;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
// Traditional WAL: durability first (for high-reliability scenarios)
|
||
|
|
// Step 1: Write operation to WAL (durability first!)
|
||
|
|
await this.writeWALEntry(entry);
|
||
|
|
try {
|
||
|
|
// Step 2: Execute the actual operation
|
||
|
|
const result = await next();
|
||
|
|
// Step 3: Mark as completed in WAL
|
||
|
|
entry.status = 'completed';
|
||
|
|
await this.writeWALEntry(entry);
|
||
|
|
this.operationCounter++;
|
||
|
|
// Check if we need to rotate log
|
||
|
|
await this.checkLogRotation();
|
||
|
|
return result;
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
// Mark as failed in WAL
|
||
|
|
entry.status = 'failed';
|
||
|
|
entry.error = error.message;
|
||
|
|
await this.writeWALEntry(entry);
|
||
|
|
this.log(`Operation ${operation} failed: ${entry.error}`, 'error');
|
||
|
|
throw error;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Asynchronous WAL entry logging (fire-and-forget for immediate writes)
|
||
|
|
*/
|
||
|
|
logAsyncWALEntry(entry) {
|
||
|
|
// Use setImmediate to defer logging without blocking the main operation
|
||
|
|
setImmediate(async () => {
|
||
|
|
try {
|
||
|
|
await this.writeWALEntry(entry);
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
// Log WAL write failures but don't throw (fire-and-forget)
|
||
|
|
this.log(`Background WAL write failed: ${error.message}`, 'warn');
|
||
|
|
}
|
||
|
|
});
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Write WAL entry to persistent storage using storage adapter
|
||
|
|
*/
|
||
|
|
async writeWALEntry(entry) {
|
||
|
|
try {
|
||
|
|
if (!this.context?.brain?.storage) {
|
||
|
|
throw new Error('Storage adapter not available');
|
||
|
|
}
|
||
|
|
const line = JSON.stringify(entry) + '\n';
|
||
|
|
// Read existing log content directly from WAL file
|
||
|
|
let existingContent = '';
|
||
|
|
try {
|
||
|
|
existingContent = await this.readWALFileDirectly(this.currentLogId);
|
||
|
|
}
|
||
|
|
catch {
|
||
|
|
// No existing log, start fresh
|
||
|
|
}
|
||
|
|
const newContent = existingContent + line;
|
||
|
|
// Write WAL directly to storage without going through embedding pipeline
|
||
|
|
// WAL files should be raw text, not embedded vectors
|
||
|
|
await this.writeWALFileDirectly(this.currentLogId, newContent);
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
// WAL write failure is critical - but don't block operations
|
||
|
|
this.log(`WAL write failed: ${error}`, 'error');
|
||
|
|
console.error('WAL write failure:', error);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Recover pending operations from all existing WAL files
|
||
|
|
*/
|
||
|
|
async recoverPendingOperations() {
|
||
|
|
if (!this.context?.brain?.storage)
|
||
|
|
return;
|
||
|
|
this.isRecovering = true;
|
||
|
|
try {
|
||
|
|
// Find all WAL files by searching for nouns with walType metadata
|
||
|
|
const walFiles = await this.findWALFiles();
|
||
|
|
if (walFiles.length === 0) {
|
||
|
|
this.log('No WAL files found for recovery');
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
this.log(`Found ${walFiles.length} WAL files for recovery`);
|
||
|
|
let totalRecovered = 0;
|
||
|
|
for (const walFile of walFiles) {
|
||
|
|
const entries = await this.readWALEntries(walFile.id);
|
||
|
|
const pending = this.findPendingOperations(entries);
|
||
|
|
if (pending.length > 0) {
|
||
|
|
this.log(`Recovering ${pending.length} pending operations from ${walFile.id}`);
|
||
|
|
for (const entry of pending) {
|
||
|
|
try {
|
||
|
|
// Attempt to replay the operation
|
||
|
|
await this.replayOperation(entry);
|
||
|
|
// Mark as recovered
|
||
|
|
entry.status = 'completed';
|
||
|
|
await this.writeWALEntry(entry);
|
||
|
|
totalRecovered++;
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
this.log(`Failed to recover operation ${entry.id}: ${error}`, 'error');
|
||
|
|
// Mark as failed
|
||
|
|
entry.status = 'failed';
|
||
|
|
entry.error = error.message;
|
||
|
|
await this.writeWALEntry(entry);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if (totalRecovered > 0) {
|
||
|
|
this.log(`Successfully recovered ${totalRecovered} operations`);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
this.log(`WAL recovery failed: ${error}`, 'error');
|
||
|
|
}
|
||
|
|
finally {
|
||
|
|
this.isRecovering = false;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Find all WAL files in storage
|
||
|
|
*/
|
||
|
|
async findWALFiles() {
|
||
|
|
if (!this.context?.brain?.storage)
|
||
|
|
return [];
|
||
|
|
const walFiles = [];
|
||
|
|
try {
|
||
|
|
// Try to search for WAL files
|
||
|
|
const extendedStorage = this.context.brain.storage;
|
||
|
|
if (extendedStorage.list && typeof extendedStorage.list === 'function') {
|
||
|
|
// Storage adapter supports listing
|
||
|
|
const allFiles = await extendedStorage.list();
|
||
|
|
for (const fileId of allFiles) {
|
||
|
|
if (fileId.startsWith(this.config.walPrefix)) {
|
||
|
|
// TODO: Update WAL file discovery to work with direct storage
|
||
|
|
// For now, just use the current log ID as the main WAL file
|
||
|
|
// This simplified approach ensures core functionality works
|
||
|
|
walFiles.push({
|
||
|
|
id: fileId,
|
||
|
|
metadata: { walType: 'log', lastUpdated: Date.now() }
|
||
|
|
});
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
this.log(`Error finding WAL files: ${error}`, 'warn');
|
||
|
|
}
|
||
|
|
return walFiles;
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Read WAL entries from a file
|
||
|
|
*/
|
||
|
|
async readWALEntries(walFileId) {
|
||
|
|
if (!this.context?.brain?.storage)
|
||
|
|
return [];
|
||
|
|
const entries = [];
|
||
|
|
try {
|
||
|
|
const walContent = await this.readWALFileDirectly(walFileId);
|
||
|
|
if (!walContent) {
|
||
|
|
return entries;
|
||
|
|
}
|
||
|
|
const lines = walContent.split('\n').filter((line) => line.trim());
|
||
|
|
for (const line of lines) {
|
||
|
|
try {
|
||
|
|
const entry = JSON.parse(line);
|
||
|
|
entries.push(entry);
|
||
|
|
}
|
||
|
|
catch {
|
||
|
|
// Skip malformed lines
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
this.log(`Error reading WAL entries from ${walFileId}: ${error}`, 'warn');
|
||
|
|
}
|
||
|
|
return entries;
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Find operations that were started but not completed
|
||
|
|
*/
|
||
|
|
findPendingOperations(entries) {
|
||
|
|
const operationMap = new Map();
|
||
|
|
for (const entry of entries) {
|
||
|
|
if (entry.status === 'pending') {
|
||
|
|
operationMap.set(entry.id, entry);
|
||
|
|
}
|
||
|
|
else if (entry.status === 'completed' || entry.status === 'failed') {
|
||
|
|
operationMap.delete(entry.id);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return Array.from(operationMap.values());
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Replay an operation during recovery
|
||
|
|
*/
|
||
|
|
async replayOperation(entry) {
|
||
|
|
if (!this.context?.brain) {
|
||
|
|
throw new Error('Brain context not available for operation replay');
|
||
|
|
}
|
||
|
|
this.log(`Replaying operation: ${entry.operation}`);
|
||
|
|
// Based on operation type, replay the operation
|
||
|
|
switch (entry.operation) {
|
||
|
|
case 'saveNoun':
|
||
|
|
case 'addNoun':
|
||
|
|
if (entry.params.noun) {
|
||
|
|
await this.context.brain.storage.saveNoun(entry.params.noun);
|
||
|
|
}
|
||
|
|
break;
|
||
|
|
case 'saveVerb':
|
||
|
|
case 'addVerb':
|
||
|
|
if (entry.params.sourceId && entry.params.targetId && entry.params.relationType) {
|
||
|
|
// Replay verb creation - would need access to verb creation logic
|
||
|
|
this.log(`Note: Verb replay not fully implemented for ${entry.operation}`);
|
||
|
|
}
|
||
|
|
break;
|
||
|
|
case 'updateMetadata':
|
||
|
|
if (entry.params.id && entry.params.metadata) {
|
||
|
|
// Would need access to metadata update logic
|
||
|
|
this.log(`Note: Metadata update replay not fully implemented for ${entry.operation}`);
|
||
|
|
}
|
||
|
|
break;
|
||
|
|
case 'delete':
|
||
|
|
if (entry.params.id) {
|
||
|
|
// Would need access to delete logic
|
||
|
|
this.log(`Note: Delete replay not fully implemented for ${entry.operation}`);
|
||
|
|
}
|
||
|
|
break;
|
||
|
|
default:
|
||
|
|
this.log(`Unknown operation type for replay: ${entry.operation}`, 'warn');
|
||
|
|
}
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Create a checkpoint to mark a point in time
|
||
|
|
*/
|
||
|
|
async createCheckpoint() {
|
||
|
|
if (!this.config.enabled)
|
||
|
|
return;
|
||
|
|
const checkpointId = uuidv4();
|
||
|
|
const entry = {
|
||
|
|
id: checkpointId,
|
||
|
|
operation: 'CHECKPOINT',
|
||
|
|
params: {
|
||
|
|
operationCount: this.operationCounter,
|
||
|
|
timestamp: Date.now(),
|
||
|
|
logId: this.currentLogId
|
||
|
|
},
|
||
|
|
timestamp: Date.now(),
|
||
|
|
status: 'completed',
|
||
|
|
checkpointId
|
||
|
|
};
|
||
|
|
await this.writeWALEntry(entry);
|
||
|
|
this.log(`Checkpoint ${checkpointId} created (${this.operationCounter} operations)`);
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Check if log rotation is needed
|
||
|
|
*/
|
||
|
|
async checkLogRotation() {
|
||
|
|
if (!this.context?.brain?.storage)
|
||
|
|
return;
|
||
|
|
try {
|
||
|
|
const walContent = await this.readWALFileDirectly(this.currentLogId);
|
||
|
|
if (walContent) {
|
||
|
|
const size = walContent.length;
|
||
|
|
if (size > this.config.maxSize) {
|
||
|
|
this.log(`Rotating WAL log (${size} bytes > ${this.config.maxSize} limit)`);
|
||
|
|
// Create new log ID
|
||
|
|
const oldLogId = this.currentLogId;
|
||
|
|
this.currentLogId = `${this.config.walPrefix}_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
|
||
|
|
// With direct file storage, we just start a new file
|
||
|
|
// The old file remains as an archived log automatically
|
||
|
|
this.log(`WAL rotated from ${oldLogId} to ${this.currentLogId}`);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
this.log(`Error checking log rotation: ${error}`, 'warn');
|
||
|
|
}
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Sanitize parameters for logging (remove large objects)
|
||
|
|
*/
|
||
|
|
sanitizeParams(params) {
|
||
|
|
if (!params)
|
||
|
|
return params;
|
||
|
|
// Create a copy and sanitize large fields
|
||
|
|
const sanitized = { ...params };
|
||
|
|
// Remove or truncate large fields
|
||
|
|
if (sanitized.vector && Array.isArray(sanitized.vector)) {
|
||
|
|
sanitized.vector = `[vector:${sanitized.vector.length}D]`;
|
||
|
|
}
|
||
|
|
if (sanitized.data && typeof sanitized.data === 'object') {
|
||
|
|
sanitized.data = '[data object]';
|
||
|
|
}
|
||
|
|
// Limit string sizes
|
||
|
|
for (const [key, value] of Object.entries(sanitized)) {
|
||
|
|
if (typeof value === 'string' && value.length > 1000) {
|
||
|
|
sanitized[key] = value.substring(0, 1000) + '...[truncated]';
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return sanitized;
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Get WAL statistics
|
||
|
|
*/
|
||
|
|
getStats() {
|
||
|
|
return {
|
||
|
|
enabled: this.config.enabled,
|
||
|
|
currentLogId: this.currentLogId,
|
||
|
|
operationCount: this.operationCounter,
|
||
|
|
logSize: 0, // Would need to calculate from storage
|
||
|
|
pendingOperations: 0, // Would need to scan current log
|
||
|
|
failedOperations: 0 // Would need to scan current log
|
||
|
|
};
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Manually trigger checkpoint
|
||
|
|
*/
|
||
|
|
async checkpoint() {
|
||
|
|
await this.createCheckpoint();
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Manually trigger log rotation
|
||
|
|
*/
|
||
|
|
async rotate() {
|
||
|
|
await this.checkLogRotation();
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Write WAL data directly to storage without embedding
|
||
|
|
* This bypasses the brain's AI processing pipeline for raw WAL data
|
||
|
|
*/
|
||
|
|
async writeWALFileDirectly(logId, content) {
|
||
|
|
try {
|
||
|
|
// Use the brain's storage adapter to write WAL file directly
|
||
|
|
// This avoids the embedding pipeline completely
|
||
|
|
if (!this.context?.brain?.storage) {
|
||
|
|
throw new Error('Storage adapter not available');
|
||
|
|
}
|
||
|
|
const storage = this.context.brain.storage;
|
||
|
|
// For filesystem storage, we can write directly to a WAL subdirectory
|
||
|
|
// For other storage types, we'll use a special WAL namespace
|
||
|
|
if (storage.constructor.name === 'FileSystemStorage') {
|
||
|
|
// Write to filesystem directly using Node.js fs
|
||
|
|
const fs = await import('fs');
|
||
|
|
const path = await import('path');
|
||
|
|
const walDir = path.join('brainy-data', 'wal');
|
||
|
|
// Ensure WAL directory exists
|
||
|
|
await fs.promises.mkdir(walDir, { recursive: true });
|
||
|
|
// Write WAL file
|
||
|
|
const walFilePath = path.join(walDir, `${logId}.wal`);
|
||
|
|
await fs.promises.writeFile(walFilePath, content, 'utf8');
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
// For other storage types, store as metadata in WAL namespace
|
||
|
|
// This is a fallback for non-filesystem storage
|
||
|
|
await storage.saveMetadata(`wal/${logId}`, {
|
||
|
|
walContent: content,
|
||
|
|
walType: 'log',
|
||
|
|
lastUpdated: Date.now()
|
||
|
|
});
|
||
|
|
}
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
this.log(`Failed to write WAL file directly: ${error}`, 'error');
|
||
|
|
throw error;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Read WAL data directly from storage without embedding
|
||
|
|
*/
|
||
|
|
async readWALFileDirectly(logId) {
|
||
|
|
try {
|
||
|
|
if (!this.context?.brain?.storage) {
|
||
|
|
throw new Error('Storage adapter not available');
|
||
|
|
}
|
||
|
|
const storage = this.context.brain.storage;
|
||
|
|
// For filesystem storage, read directly from WAL subdirectory
|
||
|
|
if (storage.constructor.name === 'FileSystemStorage') {
|
||
|
|
const fs = await import('fs');
|
||
|
|
const path = await import('path');
|
||
|
|
const walFilePath = path.join('brainy-data', 'wal', `${logId}.wal`);
|
||
|
|
try {
|
||
|
|
return await fs.promises.readFile(walFilePath, 'utf8');
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
if (error.code === 'ENOENT') {
|
||
|
|
return ''; // File doesn't exist, return empty content
|
||
|
|
}
|
||
|
|
throw error;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
// For other storage types, read from WAL namespace
|
||
|
|
try {
|
||
|
|
const metadata = await storage.getMetadata(`wal/${logId}`);
|
||
|
|
return metadata?.walContent || '';
|
||
|
|
}
|
||
|
|
catch {
|
||
|
|
return ''; // Metadata doesn't exist, return empty content
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
this.log(`Failed to read WAL file directly: ${error}`, 'error');
|
||
|
|
return ''; // Return empty content on error to allow fresh start
|
||
|
|
}
|
||
|
|
}
|
||
|
|
async onShutdown() {
|
||
|
|
if (this.checkpointTimer) {
|
||
|
|
clearInterval(this.checkpointTimer);
|
||
|
|
this.checkpointTimer = undefined;
|
||
|
|
}
|
||
|
|
// Final checkpoint before shutdown
|
||
|
|
if (this.config.enabled) {
|
||
|
|
await this.createCheckpoint();
|
||
|
|
this.log(`WAL shutdown: ${this.operationCounter} operations processed`);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
//# sourceMappingURL=walAugmentation.js.map
|