2648 lines
114 KiB
JavaScript
2648 lines
114 KiB
JavaScript
|
/**
|
||
|
|
* S3-Compatible Storage Adapter
|
||
|
|
* Uses the AWS S3 client to interact with S3-compatible storage services
|
||
|
|
* including Amazon S3, Cloudflare R2, and Google Cloud Storage
|
||
|
|
*/
|
||
|
|
import { BaseStorage, INDEX_DIR, SYSTEM_DIR, STATISTICS_KEY, getDirectoryPath } from '../baseStorage.js';
|
||
|
|
import { StorageCompatibilityLayer } from '../backwardCompatibility.js';
|
||
|
|
import { StorageOperationExecutors } from '../../utils/operationUtils.js';
|
||
|
|
import { BrainyError } from '../../errors/brainyError.js';
|
||
|
|
import { CacheManager } from '../cacheManager.js';
|
||
|
|
import { createModuleLogger, prodLog } from '../../utils/logger.js';
|
||
|
|
import { getGlobalSocketManager } from '../../utils/adaptiveSocketManager.js';
|
||
|
|
import { getGlobalBackpressure } from '../../utils/adaptiveBackpressure.js';
|
||
|
|
import { getWriteBuffer } from '../../utils/writeBuffer.js';
|
||
|
|
import { getCoalescer } from '../../utils/requestCoalescer.js';
|
||
|
|
// Export R2Storage as an alias for S3CompatibleStorage
|
||
|
|
export { S3CompatibleStorage as R2Storage };
|
||
|
|
/**
|
||
|
|
* S3-compatible storage adapter for server environments
|
||
|
|
* Uses the AWS S3 client to interact with S3-compatible storage services
|
||
|
|
* including Amazon S3, Cloudflare R2, and Google Cloud Storage
|
||
|
|
*
|
||
|
|
* To use this adapter with Amazon S3, you need to provide:
|
||
|
|
* - region: AWS region (e.g., 'us-east-1')
|
||
|
|
* - credentials: AWS credentials (accessKeyId and secretAccessKey)
|
||
|
|
* - bucketName: S3 bucket name
|
||
|
|
*
|
||
|
|
* To use this adapter with Cloudflare R2, you need to provide:
|
||
|
|
* - accountId: Cloudflare account ID
|
||
|
|
* - accessKeyId: R2 access key ID
|
||
|
|
* - secretAccessKey: R2 secret access key
|
||
|
|
* - bucketName: R2 bucket name
|
||
|
|
*
|
||
|
|
* To use this adapter with Google Cloud Storage, you need to provide:
|
||
|
|
* - region: GCS region (e.g., 'us-central1')
|
||
|
|
* - credentials: GCS credentials (accessKeyId and secretAccessKey)
|
||
|
|
* - endpoint: GCS endpoint (e.g., 'https://storage.googleapis.com')
|
||
|
|
* - bucketName: GCS bucket name
|
||
|
|
*/
|
||
|
|
export class S3CompatibleStorage extends BaseStorage {
|
||
|
|
/**
|
||
|
|
* Initialize the storage adapter
|
||
|
|
* @param options Configuration options for the S3-compatible storage
|
||
|
|
*/
|
||
|
|
constructor(options) {
|
||
|
|
super();
|
||
|
|
this.s3Client = null;
|
||
|
|
this.useDualWrite = true; // Write to both locations during migration
|
||
|
|
// Statistics caching for better performance
|
||
|
|
this.statisticsCache = null;
|
||
|
|
// Distributed locking for concurrent access control
|
||
|
|
this.lockPrefix = 'locks/';
|
||
|
|
this.activeLocks = new Set();
|
||
|
|
// Change log for efficient synchronization
|
||
|
|
this.changeLogPrefix = 'change-log/';
|
||
|
|
// Backpressure and performance management
|
||
|
|
this.pendingOperations = 0;
|
||
|
|
this.maxConcurrentOperations = 100;
|
||
|
|
this.baseBatchSize = 10;
|
||
|
|
this.currentBatchSize = 10;
|
||
|
|
this.lastMemoryCheck = 0;
|
||
|
|
this.memoryCheckInterval = 5000; // Check every 5 seconds
|
||
|
|
this.consecutiveErrors = 0;
|
||
|
|
this.lastErrorReset = Date.now();
|
||
|
|
// Adaptive socket manager for automatic optimization
|
||
|
|
this.socketManager = getGlobalSocketManager();
|
||
|
|
// Adaptive backpressure for automatic flow control
|
||
|
|
this.backpressure = getGlobalBackpressure();
|
||
|
|
// Write buffers for bulk operations
|
||
|
|
this.nounWriteBuffer = null;
|
||
|
|
this.verbWriteBuffer = null;
|
||
|
|
// Request coalescer for deduplication
|
||
|
|
this.requestCoalescer = null;
|
||
|
|
// High-volume mode detection - MUCH more aggressive
|
||
|
|
this.highVolumeMode = false;
|
||
|
|
this.lastVolumeCheck = 0;
|
||
|
|
this.volumeCheckInterval = 1000; // Check every second, not 5
|
||
|
|
this.forceHighVolumeMode = false; // Environment variable override
|
||
|
|
// Module logger
|
||
|
|
this.logger = createModuleLogger('S3Storage');
|
||
|
|
// Node cache to avoid redundant API calls
|
||
|
|
this.nodeCache = new Map();
|
||
|
|
// Batch update timer ID
|
||
|
|
this.statisticsBatchUpdateTimerId = null;
|
||
|
|
// Flag to indicate if statistics have been modified since last save
|
||
|
|
this.statisticsModified = false;
|
||
|
|
// Time of last statistics flush to storage
|
||
|
|
this.lastStatisticsFlushTime = 0;
|
||
|
|
// Minimum time between statistics flushes (5 seconds)
|
||
|
|
this.MIN_FLUSH_INTERVAL_MS = 5000;
|
||
|
|
// Maximum time to wait before flushing statistics (30 seconds)
|
||
|
|
this.MAX_FLUSH_DELAY_MS = 30000;
|
||
|
|
this.bucketName = options.bucketName;
|
||
|
|
this.region = options.region || 'auto';
|
||
|
|
this.endpoint = options.endpoint;
|
||
|
|
this.accountId = options.accountId;
|
||
|
|
this.accessKeyId = options.accessKeyId;
|
||
|
|
this.secretAccessKey = options.secretAccessKey;
|
||
|
|
this.sessionToken = options.sessionToken;
|
||
|
|
this.serviceType = options.serviceType || 's3';
|
||
|
|
this.readOnly = options.readOnly || false;
|
||
|
|
// Initialize operation executors with timeout and retry configuration
|
||
|
|
this.operationExecutors = new StorageOperationExecutors(options.operationConfig);
|
||
|
|
// Set up prefixes for different types of data using new entity-based structure
|
||
|
|
this.nounPrefix = `${getDirectoryPath('noun', 'vector')}/`;
|
||
|
|
this.verbPrefix = `${getDirectoryPath('verb', 'vector')}/`;
|
||
|
|
this.metadataPrefix = `${getDirectoryPath('noun', 'metadata')}/`; // Noun metadata
|
||
|
|
this.verbMetadataPrefix = `${getDirectoryPath('verb', 'metadata')}/`; // Verb metadata
|
||
|
|
this.indexPrefix = `${INDEX_DIR}/`; // Legacy
|
||
|
|
this.systemPrefix = `${SYSTEM_DIR}/`; // New
|
||
|
|
// Initialize cache managers
|
||
|
|
this.nounCacheManager = new CacheManager(options.cacheConfig);
|
||
|
|
this.verbCacheManager = new CacheManager(options.cacheConfig);
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Initialize the storage adapter
|
||
|
|
*/
|
||
|
|
async init() {
|
||
|
|
if (this.isInitialized) {
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
try {
|
||
|
|
// Import AWS SDK modules only when needed
|
||
|
|
const { S3Client } = await import('@aws-sdk/client-s3');
|
||
|
|
// Configure the S3 client based on the service type
|
||
|
|
const clientConfig = {
|
||
|
|
region: this.region,
|
||
|
|
credentials: {
|
||
|
|
accessKeyId: this.accessKeyId,
|
||
|
|
secretAccessKey: this.secretAccessKey
|
||
|
|
},
|
||
|
|
// Use adaptive socket manager for automatic optimization
|
||
|
|
requestHandler: this.socketManager.getHttpHandler(),
|
||
|
|
// Retry configuration for resilience
|
||
|
|
maxAttempts: 5, // Retry up to 5 times
|
||
|
|
retryMode: 'adaptive' // Use adaptive retry with backoff
|
||
|
|
};
|
||
|
|
// Add session token if provided
|
||
|
|
if (this.sessionToken) {
|
||
|
|
clientConfig.credentials.sessionToken = this.sessionToken;
|
||
|
|
}
|
||
|
|
// Add endpoint if provided (for R2, GCS, etc.)
|
||
|
|
if (this.endpoint) {
|
||
|
|
clientConfig.endpoint = this.endpoint;
|
||
|
|
}
|
||
|
|
// Special configuration for Cloudflare R2
|
||
|
|
if (this.serviceType === 'r2' && this.accountId) {
|
||
|
|
clientConfig.endpoint = `https://${this.accountId}.r2.cloudflarestorage.com`;
|
||
|
|
}
|
||
|
|
// Create the S3 client
|
||
|
|
this.s3Client = new S3Client(clientConfig);
|
||
|
|
// Ensure the bucket exists and is accessible
|
||
|
|
const { HeadBucketCommand } = await import('@aws-sdk/client-s3');
|
||
|
|
await this.s3Client.send(new HeadBucketCommand({
|
||
|
|
Bucket: this.bucketName
|
||
|
|
}));
|
||
|
|
// Create storage adapter proxies for the cache managers
|
||
|
|
const nounStorageAdapter = {
|
||
|
|
get: async (id) => this.getNoun_internal(id),
|
||
|
|
set: async (id, node) => this.saveNoun_internal(node),
|
||
|
|
delete: async (id) => this.deleteNoun_internal(id),
|
||
|
|
getMany: async (ids) => {
|
||
|
|
const result = new Map();
|
||
|
|
// Process in batches to avoid overwhelming the S3 API
|
||
|
|
const batchSize = this.getBatchSize();
|
||
|
|
const batches = [];
|
||
|
|
// Split into batches
|
||
|
|
for (let i = 0; i < ids.length; i += batchSize) {
|
||
|
|
const batch = ids.slice(i, i + batchSize);
|
||
|
|
batches.push(batch);
|
||
|
|
}
|
||
|
|
// Process each batch
|
||
|
|
for (const batch of batches) {
|
||
|
|
const batchResults = await Promise.all(batch.map(async (id) => {
|
||
|
|
const node = await this.getNoun_internal(id);
|
||
|
|
return { id, node };
|
||
|
|
}));
|
||
|
|
// Add results to map
|
||
|
|
for (const { id, node } of batchResults) {
|
||
|
|
if (node) {
|
||
|
|
result.set(id, node);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return result;
|
||
|
|
},
|
||
|
|
clear: async () => {
|
||
|
|
// No-op for now, as we don't want to clear the entire storage
|
||
|
|
// This would be implemented if needed
|
||
|
|
}
|
||
|
|
};
|
||
|
|
const verbStorageAdapter = {
|
||
|
|
get: async (id) => this.getVerb_internal(id),
|
||
|
|
set: async (id, edge) => this.saveVerb_internal(edge),
|
||
|
|
delete: async (id) => this.deleteVerb_internal(id),
|
||
|
|
getMany: async (ids) => {
|
||
|
|
const result = new Map();
|
||
|
|
// Process in batches to avoid overwhelming the S3 API
|
||
|
|
const batchSize = this.getBatchSize();
|
||
|
|
const batches = [];
|
||
|
|
// Split into batches
|
||
|
|
for (let i = 0; i < ids.length; i += batchSize) {
|
||
|
|
const batch = ids.slice(i, i + batchSize);
|
||
|
|
batches.push(batch);
|
||
|
|
}
|
||
|
|
// Process each batch
|
||
|
|
for (const batch of batches) {
|
||
|
|
const batchResults = await Promise.all(batch.map(async (id) => {
|
||
|
|
const edge = await this.getVerb_internal(id);
|
||
|
|
return { id, edge };
|
||
|
|
}));
|
||
|
|
// Add results to map
|
||
|
|
for (const { id, edge } of batchResults) {
|
||
|
|
if (edge) {
|
||
|
|
result.set(id, edge);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return result;
|
||
|
|
},
|
||
|
|
clear: async () => {
|
||
|
|
// No-op for now, as we don't want to clear the entire storage
|
||
|
|
// This would be implemented if needed
|
||
|
|
}
|
||
|
|
};
|
||
|
|
// Set storage adapters for cache managers
|
||
|
|
this.nounCacheManager.setStorageAdapters(nounStorageAdapter, nounStorageAdapter);
|
||
|
|
this.verbCacheManager.setStorageAdapters(verbStorageAdapter, verbStorageAdapter);
|
||
|
|
// Initialize write buffers for high-volume scenarios
|
||
|
|
this.initializeBuffers();
|
||
|
|
// Initialize request coalescer
|
||
|
|
this.initializeCoalescer();
|
||
|
|
// Auto-cleanup legacy /index folder on initialization
|
||
|
|
await this.cleanupLegacyIndexFolder();
|
||
|
|
this.isInitialized = true;
|
||
|
|
this.logger.info(`Initialized ${this.serviceType} storage with bucket ${this.bucketName}`);
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
this.logger.error(`Failed to initialize ${this.serviceType} storage:`, error);
|
||
|
|
throw new Error(`Failed to initialize ${this.serviceType} storage: ${error}`);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Override base class method to detect S3-specific throttling errors
|
||
|
|
*/
|
||
|
|
isThrottlingError(error) {
|
||
|
|
// First check base class detection
|
||
|
|
if (super.isThrottlingError(error)) {
|
||
|
|
return true;
|
||
|
|
}
|
||
|
|
// Additional S3-specific checks
|
||
|
|
const message = error.message?.toLowerCase() || '';
|
||
|
|
return (message.includes('please reduce your request rate') ||
|
||
|
|
message.includes('service unavailable') ||
|
||
|
|
error.Code === 'SlowDown' ||
|
||
|
|
error.Code === 'RequestLimitExceeded' ||
|
||
|
|
error.Code === 'ServiceUnavailable');
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Override to add S3-specific logging
|
||
|
|
*/
|
||
|
|
async handleThrottling(error, service) {
|
||
|
|
if (this.isThrottlingError(error)) {
|
||
|
|
prodLog.warn(`🐌 S3 storage throttling detected (${error.$metadata?.httpStatusCode || error.Code || 'timeout'}). Backing off...`);
|
||
|
|
}
|
||
|
|
// Call base class implementation
|
||
|
|
await super.handleThrottling(error, service);
|
||
|
|
if (!this.isThrottlingError(error) && this.consecutiveThrottleEvents === 0 && !this.throttlingDetected) {
|
||
|
|
prodLog.info('✅ S3 storage throttling cleared');
|
||
|
|
}
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Smart delay based on current throttling status
|
||
|
|
*/
|
||
|
|
async smartDelay() {
|
||
|
|
if (this.throttlingDetected) {
|
||
|
|
// If currently throttled, add a preventive delay
|
||
|
|
const timeSinceThrottle = Date.now() - this.lastThrottleTime;
|
||
|
|
if (timeSinceThrottle < 60000) { // Within 1 minute of throttling
|
||
|
|
await new Promise(resolve => setTimeout(resolve, Math.min(this.throttlingBackoffMs / 2, 5000)));
|
||
|
|
}
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
// Normal yield
|
||
|
|
await new Promise(resolve => setImmediate(resolve));
|
||
|
|
}
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Auto-cleanup legacy /index folder during initialization
|
||
|
|
* This removes old index data that has been migrated to _system
|
||
|
|
*/
|
||
|
|
async cleanupLegacyIndexFolder() {
|
||
|
|
try {
|
||
|
|
// Check if there are any objects in the legacy index folder
|
||
|
|
const { ListObjectsV2Command } = await import('@aws-sdk/client-s3');
|
||
|
|
const listResponse = await this.s3Client.send(new ListObjectsV2Command({
|
||
|
|
Bucket: this.bucketName,
|
||
|
|
Prefix: this.indexPrefix,
|
||
|
|
MaxKeys: 1 // Just check if anything exists
|
||
|
|
}));
|
||
|
|
// If there are objects in the legacy index folder, clean them up
|
||
|
|
if (listResponse.Contents && listResponse.Contents.length > 0) {
|
||
|
|
prodLog.info(`🧹 Cleaning up legacy /index folder during initialization...`);
|
||
|
|
// Use the existing deleteObjectsWithPrefix function logic
|
||
|
|
const { ListObjectsV2Command, DeleteObjectsCommand } = await import('@aws-sdk/client-s3');
|
||
|
|
let continuationToken = undefined;
|
||
|
|
let totalDeleted = 0;
|
||
|
|
do {
|
||
|
|
const listResponseBatch = await this.s3Client.send(new ListObjectsV2Command({
|
||
|
|
Bucket: this.bucketName,
|
||
|
|
Prefix: this.indexPrefix,
|
||
|
|
ContinuationToken: continuationToken
|
||
|
|
}));
|
||
|
|
if (listResponseBatch.Contents && listResponseBatch.Contents.length > 0) {
|
||
|
|
const objectsToDelete = listResponseBatch.Contents.map((obj) => ({
|
||
|
|
Key: obj.Key
|
||
|
|
}));
|
||
|
|
await this.s3Client.send(new DeleteObjectsCommand({
|
||
|
|
Bucket: this.bucketName,
|
||
|
|
Delete: {
|
||
|
|
Objects: objectsToDelete
|
||
|
|
}
|
||
|
|
}));
|
||
|
|
totalDeleted += objectsToDelete.length;
|
||
|
|
}
|
||
|
|
continuationToken = listResponseBatch.NextContinuationToken;
|
||
|
|
} while (continuationToken);
|
||
|
|
prodLog.info(`✅ Cleaned up ${totalDeleted} legacy index objects`);
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
prodLog.debug('No legacy /index folder found - already clean');
|
||
|
|
}
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
// Don't fail initialization if cleanup fails
|
||
|
|
prodLog.warn('Failed to cleanup legacy /index folder:', error);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Initialize write buffers for high-volume scenarios
|
||
|
|
*/
|
||
|
|
initializeBuffers() {
|
||
|
|
const storageId = `${this.serviceType}-${this.bucketName}`;
|
||
|
|
// Create noun write buffer
|
||
|
|
this.nounWriteBuffer = getWriteBuffer(`${storageId}-nouns`, 'noun', async (items) => {
|
||
|
|
// Bulk write nouns to S3
|
||
|
|
await this.bulkWriteNouns(items);
|
||
|
|
});
|
||
|
|
// Create verb write buffer
|
||
|
|
this.verbWriteBuffer = getWriteBuffer(`${storageId}-verbs`, 'verb', async (items) => {
|
||
|
|
// Bulk write verbs to S3
|
||
|
|
await this.bulkWriteVerbs(items);
|
||
|
|
});
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Initialize request coalescer
|
||
|
|
*/
|
||
|
|
initializeCoalescer() {
|
||
|
|
const storageId = `${this.serviceType}-${this.bucketName}`;
|
||
|
|
this.requestCoalescer = getCoalescer(storageId, async (batch) => {
|
||
|
|
// Process coalesced operations
|
||
|
|
await this.processCoalescedBatch(batch);
|
||
|
|
});
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Check if we should enable high-volume mode
|
||
|
|
*/
|
||
|
|
checkVolumeMode() {
|
||
|
|
const now = Date.now();
|
||
|
|
if (now - this.lastVolumeCheck < this.volumeCheckInterval) {
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
this.lastVolumeCheck = now;
|
||
|
|
// Check environment variable override
|
||
|
|
const envThreshold = process.env.BRAINY_BUFFER_THRESHOLD;
|
||
|
|
const threshold = envThreshold ? parseInt(envThreshold) : 0; // Default to 0 for immediate activation!
|
||
|
|
// Force enable from environment
|
||
|
|
if (process.env.BRAINY_FORCE_BUFFERING === 'true') {
|
||
|
|
this.forceHighVolumeMode = true;
|
||
|
|
}
|
||
|
|
// Get metrics
|
||
|
|
const backpressureStatus = this.backpressure.getStatus();
|
||
|
|
const socketMetrics = this.socketManager.getMetrics();
|
||
|
|
// Reasonable high-volume detection - only activate under real load
|
||
|
|
const isTestEnvironment = process.env.NODE_ENV === 'test';
|
||
|
|
const explicitlyDisabled = process.env.BRAINY_FORCE_BUFFERING === 'false';
|
||
|
|
// Use reasonable thresholds instead of emergency aggressive ones
|
||
|
|
const reasonableThreshold = Math.max(threshold, 10); // At least 10 pending operations
|
||
|
|
const highSocketUtilization = 0.8; // 80% socket utilization
|
||
|
|
const highRequestRate = 50; // 50 requests per second
|
||
|
|
const significantErrors = 5; // 5 consecutive errors
|
||
|
|
const shouldEnableHighVolume = !isTestEnvironment && // Disable in test environment
|
||
|
|
!explicitlyDisabled && // Allow explicit disabling
|
||
|
|
(this.forceHighVolumeMode || // Environment override
|
||
|
|
backpressureStatus.queueLength >= reasonableThreshold || // High queue backlog
|
||
|
|
socketMetrics.pendingRequests >= reasonableThreshold || // Many pending requests
|
||
|
|
this.pendingOperations >= reasonableThreshold || // Many pending ops
|
||
|
|
socketMetrics.socketUtilization >= highSocketUtilization || // High socket pressure
|
||
|
|
(socketMetrics.requestsPerSecond >= highRequestRate) || // High request rate
|
||
|
|
(this.consecutiveErrors >= significantErrors)); // Significant error pattern
|
||
|
|
if (shouldEnableHighVolume && !this.highVolumeMode) {
|
||
|
|
this.highVolumeMode = true;
|
||
|
|
this.logger.warn(`🚨 HIGH-VOLUME MODE ACTIVATED 🚨`);
|
||
|
|
this.logger.warn(` Queue Length: ${backpressureStatus.queueLength}`);
|
||
|
|
this.logger.warn(` Pending Requests: ${socketMetrics.pendingRequests}`);
|
||
|
|
this.logger.warn(` Pending Operations: ${this.pendingOperations}`);
|
||
|
|
this.logger.warn(` Socket Utilization: ${(socketMetrics.socketUtilization * 100).toFixed(1)}%`);
|
||
|
|
this.logger.warn(` Requests/sec: ${socketMetrics.requestsPerSecond}`);
|
||
|
|
this.logger.warn(` Consecutive Errors: ${this.consecutiveErrors}`);
|
||
|
|
this.logger.warn(` Threshold: ${threshold}`);
|
||
|
|
// Adjust buffer parameters for high volume
|
||
|
|
const queueLength = Math.max(backpressureStatus.queueLength, socketMetrics.pendingRequests, 100);
|
||
|
|
if (this.nounWriteBuffer) {
|
||
|
|
this.nounWriteBuffer.adjustForLoad(queueLength);
|
||
|
|
const stats = this.nounWriteBuffer.getStats();
|
||
|
|
this.logger.warn(` Noun Buffer: ${stats.bufferSize} items, ${stats.totalWrites} total writes`);
|
||
|
|
}
|
||
|
|
if (this.verbWriteBuffer) {
|
||
|
|
this.verbWriteBuffer.adjustForLoad(queueLength);
|
||
|
|
const stats = this.verbWriteBuffer.getStats();
|
||
|
|
this.logger.warn(` Verb Buffer: ${stats.bufferSize} items, ${stats.totalWrites} total writes`);
|
||
|
|
}
|
||
|
|
if (this.requestCoalescer) {
|
||
|
|
this.requestCoalescer.adjustParameters(queueLength);
|
||
|
|
const sizes = this.requestCoalescer.getQueueSizes();
|
||
|
|
this.logger.warn(` Coalescer: ${sizes.total} queued operations`);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
else if (!shouldEnableHighVolume && this.highVolumeMode && !this.forceHighVolumeMode) {
|
||
|
|
this.highVolumeMode = false;
|
||
|
|
this.logger.info('✅ High-volume mode deactivated - load normalized');
|
||
|
|
}
|
||
|
|
// Log current status every 10 checks when in high-volume mode
|
||
|
|
if (this.highVolumeMode && (now % 10000) < this.volumeCheckInterval) {
|
||
|
|
this.logger.info(`📊 High-volume mode status: Queue=${backpressureStatus.queueLength}, Pending=${socketMetrics.pendingRequests}, Sockets=${(socketMetrics.socketUtilization * 100).toFixed(1)}%`);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Bulk write nouns to S3
|
||
|
|
*/
|
||
|
|
async bulkWriteNouns(items) {
|
||
|
|
const { PutObjectCommand } = await import('@aws-sdk/client-s3');
|
||
|
|
// Process in parallel with limited concurrency
|
||
|
|
const promises = [];
|
||
|
|
const batchSize = 10; // Process 10 at a time
|
||
|
|
const entries = Array.from(items.entries());
|
||
|
|
for (let i = 0; i < entries.length; i += batchSize) {
|
||
|
|
const batch = entries.slice(i, i + batchSize);
|
||
|
|
const batchPromise = Promise.all(batch.map(async ([id, node]) => {
|
||
|
|
const serializableNode = {
|
||
|
|
...node,
|
||
|
|
connections: this.mapToObject(node.connections, (set) => Array.from(set))
|
||
|
|
};
|
||
|
|
const key = `${this.nounPrefix}${id}.json`;
|
||
|
|
const body = JSON.stringify(serializableNode, null, 2);
|
||
|
|
await this.s3Client.send(new PutObjectCommand({
|
||
|
|
Bucket: this.bucketName,
|
||
|
|
Key: key,
|
||
|
|
Body: body,
|
||
|
|
ContentType: 'application/json'
|
||
|
|
}));
|
||
|
|
})).then(() => { }); // Convert Promise<void[]> to Promise<void>
|
||
|
|
promises.push(batchPromise);
|
||
|
|
}
|
||
|
|
await Promise.all(promises);
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Bulk write verbs to S3
|
||
|
|
*/
|
||
|
|
async bulkWriteVerbs(items) {
|
||
|
|
const { PutObjectCommand } = await import('@aws-sdk/client-s3');
|
||
|
|
// Process in parallel with limited concurrency
|
||
|
|
const promises = [];
|
||
|
|
const batchSize = 10;
|
||
|
|
const entries = Array.from(items.entries());
|
||
|
|
for (let i = 0; i < entries.length; i += batchSize) {
|
||
|
|
const batch = entries.slice(i, i + batchSize);
|
||
|
|
const batchPromise = Promise.all(batch.map(async ([id, edge]) => {
|
||
|
|
const serializableEdge = {
|
||
|
|
...edge,
|
||
|
|
connections: this.mapToObject(edge.connections, (set) => Array.from(set))
|
||
|
|
};
|
||
|
|
const key = `${this.verbPrefix}${id}.json`;
|
||
|
|
const body = JSON.stringify(serializableEdge, null, 2);
|
||
|
|
await this.s3Client.send(new PutObjectCommand({
|
||
|
|
Bucket: this.bucketName,
|
||
|
|
Key: key,
|
||
|
|
Body: body,
|
||
|
|
ContentType: 'application/json'
|
||
|
|
}));
|
||
|
|
})).then(() => { }); // Convert Promise<void[]> to Promise<void>
|
||
|
|
promises.push(batchPromise);
|
||
|
|
}
|
||
|
|
await Promise.all(promises);
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Process coalesced batch of operations
|
||
|
|
*/
|
||
|
|
async processCoalescedBatch(batch) {
|
||
|
|
// Group operations by type
|
||
|
|
const writes = [];
|
||
|
|
const reads = [];
|
||
|
|
const deletes = [];
|
||
|
|
for (const op of batch) {
|
||
|
|
if (op.type === 'write') {
|
||
|
|
writes.push(op);
|
||
|
|
}
|
||
|
|
else if (op.type === 'read') {
|
||
|
|
reads.push(op);
|
||
|
|
}
|
||
|
|
else if (op.type === 'delete') {
|
||
|
|
deletes.push(op);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
// Process in order: deletes, writes, reads
|
||
|
|
if (deletes.length > 0) {
|
||
|
|
await this.processBulkDeletes(deletes);
|
||
|
|
}
|
||
|
|
if (writes.length > 0) {
|
||
|
|
await this.processBulkWrites(writes);
|
||
|
|
}
|
||
|
|
if (reads.length > 0) {
|
||
|
|
await this.processBulkReads(reads);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Process bulk deletes
|
||
|
|
*/
|
||
|
|
async processBulkDeletes(deletes) {
|
||
|
|
const { DeleteObjectCommand } = await import('@aws-sdk/client-s3');
|
||
|
|
await Promise.all(deletes.map(async (op) => {
|
||
|
|
await this.s3Client.send(new DeleteObjectCommand({
|
||
|
|
Bucket: this.bucketName,
|
||
|
|
Key: op.key
|
||
|
|
}));
|
||
|
|
}));
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Process bulk writes
|
||
|
|
*/
|
||
|
|
async processBulkWrites(writes) {
|
||
|
|
const { PutObjectCommand } = await import('@aws-sdk/client-s3');
|
||
|
|
await Promise.all(writes.map(async (op) => {
|
||
|
|
await this.s3Client.send(new PutObjectCommand({
|
||
|
|
Bucket: this.bucketName,
|
||
|
|
Key: op.key,
|
||
|
|
Body: JSON.stringify(op.data),
|
||
|
|
ContentType: 'application/json'
|
||
|
|
}));
|
||
|
|
}));
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Process bulk reads
|
||
|
|
*/
|
||
|
|
async processBulkReads(reads) {
|
||
|
|
const { GetObjectCommand } = await import('@aws-sdk/client-s3');
|
||
|
|
await Promise.all(reads.map(async (op) => {
|
||
|
|
try {
|
||
|
|
const response = await this.s3Client.send(new GetObjectCommand({
|
||
|
|
Bucket: this.bucketName,
|
||
|
|
Key: op.key
|
||
|
|
}));
|
||
|
|
if (response.Body) {
|
||
|
|
const data = await response.Body.transformToString();
|
||
|
|
op.data = JSON.parse(data);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
op.data = null;
|
||
|
|
}
|
||
|
|
}));
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Dynamically adjust batch size based on memory pressure and error rates
|
||
|
|
*/
|
||
|
|
adjustBatchSize() {
|
||
|
|
// Let the adaptive socket manager handle batch size optimization
|
||
|
|
this.currentBatchSize = this.socketManager.getBatchSize();
|
||
|
|
// Get adaptive configuration for concurrent operations
|
||
|
|
const config = this.socketManager.getConfig();
|
||
|
|
this.maxConcurrentOperations = Math.min(config.maxSockets * 2, 500);
|
||
|
|
// Track metrics for the socket manager
|
||
|
|
const now = Date.now();
|
||
|
|
// Reset error counter periodically if no recent errors
|
||
|
|
if (now - this.lastErrorReset > 60000 && this.consecutiveErrors > 0) {
|
||
|
|
this.consecutiveErrors = Math.max(0, this.consecutiveErrors - 1);
|
||
|
|
this.lastErrorReset = now;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Apply backpressure when system is under load
|
||
|
|
*/
|
||
|
|
async applyBackpressure() {
|
||
|
|
// Generate unique request ID for tracking
|
||
|
|
const requestId = `${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
|
||
|
|
try {
|
||
|
|
// Use adaptive backpressure system
|
||
|
|
await this.backpressure.requestPermission(requestId, 1);
|
||
|
|
// Track with socket manager
|
||
|
|
this.socketManager.trackRequestStart(requestId);
|
||
|
|
this.pendingOperations++;
|
||
|
|
return requestId;
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
// If backpressure rejects, throw a more informative error
|
||
|
|
const message = error instanceof Error ? error.message : String(error);
|
||
|
|
throw new Error(`System overloaded: ${message}`);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Release backpressure after operation completes
|
||
|
|
*/
|
||
|
|
releaseBackpressure(success = true, requestId) {
|
||
|
|
this.pendingOperations = Math.max(0, this.pendingOperations - 1);
|
||
|
|
if (requestId) {
|
||
|
|
// Track with socket manager
|
||
|
|
this.socketManager.trackRequestComplete(requestId, success);
|
||
|
|
// Release from backpressure system
|
||
|
|
this.backpressure.releasePermission(requestId, success);
|
||
|
|
}
|
||
|
|
if (!success) {
|
||
|
|
this.consecutiveErrors++;
|
||
|
|
}
|
||
|
|
else if (this.consecutiveErrors > 0) {
|
||
|
|
// Gradually reduce error count on success
|
||
|
|
this.consecutiveErrors = Math.max(0, this.consecutiveErrors - 0.5);
|
||
|
|
}
|
||
|
|
// Adjust batch size based on current conditions
|
||
|
|
this.adjustBatchSize();
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Get current batch size for operations
|
||
|
|
*/
|
||
|
|
getBatchSize() {
|
||
|
|
// Use adaptive socket manager's batch size
|
||
|
|
return this.socketManager.getBatchSize();
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Save a noun to storage (internal implementation)
|
||
|
|
*/
|
||
|
|
async saveNoun_internal(noun) {
|
||
|
|
return this.saveNode(noun);
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Save a node to storage
|
||
|
|
*/
|
||
|
|
async saveNode(node) {
|
||
|
|
await this.ensureInitialized();
|
||
|
|
// ALWAYS check if we should use high-volume mode (critical for detection)
|
||
|
|
this.checkVolumeMode();
|
||
|
|
// Use write buffer in high-volume mode
|
||
|
|
if (this.highVolumeMode && this.nounWriteBuffer) {
|
||
|
|
this.logger.trace(`📝 BUFFERING: Adding noun ${node.id} to write buffer (high-volume mode active)`);
|
||
|
|
await this.nounWriteBuffer.add(node.id, node);
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
else if (!this.highVolumeMode) {
|
||
|
|
this.logger.trace(`📝 DIRECT WRITE: Saving noun ${node.id} directly (high-volume mode inactive)`);
|
||
|
|
}
|
||
|
|
// Apply backpressure before starting operation
|
||
|
|
const requestId = await this.applyBackpressure();
|
||
|
|
try {
|
||
|
|
this.logger.trace(`Saving node ${node.id}`);
|
||
|
|
// Convert connections Map to a serializable format
|
||
|
|
const serializableNode = {
|
||
|
|
...node,
|
||
|
|
connections: this.mapToObject(node.connections, (set) => Array.from(set))
|
||
|
|
};
|
||
|
|
// Import the PutObjectCommand only when needed
|
||
|
|
const { PutObjectCommand } = await import('@aws-sdk/client-s3');
|
||
|
|
const key = `${this.nounPrefix}${node.id}.json`;
|
||
|
|
const body = JSON.stringify(serializableNode, null, 2);
|
||
|
|
this.logger.trace(`Saving to key: ${key}`);
|
||
|
|
// Save the node to S3-compatible storage
|
||
|
|
const result = await this.s3Client.send(new PutObjectCommand({
|
||
|
|
Bucket: this.bucketName,
|
||
|
|
Key: key,
|
||
|
|
Body: body,
|
||
|
|
ContentType: 'application/json'
|
||
|
|
}));
|
||
|
|
this.logger.debug(`Node ${node.id} saved successfully`);
|
||
|
|
// Log the change for efficient synchronization
|
||
|
|
await this.appendToChangeLog({
|
||
|
|
timestamp: Date.now(),
|
||
|
|
operation: 'add', // Could be 'update' if we track existing nodes
|
||
|
|
entityType: 'noun',
|
||
|
|
entityId: node.id,
|
||
|
|
data: {
|
||
|
|
vector: node.vector,
|
||
|
|
metadata: node.metadata
|
||
|
|
}
|
||
|
|
});
|
||
|
|
// Verify the node was saved by trying to retrieve it
|
||
|
|
const { GetObjectCommand } = await import('@aws-sdk/client-s3');
|
||
|
|
try {
|
||
|
|
const verifyResponse = await this.s3Client.send(new GetObjectCommand({
|
||
|
|
Bucket: this.bucketName,
|
||
|
|
Key: key
|
||
|
|
}));
|
||
|
|
if (verifyResponse && verifyResponse.Body) {
|
||
|
|
this.logger.trace(`Verified node ${node.id} was saved correctly`);
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
this.logger.warn(`Failed to verify node ${node.id} was saved correctly: no response or body`);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
catch (verifyError) {
|
||
|
|
this.logger.warn(`Failed to verify node ${node.id} was saved correctly:`, verifyError);
|
||
|
|
}
|
||
|
|
// Release backpressure on success
|
||
|
|
this.releaseBackpressure(true, requestId);
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
// Release backpressure on error
|
||
|
|
this.releaseBackpressure(false, requestId);
|
||
|
|
this.logger.error(`Failed to save node ${node.id}:`, error);
|
||
|
|
throw new Error(`Failed to save node ${node.id}: ${error}`);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Get a noun from storage (internal implementation)
|
||
|
|
*/
|
||
|
|
async getNoun_internal(id) {
|
||
|
|
return this.getNode(id);
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Get a node from storage
|
||
|
|
*/
|
||
|
|
async getNode(id) {
|
||
|
|
await this.ensureInitialized();
|
||
|
|
try {
|
||
|
|
// Import the GetObjectCommand only when needed
|
||
|
|
const { GetObjectCommand } = await import('@aws-sdk/client-s3');
|
||
|
|
const key = `${this.nounPrefix}${id}.json`;
|
||
|
|
this.logger.trace(`Getting node ${id} from key: ${key}`);
|
||
|
|
// Try to get the node from the nouns directory
|
||
|
|
const response = await this.s3Client.send(new GetObjectCommand({
|
||
|
|
Bucket: this.bucketName,
|
||
|
|
Key: key
|
||
|
|
}));
|
||
|
|
// Check if response is null or undefined
|
||
|
|
if (!response || !response.Body) {
|
||
|
|
this.logger.trace(`No node found for ${id}`);
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
// Convert the response body to a string
|
||
|
|
const bodyContents = await response.Body.transformToString();
|
||
|
|
this.logger.trace(`Retrieved node body for ${id}`);
|
||
|
|
// Parse the JSON string
|
||
|
|
try {
|
||
|
|
const parsedNode = JSON.parse(bodyContents);
|
||
|
|
this.logger.trace(`Parsed node data for ${id}`);
|
||
|
|
// Ensure the parsed node has the expected properties
|
||
|
|
if (!parsedNode ||
|
||
|
|
!parsedNode.id ||
|
||
|
|
!parsedNode.vector ||
|
||
|
|
!parsedNode.connections) {
|
||
|
|
this.logger.warn(`Invalid node data for ${id}`);
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
// Convert serialized connections back to Map<number, Set<string>>
|
||
|
|
const connections = new Map();
|
||
|
|
for (const [level, nodeIds] of Object.entries(parsedNode.connections)) {
|
||
|
|
connections.set(Number(level), new Set(nodeIds));
|
||
|
|
}
|
||
|
|
const node = {
|
||
|
|
id: parsedNode.id,
|
||
|
|
vector: parsedNode.vector,
|
||
|
|
connections,
|
||
|
|
level: parsedNode.level || 0
|
||
|
|
};
|
||
|
|
this.logger.trace(`Successfully retrieved node ${id}`);
|
||
|
|
return node;
|
||
|
|
}
|
||
|
|
catch (parseError) {
|
||
|
|
this.logger.error(`Failed to parse node data for ${id}:`, parseError);
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
// Node not found or other error
|
||
|
|
this.logger.trace(`Node not found for ${id}`);
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Get all nodes from storage
|
||
|
|
* @deprecated This method is deprecated and will be removed in a future version.
|
||
|
|
* It can cause memory issues with large datasets. Use getNodesWithPagination() instead.
|
||
|
|
*/
|
||
|
|
async getAllNodes() {
|
||
|
|
await this.ensureInitialized();
|
||
|
|
this.logger.warn('getAllNodes() is deprecated and will be removed in a future version. Use getNodesWithPagination() instead.');
|
||
|
|
try {
|
||
|
|
// Use the paginated method with a large limit to maintain backward compatibility
|
||
|
|
// but warn about potential issues
|
||
|
|
const result = await this.getNodesWithPagination({
|
||
|
|
limit: 1000, // Reasonable limit to avoid memory issues
|
||
|
|
useCache: true
|
||
|
|
});
|
||
|
|
if (result.hasMore) {
|
||
|
|
this.logger.warn(`Only returning the first 1000 nodes. There are more nodes available. Use getNodesWithPagination() for proper pagination.`);
|
||
|
|
}
|
||
|
|
return result.nodes;
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
this.logger.error('Failed to get all nodes:', error);
|
||
|
|
return [];
|
||
|
|
}
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Get nodes with pagination
|
||
|
|
* @param options Pagination options
|
||
|
|
* @returns Promise that resolves to a paginated result of nodes
|
||
|
|
*/
|
||
|
|
async getNodesWithPagination(options = {}) {
|
||
|
|
await this.ensureInitialized();
|
||
|
|
const limit = options.limit || 100;
|
||
|
|
const useCache = options.useCache !== false;
|
||
|
|
try {
|
||
|
|
// Import the ListObjectsV2Command and GetObjectCommand only when needed
|
||
|
|
const { ListObjectsV2Command } = await import('@aws-sdk/client-s3');
|
||
|
|
// List objects with pagination
|
||
|
|
const listResponse = await this.s3Client.send(new ListObjectsV2Command({
|
||
|
|
Bucket: this.bucketName,
|
||
|
|
Prefix: this.nounPrefix,
|
||
|
|
MaxKeys: limit,
|
||
|
|
ContinuationToken: options.cursor
|
||
|
|
}));
|
||
|
|
// If listResponse is null/undefined or there are no objects, return an empty result
|
||
|
|
if (!listResponse ||
|
||
|
|
!listResponse.Contents ||
|
||
|
|
listResponse.Contents.length === 0) {
|
||
|
|
return {
|
||
|
|
nodes: [],
|
||
|
|
hasMore: false
|
||
|
|
};
|
||
|
|
}
|
||
|
|
// Extract node IDs from the keys
|
||
|
|
const nodeIds = listResponse.Contents
|
||
|
|
.filter((object) => object && object.Key)
|
||
|
|
.map((object) => object.Key.replace(this.nounPrefix, '').replace('.json', ''));
|
||
|
|
// Use the cache manager to get nodes efficiently
|
||
|
|
const nodes = [];
|
||
|
|
if (useCache) {
|
||
|
|
// Get nodes from cache manager
|
||
|
|
const cachedNodes = await this.nounCacheManager.getMany(nodeIds);
|
||
|
|
// Add nodes to result in the same order as nodeIds
|
||
|
|
for (const id of nodeIds) {
|
||
|
|
const node = cachedNodes.get(id);
|
||
|
|
if (node) {
|
||
|
|
nodes.push(node);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
// Get nodes directly from S3 without using cache
|
||
|
|
// Process in smaller batches to reduce memory usage
|
||
|
|
const batchSize = 50;
|
||
|
|
const batches = [];
|
||
|
|
// Split into batches
|
||
|
|
for (let i = 0; i < nodeIds.length; i += batchSize) {
|
||
|
|
const batch = nodeIds.slice(i, i + batchSize);
|
||
|
|
batches.push(batch);
|
||
|
|
}
|
||
|
|
// Process each batch sequentially
|
||
|
|
for (const batch of batches) {
|
||
|
|
const batchNodes = await Promise.all(batch.map(async (id) => {
|
||
|
|
try {
|
||
|
|
return await this.getNoun_internal(id);
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
}));
|
||
|
|
// Add non-null nodes to result
|
||
|
|
for (const node of batchNodes) {
|
||
|
|
if (node) {
|
||
|
|
nodes.push(node);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
// Determine if there are more nodes
|
||
|
|
const hasMore = !!listResponse.IsTruncated;
|
||
|
|
// Set next cursor if there are more nodes
|
||
|
|
const nextCursor = listResponse.NextContinuationToken;
|
||
|
|
return {
|
||
|
|
nodes,
|
||
|
|
hasMore,
|
||
|
|
nextCursor
|
||
|
|
};
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
this.logger.error('Failed to get nodes with pagination:', error);
|
||
|
|
return {
|
||
|
|
nodes: [],
|
||
|
|
hasMore: false
|
||
|
|
};
|
||
|
|
}
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Get nouns by noun type (internal implementation)
|
||
|
|
* @param nounType The noun type to filter by
|
||
|
|
* @returns Promise that resolves to an array of nouns of the specified noun type
|
||
|
|
*/
|
||
|
|
async getNounsByNounType_internal(nounType) {
|
||
|
|
return this.getNodesByNounType(nounType);
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Get nodes by noun type
|
||
|
|
* @param nounType The noun type to filter by
|
||
|
|
* @returns Promise that resolves to an array of nodes of the specified noun type
|
||
|
|
*/
|
||
|
|
async getNodesByNounType(nounType) {
|
||
|
|
await this.ensureInitialized();
|
||
|
|
try {
|
||
|
|
const filteredNodes = [];
|
||
|
|
let hasMore = true;
|
||
|
|
let cursor = undefined;
|
||
|
|
// Use pagination to process nodes in batches
|
||
|
|
while (hasMore) {
|
||
|
|
// Get a batch of nodes
|
||
|
|
const result = await this.getNodesWithPagination({
|
||
|
|
limit: 100,
|
||
|
|
cursor,
|
||
|
|
useCache: true
|
||
|
|
});
|
||
|
|
// Filter nodes by noun type using metadata
|
||
|
|
for (const node of result.nodes) {
|
||
|
|
const metadata = await this.getMetadata(node.id);
|
||
|
|
if (metadata && metadata.noun === nounType) {
|
||
|
|
filteredNodes.push(node);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
// Update pagination state
|
||
|
|
hasMore = result.hasMore;
|
||
|
|
cursor = result.nextCursor;
|
||
|
|
// Safety check to prevent infinite loops
|
||
|
|
if (!cursor && hasMore) {
|
||
|
|
this.logger.warn('No cursor returned but hasMore is true, breaking loop');
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return filteredNodes;
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
this.logger.error(`Failed to get nodes by noun type ${nounType}:`, error);
|
||
|
|
return [];
|
||
|
|
}
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Delete a noun from storage (internal implementation)
|
||
|
|
*/
|
||
|
|
async deleteNoun_internal(id) {
|
||
|
|
return this.deleteNode(id);
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Delete a node from storage
|
||
|
|
*/
|
||
|
|
async deleteNode(id) {
|
||
|
|
await this.ensureInitialized();
|
||
|
|
try {
|
||
|
|
// Import the DeleteObjectCommand only when needed
|
||
|
|
const { DeleteObjectCommand } = await import('@aws-sdk/client-s3');
|
||
|
|
// Delete the node from S3-compatible storage
|
||
|
|
await this.s3Client.send(new DeleteObjectCommand({
|
||
|
|
Bucket: this.bucketName,
|
||
|
|
Key: `${this.nounPrefix}${id}.json`
|
||
|
|
}));
|
||
|
|
// Log the change for efficient synchronization
|
||
|
|
await this.appendToChangeLog({
|
||
|
|
timestamp: Date.now(),
|
||
|
|
operation: 'delete',
|
||
|
|
entityType: 'noun',
|
||
|
|
entityId: id
|
||
|
|
});
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
this.logger.error(`Failed to delete node ${id}:`, error);
|
||
|
|
throw new Error(`Failed to delete node ${id}: ${error}`);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Save a verb to storage (internal implementation)
|
||
|
|
*/
|
||
|
|
async saveVerb_internal(verb) {
|
||
|
|
return this.saveEdge(verb);
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Save an edge to storage
|
||
|
|
*/
|
||
|
|
async saveEdge(edge) {
|
||
|
|
await this.ensureInitialized();
|
||
|
|
// ALWAYS check if we should use high-volume mode (critical for detection)
|
||
|
|
this.checkVolumeMode();
|
||
|
|
// Use write buffer in high-volume mode
|
||
|
|
if (this.highVolumeMode && this.verbWriteBuffer) {
|
||
|
|
this.logger.trace(`📝 BUFFERING: Adding verb ${edge.id} to write buffer (high-volume mode active)`);
|
||
|
|
await this.verbWriteBuffer.add(edge.id, edge);
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
else if (!this.highVolumeMode) {
|
||
|
|
this.logger.trace(`📝 DIRECT WRITE: Saving verb ${edge.id} directly (high-volume mode inactive)`);
|
||
|
|
}
|
||
|
|
// Apply backpressure before starting operation
|
||
|
|
const requestId = await this.applyBackpressure();
|
||
|
|
try {
|
||
|
|
// Convert connections Map to a serializable format
|
||
|
|
const serializableEdge = {
|
||
|
|
...edge,
|
||
|
|
connections: this.mapToObject(edge.connections, (set) => Array.from(set))
|
||
|
|
};
|
||
|
|
// Import the PutObjectCommand only when needed
|
||
|
|
const { PutObjectCommand } = await import('@aws-sdk/client-s3');
|
||
|
|
// Save the edge to S3-compatible storage
|
||
|
|
await this.s3Client.send(new PutObjectCommand({
|
||
|
|
Bucket: this.bucketName,
|
||
|
|
Key: `${this.verbPrefix}${edge.id}.json`,
|
||
|
|
Body: JSON.stringify(serializableEdge, null, 2),
|
||
|
|
ContentType: 'application/json'
|
||
|
|
}));
|
||
|
|
// Log the change for efficient synchronization
|
||
|
|
await this.appendToChangeLog({
|
||
|
|
timestamp: Date.now(),
|
||
|
|
operation: 'add', // Could be 'update' if we track existing edges
|
||
|
|
entityType: 'verb',
|
||
|
|
entityId: edge.id,
|
||
|
|
data: {
|
||
|
|
vector: edge.vector
|
||
|
|
}
|
||
|
|
});
|
||
|
|
// Release backpressure on success
|
||
|
|
this.releaseBackpressure(true, requestId);
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
// Release backpressure on error
|
||
|
|
this.releaseBackpressure(false, requestId);
|
||
|
|
this.logger.error(`Failed to save edge ${edge.id}:`, error);
|
||
|
|
throw new Error(`Failed to save edge ${edge.id}: ${error}`);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Get a verb from storage (internal implementation)
|
||
|
|
*/
|
||
|
|
async getVerb_internal(id) {
|
||
|
|
return this.getEdge(id);
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Get an edge from storage
|
||
|
|
*/
|
||
|
|
async getEdge(id) {
|
||
|
|
await this.ensureInitialized();
|
||
|
|
try {
|
||
|
|
// Import the GetObjectCommand only when needed
|
||
|
|
const { GetObjectCommand } = await import('@aws-sdk/client-s3');
|
||
|
|
const key = `${this.verbPrefix}${id}.json`;
|
||
|
|
this.logger.trace(`Getting edge ${id} from key: ${key}`);
|
||
|
|
// Try to get the edge from the verbs directory
|
||
|
|
const response = await this.s3Client.send(new GetObjectCommand({
|
||
|
|
Bucket: this.bucketName,
|
||
|
|
Key: key
|
||
|
|
}));
|
||
|
|
// Check if response is null or undefined
|
||
|
|
if (!response || !response.Body) {
|
||
|
|
this.logger.trace(`No edge found for ${id}`);
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
// Convert the response body to a string
|
||
|
|
const bodyContents = await response.Body.transformToString();
|
||
|
|
this.logger.trace(`Retrieved edge body for ${id}`);
|
||
|
|
// Parse the JSON string
|
||
|
|
try {
|
||
|
|
const parsedEdge = JSON.parse(bodyContents);
|
||
|
|
this.logger.trace(`Parsed edge data for ${id}`);
|
||
|
|
// Ensure the parsed edge has the expected properties
|
||
|
|
if (!parsedEdge ||
|
||
|
|
!parsedEdge.id ||
|
||
|
|
!parsedEdge.vector ||
|
||
|
|
!parsedEdge.connections) {
|
||
|
|
this.logger.warn(`Invalid edge data for ${id}`);
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
// Convert serialized connections back to Map<number, Set<string>>
|
||
|
|
const connections = new Map();
|
||
|
|
for (const [level, nodeIds] of Object.entries(parsedEdge.connections)) {
|
||
|
|
connections.set(Number(level), new Set(nodeIds));
|
||
|
|
}
|
||
|
|
const edge = {
|
||
|
|
id: parsedEdge.id,
|
||
|
|
vector: parsedEdge.vector,
|
||
|
|
connections
|
||
|
|
};
|
||
|
|
this.logger.trace(`Successfully retrieved edge ${id}`);
|
||
|
|
return edge;
|
||
|
|
}
|
||
|
|
catch (parseError) {
|
||
|
|
this.logger.error(`Failed to parse edge data for ${id}:`, parseError);
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
// Edge not found or other error
|
||
|
|
this.logger.trace(`Edge not found for ${id}`);
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Get all edges from storage
|
||
|
|
* @deprecated This method is deprecated and will be removed in a future version.
|
||
|
|
* It can cause memory issues with large datasets. Use getEdgesWithPagination() instead.
|
||
|
|
*/
|
||
|
|
async getAllEdges() {
|
||
|
|
await this.ensureInitialized();
|
||
|
|
this.logger.warn('getAllEdges() is deprecated and will be removed in a future version. Use getEdgesWithPagination() instead.');
|
||
|
|
try {
|
||
|
|
// Use the paginated method with a large limit to maintain backward compatibility
|
||
|
|
// but warn about potential issues
|
||
|
|
const result = await this.getEdgesWithPagination({
|
||
|
|
limit: 1000, // Reasonable limit to avoid memory issues
|
||
|
|
useCache: true
|
||
|
|
});
|
||
|
|
if (result.hasMore) {
|
||
|
|
this.logger.warn(`Only returning the first 1000 edges. There are more edges available. Use getEdgesWithPagination() for proper pagination.`);
|
||
|
|
}
|
||
|
|
return result.edges;
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
this.logger.error('Failed to get all edges:', error);
|
||
|
|
return [];
|
||
|
|
}
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Get edges with pagination
|
||
|
|
* @param options Pagination options
|
||
|
|
* @returns Promise that resolves to a paginated result of edges
|
||
|
|
*/
|
||
|
|
async getEdgesWithPagination(options = {}) {
|
||
|
|
await this.ensureInitialized();
|
||
|
|
const limit = options.limit || 100;
|
||
|
|
const useCache = options.useCache !== false;
|
||
|
|
const filter = options.filter || {};
|
||
|
|
try {
|
||
|
|
// Import the ListObjectsV2Command only when needed
|
||
|
|
const { ListObjectsV2Command } = await import('@aws-sdk/client-s3');
|
||
|
|
// List objects with pagination
|
||
|
|
const listResponse = await this.s3Client.send(new ListObjectsV2Command({
|
||
|
|
Bucket: this.bucketName,
|
||
|
|
Prefix: this.verbPrefix,
|
||
|
|
MaxKeys: limit,
|
||
|
|
ContinuationToken: options.cursor
|
||
|
|
}));
|
||
|
|
// If listResponse is null/undefined or there are no objects, return an empty result
|
||
|
|
if (!listResponse ||
|
||
|
|
!listResponse.Contents ||
|
||
|
|
listResponse.Contents.length === 0) {
|
||
|
|
return {
|
||
|
|
edges: [],
|
||
|
|
hasMore: false
|
||
|
|
};
|
||
|
|
}
|
||
|
|
// Extract edge IDs from the keys
|
||
|
|
const edgeIds = listResponse.Contents
|
||
|
|
.filter((object) => object && object.Key)
|
||
|
|
.map((object) => object.Key.replace(this.verbPrefix, '').replace('.json', ''));
|
||
|
|
// Use the cache manager to get edges efficiently
|
||
|
|
const edges = [];
|
||
|
|
if (useCache) {
|
||
|
|
// Get edges from cache manager
|
||
|
|
const cachedEdges = await this.verbCacheManager.getMany(edgeIds);
|
||
|
|
// Add edges to result in the same order as edgeIds
|
||
|
|
for (const id of edgeIds) {
|
||
|
|
const edge = cachedEdges.get(id);
|
||
|
|
if (edge) {
|
||
|
|
// Apply filtering if needed
|
||
|
|
if (this.filterEdge(edge, filter)) {
|
||
|
|
edges.push(edge);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
// Get edges directly from S3 without using cache
|
||
|
|
// Process in smaller batches to reduce memory usage
|
||
|
|
const batchSize = 50;
|
||
|
|
const batches = [];
|
||
|
|
// Split into batches
|
||
|
|
for (let i = 0; i < edgeIds.length; i += batchSize) {
|
||
|
|
const batch = edgeIds.slice(i, i + batchSize);
|
||
|
|
batches.push(batch);
|
||
|
|
}
|
||
|
|
// Process each batch sequentially
|
||
|
|
for (const batch of batches) {
|
||
|
|
const batchEdges = await Promise.all(batch.map(async (id) => {
|
||
|
|
try {
|
||
|
|
const edge = await this.getVerb_internal(id);
|
||
|
|
// Apply filtering if needed
|
||
|
|
if (edge && this.filterEdge(edge, filter)) {
|
||
|
|
return edge;
|
||
|
|
}
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
}));
|
||
|
|
// Add non-null edges to result
|
||
|
|
for (const edge of batchEdges) {
|
||
|
|
if (edge) {
|
||
|
|
edges.push(edge);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
// Determine if there are more edges
|
||
|
|
const hasMore = !!listResponse.IsTruncated;
|
||
|
|
// Set next cursor if there are more edges
|
||
|
|
const nextCursor = listResponse.NextContinuationToken;
|
||
|
|
return {
|
||
|
|
edges,
|
||
|
|
hasMore,
|
||
|
|
nextCursor
|
||
|
|
};
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
this.logger.error('Failed to get edges with pagination:', error);
|
||
|
|
return {
|
||
|
|
edges: [],
|
||
|
|
hasMore: false
|
||
|
|
};
|
||
|
|
}
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Filter an edge based on filter criteria
|
||
|
|
* @param edge The edge to filter
|
||
|
|
* @param filter The filter criteria
|
||
|
|
* @returns True if the edge matches the filter, false otherwise
|
||
|
|
*/
|
||
|
|
filterEdge(edge, filter) {
|
||
|
|
// HNSWVerb filtering is not supported since metadata is stored separately
|
||
|
|
// This method is deprecated and should not be used with the new storage pattern
|
||
|
|
this.logger.trace('Edge filtering is deprecated and not supported with the new storage pattern');
|
||
|
|
return true; // Return all edges since filtering requires metadata
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Get verbs with pagination
|
||
|
|
* @param options Pagination options
|
||
|
|
* @returns Promise that resolves to a paginated result of verbs
|
||
|
|
*/
|
||
|
|
async getVerbsWithPagination(options = {}) {
|
||
|
|
await this.ensureInitialized();
|
||
|
|
// Convert filter to edge filter format
|
||
|
|
const edgeFilter = {};
|
||
|
|
if (options.filter) {
|
||
|
|
// Handle sourceId filter
|
||
|
|
if (options.filter.sourceId) {
|
||
|
|
edgeFilter.sourceId = Array.isArray(options.filter.sourceId)
|
||
|
|
? options.filter.sourceId[0]
|
||
|
|
: options.filter.sourceId;
|
||
|
|
}
|
||
|
|
// Handle targetId filter
|
||
|
|
if (options.filter.targetId) {
|
||
|
|
edgeFilter.targetId = Array.isArray(options.filter.targetId)
|
||
|
|
? options.filter.targetId[0]
|
||
|
|
: options.filter.targetId;
|
||
|
|
}
|
||
|
|
// Handle verbType filter
|
||
|
|
if (options.filter.verbType) {
|
||
|
|
edgeFilter.type = Array.isArray(options.filter.verbType)
|
||
|
|
? options.filter.verbType[0]
|
||
|
|
: options.filter.verbType;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
// Get edges with pagination
|
||
|
|
const result = await this.getEdgesWithPagination({
|
||
|
|
limit: options.limit,
|
||
|
|
cursor: options.cursor,
|
||
|
|
useCache: true,
|
||
|
|
filter: edgeFilter
|
||
|
|
});
|
||
|
|
// Convert HNSWVerbs to GraphVerbs by combining with metadata
|
||
|
|
const graphVerbs = [];
|
||
|
|
for (const hnswVerb of result.edges) {
|
||
|
|
const graphVerb = await this.convertHNSWVerbToGraphVerb(hnswVerb);
|
||
|
|
if (graphVerb) {
|
||
|
|
graphVerbs.push(graphVerb);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
// Apply filtering at GraphVerb level since HNSWVerb filtering is not supported
|
||
|
|
let filteredGraphVerbs = graphVerbs;
|
||
|
|
if (options.filter) {
|
||
|
|
filteredGraphVerbs = graphVerbs.filter((graphVerb) => {
|
||
|
|
// Filter by sourceId
|
||
|
|
if (options.filter.sourceId) {
|
||
|
|
const sourceIds = Array.isArray(options.filter.sourceId)
|
||
|
|
? options.filter.sourceId
|
||
|
|
: [options.filter.sourceId];
|
||
|
|
if (!sourceIds.includes(graphVerb.sourceId)) {
|
||
|
|
return false;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
// Filter by targetId
|
||
|
|
if (options.filter.targetId) {
|
||
|
|
const targetIds = Array.isArray(options.filter.targetId)
|
||
|
|
? options.filter.targetId
|
||
|
|
: [options.filter.targetId];
|
||
|
|
if (!targetIds.includes(graphVerb.targetId)) {
|
||
|
|
return false;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
// Filter by verbType (maps to type field)
|
||
|
|
if (options.filter.verbType) {
|
||
|
|
const verbTypes = Array.isArray(options.filter.verbType)
|
||
|
|
? options.filter.verbType
|
||
|
|
: [options.filter.verbType];
|
||
|
|
if (graphVerb.type && !verbTypes.includes(graphVerb.type)) {
|
||
|
|
return false;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return true;
|
||
|
|
});
|
||
|
|
}
|
||
|
|
return {
|
||
|
|
items: filteredGraphVerbs,
|
||
|
|
hasMore: result.hasMore,
|
||
|
|
nextCursor: result.nextCursor
|
||
|
|
};
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Get verbs by source (internal implementation)
|
||
|
|
*/
|
||
|
|
async getVerbsBySource_internal(sourceId) {
|
||
|
|
// Use the paginated approach to properly handle HNSWVerb to GraphVerb conversion
|
||
|
|
const result = await this.getVerbsWithPagination({
|
||
|
|
filter: { sourceId: [sourceId] },
|
||
|
|
limit: Number.MAX_SAFE_INTEGER // Get all matching results
|
||
|
|
});
|
||
|
|
return result.items;
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Get verbs by target (internal implementation)
|
||
|
|
*/
|
||
|
|
async getVerbsByTarget_internal(targetId) {
|
||
|
|
// Use the paginated approach to properly handle HNSWVerb to GraphVerb conversion
|
||
|
|
const result = await this.getVerbsWithPagination({
|
||
|
|
filter: { targetId: [targetId] },
|
||
|
|
limit: Number.MAX_SAFE_INTEGER // Get all matching results
|
||
|
|
});
|
||
|
|
return result.items;
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Get verbs by type (internal implementation)
|
||
|
|
*/
|
||
|
|
async getVerbsByType_internal(type) {
|
||
|
|
// Use the paginated approach to properly handle HNSWVerb to GraphVerb conversion
|
||
|
|
const result = await this.getVerbsWithPagination({
|
||
|
|
filter: { verbType: [type] },
|
||
|
|
limit: Number.MAX_SAFE_INTEGER // Get all matching results
|
||
|
|
});
|
||
|
|
return result.items;
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Delete a verb from storage (internal implementation)
|
||
|
|
*/
|
||
|
|
async deleteVerb_internal(id) {
|
||
|
|
return this.deleteEdge(id);
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Delete an edge from storage
|
||
|
|
*/
|
||
|
|
async deleteEdge(id) {
|
||
|
|
await this.ensureInitialized();
|
||
|
|
try {
|
||
|
|
// Import the DeleteObjectCommand only when needed
|
||
|
|
const { DeleteObjectCommand } = await import('@aws-sdk/client-s3');
|
||
|
|
// Delete the edge from S3-compatible storage
|
||
|
|
await this.s3Client.send(new DeleteObjectCommand({
|
||
|
|
Bucket: this.bucketName,
|
||
|
|
Key: `${this.verbPrefix}${id}.json`
|
||
|
|
}));
|
||
|
|
// Log the change for efficient synchronization
|
||
|
|
await this.appendToChangeLog({
|
||
|
|
timestamp: Date.now(),
|
||
|
|
operation: 'delete',
|
||
|
|
entityType: 'verb',
|
||
|
|
entityId: id
|
||
|
|
});
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
this.logger.error(`Failed to delete edge ${id}:`, error);
|
||
|
|
throw new Error(`Failed to delete edge ${id}: ${error}`);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Save metadata to storage
|
||
|
|
*/
|
||
|
|
async saveMetadata(id, metadata) {
|
||
|
|
await this.ensureInitialized();
|
||
|
|
// Apply backpressure before starting operation
|
||
|
|
const requestId = await this.applyBackpressure();
|
||
|
|
try {
|
||
|
|
// Import the PutObjectCommand only when needed
|
||
|
|
const { PutObjectCommand } = await import('@aws-sdk/client-s3');
|
||
|
|
const key = `${this.metadataPrefix}${id}.json`;
|
||
|
|
const body = JSON.stringify(metadata, null, 2);
|
||
|
|
this.logger.trace(`Saving metadata for ${id} to key: ${key}`);
|
||
|
|
// Save the metadata to S3-compatible storage
|
||
|
|
const result = await this.s3Client.send(new PutObjectCommand({
|
||
|
|
Bucket: this.bucketName,
|
||
|
|
Key: key,
|
||
|
|
Body: body,
|
||
|
|
ContentType: 'application/json'
|
||
|
|
}));
|
||
|
|
this.logger.debug(`Metadata for ${id} saved successfully`);
|
||
|
|
// Log the change for efficient synchronization
|
||
|
|
await this.appendToChangeLog({
|
||
|
|
timestamp: Date.now(),
|
||
|
|
operation: 'add', // Could be 'update' if we track existing metadata
|
||
|
|
entityType: 'metadata',
|
||
|
|
entityId: id,
|
||
|
|
data: metadata
|
||
|
|
});
|
||
|
|
// Verify the metadata was saved by trying to retrieve it
|
||
|
|
const { GetObjectCommand } = await import('@aws-sdk/client-s3');
|
||
|
|
try {
|
||
|
|
const verifyResponse = await this.s3Client.send(new GetObjectCommand({
|
||
|
|
Bucket: this.bucketName,
|
||
|
|
Key: key
|
||
|
|
}));
|
||
|
|
if (verifyResponse && verifyResponse.Body) {
|
||
|
|
this.logger.trace(`Verified metadata for ${id} was saved correctly`);
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
this.logger.warn(`Failed to verify metadata for ${id} was saved correctly: no response or body`);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
catch (verifyError) {
|
||
|
|
this.logger.warn(`Failed to verify metadata for ${id} was saved correctly:`, verifyError);
|
||
|
|
}
|
||
|
|
// Release backpressure on success
|
||
|
|
this.releaseBackpressure(true, requestId);
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
// Release backpressure on error
|
||
|
|
this.releaseBackpressure(false, requestId);
|
||
|
|
this.logger.error(`Failed to save metadata for ${id}:`, error);
|
||
|
|
throw new Error(`Failed to save metadata for ${id}: ${error}`);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Save verb metadata to storage
|
||
|
|
*/
|
||
|
|
async saveVerbMetadata(id, metadata) {
|
||
|
|
await this.ensureInitialized();
|
||
|
|
try {
|
||
|
|
// Import the PutObjectCommand only when needed
|
||
|
|
const { PutObjectCommand } = await import('@aws-sdk/client-s3');
|
||
|
|
const key = `${this.verbMetadataPrefix}${id}.json`;
|
||
|
|
const body = JSON.stringify(metadata, null, 2);
|
||
|
|
this.logger.trace(`Saving verb metadata for ${id} to key: ${key}`);
|
||
|
|
// Save the verb metadata to S3-compatible storage
|
||
|
|
const result = await this.s3Client.send(new PutObjectCommand({
|
||
|
|
Bucket: this.bucketName,
|
||
|
|
Key: key,
|
||
|
|
Body: body,
|
||
|
|
ContentType: 'application/json'
|
||
|
|
}));
|
||
|
|
this.logger.debug(`Verb metadata for ${id} saved successfully`);
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
this.logger.error(`Failed to save verb metadata for ${id}:`, error);
|
||
|
|
throw new Error(`Failed to save verb metadata for ${id}: ${error}`);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Get verb metadata from storage
|
||
|
|
*/
|
||
|
|
async getVerbMetadata(id) {
|
||
|
|
await this.ensureInitialized();
|
||
|
|
try {
|
||
|
|
// Import the GetObjectCommand only when needed
|
||
|
|
const { GetObjectCommand } = await import('@aws-sdk/client-s3');
|
||
|
|
const key = `${this.verbMetadataPrefix}${id}.json`;
|
||
|
|
this.logger.trace(`Getting verb metadata for ${id} from key: ${key}`);
|
||
|
|
// Try to get the verb metadata
|
||
|
|
const response = await this.s3Client.send(new GetObjectCommand({
|
||
|
|
Bucket: this.bucketName,
|
||
|
|
Key: key
|
||
|
|
}));
|
||
|
|
// Check if response is null or undefined
|
||
|
|
if (!response || !response.Body) {
|
||
|
|
this.logger.trace(`No verb metadata found for ${id}`);
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
// Convert the response body to a string
|
||
|
|
const bodyContents = await response.Body.transformToString();
|
||
|
|
this.logger.trace(`Retrieved verb metadata body for ${id}`);
|
||
|
|
// Parse the JSON string
|
||
|
|
try {
|
||
|
|
const parsedMetadata = JSON.parse(bodyContents);
|
||
|
|
this.logger.trace(`Successfully retrieved verb metadata for ${id}`);
|
||
|
|
return parsedMetadata;
|
||
|
|
}
|
||
|
|
catch (parseError) {
|
||
|
|
this.logger.error(`Failed to parse verb metadata for ${id}:`, parseError);
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
// Check if this is a "NoSuchKey" error (object doesn't exist)
|
||
|
|
if (error.name === 'NoSuchKey' ||
|
||
|
|
(error.message &&
|
||
|
|
(error.message.includes('NoSuchKey') ||
|
||
|
|
error.message.includes('not found') ||
|
||
|
|
error.message.includes('does not exist')))) {
|
||
|
|
this.logger.trace(`Verb metadata not found for ${id}`);
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
// For other types of errors, convert to BrainyError for better classification
|
||
|
|
throw BrainyError.fromError(error, `getVerbMetadata(${id})`);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Save noun metadata to storage
|
||
|
|
*/
|
||
|
|
async saveNounMetadata(id, metadata) {
|
||
|
|
await this.ensureInitialized();
|
||
|
|
try {
|
||
|
|
// Import the PutObjectCommand only when needed
|
||
|
|
const { PutObjectCommand } = await import('@aws-sdk/client-s3');
|
||
|
|
const key = `${this.metadataPrefix}${id}.json`;
|
||
|
|
const body = JSON.stringify(metadata, null, 2);
|
||
|
|
this.logger.trace(`Saving noun metadata for ${id} to key: ${key}`);
|
||
|
|
// Save the noun metadata to S3-compatible storage
|
||
|
|
const result = await this.s3Client.send(new PutObjectCommand({
|
||
|
|
Bucket: this.bucketName,
|
||
|
|
Key: key,
|
||
|
|
Body: body,
|
||
|
|
ContentType: 'application/json'
|
||
|
|
}));
|
||
|
|
this.logger.debug(`Noun metadata for ${id} saved successfully`);
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
this.logger.error(`Failed to save noun metadata for ${id}:`, error);
|
||
|
|
throw new Error(`Failed to save noun metadata for ${id}: ${error}`);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Get multiple metadata objects in batches (CRITICAL: Prevents socket exhaustion)
|
||
|
|
* This is the solution to the metadata reading socket exhaustion during initialization
|
||
|
|
*/
|
||
|
|
async getMetadataBatch(ids) {
|
||
|
|
await this.ensureInitialized();
|
||
|
|
const results = new Map();
|
||
|
|
const batchSize = Math.min(this.getBatchSize(), 10); // Smaller batches for metadata to prevent socket exhaustion
|
||
|
|
// Process in smaller batches to avoid socket exhaustion
|
||
|
|
for (let i = 0; i < ids.length; i += batchSize) {
|
||
|
|
const batch = ids.slice(i, i + batchSize);
|
||
|
|
// Process batch with concurrency control and enhanced retry logic
|
||
|
|
const batchPromises = batch.map(async (id) => {
|
||
|
|
try {
|
||
|
|
// Add timeout wrapper for individual metadata reads
|
||
|
|
const metadata = await Promise.race([
|
||
|
|
this.getMetadata(id),
|
||
|
|
new Promise((_, reject) => setTimeout(() => reject(new Error('Metadata read timeout')), 5000) // 5 second timeout
|
||
|
|
)
|
||
|
|
]);
|
||
|
|
return { id, metadata };
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
// Handle throttling and enhanced error handling
|
||
|
|
await this.handleThrottling(error);
|
||
|
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
||
|
|
if (this.isThrottlingError(error)) {
|
||
|
|
// Throttling errors are already logged in handleThrottling
|
||
|
|
}
|
||
|
|
else if (errorMessage.includes('timeout') || errorMessage.includes('ECONNRESET')) {
|
||
|
|
this.logger.debug(`⏰ Metadata timeout for ${id} (normal during initial indexing):`, errorMessage);
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
this.logger.debug(`Failed to read metadata for ${id}:`, error);
|
||
|
|
}
|
||
|
|
return { id, metadata: null };
|
||
|
|
}
|
||
|
|
});
|
||
|
|
const batchResults = await Promise.all(batchPromises);
|
||
|
|
// Track error rates to adjust delays
|
||
|
|
let errorCount = 0;
|
||
|
|
for (const { id, metadata } of batchResults) {
|
||
|
|
if (metadata !== null) {
|
||
|
|
results.set(id, metadata);
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
errorCount++;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
// Smart delay based on error rates and throttling status
|
||
|
|
const errorRate = errorCount / batch.length;
|
||
|
|
if (errorRate > 0.5) {
|
||
|
|
// High error rate - use smart delay with throttling awareness
|
||
|
|
await this.smartDelay();
|
||
|
|
await new Promise(resolve => setTimeout(resolve, 2000)); // Extra delay for high error rates
|
||
|
|
prodLog.debug(`🐌 High error rate (${(errorRate * 100).toFixed(1)}%) - adding smart delay`);
|
||
|
|
}
|
||
|
|
else if (errorRate > 0.2) {
|
||
|
|
// Moderate error rate - smart delay
|
||
|
|
await this.smartDelay();
|
||
|
|
await new Promise(resolve => setTimeout(resolve, 500)); // Modest extra delay
|
||
|
|
prodLog.debug(`⚡ Moderate error rate (${(errorRate * 100).toFixed(1)}%) - adding smart delay`);
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
// Low error rate - just smart delay (respects throttling status)
|
||
|
|
await this.smartDelay();
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return results;
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Get multiple verb metadata objects in batches (prevents socket exhaustion)
|
||
|
|
*/
|
||
|
|
async getVerbMetadataBatch(ids) {
|
||
|
|
await this.ensureInitialized();
|
||
|
|
const results = new Map();
|
||
|
|
const batchSize = Math.min(this.getBatchSize(), 10); // Smaller batches for metadata to prevent socket exhaustion
|
||
|
|
// Process in smaller batches to avoid socket exhaustion
|
||
|
|
for (let i = 0; i < ids.length; i += batchSize) {
|
||
|
|
const batch = ids.slice(i, i + batchSize);
|
||
|
|
// Process batch with concurrency control
|
||
|
|
const batchPromises = batch.map(async (id) => {
|
||
|
|
try {
|
||
|
|
const metadata = await this.getVerbMetadata(id);
|
||
|
|
return { id, metadata };
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
// Don't fail entire batch if one metadata read fails
|
||
|
|
this.logger.debug(`Failed to read verb metadata for ${id}:`, error);
|
||
|
|
return { id, metadata: null };
|
||
|
|
}
|
||
|
|
});
|
||
|
|
const batchResults = await Promise.all(batchPromises);
|
||
|
|
// Add results to map
|
||
|
|
for (const { id, metadata } of batchResults) {
|
||
|
|
if (metadata !== null) {
|
||
|
|
results.set(id, metadata);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
// Yield to prevent socket exhaustion between batches
|
||
|
|
await new Promise(resolve => setImmediate(resolve));
|
||
|
|
}
|
||
|
|
return results;
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Get noun metadata from storage
|
||
|
|
*/
|
||
|
|
async getNounMetadata(id) {
|
||
|
|
await this.ensureInitialized();
|
||
|
|
try {
|
||
|
|
// Import the GetObjectCommand only when needed
|
||
|
|
const { GetObjectCommand } = await import('@aws-sdk/client-s3');
|
||
|
|
const key = `${this.metadataPrefix}${id}.json`;
|
||
|
|
this.logger.trace(`Getting noun metadata for ${id} from key: ${key}`);
|
||
|
|
// Try to get the noun metadata
|
||
|
|
const response = await this.s3Client.send(new GetObjectCommand({
|
||
|
|
Bucket: this.bucketName,
|
||
|
|
Key: key
|
||
|
|
}));
|
||
|
|
// Check if response is null or undefined
|
||
|
|
if (!response || !response.Body) {
|
||
|
|
this.logger.trace(`No noun metadata found for ${id}`);
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
// Convert the response body to a string
|
||
|
|
const bodyContents = await response.Body.transformToString();
|
||
|
|
this.logger.trace(`Retrieved noun metadata body for ${id}`);
|
||
|
|
// Parse the JSON string
|
||
|
|
try {
|
||
|
|
const parsedMetadata = JSON.parse(bodyContents);
|
||
|
|
this.logger.trace(`Successfully retrieved noun metadata for ${id}`);
|
||
|
|
return parsedMetadata;
|
||
|
|
}
|
||
|
|
catch (parseError) {
|
||
|
|
this.logger.error(`Failed to parse noun metadata for ${id}:`, parseError);
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
// Check if this is a "NoSuchKey" error (object doesn't exist)
|
||
|
|
if (error.name === 'NoSuchKey' ||
|
||
|
|
(error.message &&
|
||
|
|
(error.message.includes('NoSuchKey') ||
|
||
|
|
error.message.includes('not found') ||
|
||
|
|
error.message.includes('does not exist')))) {
|
||
|
|
this.logger.trace(`Noun metadata not found for ${id}`);
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
// For other types of errors, convert to BrainyError for better classification
|
||
|
|
throw BrainyError.fromError(error, `getNounMetadata(${id})`);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Get metadata from storage
|
||
|
|
*/
|
||
|
|
async getMetadata(id) {
|
||
|
|
await this.ensureInitialized();
|
||
|
|
return this.operationExecutors.executeGet(async () => {
|
||
|
|
try {
|
||
|
|
// Import the GetObjectCommand only when needed
|
||
|
|
const { GetObjectCommand } = await import('@aws-sdk/client-s3');
|
||
|
|
prodLog.debug(`Getting metadata for ${id} from bucket ${this.bucketName}`);
|
||
|
|
const key = `${this.metadataPrefix}${id}.json`;
|
||
|
|
prodLog.debug(`Looking for metadata at key: ${key}`);
|
||
|
|
// Try to get the metadata from the metadata directory
|
||
|
|
const response = await this.s3Client.send(new GetObjectCommand({
|
||
|
|
Bucket: this.bucketName,
|
||
|
|
Key: key
|
||
|
|
}));
|
||
|
|
// Check if response is null or undefined (can happen in mock implementations)
|
||
|
|
if (!response || !response.Body) {
|
||
|
|
prodLog.debug(`No metadata found for ${id}`);
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
// Convert the response body to a string
|
||
|
|
const bodyContents = await response.Body.transformToString();
|
||
|
|
prodLog.debug(`Retrieved metadata body: ${bodyContents}`);
|
||
|
|
// Parse the JSON string
|
||
|
|
try {
|
||
|
|
const parsedMetadata = JSON.parse(bodyContents);
|
||
|
|
prodLog.debug(`Successfully retrieved metadata for ${id}:`, parsedMetadata);
|
||
|
|
return parsedMetadata;
|
||
|
|
}
|
||
|
|
catch (parseError) {
|
||
|
|
prodLog.error(`Failed to parse metadata for ${id}:`, parseError);
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
// Check if this is a "NoSuchKey" error (object doesn't exist)
|
||
|
|
// In AWS SDK, this would be error.name === 'NoSuchKey'
|
||
|
|
// In our mock, we might get different error types
|
||
|
|
if (error.name === 'NoSuchKey' ||
|
||
|
|
(error.message &&
|
||
|
|
(error.message.includes('NoSuchKey') ||
|
||
|
|
error.message.includes('not found') ||
|
||
|
|
error.message.includes('does not exist')))) {
|
||
|
|
prodLog.debug(`Metadata not found for ${id}`);
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
// For other types of errors, convert to BrainyError for better classification
|
||
|
|
throw BrainyError.fromError(error, `getMetadata(${id})`);
|
||
|
|
}
|
||
|
|
}, `getMetadata(${id})`);
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Clear all data from storage
|
||
|
|
*/
|
||
|
|
async clear() {
|
||
|
|
await this.ensureInitialized();
|
||
|
|
try {
|
||
|
|
// Import the ListObjectsV2Command and DeleteObjectCommand only when needed
|
||
|
|
const { ListObjectsV2Command, DeleteObjectCommand } = await import('@aws-sdk/client-s3');
|
||
|
|
// Helper function to delete all objects with a given prefix
|
||
|
|
const deleteObjectsWithPrefix = async (prefix) => {
|
||
|
|
// List all objects with the given prefix
|
||
|
|
const listResponse = await this.s3Client.send(new ListObjectsV2Command({
|
||
|
|
Bucket: this.bucketName,
|
||
|
|
Prefix: prefix
|
||
|
|
}));
|
||
|
|
// If there are no objects or Contents is undefined, return
|
||
|
|
if (!listResponse ||
|
||
|
|
!listResponse.Contents ||
|
||
|
|
listResponse.Contents.length === 0) {
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
// Delete each object
|
||
|
|
for (const object of listResponse.Contents) {
|
||
|
|
if (object && object.Key) {
|
||
|
|
await this.s3Client.send(new DeleteObjectCommand({
|
||
|
|
Bucket: this.bucketName,
|
||
|
|
Key: object.Key
|
||
|
|
}));
|
||
|
|
}
|
||
|
|
}
|
||
|
|
};
|
||
|
|
// Delete all objects in the nouns directory
|
||
|
|
await deleteObjectsWithPrefix(this.nounPrefix);
|
||
|
|
// Delete all objects in the verbs directory
|
||
|
|
await deleteObjectsWithPrefix(this.verbPrefix);
|
||
|
|
// Delete all objects in the noun metadata directory
|
||
|
|
await deleteObjectsWithPrefix(this.metadataPrefix);
|
||
|
|
// Delete all objects in the verb metadata directory
|
||
|
|
await deleteObjectsWithPrefix(this.verbMetadataPrefix);
|
||
|
|
// Delete all objects in the index directory
|
||
|
|
await deleteObjectsWithPrefix(this.indexPrefix);
|
||
|
|
// Clear the statistics cache
|
||
|
|
this.statisticsCache = null;
|
||
|
|
this.statisticsModified = false;
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
prodLog.error('Failed to clear storage:', error);
|
||
|
|
throw new Error(`Failed to clear storage: ${error}`);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Get information about storage usage and capacity
|
||
|
|
* Optimized version that uses cached statistics instead of expensive full scans
|
||
|
|
*/
|
||
|
|
async getStorageStatus() {
|
||
|
|
await this.ensureInitialized();
|
||
|
|
try {
|
||
|
|
// Use cached statistics instead of expensive ListObjects scans
|
||
|
|
const stats = await this.getStatisticsData();
|
||
|
|
let totalSize = 0;
|
||
|
|
let nodeCount = 0;
|
||
|
|
let edgeCount = 0;
|
||
|
|
let metadataCount = 0;
|
||
|
|
if (stats) {
|
||
|
|
// Calculate counts from statistics cache (fast)
|
||
|
|
nodeCount = Object.values(stats.nounCount).reduce((sum, count) => sum + count, 0);
|
||
|
|
edgeCount = Object.values(stats.verbCount).reduce((sum, count) => sum + count, 0);
|
||
|
|
metadataCount = Object.values(stats.metadataCount).reduce((sum, count) => sum + count, 0);
|
||
|
|
// Estimate size based on counts (much faster than scanning)
|
||
|
|
// Use conservative estimates: 1KB per noun, 0.5KB per verb, 0.2KB per metadata
|
||
|
|
const estimatedNounSize = nodeCount * 1024; // 1KB per noun
|
||
|
|
const estimatedVerbSize = edgeCount * 512; // 0.5KB per verb
|
||
|
|
const estimatedMetadataSize = metadataCount * 204; // 0.2KB per metadata
|
||
|
|
const estimatedIndexSize = stats.hnswIndexSize || (nodeCount * 50); // Estimate index overhead
|
||
|
|
totalSize = estimatedNounSize + estimatedVerbSize + estimatedMetadataSize + estimatedIndexSize;
|
||
|
|
}
|
||
|
|
// If no stats available, fall back to minimal sample-based estimation
|
||
|
|
if (!stats || totalSize === 0) {
|
||
|
|
const sampleResult = await this.getSampleBasedStorageEstimate();
|
||
|
|
totalSize = sampleResult.estimatedSize;
|
||
|
|
nodeCount = sampleResult.nodeCount;
|
||
|
|
edgeCount = sampleResult.edgeCount;
|
||
|
|
metadataCount = sampleResult.metadataCount;
|
||
|
|
}
|
||
|
|
// Ensure we have a minimum size if we have objects
|
||
|
|
if (totalSize === 0 &&
|
||
|
|
(nodeCount > 0 || edgeCount > 0 || metadataCount > 0)) {
|
||
|
|
// Setting minimum size for objects
|
||
|
|
totalSize = (nodeCount + edgeCount + metadataCount) * 100; // Arbitrary size per object
|
||
|
|
}
|
||
|
|
// For testing purposes, always ensure we have a positive size if we have any objects
|
||
|
|
if (nodeCount > 0 || edgeCount > 0 || metadataCount > 0) {
|
||
|
|
// Ensuring positive size for storage status
|
||
|
|
totalSize = Math.max(totalSize, 1);
|
||
|
|
}
|
||
|
|
// Use service breakdown from statistics instead of expensive metadata scans
|
||
|
|
const nounTypeCounts = stats?.nounCount || {};
|
||
|
|
return {
|
||
|
|
type: this.serviceType,
|
||
|
|
used: totalSize,
|
||
|
|
quota: null, // S3-compatible services typically don't provide quota information through the API
|
||
|
|
details: {
|
||
|
|
bucketName: this.bucketName,
|
||
|
|
region: this.region,
|
||
|
|
endpoint: this.endpoint,
|
||
|
|
nodeCount,
|
||
|
|
edgeCount,
|
||
|
|
metadataCount,
|
||
|
|
nounTypes: nounTypeCounts
|
||
|
|
}
|
||
|
|
};
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
this.logger.error('Failed to get storage status:', error);
|
||
|
|
return {
|
||
|
|
type: this.serviceType,
|
||
|
|
used: 0,
|
||
|
|
quota: null,
|
||
|
|
details: { error: String(error) }
|
||
|
|
};
|
||
|
|
}
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Get the statistics key for a specific date
|
||
|
|
* @param date The date to get the key for
|
||
|
|
* @returns The statistics key for the specified date
|
||
|
|
*/
|
||
|
|
getStatisticsKeyForDate(date) {
|
||
|
|
const year = date.getUTCFullYear();
|
||
|
|
const month = String(date.getUTCMonth() + 1).padStart(2, '0');
|
||
|
|
const day = String(date.getUTCDate()).padStart(2, '0');
|
||
|
|
return `${this.systemPrefix}${STATISTICS_KEY}_${year}${month}${day}.json`;
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Get the current statistics key
|
||
|
|
* @returns The current statistics key
|
||
|
|
*/
|
||
|
|
getCurrentStatisticsKey() {
|
||
|
|
return this.getStatisticsKeyForDate(new Date());
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Get the legacy statistics key (DEPRECATED - /index folder is auto-cleaned)
|
||
|
|
* @returns The legacy statistics key
|
||
|
|
* @deprecated Legacy /index folder is automatically cleaned on initialization
|
||
|
|
*/
|
||
|
|
getLegacyStatisticsKey() {
|
||
|
|
return `${this.indexPrefix}${STATISTICS_KEY}.json`;
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Schedule a batch update of statistics
|
||
|
|
*/
|
||
|
|
scheduleBatchUpdate() {
|
||
|
|
// Mark statistics as modified
|
||
|
|
this.statisticsModified = true;
|
||
|
|
// If we're in read-only mode, don't update statistics
|
||
|
|
if (this.readOnly) {
|
||
|
|
this.logger.trace('Skipping statistics update in read-only mode');
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
// If a timer is already set, don't set another one
|
||
|
|
if (this.statisticsBatchUpdateTimerId !== null) {
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
// Calculate time since last flush
|
||
|
|
const now = Date.now();
|
||
|
|
const timeSinceLastFlush = now - this.lastStatisticsFlushTime;
|
||
|
|
// If we've recently flushed, wait longer before the next flush
|
||
|
|
const delayMs = timeSinceLastFlush < this.MIN_FLUSH_INTERVAL_MS
|
||
|
|
? this.MAX_FLUSH_DELAY_MS
|
||
|
|
: this.MIN_FLUSH_INTERVAL_MS;
|
||
|
|
// Schedule the batch update
|
||
|
|
this.statisticsBatchUpdateTimerId = setTimeout(() => {
|
||
|
|
this.flushStatistics();
|
||
|
|
}, delayMs);
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Flush statistics to storage with distributed locking
|
||
|
|
*/
|
||
|
|
async flushStatistics() {
|
||
|
|
// Clear the timer
|
||
|
|
if (this.statisticsBatchUpdateTimerId !== null) {
|
||
|
|
clearTimeout(this.statisticsBatchUpdateTimerId);
|
||
|
|
this.statisticsBatchUpdateTimerId = null;
|
||
|
|
}
|
||
|
|
// If statistics haven't been modified, no need to flush
|
||
|
|
if (!this.statisticsModified || !this.statisticsCache) {
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
const lockKey = 'statistics-flush';
|
||
|
|
const lockValue = `${Date.now()}_${Math.random()}_${process.pid || 'browser'}`;
|
||
|
|
// Try to acquire lock for statistics update
|
||
|
|
const lockAcquired = await this.acquireLock(lockKey, 15000); // 15 second timeout
|
||
|
|
if (!lockAcquired) {
|
||
|
|
// Another instance is updating statistics, skip this flush
|
||
|
|
// but keep the modified flag so we'll try again later
|
||
|
|
this.logger.debug('Statistics flush skipped - another instance is updating');
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
try {
|
||
|
|
// Re-check if statistics are still modified after acquiring lock
|
||
|
|
if (!this.statisticsModified || !this.statisticsCache) {
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
// Import the PutObjectCommand and GetObjectCommand only when needed
|
||
|
|
const { PutObjectCommand, GetObjectCommand } = await import('@aws-sdk/client-s3');
|
||
|
|
// Get the current statistics key
|
||
|
|
const key = this.getCurrentStatisticsKey();
|
||
|
|
// Read current statistics from storage to merge with local changes
|
||
|
|
let currentStorageStats = null;
|
||
|
|
try {
|
||
|
|
currentStorageStats = await this.tryGetStatisticsFromKey(key);
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
// If we can't read current stats, proceed with local cache
|
||
|
|
this.logger.warn('Could not read current statistics from storage, using local cache:', error);
|
||
|
|
}
|
||
|
|
// Merge local statistics with storage statistics
|
||
|
|
let mergedStats = this.statisticsCache;
|
||
|
|
if (currentStorageStats) {
|
||
|
|
mergedStats = this.mergeStatistics(currentStorageStats, this.statisticsCache);
|
||
|
|
}
|
||
|
|
const body = JSON.stringify(mergedStats, null, 2);
|
||
|
|
// Save the merged statistics to S3-compatible storage
|
||
|
|
await this.s3Client.send(new PutObjectCommand({
|
||
|
|
Bucket: this.bucketName,
|
||
|
|
Key: key,
|
||
|
|
Body: body,
|
||
|
|
ContentType: 'application/json',
|
||
|
|
Metadata: {
|
||
|
|
'last-updated': Date.now().toString(),
|
||
|
|
'updated-by': process.pid?.toString() || 'browser'
|
||
|
|
}
|
||
|
|
}));
|
||
|
|
// Update the last flush time
|
||
|
|
this.lastStatisticsFlushTime = Date.now();
|
||
|
|
// Reset the modified flag
|
||
|
|
this.statisticsModified = false;
|
||
|
|
// Update local cache with merged data
|
||
|
|
this.statisticsCache = mergedStats;
|
||
|
|
// During migration period, also update the legacy location
|
||
|
|
// for backward compatibility with older services
|
||
|
|
if (this.useDualWrite) {
|
||
|
|
try {
|
||
|
|
const legacyKey = this.getLegacyStatisticsKey();
|
||
|
|
await this.s3Client.send(new PutObjectCommand({
|
||
|
|
Bucket: this.bucketName,
|
||
|
|
Key: legacyKey,
|
||
|
|
Body: body,
|
||
|
|
ContentType: 'application/json',
|
||
|
|
Metadata: {
|
||
|
|
'migration-note': 'dual-write-for-compatibility',
|
||
|
|
'schema-version': '2'
|
||
|
|
}
|
||
|
|
}));
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
StorageCompatibilityLayer.logMigrationEvent('Failed to write statistics to legacy S3 location', { error });
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
this.logger.error('Failed to flush statistics data:', error);
|
||
|
|
// Mark as still modified so we'll try again later
|
||
|
|
this.statisticsModified = true;
|
||
|
|
// Don't throw the error to avoid disrupting the application
|
||
|
|
}
|
||
|
|
finally {
|
||
|
|
// Always release the lock
|
||
|
|
await this.releaseLock(lockKey, lockValue);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Merge statistics from storage with local statistics
|
||
|
|
* @param storageStats Statistics from storage
|
||
|
|
* @param localStats Local statistics to merge
|
||
|
|
* @returns Merged statistics data
|
||
|
|
*/
|
||
|
|
mergeStatistics(storageStats, localStats) {
|
||
|
|
// Merge noun counts by taking the maximum of each type
|
||
|
|
const mergedNounCount = {
|
||
|
|
...storageStats.nounCount
|
||
|
|
};
|
||
|
|
for (const [type, count] of Object.entries(localStats.nounCount)) {
|
||
|
|
mergedNounCount[type] = Math.max(mergedNounCount[type] || 0, count);
|
||
|
|
}
|
||
|
|
// Merge verb counts by taking the maximum of each type
|
||
|
|
const mergedVerbCount = {
|
||
|
|
...storageStats.verbCount
|
||
|
|
};
|
||
|
|
for (const [type, count] of Object.entries(localStats.verbCount)) {
|
||
|
|
mergedVerbCount[type] = Math.max(mergedVerbCount[type] || 0, count);
|
||
|
|
}
|
||
|
|
// Merge metadata counts by taking the maximum of each type
|
||
|
|
const mergedMetadataCount = {
|
||
|
|
...storageStats.metadataCount
|
||
|
|
};
|
||
|
|
for (const [type, count] of Object.entries(localStats.metadataCount)) {
|
||
|
|
mergedMetadataCount[type] = Math.max(mergedMetadataCount[type] || 0, count);
|
||
|
|
}
|
||
|
|
return {
|
||
|
|
nounCount: mergedNounCount,
|
||
|
|
verbCount: mergedVerbCount,
|
||
|
|
metadataCount: mergedMetadataCount,
|
||
|
|
hnswIndexSize: Math.max(storageStats.hnswIndexSize, localStats.hnswIndexSize),
|
||
|
|
lastUpdated: new Date(Math.max(new Date(storageStats.lastUpdated).getTime(), new Date(localStats.lastUpdated).getTime())).toISOString()
|
||
|
|
};
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Save statistics data to storage
|
||
|
|
* @param statistics The statistics data to save
|
||
|
|
*/
|
||
|
|
async saveStatisticsData(statistics) {
|
||
|
|
await this.ensureInitialized();
|
||
|
|
try {
|
||
|
|
// Update the cache with a deep copy to avoid reference issues
|
||
|
|
this.statisticsCache = {
|
||
|
|
nounCount: { ...statistics.nounCount },
|
||
|
|
verbCount: { ...statistics.verbCount },
|
||
|
|
metadataCount: { ...statistics.metadataCount },
|
||
|
|
hnswIndexSize: statistics.hnswIndexSize,
|
||
|
|
lastUpdated: statistics.lastUpdated
|
||
|
|
};
|
||
|
|
// Schedule a batch update instead of saving immediately
|
||
|
|
this.scheduleBatchUpdate();
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
this.logger.error('Failed to save statistics data:', error);
|
||
|
|
throw new Error(`Failed to save statistics data: ${error}`);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Get statistics data from storage
|
||
|
|
* @returns Promise that resolves to the statistics data or null if not found
|
||
|
|
*/
|
||
|
|
async getStatisticsData() {
|
||
|
|
await this.ensureInitialized();
|
||
|
|
// Enhanced cache strategy: use cache for 5 minutes to avoid expensive lookups
|
||
|
|
const CACHE_TTL = 5 * 60 * 1000; // 5 minutes
|
||
|
|
const timeSinceFlush = Date.now() - this.lastStatisticsFlushTime;
|
||
|
|
const shouldUseCache = this.statisticsCache && timeSinceFlush < CACHE_TTL;
|
||
|
|
if (shouldUseCache && this.statisticsCache) {
|
||
|
|
// Use cached statistics without logging since loggingConfig not available in storage adapter
|
||
|
|
return {
|
||
|
|
nounCount: { ...this.statisticsCache.nounCount },
|
||
|
|
verbCount: { ...this.statisticsCache.verbCount },
|
||
|
|
metadataCount: { ...this.statisticsCache.metadataCount },
|
||
|
|
hnswIndexSize: this.statisticsCache.hnswIndexSize,
|
||
|
|
lastUpdated: this.statisticsCache.lastUpdated
|
||
|
|
};
|
||
|
|
}
|
||
|
|
try {
|
||
|
|
// Fetching fresh statistics from storage
|
||
|
|
// Import the GetObjectCommand only when needed
|
||
|
|
const { GetObjectCommand } = await import('@aws-sdk/client-s3');
|
||
|
|
// Try statistics locations in order of preference (but with timeout)
|
||
|
|
// NOTE: Legacy /index folder is auto-cleaned on init, so only check _system
|
||
|
|
const keys = [
|
||
|
|
this.getCurrentStatisticsKey(),
|
||
|
|
// Only try yesterday if it's within 2 hours of midnight to avoid unnecessary calls
|
||
|
|
...(this.shouldTryYesterday() ? [this.getStatisticsKeyForDate(this.getYesterday())] : [])
|
||
|
|
// Legacy fallback removed - /index folder is auto-cleaned on initialization
|
||
|
|
];
|
||
|
|
let statistics = null;
|
||
|
|
// Try each key with a timeout to prevent hanging
|
||
|
|
for (const key of keys) {
|
||
|
|
try {
|
||
|
|
statistics = await Promise.race([
|
||
|
|
this.tryGetStatisticsFromKey(key),
|
||
|
|
new Promise((_, reject) => setTimeout(() => reject(new Error('Timeout')), 2000) // 2 second timeout per key
|
||
|
|
)
|
||
|
|
]);
|
||
|
|
if (statistics)
|
||
|
|
break; // Found statistics, stop trying other keys
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
// Continue to next key on timeout or error
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
// If we found statistics, update the cache
|
||
|
|
if (statistics) {
|
||
|
|
// Update the cache with a deep copy
|
||
|
|
this.statisticsCache = {
|
||
|
|
nounCount: { ...statistics.nounCount },
|
||
|
|
verbCount: { ...statistics.verbCount },
|
||
|
|
metadataCount: { ...statistics.metadataCount },
|
||
|
|
hnswIndexSize: statistics.hnswIndexSize,
|
||
|
|
lastUpdated: statistics.lastUpdated
|
||
|
|
};
|
||
|
|
}
|
||
|
|
// Successfully loaded statistics from storage
|
||
|
|
return statistics;
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
this.logger.warn('Error getting statistics data, returning cached or null:', error);
|
||
|
|
// Return cached data if available, even if stale, rather than throwing
|
||
|
|
return this.statisticsCache || null;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Check if we should try yesterday's statistics file
|
||
|
|
* Only try within 2 hours of midnight to avoid unnecessary calls
|
||
|
|
*/
|
||
|
|
shouldTryYesterday() {
|
||
|
|
const now = new Date();
|
||
|
|
const hour = now.getHours();
|
||
|
|
// Only try yesterday's file between 10 PM and 2 AM
|
||
|
|
return hour >= 22 || hour <= 2;
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Get yesterday's date
|
||
|
|
*/
|
||
|
|
getYesterday() {
|
||
|
|
const yesterday = new Date();
|
||
|
|
yesterday.setDate(yesterday.getDate() - 1);
|
||
|
|
return yesterday;
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Try to get statistics from a specific key
|
||
|
|
* @param key The key to try to get statistics from
|
||
|
|
* @returns The statistics data or null if not found
|
||
|
|
*/
|
||
|
|
async tryGetStatisticsFromKey(key) {
|
||
|
|
try {
|
||
|
|
// Import the GetObjectCommand only when needed
|
||
|
|
const { GetObjectCommand } = await import('@aws-sdk/client-s3');
|
||
|
|
// Try to get the statistics from the specified key
|
||
|
|
const response = await this.s3Client.send(new GetObjectCommand({
|
||
|
|
Bucket: this.bucketName,
|
||
|
|
Key: key
|
||
|
|
}));
|
||
|
|
// Check if response is null or undefined
|
||
|
|
if (!response || !response.Body) {
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
// Convert the response body to a string
|
||
|
|
const bodyContents = await response.Body.transformToString();
|
||
|
|
// Parse the JSON string
|
||
|
|
return JSON.parse(bodyContents);
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
// Check if this is a "NoSuchKey" error (object doesn't exist)
|
||
|
|
if (error.name === 'NoSuchKey' ||
|
||
|
|
(error.message &&
|
||
|
|
(error.message.includes('NoSuchKey') ||
|
||
|
|
error.message.includes('not found') ||
|
||
|
|
error.message.includes('does not exist')))) {
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
// For other errors, propagate them
|
||
|
|
throw error;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Append an entry to the change log for efficient synchronization
|
||
|
|
* @param entry The change log entry to append
|
||
|
|
*/
|
||
|
|
async appendToChangeLog(entry) {
|
||
|
|
try {
|
||
|
|
// Import the PutObjectCommand only when needed
|
||
|
|
const { PutObjectCommand } = await import('@aws-sdk/client-s3');
|
||
|
|
// Create a unique key for this change log entry
|
||
|
|
const changeLogKey = `${this.changeLogPrefix}${entry.timestamp}-${Math.random().toString(36).substr(2, 9)}.json`;
|
||
|
|
// Add instance ID for tracking
|
||
|
|
const entryWithInstance = {
|
||
|
|
...entry,
|
||
|
|
instanceId: process.pid?.toString() || 'browser'
|
||
|
|
};
|
||
|
|
// Save the change log entry
|
||
|
|
await this.s3Client.send(new PutObjectCommand({
|
||
|
|
Bucket: this.bucketName,
|
||
|
|
Key: changeLogKey,
|
||
|
|
Body: JSON.stringify(entryWithInstance),
|
||
|
|
ContentType: 'application/json',
|
||
|
|
Metadata: {
|
||
|
|
timestamp: entry.timestamp.toString(),
|
||
|
|
operation: entry.operation,
|
||
|
|
'entity-type': entry.entityType,
|
||
|
|
'entity-id': entry.entityId
|
||
|
|
}
|
||
|
|
}));
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
this.logger.warn('Failed to append to change log:', error);
|
||
|
|
// Don't throw error to avoid disrupting main operations
|
||
|
|
}
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Get changes from the change log since a specific timestamp
|
||
|
|
* @param sinceTimestamp Timestamp to get changes since
|
||
|
|
* @param maxEntries Maximum number of entries to return (default: 1000)
|
||
|
|
* @returns Array of change log entries
|
||
|
|
*/
|
||
|
|
async getChangesSince(sinceTimestamp, maxEntries = 1000) {
|
||
|
|
await this.ensureInitialized();
|
||
|
|
try {
|
||
|
|
// Import the ListObjectsV2Command and GetObjectCommand only when needed
|
||
|
|
const { ListObjectsV2Command, GetObjectCommand } = await import('@aws-sdk/client-s3');
|
||
|
|
// List change log objects
|
||
|
|
const response = await this.s3Client.send(new ListObjectsV2Command({
|
||
|
|
Bucket: this.bucketName,
|
||
|
|
Prefix: this.changeLogPrefix,
|
||
|
|
MaxKeys: maxEntries * 2 // Get more than needed to filter by timestamp
|
||
|
|
}));
|
||
|
|
if (!response.Contents) {
|
||
|
|
return [];
|
||
|
|
}
|
||
|
|
const changes = [];
|
||
|
|
// Process each change log entry
|
||
|
|
for (const object of response.Contents) {
|
||
|
|
if (!object.Key || changes.length >= maxEntries)
|
||
|
|
break;
|
||
|
|
try {
|
||
|
|
// Get the change log entry
|
||
|
|
const getResponse = await this.s3Client.send(new GetObjectCommand({
|
||
|
|
Bucket: this.bucketName,
|
||
|
|
Key: object.Key
|
||
|
|
}));
|
||
|
|
if (getResponse.Body) {
|
||
|
|
const entryData = await getResponse.Body.transformToString();
|
||
|
|
const entry = JSON.parse(entryData);
|
||
|
|
// Only include entries newer than the specified timestamp
|
||
|
|
if (entry.timestamp > sinceTimestamp) {
|
||
|
|
changes.push(entry);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
this.logger.warn(`Failed to read change log entry ${object.Key}:`, error);
|
||
|
|
// Continue processing other entries
|
||
|
|
}
|
||
|
|
}
|
||
|
|
// Sort by timestamp (oldest first)
|
||
|
|
changes.sort((a, b) => a.timestamp - b.timestamp);
|
||
|
|
return changes.slice(0, maxEntries);
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
this.logger.error('Failed to get changes from change log:', error);
|
||
|
|
return [];
|
||
|
|
}
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Clean up old change log entries to prevent unlimited growth
|
||
|
|
* @param olderThanTimestamp Remove entries older than this timestamp
|
||
|
|
*/
|
||
|
|
async cleanupOldChangeLogs(olderThanTimestamp) {
|
||
|
|
await this.ensureInitialized();
|
||
|
|
try {
|
||
|
|
// Import the ListObjectsV2Command and DeleteObjectCommand only when needed
|
||
|
|
const { ListObjectsV2Command, DeleteObjectCommand } = await import('@aws-sdk/client-s3');
|
||
|
|
// List change log objects
|
||
|
|
const response = await this.s3Client.send(new ListObjectsV2Command({
|
||
|
|
Bucket: this.bucketName,
|
||
|
|
Prefix: this.changeLogPrefix,
|
||
|
|
MaxKeys: 1000
|
||
|
|
}));
|
||
|
|
if (!response.Contents) {
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
const entriesToDelete = [];
|
||
|
|
// Check each change log entry for age
|
||
|
|
for (const object of response.Contents) {
|
||
|
|
if (!object.Key)
|
||
|
|
continue;
|
||
|
|
// Extract timestamp from the key (format: change-log/timestamp-randomid.json)
|
||
|
|
const keyParts = object.Key.split('/');
|
||
|
|
if (keyParts.length >= 2) {
|
||
|
|
const filename = keyParts[keyParts.length - 1];
|
||
|
|
const timestampStr = filename.split('-')[0];
|
||
|
|
const timestamp = parseInt(timestampStr);
|
||
|
|
if (!isNaN(timestamp) && timestamp < olderThanTimestamp) {
|
||
|
|
entriesToDelete.push(object.Key);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
// Delete old entries
|
||
|
|
for (const key of entriesToDelete) {
|
||
|
|
try {
|
||
|
|
await this.s3Client.send(new DeleteObjectCommand({
|
||
|
|
Bucket: this.bucketName,
|
||
|
|
Key: key
|
||
|
|
}));
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
this.logger.warn(`Failed to delete old change log entry ${key}:`, error);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if (entriesToDelete.length > 0) {
|
||
|
|
this.logger.debug(`Cleaned up ${entriesToDelete.length} old change log entries`);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
this.logger.warn('Failed to cleanup old change logs:', error);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Sample-based storage estimation as fallback when statistics unavailable
|
||
|
|
* Much faster than full scans - samples first 50 objects per prefix
|
||
|
|
*/
|
||
|
|
async getSampleBasedStorageEstimate() {
|
||
|
|
try {
|
||
|
|
const { ListObjectsV2Command } = await import('@aws-sdk/client-s3');
|
||
|
|
const sampleSize = 50; // Sample first 50 objects per prefix
|
||
|
|
const prefixes = [
|
||
|
|
{ prefix: this.nounPrefix, type: 'noun' },
|
||
|
|
{ prefix: this.verbPrefix, type: 'verb' },
|
||
|
|
{ prefix: this.metadataPrefix, type: 'metadata' }
|
||
|
|
];
|
||
|
|
let totalSampleSize = 0;
|
||
|
|
const counts = { noun: 0, verb: 0, metadata: 0 };
|
||
|
|
for (const { prefix, type } of prefixes) {
|
||
|
|
// Get small sample of objects
|
||
|
|
const listResponse = await this.s3Client.send(new ListObjectsV2Command({
|
||
|
|
Bucket: this.bucketName,
|
||
|
|
Prefix: prefix,
|
||
|
|
MaxKeys: sampleSize
|
||
|
|
}));
|
||
|
|
if (listResponse.Contents && listResponse.Contents.length > 0) {
|
||
|
|
let sampleSize = 0;
|
||
|
|
let sampleCount = listResponse.Contents.length;
|
||
|
|
// Calculate size from first few objects in sample
|
||
|
|
for (let i = 0; i < Math.min(10, sampleCount); i++) {
|
||
|
|
const obj = listResponse.Contents[i];
|
||
|
|
if (obj && obj.Size) {
|
||
|
|
sampleSize += typeof obj.Size === 'number' ? obj.Size : parseInt(obj.Size.toString(), 10);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
// Estimate total count (if we got MaxKeys, there are probably more)
|
||
|
|
let estimatedCount = sampleCount;
|
||
|
|
if (sampleCount === sampleSize && listResponse.IsTruncated) {
|
||
|
|
// Rough estimate: if we got exactly MaxKeys and truncated, multiply by 10
|
||
|
|
estimatedCount = sampleCount * 10;
|
||
|
|
}
|
||
|
|
// Estimate average object size and total size
|
||
|
|
const avgSize = sampleSize / Math.min(10, sampleCount) || 512; // Default 512 bytes
|
||
|
|
const estimatedTotalSize = avgSize * estimatedCount;
|
||
|
|
totalSampleSize += estimatedTotalSize;
|
||
|
|
counts[type] = estimatedCount;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return {
|
||
|
|
estimatedSize: totalSampleSize,
|
||
|
|
nodeCount: counts.noun,
|
||
|
|
edgeCount: counts.verb,
|
||
|
|
metadataCount: counts.metadata
|
||
|
|
};
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
// If even sampling fails, return minimal estimates
|
||
|
|
return {
|
||
|
|
estimatedSize: 1024, // 1KB minimum
|
||
|
|
nodeCount: 0,
|
||
|
|
edgeCount: 0,
|
||
|
|
metadataCount: 0
|
||
|
|
};
|
||
|
|
}
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Acquire a distributed lock for coordinating operations across multiple instances
|
||
|
|
* @param lockKey The key to lock on
|
||
|
|
* @param ttl Time to live for the lock in milliseconds (default: 30 seconds)
|
||
|
|
* @returns Promise that resolves to true if lock was acquired, false otherwise
|
||
|
|
*/
|
||
|
|
async acquireLock(lockKey, ttl = 30000) {
|
||
|
|
await this.ensureInitialized();
|
||
|
|
const lockObject = `${this.lockPrefix}${lockKey}`;
|
||
|
|
const lockValue = `${Date.now()}_${Math.random()}_${process.pid || 'browser'}`;
|
||
|
|
const expiresAt = Date.now() + ttl;
|
||
|
|
try {
|
||
|
|
// Import the PutObjectCommand and HeadObjectCommand only when needed
|
||
|
|
const { PutObjectCommand, HeadObjectCommand } = await import('@aws-sdk/client-s3');
|
||
|
|
// First check if lock already exists and is still valid
|
||
|
|
try {
|
||
|
|
const headResponse = await this.s3Client.send(new HeadObjectCommand({
|
||
|
|
Bucket: this.bucketName,
|
||
|
|
Key: lockObject
|
||
|
|
}));
|
||
|
|
// Check if existing lock has expired
|
||
|
|
const existingExpiresAt = headResponse.Metadata?.['expires-at'];
|
||
|
|
if (existingExpiresAt && parseInt(existingExpiresAt) > Date.now()) {
|
||
|
|
// Lock exists and is still valid
|
||
|
|
return false;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
// If HeadObject fails with NoSuchKey or NotFound, the lock doesn't exist, which is good
|
||
|
|
if (error.name !== 'NoSuchKey' &&
|
||
|
|
!error.message?.includes('NoSuchKey') &&
|
||
|
|
error.name !== 'NotFound' &&
|
||
|
|
!error.message?.includes('NotFound')) {
|
||
|
|
throw error;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
// Try to create the lock
|
||
|
|
await this.s3Client.send(new PutObjectCommand({
|
||
|
|
Bucket: this.bucketName,
|
||
|
|
Key: lockObject,
|
||
|
|
Body: lockValue,
|
||
|
|
ContentType: 'text/plain',
|
||
|
|
Metadata: {
|
||
|
|
'expires-at': expiresAt.toString(),
|
||
|
|
'lock-value': lockValue
|
||
|
|
}
|
||
|
|
}));
|
||
|
|
// Add to active locks for cleanup
|
||
|
|
this.activeLocks.add(lockKey);
|
||
|
|
// Schedule automatic cleanup when lock expires
|
||
|
|
setTimeout(() => {
|
||
|
|
this.releaseLock(lockKey, lockValue).catch((error) => {
|
||
|
|
this.logger.warn(`Failed to auto-release expired lock ${lockKey}:`, error);
|
||
|
|
});
|
||
|
|
}, ttl);
|
||
|
|
return true;
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
this.logger.warn(`Failed to acquire lock ${lockKey}:`, error);
|
||
|
|
return false;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Release a distributed lock
|
||
|
|
* @param lockKey The key to unlock
|
||
|
|
* @param lockValue The value used when acquiring the lock (for verification)
|
||
|
|
* @returns Promise that resolves when lock is released
|
||
|
|
*/
|
||
|
|
async releaseLock(lockKey, lockValue) {
|
||
|
|
await this.ensureInitialized();
|
||
|
|
const lockObject = `${this.lockPrefix}${lockKey}`;
|
||
|
|
try {
|
||
|
|
// Import the DeleteObjectCommand and GetObjectCommand only when needed
|
||
|
|
const { DeleteObjectCommand, GetObjectCommand } = await import('@aws-sdk/client-s3');
|
||
|
|
// If lockValue is provided, verify it matches before releasing
|
||
|
|
if (lockValue) {
|
||
|
|
try {
|
||
|
|
const response = await this.s3Client.send(new GetObjectCommand({
|
||
|
|
Bucket: this.bucketName,
|
||
|
|
Key: lockObject
|
||
|
|
}));
|
||
|
|
const existingValue = await response.Body?.transformToString();
|
||
|
|
if (existingValue !== lockValue) {
|
||
|
|
// Lock was acquired by someone else, don't release it
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
// If lock doesn't exist, that's fine
|
||
|
|
if (error.name === 'NoSuchKey' ||
|
||
|
|
error.message?.includes('NoSuchKey') ||
|
||
|
|
error.name === 'NotFound' ||
|
||
|
|
error.message?.includes('NotFound')) {
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
throw error;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
// Delete the lock object
|
||
|
|
await this.s3Client.send(new DeleteObjectCommand({
|
||
|
|
Bucket: this.bucketName,
|
||
|
|
Key: lockObject
|
||
|
|
}));
|
||
|
|
// Remove from active locks
|
||
|
|
this.activeLocks.delete(lockKey);
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
this.logger.warn(`Failed to release lock ${lockKey}:`, error);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Clean up expired locks to prevent lock leakage
|
||
|
|
* This method should be called periodically
|
||
|
|
*/
|
||
|
|
async cleanupExpiredLocks() {
|
||
|
|
await this.ensureInitialized();
|
||
|
|
try {
|
||
|
|
// Import the ListObjectsV2Command and DeleteObjectCommand only when needed
|
||
|
|
const { ListObjectsV2Command, DeleteObjectCommand, HeadObjectCommand } = await import('@aws-sdk/client-s3');
|
||
|
|
// List all lock objects
|
||
|
|
const response = await this.s3Client.send(new ListObjectsV2Command({
|
||
|
|
Bucket: this.bucketName,
|
||
|
|
Prefix: this.lockPrefix,
|
||
|
|
MaxKeys: 1000
|
||
|
|
}));
|
||
|
|
if (!response.Contents) {
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
const now = Date.now();
|
||
|
|
const expiredLocks = [];
|
||
|
|
// Check each lock for expiration
|
||
|
|
for (const object of response.Contents) {
|
||
|
|
if (!object.Key)
|
||
|
|
continue;
|
||
|
|
try {
|
||
|
|
const headResponse = await this.s3Client.send(new HeadObjectCommand({
|
||
|
|
Bucket: this.bucketName,
|
||
|
|
Key: object.Key
|
||
|
|
}));
|
||
|
|
const expiresAt = headResponse.Metadata?.['expires-at'];
|
||
|
|
if (expiresAt && parseInt(expiresAt) < now) {
|
||
|
|
expiredLocks.push(object.Key);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
// If we can't read the lock metadata, consider it expired
|
||
|
|
expiredLocks.push(object.Key);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
// Delete expired locks
|
||
|
|
for (const lockKey of expiredLocks) {
|
||
|
|
try {
|
||
|
|
await this.s3Client.send(new DeleteObjectCommand({
|
||
|
|
Bucket: this.bucketName,
|
||
|
|
Key: lockKey
|
||
|
|
}));
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
this.logger.warn(`Failed to delete expired lock ${lockKey}:`, error);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if (expiredLocks.length > 0) {
|
||
|
|
this.logger.debug(`Cleaned up ${expiredLocks.length} expired locks`);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
catch (error) {
|
||
|
|
this.logger.warn('Failed to cleanup expired locks:', error);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Get nouns with pagination support
|
||
|
|
* @param options Pagination options
|
||
|
|
* @returns Promise that resolves to a paginated result of nouns
|
||
|
|
*/
|
||
|
|
async getNounsWithPagination(options = {}) {
|
||
|
|
await this.ensureInitialized();
|
||
|
|
const limit = options.limit || 100;
|
||
|
|
const cursor = options.cursor;
|
||
|
|
// Get paginated nodes
|
||
|
|
const result = await this.getNodesWithPagination({
|
||
|
|
limit,
|
||
|
|
cursor,
|
||
|
|
useCache: true
|
||
|
|
});
|
||
|
|
// Apply filters if provided
|
||
|
|
let filteredNodes = result.nodes;
|
||
|
|
if (options.filter) {
|
||
|
|
// Filter by noun type
|
||
|
|
if (options.filter.nounType) {
|
||
|
|
const nounTypes = Array.isArray(options.filter.nounType)
|
||
|
|
? options.filter.nounType
|
||
|
|
: [options.filter.nounType];
|
||
|
|
const filteredByType = [];
|
||
|
|
for (const node of filteredNodes) {
|
||
|
|
const metadata = await this.getNounMetadata(node.id);
|
||
|
|
if (metadata && nounTypes.includes(metadata.type || metadata.noun)) {
|
||
|
|
filteredByType.push(node);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
filteredNodes = filteredByType;
|
||
|
|
}
|
||
|
|
// Filter by service
|
||
|
|
if (options.filter.service) {
|
||
|
|
const services = Array.isArray(options.filter.service)
|
||
|
|
? options.filter.service
|
||
|
|
: [options.filter.service];
|
||
|
|
const filteredByService = [];
|
||
|
|
for (const node of filteredNodes) {
|
||
|
|
const metadata = await this.getNounMetadata(node.id);
|
||
|
|
if (metadata && services.includes(metadata.service)) {
|
||
|
|
filteredByService.push(node);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
filteredNodes = filteredByService;
|
||
|
|
}
|
||
|
|
// Filter by metadata
|
||
|
|
if (options.filter.metadata) {
|
||
|
|
const metadataFilter = options.filter.metadata;
|
||
|
|
const filteredByMetadata = [];
|
||
|
|
for (const node of filteredNodes) {
|
||
|
|
const metadata = await this.getNounMetadata(node.id);
|
||
|
|
if (metadata) {
|
||
|
|
const matches = Object.entries(metadataFilter).every(([key, value]) => metadata[key] === value);
|
||
|
|
if (matches) {
|
||
|
|
filteredByMetadata.push(node);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
filteredNodes = filteredByMetadata;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return {
|
||
|
|
items: filteredNodes,
|
||
|
|
hasMore: result.hasMore,
|
||
|
|
nextCursor: result.nextCursor
|
||
|
|
};
|
||
|
|
}
|
||
|
|
}
|
||
|
|
//# sourceMappingURL=s3CompatibleStorage.js.map
|