Initial commit: Brainy - Multi-Dimensional AI Database

Open source vector database with HNSW indexing, graph relationships,
and metadata facets. Features CLI with professional augmentation registry
integration for discovering extensions and capabilities.

dist/storage/adapters/baseStorageAdapter.d.ts

@@ -0,0 +1,214 @@
+/**
+ * Base Storage Adapter
+ * Provides common functionality for all storage adapters, including statistics tracking
+ */
+import { StatisticsData, StorageAdapter } from '../../coreTypes.js';
+/**
+ * Base class for storage adapters that implements statistics tracking
+ */
+export declare abstract class BaseStorageAdapter implements StorageAdapter {
+    abstract init(): Promise<void>;
+    abstract saveNoun(noun: any): Promise<void>;
+    abstract getNoun(id: string): Promise<any | null>;
+    abstract getNounsByNounType(nounType: string): Promise<any[]>;
+    abstract deleteNoun(id: string): Promise<void>;
+    abstract saveVerb(verb: any): Promise<void>;
+    abstract getVerb(id: string): Promise<any | null>;
+    abstract getVerbsBySource(sourceId: string): Promise<any[]>;
+    abstract getVerbsByTarget(targetId: string): Promise<any[]>;
+    abstract getVerbsByType(type: string): Promise<any[]>;
+    abstract deleteVerb(id: string): Promise<void>;
+    abstract saveMetadata(id: string, metadata: any): Promise<void>;
+    abstract getMetadata(id: string): Promise<any | null>;
+    abstract saveVerbMetadata(id: string, metadata: any): Promise<void>;
+    abstract getVerbMetadata(id: string): Promise<any | null>;
+    abstract clear(): Promise<void>;
+    abstract getStorageStatus(): Promise<{
+        type: string;
+        used: number;
+        quota: number | null;
+        details?: Record<string, any>;
+    }>;
+    /**
+     * Get nouns with pagination and filtering
+     * @param options Pagination and filtering options
+     * @returns Promise that resolves to a paginated result of nouns
+     */
+    abstract getNouns(options?: {
+        pagination?: {
+            offset?: number;
+            limit?: number;
+            cursor?: string;
+        };
+        filter?: {
+            nounType?: string | string[];
+            service?: string | string[];
+            metadata?: Record<string, any>;
+        };
+    }): Promise<{
+        items: any[];
+        totalCount?: number;
+        hasMore: boolean;
+        nextCursor?: string;
+    }>;
+    /**
+     * Get verbs with pagination and filtering
+     * @param options Pagination and filtering options
+     * @returns Promise that resolves to a paginated result of verbs
+     */
+    abstract getVerbs(options?: {
+        pagination?: {
+            offset?: number;
+            limit?: number;
+            cursor?: string;
+        };
+        filter?: {
+            verbType?: string | string[];
+            sourceId?: string | string[];
+            targetId?: string | string[];
+            service?: string | string[];
+            metadata?: Record<string, any>;
+        };
+    }): Promise<{
+        items: any[];
+        totalCount?: number;
+        hasMore: boolean;
+        nextCursor?: string;
+    }>;
+    protected statisticsCache: StatisticsData | null;
+    protected statisticsBatchUpdateTimerId: NodeJS.Timeout | null;
+    protected statisticsModified: boolean;
+    protected lastStatisticsFlushTime: number;
+    protected readonly MIN_FLUSH_INTERVAL_MS = 5000;
+    protected readonly MAX_FLUSH_DELAY_MS = 30000;
+    protected throttlingDetected: boolean;
+    protected throttlingBackoffMs: number;
+    protected maxBackoffMs: number;
+    protected consecutiveThrottleEvents: number;
+    protected lastThrottleTime: number;
+    protected totalThrottleEvents: number;
+    protected throttleEventsByHour: number[];
+    protected throttleReasons: Record<string, number>;
+    protected lastThrottleHourIndex: number;
+    protected delayedOperations: number;
+    protected retriedOperations: number;
+    protected failedDueToThrottling: number;
+    protected totalDelayMs: number;
+    protected serviceThrottling: Map<string, {
+        throttleCount: number;
+        lastThrottle: number;
+        status: 'normal' | 'throttled' | 'recovering';
+    }>;
+    protected abstract saveStatisticsData(statistics: StatisticsData): Promise<void>;
+    protected abstract getStatisticsData(): Promise<StatisticsData | null>;
+    /**
+     * Save statistics data
+     * @param statistics The statistics data to save
+     */
+    saveStatistics(statistics: StatisticsData): Promise<void>;
+    /**
+     * Get statistics data
+     * @returns Promise that resolves to the statistics data
+     */
+    getStatistics(): Promise<StatisticsData | null>;
+    /**
+     * Schedule a batch update of statistics
+     */
+    protected scheduleBatchUpdate(): void;
+    /**
+     * Flush statistics to storage
+     */
+    protected flushStatistics(): Promise<void>;
+    /**
+     * Increment a statistic counter
+     * @param type The type of statistic to increment ('noun', 'verb', 'metadata')
+     * @param service The service that inserted the data
+     * @param amount The amount to increment by (default: 1)
+     */
+    incrementStatistic(type: 'noun' | 'verb' | 'metadata', service: string, amount?: number): Promise<void>;
+    /**
+     * Track service activity (first/last activity, operation counts)
+     * @param service The service name
+     * @param operation The operation type
+     */
+    protected trackServiceActivity(service: string, operation: 'add' | 'update' | 'delete'): void;
+    /**
+     * Decrement a statistic counter
+     * @param type The type of statistic to decrement ('noun', 'verb', 'metadata')
+     * @param service The service that inserted the data
+     * @param amount The amount to decrement by (default: 1)
+     */
+    decrementStatistic(type: 'noun' | 'verb' | 'metadata', service: string, amount?: number): Promise<void>;
+    /**
+     * Update the HNSW index size statistic
+     * @param size The new size of the HNSW index
+     */
+    updateHnswIndexSize(size: number): Promise<void>;
+    /**
+     * Force an immediate flush of statistics to storage
+     * This ensures that any pending statistics updates are written to persistent storage
+     */
+    flushStatisticsToStorage(): Promise<void>;
+    /**
+     * Track field names from a JSON document
+     * @param jsonDocument The JSON document to extract field names from
+     * @param service The service that inserted the data
+     */
+    trackFieldNames(jsonDocument: any, service: string): Promise<void>;
+    /**
+     * Get available field names by service
+     * @returns Record of field names by service
+     */
+    getAvailableFieldNames(): Promise<Record<string, string[]>>;
+    /**
+     * Get standard field mappings
+     * @returns Record of standard field mappings
+     */
+    getStandardFieldMappings(): Promise<Record<string, Record<string, string[]>>>;
+    /**
+     * Create default statistics data
+     * @returns Default statistics data
+     */
+    protected createDefaultStatistics(): StatisticsData;
+    /**
+     * Detect if an error is a throttling error
+     * Override this method in specific adapters for custom detection
+     */
+    protected isThrottlingError(error: any): boolean;
+    /**
+     * Track a throttling event
+     * @param error The error that caused throttling
+     * @param service Optional service that was throttled
+     */
+    protected trackThrottlingEvent(error: any, service?: string): void;
+    /**
+     * Get the reason for throttling from an error
+     */
+    protected getThrottleReason(error: any): string;
+    /**
+     * Clear throttling state after successful operations
+     */
+    protected clearThrottlingState(): void;
+    /**
+     * Handle throttling by implementing exponential backoff
+     * @param error The error that triggered throttling
+     * @param service Optional service that was throttled
+     */
+    handleThrottling(error: any, service?: string): Promise<void>;
+    /**
+     * Track a retried operation
+     */
+    protected trackRetriedOperation(): void;
+    /**
+     * Track an operation that failed due to throttling
+     */
+    protected trackFailedDueToThrottling(): void;
+    /**
+     * Get current throttling metrics
+     */
+    protected getThrottlingMetrics(): StatisticsData['throttlingMetrics'];
+    /**
+     * Include throttling metrics in statistics
+     */
+    getStatisticsWithThrottling(): Promise<StatisticsData | null>;
+}

dist/storage/adapters/baseStorageAdapter.js

@@ -0,0 +1,613 @@
+/**
+ * Base Storage Adapter
+ * Provides common functionality for all storage adapters, including statistics tracking
+ */
+import { extractFieldNamesFromJson, mapToStandardField } from '../../utils/fieldNameTracking.js';
+/**
+ * Base class for storage adapters that implements statistics tracking
+ */
+export class BaseStorageAdapter {
+    constructor() {
+        // Statistics cache
+        this.statisticsCache = null;
+        // Batch update timer ID
+        this.statisticsBatchUpdateTimerId = null;
+        // Flag to indicate if statistics have been modified since last save
+        this.statisticsModified = false;
+        // Time of last statistics flush to storage
+        this.lastStatisticsFlushTime = 0;
+        // Minimum time between statistics flushes (5 seconds)
+        this.MIN_FLUSH_INTERVAL_MS = 5000;
+        // Maximum time to wait before flushing statistics (30 seconds)
+        this.MAX_FLUSH_DELAY_MS = 30000;
+        // Throttling tracking properties
+        this.throttlingDetected = false;
+        this.throttlingBackoffMs = 1000; // Start with 1 second
+        this.maxBackoffMs = 30000; // Max 30 seconds
+        this.consecutiveThrottleEvents = 0;
+        this.lastThrottleTime = 0;
+        this.totalThrottleEvents = 0;
+        this.throttleEventsByHour = new Array(24).fill(0);
+        this.throttleReasons = {};
+        this.lastThrottleHourIndex = -1;
+        // Operation impact tracking
+        this.delayedOperations = 0;
+        this.retriedOperations = 0;
+        this.failedDueToThrottling = 0;
+        this.totalDelayMs = 0;
+        // Service-level throttling
+        this.serviceThrottling = new Map();
+    }
+    /**
+     * Save statistics data
+     * @param statistics The statistics data to save
+     */
+    async saveStatistics(statistics) {
+        // Update the cache with a deep copy to avoid reference issues
+        this.statisticsCache = {
+            nounCount: { ...statistics.nounCount },
+            verbCount: { ...statistics.verbCount },
+            metadataCount: { ...statistics.metadataCount },
+            hnswIndexSize: statistics.hnswIndexSize,
+            lastUpdated: statistics.lastUpdated,
+            // Include serviceActivity if present
+            ...(statistics.serviceActivity && {
+                serviceActivity: Object.fromEntries(Object.entries(statistics.serviceActivity).map(([k, v]) => [k, { ...v }]))
+            }),
+            // Include services if present
+            ...(statistics.services && {
+                services: statistics.services.map(s => ({ ...s }))
+            })
+        };
+        // Schedule a batch update instead of saving immediately
+        this.scheduleBatchUpdate();
+    }
+    /**
+     * Get statistics data
+     * @returns Promise that resolves to the statistics data
+     */
+    async getStatistics() {
+        // If we have cached statistics, return a deep copy
+        if (this.statisticsCache) {
+            return {
+                nounCount: { ...this.statisticsCache.nounCount },
+                verbCount: { ...this.statisticsCache.verbCount },
+                metadataCount: { ...this.statisticsCache.metadataCount },
+                hnswIndexSize: this.statisticsCache.hnswIndexSize,
+                lastUpdated: this.statisticsCache.lastUpdated
+            };
+        }
+        // Otherwise, get from storage
+        const statistics = await this.getStatisticsData();
+        // If we found statistics, update the cache
+        if (statistics) {
+            // Update the cache with a deep copy
+            this.statisticsCache = {
+                nounCount: { ...statistics.nounCount },
+                verbCount: { ...statistics.verbCount },
+                metadataCount: { ...statistics.metadataCount },
+                hnswIndexSize: statistics.hnswIndexSize,
+                lastUpdated: statistics.lastUpdated
+            };
+        }
+        return statistics;
+    }
+    /**
+     * Schedule a batch update of statistics
+     */
+    scheduleBatchUpdate() {
+        // Mark statistics as modified
+        this.statisticsModified = true;
+        // If a timer is already set, don't set another one
+        if (this.statisticsBatchUpdateTimerId !== null) {
+            return;
+        }
+        // Calculate time since last flush
+        const now = Date.now();
+        const timeSinceLastFlush = now - this.lastStatisticsFlushTime;
+        // If we've recently flushed, wait longer before the next flush
+        const delayMs = timeSinceLastFlush < this.MIN_FLUSH_INTERVAL_MS
+            ? this.MAX_FLUSH_DELAY_MS
+            : this.MIN_FLUSH_INTERVAL_MS;
+        // Schedule the batch update
+        this.statisticsBatchUpdateTimerId = setTimeout(() => {
+            this.flushStatistics();
+        }, delayMs);
+    }
+    /**
+     * Flush statistics to storage
+     */
+    async flushStatistics() {
+        // Clear the timer
+        if (this.statisticsBatchUpdateTimerId !== null) {
+            clearTimeout(this.statisticsBatchUpdateTimerId);
+            this.statisticsBatchUpdateTimerId = null;
+        }
+        // If statistics haven't been modified, no need to flush
+        if (!this.statisticsModified || !this.statisticsCache) {
+            return;
+        }
+        try {
+            // Save the statistics to storage
+            await this.saveStatisticsData(this.statisticsCache);
+            // Update the last flush time
+            this.lastStatisticsFlushTime = Date.now();
+            // Reset the modified flag
+            this.statisticsModified = false;
+        }
+        catch (error) {
+            console.error('Failed to flush statistics data:', error);
+            // Mark as still modified so we'll try again later
+            this.statisticsModified = true;
+            // Don't throw the error to avoid disrupting the application
+        }
+    }
+    /**
+     * Increment a statistic counter
+     * @param type The type of statistic to increment ('noun', 'verb', 'metadata')
+     * @param service The service that inserted the data
+     * @param amount The amount to increment by (default: 1)
+     */
+    async incrementStatistic(type, service, amount = 1) {
+        // Get current statistics from cache or storage
+        let statistics = this.statisticsCache;
+        if (!statistics) {
+            statistics = await this.getStatisticsData();
+            if (!statistics) {
+                statistics = this.createDefaultStatistics();
+            }
+            // Update the cache
+            this.statisticsCache = {
+                nounCount: { ...statistics.nounCount },
+                verbCount: { ...statistics.verbCount },
+                metadataCount: { ...statistics.metadataCount },
+                hnswIndexSize: statistics.hnswIndexSize,
+                lastUpdated: statistics.lastUpdated,
+                // Include serviceActivity if present
+                ...(statistics.serviceActivity && {
+                    serviceActivity: Object.fromEntries(Object.entries(statistics.serviceActivity).map(([k, v]) => [k, { ...v }]))
+                }),
+                // Include services if present
+                ...(statistics.services && {
+                    services: statistics.services.map(s => ({ ...s }))
+                })
+            };
+        }
+        // Increment the appropriate counter
+        const counterMap = {
+            noun: this.statisticsCache.nounCount,
+            verb: this.statisticsCache.verbCount,
+            metadata: this.statisticsCache.metadataCount
+        };
+        const counter = counterMap[type];
+        counter[service] = (counter[service] || 0) + amount;
+        // Track service activity
+        this.trackServiceActivity(service, 'add');
+        // Update timestamp
+        this.statisticsCache.lastUpdated = new Date().toISOString();
+        // Schedule a batch update instead of saving immediately
+        this.scheduleBatchUpdate();
+    }
+    /**
+     * Track service activity (first/last activity, operation counts)
+     * @param service The service name
+     * @param operation The operation type
+     */
+    trackServiceActivity(service, operation) {
+        if (!this.statisticsCache) {
+            return;
+        }
+        // Initialize serviceActivity if it doesn't exist
+        if (!this.statisticsCache.serviceActivity) {
+            this.statisticsCache.serviceActivity = {};
+        }
+        const now = new Date().toISOString();
+        const activity = this.statisticsCache.serviceActivity[service];
+        if (!activity) {
+            // First activity for this service
+            this.statisticsCache.serviceActivity[service] = {
+                firstActivity: now,
+                lastActivity: now,
+                totalOperations: 1
+            };
+        }
+        else {
+            // Update existing activity
+            activity.lastActivity = now;
+            activity.totalOperations++;
+        }
+    }
+    /**
+     * Decrement a statistic counter
+     * @param type The type of statistic to decrement ('noun', 'verb', 'metadata')
+     * @param service The service that inserted the data
+     * @param amount The amount to decrement by (default: 1)
+     */
+    async decrementStatistic(type, service, amount = 1) {
+        // Get current statistics from cache or storage
+        let statistics = this.statisticsCache;
+        if (!statistics) {
+            statistics = await this.getStatisticsData();
+            if (!statistics) {
+                statistics = this.createDefaultStatistics();
+            }
+            // Update the cache
+            this.statisticsCache = {
+                nounCount: { ...statistics.nounCount },
+                verbCount: { ...statistics.verbCount },
+                metadataCount: { ...statistics.metadataCount },
+                hnswIndexSize: statistics.hnswIndexSize,
+                lastUpdated: statistics.lastUpdated,
+                // Include serviceActivity if present
+                ...(statistics.serviceActivity && {
+                    serviceActivity: Object.fromEntries(Object.entries(statistics.serviceActivity).map(([k, v]) => [k, { ...v }]))
+                }),
+                // Include services if present
+                ...(statistics.services && {
+                    services: statistics.services.map(s => ({ ...s }))
+                })
+            };
+        }
+        // Decrement the appropriate counter
+        const counterMap = {
+            noun: this.statisticsCache.nounCount,
+            verb: this.statisticsCache.verbCount,
+            metadata: this.statisticsCache.metadataCount
+        };
+        const counter = counterMap[type];
+        counter[service] = Math.max(0, (counter[service] || 0) - amount);
+        // Track service activity
+        this.trackServiceActivity(service, 'delete');
+        // Update timestamp
+        this.statisticsCache.lastUpdated = new Date().toISOString();
+        // Schedule a batch update instead of saving immediately
+        this.scheduleBatchUpdate();
+    }
+    /**
+     * Update the HNSW index size statistic
+     * @param size The new size of the HNSW index
+     */
+    async updateHnswIndexSize(size) {
+        // Get current statistics from cache or storage
+        let statistics = this.statisticsCache;
+        if (!statistics) {
+            statistics = await this.getStatisticsData();
+            if (!statistics) {
+                statistics = this.createDefaultStatistics();
+            }
+            // Update the cache
+            this.statisticsCache = {
+                nounCount: { ...statistics.nounCount },
+                verbCount: { ...statistics.verbCount },
+                metadataCount: { ...statistics.metadataCount },
+                hnswIndexSize: statistics.hnswIndexSize,
+                lastUpdated: statistics.lastUpdated,
+                // Include serviceActivity if present
+                ...(statistics.serviceActivity && {
+                    serviceActivity: Object.fromEntries(Object.entries(statistics.serviceActivity).map(([k, v]) => [k, { ...v }]))
+                }),
+                // Include services if present
+                ...(statistics.services && {
+                    services: statistics.services.map(s => ({ ...s }))
+                })
+            };
+        }
+        // Update HNSW index size
+        this.statisticsCache.hnswIndexSize = size;
+        // Update timestamp
+        this.statisticsCache.lastUpdated = new Date().toISOString();
+        // Schedule a batch update instead of saving immediately
+        this.scheduleBatchUpdate();
+    }
+    /**
+     * Force an immediate flush of statistics to storage
+     * This ensures that any pending statistics updates are written to persistent storage
+     */
+    async flushStatisticsToStorage() {
+        // If there are no statistics in cache or they haven't been modified, nothing to flush
+        if (!this.statisticsCache || !this.statisticsModified) {
+            return;
+        }
+        // Call the protected flushStatistics method to immediately write to storage
+        await this.flushStatistics();
+    }
+    /**
+     * Track field names from a JSON document
+     * @param jsonDocument The JSON document to extract field names from
+     * @param service The service that inserted the data
+     */
+    async trackFieldNames(jsonDocument, service) {
+        // Skip if not a JSON object
+        if (typeof jsonDocument !== 'object' || jsonDocument === null || Array.isArray(jsonDocument)) {
+            return;
+        }
+        // Get current statistics from cache or storage
+        let statistics = this.statisticsCache;
+        if (!statistics) {
+            statistics = await this.getStatisticsData();
+            if (!statistics) {
+                statistics = this.createDefaultStatistics();
+            }
+            // Update the cache
+            this.statisticsCache = {
+                ...statistics,
+                nounCount: { ...statistics.nounCount },
+                verbCount: { ...statistics.verbCount },
+                metadataCount: { ...statistics.metadataCount },
+                fieldNames: { ...statistics.fieldNames },
+                standardFieldMappings: { ...statistics.standardFieldMappings }
+            };
+        }
+        // Ensure fieldNames exists
+        if (!this.statisticsCache.fieldNames) {
+            this.statisticsCache.fieldNames = {};
+        }
+        // Ensure standardFieldMappings exists
+        if (!this.statisticsCache.standardFieldMappings) {
+            this.statisticsCache.standardFieldMappings = {};
+        }
+        // Extract field names from the JSON document
+        const fieldNames = extractFieldNamesFromJson(jsonDocument);
+        // Initialize service entry if it doesn't exist
+        if (!this.statisticsCache.fieldNames[service]) {
+            this.statisticsCache.fieldNames[service] = [];
+        }
+        // Add new field names to the service's list
+        for (const fieldName of fieldNames) {
+            if (!this.statisticsCache.fieldNames[service].includes(fieldName)) {
+                this.statisticsCache.fieldNames[service].push(fieldName);
+            }
+            // Map to standard field if possible
+            const standardField = mapToStandardField(fieldName);
+            if (standardField) {
+                // Initialize standard field entry if it doesn't exist
+                if (!this.statisticsCache.standardFieldMappings[standardField]) {
+                    this.statisticsCache.standardFieldMappings[standardField] = {};
+                }
+                // Initialize service entry if it doesn't exist
+                if (!this.statisticsCache.standardFieldMappings[standardField][service]) {
+                    this.statisticsCache.standardFieldMappings[standardField][service] = [];
+                }
+                // Add field name to standard field mapping if not already there
+                if (!this.statisticsCache.standardFieldMappings[standardField][service].includes(fieldName)) {
+                    this.statisticsCache.standardFieldMappings[standardField][service].push(fieldName);
+                }
+            }
+        }
+        // Update timestamp
+        this.statisticsCache.lastUpdated = new Date().toISOString();
+        // Schedule a batch update
+        this.statisticsModified = true;
+        this.scheduleBatchUpdate();
+    }
+    /**
+     * Get available field names by service
+     * @returns Record of field names by service
+     */
+    async getAvailableFieldNames() {
+        // Get current statistics from cache or storage
+        let statistics = this.statisticsCache;
+        if (!statistics) {
+            statistics = await this.getStatisticsData();
+            if (!statistics) {
+                return {};
+            }
+        }
+        // Return field names by service
+        return statistics.fieldNames || {};
+    }
+    /**
+     * Get standard field mappings
+     * @returns Record of standard field mappings
+     */
+    async getStandardFieldMappings() {
+        // Get current statistics from cache or storage
+        let statistics = this.statisticsCache;
+        if (!statistics) {
+            statistics = await this.getStatisticsData();
+            if (!statistics) {
+                return {};
+            }
+        }
+        // Return standard field mappings
+        return statistics.standardFieldMappings || {};
+    }
+    /**
+     * Create default statistics data
+     * @returns Default statistics data
+     */
+    createDefaultStatistics() {
+        return {
+            nounCount: {},
+            verbCount: {},
+            metadataCount: {},
+            hnswIndexSize: 0,
+            fieldNames: {},
+            standardFieldMappings: {},
+            lastUpdated: new Date().toISOString()
+        };
+    }
+    /**
+     * Detect if an error is a throttling error
+     * Override this method in specific adapters for custom detection
+     */
+    isThrottlingError(error) {
+        const statusCode = error.$metadata?.httpStatusCode || error.statusCode || error.code;
+        const message = error.message?.toLowerCase() || '';
+        return (statusCode === 429 || // Too Many Requests
+            statusCode === 503 || // Service Unavailable / Slow Down
+            statusCode === 'ECONNRESET' || // Connection reset
+            statusCode === 'ETIMEDOUT' || // Timeout
+            message.includes('throttl') ||
+            message.includes('slow down') ||
+            message.includes('rate limit') ||
+            message.includes('too many requests') ||
+            message.includes('quota exceeded'));
+    }
+    /**
+     * Track a throttling event
+     * @param error The error that caused throttling
+     * @param service Optional service that was throttled
+     */
+    trackThrottlingEvent(error, service) {
+        this.throttlingDetected = true;
+        this.consecutiveThrottleEvents++;
+        this.lastThrottleTime = Date.now();
+        this.totalThrottleEvents++;
+        // Track by hour
+        const hourIndex = new Date().getHours();
+        if (hourIndex !== this.lastThrottleHourIndex) {
+            // Reset hour tracking if we've moved to a new hour
+            this.throttleEventsByHour = new Array(24).fill(0);
+            this.lastThrottleHourIndex = hourIndex;
+        }
+        this.throttleEventsByHour[hourIndex]++;
+        // Track throttle reason
+        const reason = this.getThrottleReason(error);
+        this.throttleReasons[reason] = (this.throttleReasons[reason] || 0) + 1;
+        // Track service-level throttling
+        if (service) {
+            const serviceInfo = this.serviceThrottling.get(service) || {
+                throttleCount: 0,
+                lastThrottle: 0,
+                status: 'normal'
+            };
+            serviceInfo.throttleCount++;
+            serviceInfo.lastThrottle = Date.now();
+            serviceInfo.status = 'throttled';
+            this.serviceThrottling.set(service, serviceInfo);
+        }
+        // Exponential backoff
+        this.throttlingBackoffMs = Math.min(this.throttlingBackoffMs * 2, this.maxBackoffMs);
+    }
+    /**
+     * Get the reason for throttling from an error
+     */
+    getThrottleReason(error) {
+        const statusCode = error.$metadata?.httpStatusCode || error.statusCode || error.code;
+        if (statusCode === 429)
+            return '429_TooManyRequests';
+        if (statusCode === 503)
+            return '503_ServiceUnavailable';
+        if (statusCode === 'ECONNRESET')
+            return 'ConnectionReset';
+        if (statusCode === 'ETIMEDOUT')
+            return 'Timeout';
+        const message = error.message?.toLowerCase() || '';
+        if (message.includes('throttl'))
+            return 'Throttled';
+        if (message.includes('slow down'))
+            return 'SlowDown';
+        if (message.includes('rate limit'))
+            return 'RateLimit';
+        if (message.includes('quota exceeded'))
+            return 'QuotaExceeded';
+        return 'Unknown';
+    }
+    /**
+     * Clear throttling state after successful operations
+     */
+    clearThrottlingState() {
+        if (this.consecutiveThrottleEvents > 0) {
+            this.consecutiveThrottleEvents = 0;
+            this.throttlingBackoffMs = 1000; // Reset to initial backoff
+            if (this.throttlingDetected) {
+                this.throttlingDetected = false;
+                // Update service statuses
+                for (const [service, info] of this.serviceThrottling) {
+                    if (info.status === 'throttled') {
+                        info.status = 'recovering';
+                    }
+                    else if (info.status === 'recovering') {
+                        const timeSinceThrottle = Date.now() - info.lastThrottle;
+                        if (timeSinceThrottle > 60000) { // 1 minute recovery period
+                            info.status = 'normal';
+                        }
+                    }
+                }
+            }
+        }
+    }
+    /**
+     * Handle throttling by implementing exponential backoff
+     * @param error The error that triggered throttling
+     * @param service Optional service that was throttled
+     */
+    async handleThrottling(error, service) {
+        if (this.isThrottlingError(error)) {
+            this.trackThrottlingEvent(error, service);
+            // Add delay for retry
+            const delayMs = this.throttlingBackoffMs;
+            this.totalDelayMs += delayMs;
+            this.delayedOperations++;
+            await new Promise(resolve => setTimeout(resolve, delayMs));
+        }
+        else {
+            // Clear throttling state on non-throttling errors
+            this.clearThrottlingState();
+        }
+    }
+    /**
+     * Track a retried operation
+     */
+    trackRetriedOperation() {
+        this.retriedOperations++;
+    }
+    /**
+     * Track an operation that failed due to throttling
+     */
+    trackFailedDueToThrottling() {
+        this.failedDueToThrottling++;
+    }
+    /**
+     * Get current throttling metrics
+     */
+    getThrottlingMetrics() {
+        const averageDelayMs = this.delayedOperations > 0
+            ? this.totalDelayMs / this.delayedOperations
+            : 0;
+        // Convert service throttling map to record
+        const serviceThrottlingRecord = {};
+        for (const [service, info] of this.serviceThrottling) {
+            serviceThrottlingRecord[service] = {
+                throttleCount: info.throttleCount,
+                lastThrottle: new Date(info.lastThrottle).toISOString(),
+                status: info.status
+            };
+        }
+        return {
+            storage: {
+                currentlyThrottled: this.throttlingDetected,
+                lastThrottleTime: this.lastThrottleTime > 0
+                    ? new Date(this.lastThrottleTime).toISOString()
+                    : undefined,
+                consecutiveThrottleEvents: this.consecutiveThrottleEvents,
+                currentBackoffMs: this.throttlingBackoffMs,
+                totalThrottleEvents: this.totalThrottleEvents,
+                throttleEventsByHour: [...this.throttleEventsByHour],
+                throttleReasons: { ...this.throttleReasons }
+            },
+            operationImpact: {
+                delayedOperations: this.delayedOperations,
+                retriedOperations: this.retriedOperations,
+                failedDueToThrottling: this.failedDueToThrottling,
+                averageDelayMs,
+                totalDelayMs: this.totalDelayMs
+            },
+            serviceThrottling: Object.keys(serviceThrottlingRecord).length > 0
+                ? serviceThrottlingRecord
+                : undefined
+        };
+    }
+    /**
+     * Include throttling metrics in statistics
+     */
+    async getStatisticsWithThrottling() {
+        const stats = await this.getStatistics();
+        if (stats) {
+            stats.throttlingMetrics = this.getThrottlingMetrics();
+        }
+        return stats;
+    }
+}
+//# sourceMappingURL=baseStorageAdapter.js.map

dist/storage/adapters/batchS3Operations.d.ts

@@ -0,0 +1,71 @@
+/**
+ * Enhanced Batch S3 Operations for High-Performance Vector Retrieval
+ * Implements optimized batch operations to reduce S3 API calls and latency
+ */
+import { HNSWNoun } from '../../coreTypes.js';
+type S3Client = any;
+export interface BatchRetrievalOptions {
+    maxConcurrency?: number;
+    prefetchSize?: number;
+    useS3Select?: boolean;
+    compressionEnabled?: boolean;
+}
+export interface BatchResult<T> {
+    items: Map<string, T>;
+    errors: Map<string, Error>;
+    statistics: {
+        totalRequested: number;
+        totalRetrieved: number;
+        totalErrors: number;
+        duration: number;
+        apiCalls: number;
+    };
+}
+/**
+ * High-performance batch operations for S3-compatible storage
+ * Optimizes retrieval patterns for HNSW search operations
+ */
+export declare class BatchS3Operations {
+    private s3Client;
+    private bucketName;
+    private options;
+    constructor(s3Client: S3Client, bucketName: string, options?: BatchRetrievalOptions);
+    /**
+     * Batch retrieve HNSW nodes with intelligent prefetching
+     */
+    batchGetNodes(nodeIds: string[], prefix?: string): Promise<BatchResult<HNSWNoun>>;
+    /**
+     * Parallel GetObject operations for small batches
+     */
+    private parallelGetObjects;
+    /**
+     * Chunked parallel retrieval with intelligent batching
+     */
+    private chunkedParallelGet;
+    /**
+     * List-based batch retrieval for large datasets
+     * Uses S3 ListObjects to reduce API calls
+     */
+    private listBasedBatchGet;
+    /**
+     * Intelligent prefetch based on HNSW graph connectivity
+     */
+    prefetchConnectedNodes(currentNodeIds: string[], connectionMap: Map<string, Set<string>>, prefix?: string): Promise<BatchResult<HNSWNoun>>;
+    /**
+     * S3 Select-based retrieval for filtered queries
+     */
+    selectiveRetrieve(prefix: string, filter: {
+        vectorDimension?: number;
+        metadataKey?: string;
+        metadataValue?: any;
+    }): Promise<BatchResult<HNSWNoun>>;
+    /**
+     * Parse stored object from JSON string
+     */
+    private parseStoredObject;
+    /**
+     * Utility function to chunk arrays
+     */
+    private chunkArray;
+}
+export {};

dist/storage/adapters/batchS3Operations.js

@@ -0,0 +1,287 @@
+/**
+ * Enhanced Batch S3 Operations for High-Performance Vector Retrieval
+ * Implements optimized batch operations to reduce S3 API calls and latency
+ */
+/**
+ * High-performance batch operations for S3-compatible storage
+ * Optimizes retrieval patterns for HNSW search operations
+ */
+export class BatchS3Operations {
+    constructor(s3Client, bucketName, options = {}) {
+        this.s3Client = s3Client;
+        this.bucketName = bucketName;
+        this.options = {
+            maxConcurrency: 50, // AWS S3 rate limit friendly
+            prefetchSize: 100,
+            useS3Select: false,
+            compressionEnabled: false,
+            ...options
+        };
+    }
+    /**
+     * Batch retrieve HNSW nodes with intelligent prefetching
+     */
+    async batchGetNodes(nodeIds, prefix = 'nodes/') {
+        const startTime = Date.now();
+        const result = {
+            items: new Map(),
+            errors: new Map(),
+            statistics: {
+                totalRequested: nodeIds.length,
+                totalRetrieved: 0,
+                totalErrors: 0,
+                duration: 0,
+                apiCalls: 0
+            }
+        };
+        if (nodeIds.length === 0) {
+            result.statistics.duration = Date.now() - startTime;
+            return result;
+        }
+        // Use different strategies based on request size
+        if (nodeIds.length <= 10) {
+            // Small batch - use parallel GetObject
+            await this.parallelGetObjects(nodeIds, prefix, result);
+        }
+        else if (nodeIds.length <= 1000) {
+            // Medium batch - use chunked parallel with prefetching
+            await this.chunkedParallelGet(nodeIds, prefix, result);
+        }
+        else {
+            // Large batch - use S3 list-based approach with filtering
+            await this.listBasedBatchGet(nodeIds, prefix, result);
+        }
+        result.statistics.duration = Date.now() - startTime;
+        return result;
+    }
+    /**
+     * Parallel GetObject operations for small batches
+     */
+    async parallelGetObjects(ids, prefix, result) {
+        const { GetObjectCommand } = await import('@aws-sdk/client-s3');
+        const semaphore = new Semaphore(this.options.maxConcurrency);
+        const promises = ids.map(async (id) => {
+            await semaphore.acquire();
+            try {
+                result.statistics.apiCalls++;
+                const response = await this.s3Client.send(new GetObjectCommand({
+                    Bucket: this.bucketName,
+                    Key: `${prefix}${id}.json`
+                }));
+                if (response.Body) {
+                    const content = await response.Body.transformToString();
+                    const item = this.parseStoredObject(content);
+                    if (item) {
+                        result.items.set(id, item);
+                        result.statistics.totalRetrieved++;
+                    }
+                }
+            }
+            catch (error) {
+                result.errors.set(id, error);
+                result.statistics.totalErrors++;
+            }
+            finally {
+                semaphore.release();
+            }
+        });
+        await Promise.all(promises);
+    }
+    /**
+     * Chunked parallel retrieval with intelligent batching
+     */
+    async chunkedParallelGet(ids, prefix, result) {
+        const chunkSize = Math.min(50, Math.ceil(ids.length / 10));
+        const chunks = this.chunkArray(ids, chunkSize);
+        // Process chunks with controlled concurrency
+        const semaphore = new Semaphore(Math.min(5, chunks.length));
+        const chunkPromises = chunks.map(async (chunk) => {
+            await semaphore.acquire();
+            try {
+                await this.parallelGetObjects(chunk, prefix, result);
+            }
+            finally {
+                semaphore.release();
+            }
+        });
+        await Promise.all(chunkPromises);
+    }
+    /**
+     * List-based batch retrieval for large datasets
+     * Uses S3 ListObjects to reduce API calls
+     */
+    async listBasedBatchGet(ids, prefix, result) {
+        const { ListObjectsV2Command, GetObjectCommand } = await import('@aws-sdk/client-s3');
+        // Create a set for O(1) lookup
+        const idSet = new Set(ids);
+        // List objects with the prefix
+        let continuationToken;
+        const maxKeys = 1000;
+        do {
+            result.statistics.apiCalls++;
+            const listResponse = await this.s3Client.send(new ListObjectsV2Command({
+                Bucket: this.bucketName,
+                Prefix: prefix,
+                MaxKeys: maxKeys,
+                ContinuationToken: continuationToken
+            }));
+            if (listResponse.Contents) {
+                // Filter objects that match our requested IDs
+                const matchingObjects = listResponse.Contents.filter((obj) => {
+                    if (!obj.Key)
+                        return false;
+                    const id = obj.Key.replace(prefix, '').replace('.json', '');
+                    return idSet.has(id);
+                });
+                // Batch retrieve matching objects
+                const semaphore = new Semaphore(this.options.maxConcurrency);
+                const retrievalPromises = matchingObjects.map(async (obj) => {
+                    if (!obj.Key)
+                        return;
+                    await semaphore.acquire();
+                    try {
+                        result.statistics.apiCalls++;
+                        const response = await this.s3Client.send(new GetObjectCommand({
+                            Bucket: this.bucketName,
+                            Key: obj.Key
+                        }));
+                        if (response.Body) {
+                            const content = await response.Body.transformToString();
+                            const item = this.parseStoredObject(content);
+                            if (item) {
+                                const id = obj.Key.replace(prefix, '').replace('.json', '');
+                                result.items.set(id, item);
+                                result.statistics.totalRetrieved++;
+                            }
+                        }
+                    }
+                    catch (error) {
+                        const id = obj.Key.replace(prefix, '').replace('.json', '');
+                        result.errors.set(id, error);
+                        result.statistics.totalErrors++;
+                    }
+                    finally {
+                        semaphore.release();
+                    }
+                });
+                await Promise.all(retrievalPromises);
+            }
+            continuationToken = listResponse.NextContinuationToken;
+        } while (continuationToken && result.items.size < ids.length);
+    }
+    /**
+     * Intelligent prefetch based on HNSW graph connectivity
+     */
+    async prefetchConnectedNodes(currentNodeIds, connectionMap, prefix = 'nodes/') {
+        // Analyze connection patterns to predict next nodes
+        const predictedNodes = new Set();
+        for (const nodeId of currentNodeIds) {
+            const connections = connectionMap.get(nodeId);
+            if (connections) {
+                // Add immediate neighbors
+                connections.forEach(connId => predictedNodes.add(connId));
+                // Add second-degree neighbors (limited)
+                let count = 0;
+                for (const connId of connections) {
+                    if (count >= 5)
+                        break; // Limit prefetch scope
+                    const secondDegree = connectionMap.get(connId);
+                    if (secondDegree) {
+                        secondDegree.forEach(id => {
+                            if (count < 20) {
+                                predictedNodes.add(id);
+                                count++;
+                            }
+                        });
+                    }
+                }
+            }
+        }
+        // Remove nodes we already have
+        const nodesToPrefetch = Array.from(predictedNodes).filter(id => !currentNodeIds.includes(id));
+        return this.batchGetNodes(nodesToPrefetch.slice(0, this.options.prefetchSize), prefix);
+    }
+    /**
+     * S3 Select-based retrieval for filtered queries
+     */
+    async selectiveRetrieve(prefix, filter) {
+        // This would use S3 Select to filter objects server-side
+        // Reducing data transfer for large-scale operations
+        const startTime = Date.now();
+        const result = {
+            items: new Map(),
+            errors: new Map(),
+            statistics: {
+                totalRequested: 0,
+                totalRetrieved: 0,
+                totalErrors: 0,
+                duration: 0,
+                apiCalls: 0
+            }
+        };
+        // S3 Select implementation would go here
+        // For now, fall back to list-based approach
+        console.warn('S3 Select not implemented, falling back to list-based retrieval');
+        result.statistics.duration = Date.now() - startTime;
+        return result;
+    }
+    /**
+     * Parse stored object from JSON string
+     */
+    parseStoredObject(content) {
+        try {
+            const parsed = JSON.parse(content);
+            // Reconstruct HNSW node structure
+            if (parsed.connections && typeof parsed.connections === 'object') {
+                const connections = new Map();
+                for (const [level, nodeIds] of Object.entries(parsed.connections)) {
+                    connections.set(Number(level), new Set(nodeIds));
+                }
+                parsed.connections = connections;
+            }
+            return parsed;
+        }
+        catch (error) {
+            console.error('Failed to parse stored object:', error);
+            return null;
+        }
+    }
+    /**
+     * Utility function to chunk arrays
+     */
+    chunkArray(array, chunkSize) {
+        const chunks = [];
+        for (let i = 0; i < array.length; i += chunkSize) {
+            chunks.push(array.slice(i, i + chunkSize));
+        }
+        return chunks;
+    }
+}
+/**
+ * Simple semaphore implementation for concurrency control
+ */
+class Semaphore {
+    constructor(permits) {
+        this.waiting = [];
+        this.permits = permits;
+    }
+    async acquire() {
+        if (this.permits > 0) {
+            this.permits--;
+            return Promise.resolve();
+        }
+        return new Promise((resolve) => {
+            this.waiting.push(resolve);
+        });
+    }
+    release() {
+        if (this.waiting.length > 0) {
+            const resolve = this.waiting.shift();
+            resolve();
+        }
+        else {
+            this.permits++;
+        }
+    }
+}
+//# sourceMappingURL=batchS3Operations.js.map

dist/storage/adapters/fileSystemStorage.d.ts

@@ -0,0 +1,226 @@
+/**
+ * File System Storage Adapter
+ * File system storage adapter for Node.js environments
+ */
+import { GraphVerb, HNSWNoun, HNSWVerb, StatisticsData } from '../../coreTypes.js';
+import { BaseStorage } from '../baseStorage.js';
+type HNSWNode = HNSWNoun;
+type Edge = HNSWVerb;
+/**
+ * File system storage adapter for Node.js environments
+ * Uses the file system to store data in the specified directory structure
+ */
+export declare class FileSystemStorage extends BaseStorage {
+    private rootDir;
+    private nounsDir;
+    private verbsDir;
+    private metadataDir;
+    private nounMetadataDir;
+    private verbMetadataDir;
+    private indexDir;
+    private systemDir;
+    private lockDir;
+    private useDualWrite;
+    private activeLocks;
+    /**
+     * Initialize the storage adapter
+     * @param rootDirectory The root directory for storage
+     */
+    constructor(rootDirectory: string);
+    /**
+     * Initialize the storage adapter
+     */
+    init(): Promise<void>;
+    /**
+     * Check if a directory exists
+     */
+    private directoryExists;
+    /**
+     * Ensure a directory exists, creating it if necessary
+     */
+    private ensureDirectoryExists;
+    /**
+     * Save a node to storage
+     */
+    protected saveNode(node: HNSWNode): Promise<void>;
+    /**
+     * Get a node from storage
+     */
+    protected getNode(id: string): Promise<HNSWNode | null>;
+    /**
+     * Get all nodes from storage
+     */
+    protected getAllNodes(): Promise<HNSWNode[]>;
+    /**
+     * Get nodes by noun type
+     * @param nounType The noun type to filter by
+     * @returns Promise that resolves to an array of nodes of the specified noun type
+     */
+    protected getNodesByNounType(nounType: string): Promise<HNSWNode[]>;
+    /**
+     * Delete a node from storage
+     */
+    protected deleteNode(id: string): Promise<void>;
+    /**
+     * Save an edge to storage
+     */
+    protected saveEdge(edge: Edge): Promise<void>;
+    /**
+     * Get an edge from storage
+     */
+    protected getEdge(id: string): Promise<Edge | null>;
+    /**
+     * Get all edges from storage
+     */
+    protected getAllEdges(): Promise<Edge[]>;
+    /**
+     * Get edges by source
+     */
+    protected getEdgesBySource(sourceId: string): Promise<Edge[]>;
+    /**
+     * Get edges by target
+     */
+    protected getEdgesByTarget(targetId: string): Promise<Edge[]>;
+    /**
+     * Get edges by type
+     */
+    protected getEdgesByType(type: string): Promise<Edge[]>;
+    /**
+     * Delete an edge from storage
+     */
+    protected deleteEdge(id: string): Promise<void>;
+    /**
+     * Save metadata to storage
+     */
+    saveMetadata(id: string, metadata: any): Promise<void>;
+    /**
+     * Get metadata from storage
+     */
+    getMetadata(id: string): Promise<any | null>;
+    /**
+     * Get multiple metadata objects in batches (CRITICAL: Prevents socket exhaustion)
+     * FileSystem implementation uses controlled concurrency to prevent too many file reads
+     */
+    getMetadataBatch(ids: string[]): Promise<Map<string, any>>;
+    /**
+     * Save noun metadata to storage
+     */
+    saveNounMetadata(id: string, metadata: any): Promise<void>;
+    /**
+     * Get noun metadata from storage
+     */
+    getNounMetadata(id: string): Promise<any | null>;
+    /**
+     * Save verb metadata to storage
+     */
+    saveVerbMetadata(id: string, metadata: any): Promise<void>;
+    /**
+     * Get verb metadata from storage
+     */
+    getVerbMetadata(id: string): Promise<any | null>;
+    /**
+     * Get nouns with pagination support
+     * @param options Pagination options
+     */
+    getNounsWithPagination(options?: {
+        limit?: number;
+        cursor?: string;
+        filter?: any;
+    }): Promise<{
+        items: HNSWNoun[];
+        totalCount: number;
+        hasMore: boolean;
+        nextCursor?: string;
+    }>;
+    /**
+     * Clear all data from storage
+     */
+    clear(): Promise<void>;
+    /**
+     * Get information about storage usage and capacity
+     */
+    getStorageStatus(): Promise<{
+        type: string;
+        used: number;
+        quota: number | null;
+        details?: Record<string, any>;
+    }>;
+    /**
+     * Implementation of abstract methods from BaseStorage
+     */
+    /**
+     * Save a noun to storage
+     */
+    protected saveNoun_internal(noun: HNSWNoun): Promise<void>;
+    /**
+     * Get a noun from storage
+     */
+    protected getNoun_internal(id: string): Promise<HNSWNoun | null>;
+    /**
+     * Get nouns by noun type
+     */
+    protected getNounsByNounType_internal(nounType: string): Promise<HNSWNoun[]>;
+    /**
+     * Delete a noun from storage
+     */
+    protected deleteNoun_internal(id: string): Promise<void>;
+    /**
+     * Save a verb to storage
+     */
+    protected saveVerb_internal(verb: HNSWVerb): Promise<void>;
+    /**
+     * Get a verb from storage
+     */
+    protected getVerb_internal(id: string): Promise<HNSWVerb | null>;
+    /**
+     * Get verbs by source
+     */
+    protected getVerbsBySource_internal(sourceId: string): Promise<GraphVerb[]>;
+    /**
+     * Get verbs by target
+     */
+    protected getVerbsByTarget_internal(targetId: string): Promise<GraphVerb[]>;
+    /**
+     * Get verbs by type
+     */
+    protected getVerbsByType_internal(type: string): Promise<GraphVerb[]>;
+    /**
+     * Delete a verb from storage
+     */
+    protected deleteVerb_internal(id: string): Promise<void>;
+    /**
+     * Acquire a file-based lock for coordinating operations across multiple processes
+     * @param lockKey The key to lock on
+     * @param ttl Time to live for the lock in milliseconds (default: 30 seconds)
+     * @returns Promise that resolves to true if lock was acquired, false otherwise
+     */
+    private acquireLock;
+    /**
+     * Release a file-based lock
+     * @param lockKey The key to unlock
+     * @param lockValue The value used when acquiring the lock (for verification)
+     * @returns Promise that resolves when lock is released
+     */
+    private releaseLock;
+    /**
+     * Clean up expired lock files
+     */
+    private cleanupExpiredLocks;
+    /**
+     * Save statistics data to storage with file-based locking
+     */
+    protected saveStatisticsData(statistics: StatisticsData): Promise<void>;
+    /**
+     * Get statistics data from storage
+     */
+    protected getStatisticsData(): Promise<StatisticsData | null>;
+    /**
+     * Save statistics with backward compatibility (dual write)
+     */
+    private saveStatisticsWithBackwardCompat;
+    /**
+     * Get statistics with backward compatibility (dual read)
+     */
+    private getStatisticsWithBackwardCompat;
+}
+export {};

dist/storage/adapters/memoryStorage.d.ts

@@ -0,0 +1,172 @@
+/**
+ * Memory Storage Adapter
+ * In-memory storage adapter for environments where persistent storage is not available or needed
+ */
+import { GraphVerb, HNSWNoun, HNSWVerb, StatisticsData } from '../../coreTypes.js';
+import { BaseStorage } from '../baseStorage.js';
+import { PaginatedResult } from '../../types/paginationTypes.js';
+/**
+ * In-memory storage adapter
+ * Uses Maps to store data in memory
+ */
+export declare class MemoryStorage extends BaseStorage {
+    private nouns;
+    private verbs;
+    private metadata;
+    private nounMetadata;
+    private verbMetadata;
+    private statistics;
+    constructor();
+    /**
+     * Initialize the storage adapter
+     * Nothing to initialize for in-memory storage
+     */
+    init(): Promise<void>;
+    /**
+     * Save a noun to storage
+     */
+    protected saveNoun_internal(noun: HNSWNoun): Promise<void>;
+    /**
+     * Get a noun from storage
+     */
+    protected getNoun_internal(id: string): Promise<HNSWNoun | null>;
+    /**
+     * Get nouns with pagination and filtering
+     * @param options Pagination and filtering options
+     * @returns Promise that resolves to a paginated result of nouns
+     */
+    getNouns(options?: {
+        pagination?: {
+            offset?: number;
+            limit?: number;
+            cursor?: string;
+        };
+        filter?: {
+            nounType?: string | string[];
+            service?: string | string[];
+            metadata?: Record<string, any>;
+        };
+    }): Promise<PaginatedResult<HNSWNoun>>;
+    /**
+     * Get nouns with pagination - simplified interface for compatibility
+     */
+    getNounsWithPagination(options?: {
+        limit?: number;
+        cursor?: string;
+        filter?: any;
+    }): Promise<{
+        items: HNSWNoun[];
+        totalCount: number;
+        hasMore: boolean;
+        nextCursor?: string;
+    }>;
+    /**
+     * Get nouns by noun type
+     * @param nounType The noun type to filter by
+     * @returns Promise that resolves to an array of nouns of the specified noun type
+     * @deprecated Use getNouns() with filter.nounType instead
+     */
+    protected getNounsByNounType_internal(nounType: string): Promise<HNSWNoun[]>;
+    /**
+     * Delete a noun from storage
+     */
+    protected deleteNoun_internal(id: string): Promise<void>;
+    /**
+     * Save a verb to storage
+     */
+    protected saveVerb_internal(verb: HNSWVerb): Promise<void>;
+    /**
+     * Get a verb from storage
+     */
+    protected getVerb_internal(id: string): Promise<HNSWVerb | null>;
+    /**
+     * Get verbs with pagination and filtering
+     * @param options Pagination and filtering options
+     * @returns Promise that resolves to a paginated result of verbs
+     */
+    getVerbs(options?: {
+        pagination?: {
+            offset?: number;
+            limit?: number;
+            cursor?: string;
+        };
+        filter?: {
+            verbType?: string | string[];
+            sourceId?: string | string[];
+            targetId?: string | string[];
+            service?: string | string[];
+            metadata?: Record<string, any>;
+        };
+    }): Promise<PaginatedResult<GraphVerb>>;
+    /**
+     * Get verbs by source
+     * @deprecated Use getVerbs() with filter.sourceId instead
+     */
+    protected getVerbsBySource_internal(sourceId: string): Promise<GraphVerb[]>;
+    /**
+     * Get verbs by target
+     * @deprecated Use getVerbs() with filter.targetId instead
+     */
+    protected getVerbsByTarget_internal(targetId: string): Promise<GraphVerb[]>;
+    /**
+     * Get verbs by type
+     * @deprecated Use getVerbs() with filter.verbType instead
+     */
+    protected getVerbsByType_internal(type: string): Promise<GraphVerb[]>;
+    /**
+     * Delete a verb from storage
+     */
+    protected deleteVerb_internal(id: string): Promise<void>;
+    /**
+     * Save metadata to storage
+     */
+    saveMetadata(id: string, metadata: any): Promise<void>;
+    /**
+     * Get metadata from storage
+     */
+    getMetadata(id: string): Promise<any | null>;
+    /**
+     * Get multiple metadata objects in batches (CRITICAL: Prevents socket exhaustion)
+     * Memory storage implementation is simple since all data is already in memory
+     */
+    getMetadataBatch(ids: string[]): Promise<Map<string, any>>;
+    /**
+     * Save noun metadata to storage
+     */
+    saveNounMetadata(id: string, metadata: any): Promise<void>;
+    /**
+     * Get noun metadata from storage
+     */
+    getNounMetadata(id: string): Promise<any | null>;
+    /**
+     * Save verb metadata to storage
+     */
+    saveVerbMetadata(id: string, metadata: any): Promise<void>;
+    /**
+     * Get verb metadata from storage
+     */
+    getVerbMetadata(id: string): Promise<any | null>;
+    /**
+     * Clear all data from storage
+     */
+    clear(): Promise<void>;
+    /**
+     * Get information about storage usage and capacity
+     */
+    getStorageStatus(): Promise<{
+        type: string;
+        used: number;
+        quota: number | null;
+        details?: Record<string, any>;
+    }>;
+    /**
+     * Save statistics data to storage
+     * @param statistics The statistics data to save
+     */
+    protected saveStatisticsData(statistics: StatisticsData): Promise<void>;
+    /**
+     * Get statistics data from storage
+     * @returns Promise that resolves to the statistics data or null if not found
+     */
+    protected getStatisticsData(): Promise<StatisticsData | null>;
+}

dist/storage/adapters/memoryStorage.js

@@ -0,0 +1,548 @@
+/**
+ * Memory Storage Adapter
+ * In-memory storage adapter for environments where persistent storage is not available or needed
+ */
+import { BaseStorage } from '../baseStorage.js';
+// No type aliases needed - using the original types directly
+/**
+ * In-memory storage adapter
+ * Uses Maps to store data in memory
+ */
+export class MemoryStorage extends BaseStorage {
+    constructor() {
+        super();
+        // Single map of noun ID to noun
+        this.nouns = new Map();
+        this.verbs = new Map();
+        this.metadata = new Map();
+        this.nounMetadata = new Map();
+        this.verbMetadata = new Map();
+        this.statistics = null;
+    }
+    /**
+     * Initialize the storage adapter
+     * Nothing to initialize for in-memory storage
+     */
+    async init() {
+        this.isInitialized = true;
+    }
+    /**
+     * Save a noun to storage
+     */
+    async saveNoun_internal(noun) {
+        // Create a deep copy to avoid reference issues
+        const nounCopy = {
+            id: noun.id,
+            vector: [...noun.vector],
+            connections: new Map(),
+            level: noun.level || 0
+        };
+        // Copy connections
+        for (const [level, connections] of noun.connections.entries()) {
+            nounCopy.connections.set(level, new Set(connections));
+        }
+        // Save the noun directly in the nouns map
+        this.nouns.set(noun.id, nounCopy);
+    }
+    /**
+     * Get a noun from storage
+     */
+    async getNoun_internal(id) {
+        // Get the noun directly from the nouns map
+        const noun = this.nouns.get(id);
+        // If not found, return null
+        if (!noun) {
+            return null;
+        }
+        // Return a deep copy to avoid reference issues
+        const nounCopy = {
+            id: noun.id,
+            vector: [...noun.vector],
+            connections: new Map(),
+            level: noun.level || 0
+        };
+        // Copy connections
+        for (const [level, connections] of noun.connections.entries()) {
+            nounCopy.connections.set(level, new Set(connections));
+        }
+        return nounCopy;
+    }
+    /**
+     * Get nouns with pagination and filtering
+     * @param options Pagination and filtering options
+     * @returns Promise that resolves to a paginated result of nouns
+     */
+    async getNouns(options = {}) {
+        const pagination = options.pagination || {};
+        const filter = options.filter || {};
+        // Default values
+        const offset = pagination.offset || 0;
+        const limit = pagination.limit || 100;
+        // Convert string types to arrays for consistent handling
+        const nounTypes = filter.nounType
+            ? Array.isArray(filter.nounType) ? filter.nounType : [filter.nounType]
+            : undefined;
+        const services = filter.service
+            ? Array.isArray(filter.service) ? filter.service : [filter.service]
+            : undefined;
+        // First, collect all noun IDs that match the filter criteria
+        const matchingIds = [];
+        // Iterate through all nouns to find matches
+        for (const [nounId, noun] of this.nouns.entries()) {
+            // Get the metadata to check filters
+            const metadata = await this.getMetadata(nounId);
+            if (!metadata)
+                continue;
+            // Filter by noun type if specified
+            if (nounTypes && !nounTypes.includes(metadata.noun)) {
+                continue;
+            }
+            // Filter by service if specified
+            if (services && metadata.service && !services.includes(metadata.service)) {
+                continue;
+            }
+            // Filter by metadata fields if specified
+            if (filter.metadata) {
+                let metadataMatch = true;
+                for (const [key, value] of Object.entries(filter.metadata)) {
+                    if (metadata[key] !== value) {
+                        metadataMatch = false;
+                        break;
+                    }
+                }
+                if (!metadataMatch)
+                    continue;
+            }
+            // If we got here, the noun matches all filters
+            matchingIds.push(nounId);
+        }
+        // Calculate pagination
+        const totalCount = matchingIds.length;
+        const paginatedIds = matchingIds.slice(offset, offset + limit);
+        const hasMore = offset + limit < totalCount;
+        // Create cursor for next page if there are more results
+        const nextCursor = hasMore ? `${offset + limit}` : undefined;
+        // Fetch the actual nouns for the current page
+        const items = [];
+        for (const id of paginatedIds) {
+            const noun = this.nouns.get(id);
+            if (!noun)
+                continue;
+            // Create a deep copy to avoid reference issues
+            const nounCopy = {
+                id: noun.id,
+                vector: [...noun.vector],
+                connections: new Map(),
+                level: noun.level || 0
+            };
+            // Copy connections
+            for (const [level, connections] of noun.connections.entries()) {
+                nounCopy.connections.set(level, new Set(connections));
+            }
+            items.push(nounCopy);
+        }
+        return {
+            items,
+            totalCount,
+            hasMore,
+            nextCursor
+        };
+    }
+    /**
+     * Get nouns with pagination - simplified interface for compatibility
+     */
+    async getNounsWithPagination(options = {}) {
+        // Convert to the getNouns format
+        const result = await this.getNouns({
+            pagination: {
+                offset: options.cursor ? parseInt(options.cursor) : 0,
+                limit: options.limit || 100
+            },
+            filter: options.filter
+        });
+        return {
+            items: result.items,
+            totalCount: result.totalCount || 0,
+            hasMore: result.hasMore,
+            nextCursor: result.nextCursor
+        };
+    }
+    /**
+     * Get nouns by noun type
+     * @param nounType The noun type to filter by
+     * @returns Promise that resolves to an array of nouns of the specified noun type
+     * @deprecated Use getNouns() with filter.nounType instead
+     */
+    async getNounsByNounType_internal(nounType) {
+        const result = await this.getNouns({
+            filter: {
+                nounType
+            }
+        });
+        return result.items;
+    }
+    /**
+     * Delete a noun from storage
+     */
+    async deleteNoun_internal(id) {
+        this.nouns.delete(id);
+    }
+    /**
+     * Save a verb to storage
+     */
+    async saveVerb_internal(verb) {
+        // Create a deep copy to avoid reference issues
+        const verbCopy = {
+            id: verb.id,
+            vector: [...verb.vector],
+            connections: new Map()
+        };
+        // Copy connections
+        for (const [level, connections] of verb.connections.entries()) {
+            verbCopy.connections.set(level, new Set(connections));
+        }
+        // Save the verb directly in the verbs map
+        this.verbs.set(verb.id, verbCopy);
+    }
+    /**
+     * Get a verb from storage
+     */
+    async getVerb_internal(id) {
+        // Get the verb directly from the verbs map
+        const verb = this.verbs.get(id);
+        // If not found, return null
+        if (!verb) {
+            return null;
+        }
+        // Create default timestamp if not present
+        const defaultTimestamp = {
+            seconds: Math.floor(Date.now() / 1000),
+            nanoseconds: (Date.now() % 1000) * 1000000
+        };
+        // Create default createdBy if not present
+        const defaultCreatedBy = {
+            augmentation: 'unknown',
+            version: '1.0'
+        };
+        // Return a deep copy of the HNSWVerb
+        const verbCopy = {
+            id: verb.id,
+            vector: [...verb.vector],
+            connections: new Map()
+        };
+        // Copy connections
+        for (const [level, connections] of verb.connections.entries()) {
+            verbCopy.connections.set(level, new Set(connections));
+        }
+        return verbCopy;
+    }
+    /**
+     * Get verbs with pagination and filtering
+     * @param options Pagination and filtering options
+     * @returns Promise that resolves to a paginated result of verbs
+     */
+    async getVerbs(options = {}) {
+        const pagination = options.pagination || {};
+        const filter = options.filter || {};
+        // Default values
+        const offset = pagination.offset || 0;
+        const limit = pagination.limit || 100;
+        // Convert string types to arrays for consistent handling
+        const verbTypes = filter.verbType
+            ? Array.isArray(filter.verbType) ? filter.verbType : [filter.verbType]
+            : undefined;
+        const sourceIds = filter.sourceId
+            ? Array.isArray(filter.sourceId) ? filter.sourceId : [filter.sourceId]
+            : undefined;
+        const targetIds = filter.targetId
+            ? Array.isArray(filter.targetId) ? filter.targetId : [filter.targetId]
+            : undefined;
+        const services = filter.service
+            ? Array.isArray(filter.service) ? filter.service : [filter.service]
+            : undefined;
+        // First, collect all verb IDs that match the filter criteria
+        const matchingIds = [];
+        // Iterate through all verbs to find matches
+        for (const [verbId, hnswVerb] of this.verbs.entries()) {
+            // Get the metadata for this verb to do filtering
+            const metadata = this.verbMetadata.get(verbId);
+            // Filter by verb type if specified
+            if (verbTypes && metadata && !verbTypes.includes(metadata.type || metadata.verb || '')) {
+                continue;
+            }
+            // Filter by source ID if specified
+            if (sourceIds && metadata && !sourceIds.includes(metadata.sourceId || metadata.source || '')) {
+                continue;
+            }
+            // Filter by target ID if specified
+            if (targetIds && metadata && !targetIds.includes(metadata.targetId || metadata.target || '')) {
+                continue;
+            }
+            // Filter by metadata fields if specified
+            if (filter.metadata && metadata && metadata.data) {
+                let metadataMatch = true;
+                for (const [key, value] of Object.entries(filter.metadata)) {
+                    if (metadata.data[key] !== value) {
+                        metadataMatch = false;
+                        break;
+                    }
+                }
+                if (!metadataMatch)
+                    continue;
+            }
+            // Filter by service if specified
+            if (services && metadata && metadata.createdBy && metadata.createdBy.augmentation &&
+                !services.includes(metadata.createdBy.augmentation)) {
+                continue;
+            }
+            // If we got here, the verb matches all filters
+            matchingIds.push(verbId);
+        }
+        // Calculate pagination
+        const totalCount = matchingIds.length;
+        const paginatedIds = matchingIds.slice(offset, offset + limit);
+        const hasMore = offset + limit < totalCount;
+        // Create cursor for next page if there are more results
+        const nextCursor = hasMore ? `${offset + limit}` : undefined;
+        // Fetch the actual verbs for the current page
+        const items = [];
+        for (const id of paginatedIds) {
+            const hnswVerb = this.verbs.get(id);
+            const metadata = this.verbMetadata.get(id);
+            if (!hnswVerb)
+                continue;
+            if (!metadata) {
+                console.warn(`Verb ${id} found but no metadata - creating minimal GraphVerb`);
+                // Return minimal GraphVerb if metadata is missing
+                items.push({
+                    id: hnswVerb.id,
+                    vector: hnswVerb.vector,
+                    sourceId: '',
+                    targetId: ''
+                });
+                continue;
+            }
+            // Create a complete GraphVerb by combining HNSWVerb with metadata
+            const graphVerb = {
+                id: hnswVerb.id,
+                vector: [...hnswVerb.vector],
+                sourceId: metadata.sourceId,
+                targetId: metadata.targetId,
+                source: metadata.source,
+                target: metadata.target,
+                verb: metadata.verb,
+                type: metadata.type,
+                weight: metadata.weight,
+                createdAt: metadata.createdAt,
+                updatedAt: metadata.updatedAt,
+                createdBy: metadata.createdBy,
+                data: metadata.data,
+                metadata: metadata.data // Alias for backward compatibility
+            };
+            items.push(graphVerb);
+        }
+        return {
+            items,
+            totalCount,
+            hasMore,
+            nextCursor
+        };
+    }
+    /**
+     * Get verbs by source
+     * @deprecated Use getVerbs() with filter.sourceId instead
+     */
+    async getVerbsBySource_internal(sourceId) {
+        const result = await this.getVerbs({
+            filter: {
+                sourceId
+            }
+        });
+        return result.items;
+    }
+    /**
+     * Get verbs by target
+     * @deprecated Use getVerbs() with filter.targetId instead
+     */
+    async getVerbsByTarget_internal(targetId) {
+        const result = await this.getVerbs({
+            filter: {
+                targetId
+            }
+        });
+        return result.items;
+    }
+    /**
+     * Get verbs by type
+     * @deprecated Use getVerbs() with filter.verbType instead
+     */
+    async getVerbsByType_internal(type) {
+        const result = await this.getVerbs({
+            filter: {
+                verbType: type
+            }
+        });
+        return result.items;
+    }
+    /**
+     * Delete a verb from storage
+     */
+    async deleteVerb_internal(id) {
+        // Delete the verb directly from the verbs map
+        this.verbs.delete(id);
+    }
+    /**
+     * Save metadata to storage
+     */
+    async saveMetadata(id, metadata) {
+        this.metadata.set(id, JSON.parse(JSON.stringify(metadata)));
+    }
+    /**
+     * Get metadata from storage
+     */
+    async getMetadata(id) {
+        const metadata = this.metadata.get(id);
+        if (!metadata) {
+            return null;
+        }
+        return JSON.parse(JSON.stringify(metadata));
+    }
+    /**
+     * Get multiple metadata objects in batches (CRITICAL: Prevents socket exhaustion)
+     * Memory storage implementation is simple since all data is already in memory
+     */
+    async getMetadataBatch(ids) {
+        const results = new Map();
+        // Memory storage can handle all IDs at once since it's in-memory
+        for (const id of ids) {
+            const metadata = this.metadata.get(id);
+            if (metadata) {
+                // Deep clone to prevent mutation
+                results.set(id, JSON.parse(JSON.stringify(metadata)));
+            }
+        }
+        return results;
+    }
+    /**
+     * Save noun metadata to storage
+     */
+    async saveNounMetadata(id, metadata) {
+        this.nounMetadata.set(id, JSON.parse(JSON.stringify(metadata)));
+    }
+    /**
+     * Get noun metadata from storage
+     */
+    async getNounMetadata(id) {
+        const metadata = this.nounMetadata.get(id);
+        if (!metadata) {
+            return null;
+        }
+        return JSON.parse(JSON.stringify(metadata));
+    }
+    /**
+     * Save verb metadata to storage
+     */
+    async saveVerbMetadata(id, metadata) {
+        this.verbMetadata.set(id, JSON.parse(JSON.stringify(metadata)));
+    }
+    /**
+     * Get verb metadata from storage
+     */
+    async getVerbMetadata(id) {
+        const metadata = this.verbMetadata.get(id);
+        if (!metadata) {
+            return null;
+        }
+        return JSON.parse(JSON.stringify(metadata));
+    }
+    /**
+     * Clear all data from storage
+     */
+    async clear() {
+        this.nouns.clear();
+        this.verbs.clear();
+        this.metadata.clear();
+        this.nounMetadata.clear();
+        this.verbMetadata.clear();
+        this.statistics = null;
+        // Clear the statistics cache
+        this.statisticsCache = null;
+        this.statisticsModified = false;
+    }
+    /**
+     * Get information about storage usage and capacity
+     */
+    async getStorageStatus() {
+        return {
+            type: 'memory',
+            used: 0, // In-memory storage doesn't have a meaningful size
+            quota: null, // In-memory storage doesn't have a quota
+            details: {
+                nodeCount: this.nouns.size,
+                edgeCount: this.verbs.size,
+                metadataCount: this.metadata.size
+            }
+        };
+    }
+    /**
+     * Save statistics data to storage
+     * @param statistics The statistics data to save
+     */
+    async saveStatisticsData(statistics) {
+        // For memory storage, we just need to store the statistics in memory
+        // Create a deep copy to avoid reference issues
+        this.statistics = {
+            nounCount: { ...statistics.nounCount },
+            verbCount: { ...statistics.verbCount },
+            metadataCount: { ...statistics.metadataCount },
+            hnswIndexSize: statistics.hnswIndexSize,
+            lastUpdated: statistics.lastUpdated,
+            // Include serviceActivity if present
+            ...(statistics.serviceActivity && {
+                serviceActivity: Object.fromEntries(Object.entries(statistics.serviceActivity).map(([k, v]) => [k, { ...v }]))
+            }),
+            // Include services if present
+            ...(statistics.services && {
+                services: statistics.services.map(s => ({ ...s }))
+            }),
+            // Include distributedConfig if present
+            ...(statistics.distributedConfig && {
+                distributedConfig: JSON.parse(JSON.stringify(statistics.distributedConfig))
+            })
+        };
+        // Since this is in-memory, there's no need for time-based partitioning
+        // or legacy file handling
+    }
+    /**
+     * Get statistics data from storage
+     * @returns Promise that resolves to the statistics data or null if not found
+     */
+    async getStatisticsData() {
+        if (!this.statistics) {
+            return null;
+        }
+        // Return a deep copy to avoid reference issues
+        return {
+            nounCount: { ...this.statistics.nounCount },
+            verbCount: { ...this.statistics.verbCount },
+            metadataCount: { ...this.statistics.metadataCount },
+            hnswIndexSize: this.statistics.hnswIndexSize,
+            lastUpdated: this.statistics.lastUpdated,
+            // Include serviceActivity if present
+            ...(this.statistics.serviceActivity && {
+                serviceActivity: Object.fromEntries(Object.entries(this.statistics.serviceActivity).map(([k, v]) => [k, { ...v }]))
+            }),
+            // Include services if present
+            ...(this.statistics.services && {
+                services: this.statistics.services.map(s => ({ ...s }))
+            }),
+            // Include distributedConfig if present
+            ...(this.statistics.distributedConfig && {
+                distributedConfig: JSON.parse(JSON.stringify(this.statistics.distributedConfig))
+            })
+        };
+        // Since this is in-memory, there's no need for fallback mechanisms
+        // to check multiple storage locations
+    }
+}
+//# sourceMappingURL=memoryStorage.js.map

dist/storage/adapters/opfsStorage.d.ts

@@ -0,0 +1,258 @@
+/**
+ * OPFS (Origin Private File System) Storage Adapter
+ * Provides persistent storage for the vector database using the Origin Private File System API
+ */
+import { GraphVerb, HNSWNoun, HNSWVerb, StatisticsData } from '../../coreTypes.js';
+import { BaseStorage } from '../baseStorage.js';
+import '../../types/fileSystemTypes.js';
+type HNSWNode = HNSWNoun;
+/**
+ * Type alias for HNSWVerb to make the code more readable
+ */
+type Edge = HNSWVerb;
+type HNSWNoun_internal = HNSWNoun;
+/**
+ * OPFS storage adapter for browser environments
+ * Uses the Origin Private File System API to store data persistently
+ */
+export declare class OPFSStorage extends BaseStorage {
+    private rootDir;
+    private nounsDir;
+    private verbsDir;
+    private metadataDir;
+    private nounMetadataDir;
+    private verbMetadataDir;
+    private indexDir;
+    private isAvailable;
+    private isPersistentRequested;
+    private isPersistentGranted;
+    private statistics;
+    private activeLocks;
+    private lockPrefix;
+    constructor();
+    /**
+     * Initialize the storage adapter
+     */
+    init(): Promise<void>;
+    /**
+     * Check if OPFS is available in the current environment
+     */
+    isOPFSAvailable(): boolean;
+    /**
+     * Request persistent storage permission from the user
+     * @returns Promise that resolves to true if permission was granted, false otherwise
+     */
+    requestPersistentStorage(): Promise<boolean>;
+    /**
+     * Check if persistent storage is granted
+     * @returns Promise that resolves to true if persistent storage is granted, false otherwise
+     */
+    isPersistent(): Promise<boolean>;
+    /**
+     * Save a noun to storage
+     */
+    protected saveNoun_internal(noun: HNSWNoun_internal): Promise<void>;
+    /**
+     * Get a noun from storage
+     */
+    protected getNoun_internal(id: string): Promise<HNSWNoun_internal | null>;
+    /**
+     * Get nouns by noun type (internal implementation)
+     * @param nounType The noun type to filter by
+     * @returns Promise that resolves to an array of nouns of the specified noun type
+     */
+    protected getNounsByNounType_internal(nounType: string): Promise<HNSWNoun[]>;
+    /**
+     * Get nodes by noun type
+     * @param nounType The noun type to filter by
+     * @returns Promise that resolves to an array of nodes of the specified noun type
+     */
+    protected getNodesByNounType(nounType: string): Promise<HNSWNode[]>;
+    /**
+     * Delete a noun from storage (internal implementation)
+     */
+    protected deleteNoun_internal(id: string): Promise<void>;
+    /**
+     * Delete a node from storage
+     */
+    protected deleteNode(id: string): Promise<void>;
+    /**
+     * Save a verb to storage (internal implementation)
+     */
+    protected saveVerb_internal(verb: HNSWVerb): Promise<void>;
+    /**
+     * Save an edge to storage
+     */
+    protected saveEdge(edge: Edge): Promise<void>;
+    /**
+     * Get a verb from storage (internal implementation)
+     */
+    protected getVerb_internal(id: string): Promise<HNSWVerb | null>;
+    /**
+     * Get an edge from storage
+     */
+    protected getEdge(id: string): Promise<Edge | null>;
+    /**
+     * Get all edges from storage
+     */
+    protected getAllEdges(): Promise<Edge[]>;
+    /**
+     * Get verbs by source (internal implementation)
+     */
+    protected getVerbsBySource_internal(sourceId: string): Promise<GraphVerb[]>;
+    /**
+     * Get edges by source
+     */
+    protected getEdgesBySource(sourceId: string): Promise<GraphVerb[]>;
+    /**
+     * Get verbs by target (internal implementation)
+     */
+    protected getVerbsByTarget_internal(targetId: string): Promise<GraphVerb[]>;
+    /**
+     * Get edges by target
+     */
+    protected getEdgesByTarget(targetId: string): Promise<GraphVerb[]>;
+    /**
+     * Get verbs by type (internal implementation)
+     */
+    protected getVerbsByType_internal(type: string): Promise<GraphVerb[]>;
+    /**
+     * Get edges by type
+     */
+    protected getEdgesByType(type: string): Promise<GraphVerb[]>;
+    /**
+     * Delete a verb from storage (internal implementation)
+     */
+    protected deleteVerb_internal(id: string): Promise<void>;
+    /**
+     * Delete an edge from storage
+     */
+    protected deleteEdge(id: string): Promise<void>;
+    /**
+     * Save metadata to storage
+     */
+    saveMetadata(id: string, metadata: any): Promise<void>;
+    /**
+     * Get metadata from storage
+     */
+    getMetadata(id: string): Promise<any | null>;
+    /**
+     * Get multiple metadata objects in batches (CRITICAL: Prevents socket exhaustion)
+     * OPFS implementation uses controlled concurrency for file operations
+     */
+    getMetadataBatch(ids: string[]): Promise<Map<string, any>>;
+    /**
+     * Save verb metadata to storage
+     */
+    saveVerbMetadata(id: string, metadata: any): Promise<void>;
+    /**
+     * Get verb metadata from storage
+     */
+    getVerbMetadata(id: string): Promise<any | null>;
+    /**
+     * Save noun metadata to storage
+     */
+    saveNounMetadata(id: string, metadata: any): Promise<void>;
+    /**
+     * Get noun metadata from storage
+     */
+    getNounMetadata(id: string): Promise<any | null>;
+    /**
+     * Clear all data from storage
+     */
+    clear(): Promise<void>;
+    /**
+     * Get information about storage usage and capacity
+     */
+    getStorageStatus(): Promise<{
+        type: string;
+        used: number;
+        quota: number | null;
+        details?: Record<string, any>;
+    }>;
+    /**
+     * Get the statistics key for a specific date
+     * @param date The date to get the key for
+     * @returns The statistics key for the specified date
+     */
+    private getStatisticsKeyForDate;
+    /**
+     * Get the current statistics key
+     * @returns The current statistics key
+     */
+    private getCurrentStatisticsKey;
+    /**
+     * Get the legacy statistics key (for backward compatibility)
+     * @returns The legacy statistics key
+     */
+    private getLegacyStatisticsKey;
+    /**
+     * Acquire a browser-based lock for coordinating operations across multiple tabs
+     * @param lockKey The key to lock on
+     * @param ttl Time to live for the lock in milliseconds (default: 30 seconds)
+     * @returns Promise that resolves to true if lock was acquired, false otherwise
+     */
+    private acquireLock;
+    /**
+     * Release a browser-based lock
+     * @param lockKey The key to unlock
+     * @param lockValue The value used when acquiring the lock (for verification)
+     * @returns Promise that resolves when lock is released
+     */
+    private releaseLock;
+    /**
+     * Clean up expired locks from localStorage
+     */
+    private cleanupExpiredLocks;
+    /**
+     * Save statistics data to storage with browser-based locking
+     * @param statistics The statistics data to save
+     */
+    protected saveStatisticsData(statistics: StatisticsData): Promise<void>;
+    /**
+     * Get statistics data from storage
+     * @returns Promise that resolves to the statistics data or null if not found
+     */
+    protected getStatisticsData(): Promise<StatisticsData | null>;
+    /**
+     * Get nouns with pagination support
+     * @param options Pagination and filter options
+     * @returns Promise that resolves to a paginated result of nouns
+     */
+    getNounsWithPagination(options?: {
+        limit?: number;
+        cursor?: string;
+        filter?: {
+            nounType?: string | string[];
+            service?: string | string[];
+            metadata?: Record<string, any>;
+        };
+    }): Promise<{
+        items: HNSWNoun[];
+        totalCount?: number;
+        hasMore: boolean;
+        nextCursor?: string;
+    }>;
+    /**
+     * Get verbs with pagination support
+     * @param options Pagination and filter options
+     * @returns Promise that resolves to a paginated result of verbs
+     */
+    getVerbsWithPagination(options?: {
+        limit?: number;
+        cursor?: string;
+        filter?: {
+            verbType?: string | string[];
+            sourceId?: string | string[];
+            targetId?: string | string[];
+            service?: string | string[];
+            metadata?: Record<string, any>;
+        };
+    }): Promise<{
+        items: GraphVerb[];
+        totalCount?: number;
+        hasMore: boolean;
+        nextCursor?: string;
+    }>;
+}
+export {};

dist/storage/adapters/optimizedS3Search.d.ts

@@ -0,0 +1,79 @@
+/**
+ * Optimized S3 Search and Pagination
+ * Provides efficient search and pagination capabilities for S3-compatible storage
+ */
+import { HNSWNoun, GraphVerb } from '../../coreTypes.js';
+/**
+ * Pagination result interface
+ */
+export interface PaginationResult<T> {
+    items: T[];
+    totalCount?: number;
+    hasMore: boolean;
+    nextCursor?: string;
+}
+/**
+ * Filter interface for nouns
+ */
+export interface NounFilter {
+    nounType?: string | string[];
+    service?: string | string[];
+    metadata?: Record<string, any>;
+}
+/**
+ * Filter interface for verbs
+ */
+export interface VerbFilter {
+    verbType?: string | string[];
+    sourceId?: string | string[];
+    targetId?: string | string[];
+    service?: string | string[];
+    metadata?: Record<string, any>;
+}
+/**
+ * Interface for storage operations needed by optimized search
+ */
+export interface StorageOperations {
+    listObjectKeys(prefix: string, limit: number, cursor?: string): Promise<{
+        keys: string[];
+        hasMore: boolean;
+        nextCursor?: string;
+    }>;
+    getObject<T>(key: string): Promise<T | null>;
+    getMetadata(id: string, type: 'noun' | 'verb'): Promise<any | null>;
+}
+/**
+ * Optimized search implementation for S3-compatible storage
+ */
+export declare class OptimizedS3Search {
+    private storage;
+    constructor(storage: StorageOperations);
+    /**
+     * Get nouns with optimized pagination and filtering
+     */
+    getNounsWithPagination(options?: {
+        limit?: number;
+        cursor?: string;
+        filter?: NounFilter;
+    }): Promise<PaginationResult<HNSWNoun>>;
+    /**
+     * Get verbs with optimized pagination and filtering
+     */
+    getVerbsWithPagination(options?: {
+        limit?: number;
+        cursor?: string;
+        filter?: VerbFilter;
+    }): Promise<PaginationResult<GraphVerb>>;
+    /**
+     * Check if a noun matches the filter criteria
+     */
+    private matchesNounFilter;
+    /**
+     * Check if a verb matches the filter criteria
+     */
+    private matchesVerbFilter;
+    /**
+     * Combine HNSWVerb data with metadata to create GraphVerb
+     */
+    private combineVerbWithMetadata;
+}

dist/storage/adapters/optimizedS3Search.js

@@ -0,0 +1,249 @@
+/**
+ * Optimized S3 Search and Pagination
+ * Provides efficient search and pagination capabilities for S3-compatible storage
+ */
+import { createModuleLogger } from '../../utils/logger.js';
+import { getDirectoryPath } from '../baseStorage.js';
+const logger = createModuleLogger('OptimizedS3Search');
+/**
+ * Optimized search implementation for S3-compatible storage
+ */
+export class OptimizedS3Search {
+    constructor(storage) {
+        this.storage = storage;
+    }
+    /**
+     * Get nouns with optimized pagination and filtering
+     */
+    async getNounsWithPagination(options = {}) {
+        const limit = options.limit || 100;
+        const cursor = options.cursor;
+        try {
+            // List noun objects with pagination
+            const listResult = await this.storage.listObjectKeys(`${getDirectoryPath('noun', 'vector')}/`, limit * 2, cursor);
+            if (!listResult.keys.length) {
+                return {
+                    items: [],
+                    hasMore: false
+                };
+            }
+            // Load nouns in parallel batches
+            const nouns = [];
+            const batchSize = 10;
+            for (let i = 0; i < listResult.keys.length && nouns.length < limit; i += batchSize) {
+                const batch = listResult.keys.slice(i, i + batchSize);
+                const batchPromises = batch.map(key => this.storage.getObject(key));
+                const batchResults = await Promise.all(batchPromises);
+                for (const noun of batchResults) {
+                    if (!noun)
+                        continue;
+                    // Apply filters
+                    if (options.filter && !(await this.matchesNounFilter(noun, options.filter))) {
+                        continue;
+                    }
+                    nouns.push(noun);
+                    if (nouns.length >= limit) {
+                        break;
+                    }
+                }
+            }
+            // Determine if there are more items
+            const hasMore = listResult.hasMore || nouns.length >= limit;
+            // Set next cursor
+            let nextCursor;
+            if (hasMore && nouns.length > 0) {
+                nextCursor = nouns[nouns.length - 1].id;
+            }
+            return {
+                items: nouns.slice(0, limit),
+                hasMore,
+                nextCursor
+            };
+        }
+        catch (error) {
+            logger.error('Failed to get nouns with pagination:', error);
+            return {
+                items: [],
+                hasMore: false
+            };
+        }
+    }
+    /**
+     * Get verbs with optimized pagination and filtering
+     */
+    async getVerbsWithPagination(options = {}) {
+        const limit = options.limit || 100;
+        const cursor = options.cursor;
+        try {
+            // List verb objects with pagination
+            const listResult = await this.storage.listObjectKeys(`${getDirectoryPath('verb', 'vector')}/`, limit * 2, cursor);
+            if (!listResult.keys.length) {
+                return {
+                    items: [],
+                    hasMore: false
+                };
+            }
+            // Load verbs in parallel batches
+            const verbs = [];
+            const batchSize = 10;
+            for (let i = 0; i < listResult.keys.length && verbs.length < limit; i += batchSize) {
+                const batch = listResult.keys.slice(i, i + batchSize);
+                // Load verbs and their metadata in parallel
+                const batchPromises = batch.map(async (key) => {
+                    const verbData = await this.storage.getObject(key);
+                    if (!verbData)
+                        return null;
+                    // Get metadata
+                    const verbId = key.replace(`${getDirectoryPath('verb', 'vector')}/`, '').replace('.json', '');
+                    const metadata = await this.storage.getMetadata(verbId, 'verb');
+                    // Combine into GraphVerb
+                    return this.combineVerbWithMetadata(verbData, metadata);
+                });
+                const batchResults = await Promise.all(batchPromises);
+                for (const verb of batchResults) {
+                    if (!verb)
+                        continue;
+                    // Apply filters
+                    if (options.filter && !this.matchesVerbFilter(verb, options.filter)) {
+                        continue;
+                    }
+                    verbs.push(verb);
+                    if (verbs.length >= limit) {
+                        break;
+                    }
+                }
+            }
+            // Determine if there are more items
+            const hasMore = listResult.hasMore || verbs.length >= limit;
+            // Set next cursor
+            let nextCursor;
+            if (hasMore && verbs.length > 0) {
+                nextCursor = verbs[verbs.length - 1].id;
+            }
+            return {
+                items: verbs.slice(0, limit),
+                hasMore,
+                nextCursor
+            };
+        }
+        catch (error) {
+            logger.error('Failed to get verbs with pagination:', error);
+            return {
+                items: [],
+                hasMore: false
+            };
+        }
+    }
+    /**
+     * Check if a noun matches the filter criteria
+     */
+    async matchesNounFilter(noun, filter) {
+        // Get metadata for filtering
+        const metadata = await this.storage.getMetadata(noun.id, 'noun');
+        // Filter by noun type
+        if (filter.nounType) {
+            const nounTypes = Array.isArray(filter.nounType) ? filter.nounType : [filter.nounType];
+            const nounType = metadata?.type || metadata?.noun;
+            if (!nounType || !nounTypes.includes(nounType)) {
+                return false;
+            }
+        }
+        // Filter by service
+        if (filter.service) {
+            const services = Array.isArray(filter.service) ? filter.service : [filter.service];
+            if (!metadata?.service || !services.includes(metadata.service)) {
+                return false;
+            }
+        }
+        // Filter by metadata
+        if (filter.metadata) {
+            if (!metadata)
+                return false;
+            for (const [key, value] of Object.entries(filter.metadata)) {
+                if (metadata[key] !== value) {
+                    return false;
+                }
+            }
+        }
+        return true;
+    }
+    /**
+     * Check if a verb matches the filter criteria
+     */
+    matchesVerbFilter(verb, filter) {
+        // Filter by verb type
+        if (filter.verbType) {
+            const verbTypes = Array.isArray(filter.verbType) ? filter.verbType : [filter.verbType];
+            if (!verb.type || !verbTypes.includes(verb.type)) {
+                return false;
+            }
+        }
+        // Filter by source ID
+        if (filter.sourceId) {
+            const sourceIds = Array.isArray(filter.sourceId) ? filter.sourceId : [filter.sourceId];
+            if (!verb.sourceId || !sourceIds.includes(verb.sourceId)) {
+                return false;
+            }
+        }
+        // Filter by target ID
+        if (filter.targetId) {
+            const targetIds = Array.isArray(filter.targetId) ? filter.targetId : [filter.targetId];
+            if (!verb.targetId || !targetIds.includes(verb.targetId)) {
+                return false;
+            }
+        }
+        // Filter by service
+        if (filter.service) {
+            const services = Array.isArray(filter.service) ? filter.service : [filter.service];
+            if (!verb.metadata?.service || !services.includes(verb.metadata.service)) {
+                return false;
+            }
+        }
+        // Filter by metadata
+        if (filter.metadata) {
+            if (!verb.metadata)
+                return false;
+            for (const [key, value] of Object.entries(filter.metadata)) {
+                if (verb.metadata[key] !== value) {
+                    return false;
+                }
+            }
+        }
+        return true;
+    }
+    /**
+     * Combine HNSWVerb data with metadata to create GraphVerb
+     */
+    combineVerbWithMetadata(verbData, metadata) {
+        if (!verbData || !metadata)
+            return null;
+        // Create default timestamp if not present
+        const defaultTimestamp = {
+            seconds: Math.floor(Date.now() / 1000),
+            nanoseconds: (Date.now() % 1000) * 1000000
+        };
+        // Create default createdBy if not present
+        const defaultCreatedBy = {
+            augmentation: 'unknown',
+            version: '1.0'
+        };
+        return {
+            id: verbData.id,
+            vector: verbData.vector,
+            sourceId: metadata.sourceId,
+            targetId: metadata.targetId,
+            source: metadata.source,
+            target: metadata.target,
+            verb: metadata.verb,
+            type: metadata.type,
+            weight: metadata.weight || 1.0,
+            metadata: metadata.metadata || {},
+            createdAt: metadata.createdAt || defaultTimestamp,
+            updatedAt: metadata.updatedAt || defaultTimestamp,
+            createdBy: metadata.createdBy || defaultCreatedBy,
+            data: metadata.data,
+            embedding: verbData.vector
+        };
+    }
+}
+//# sourceMappingURL=optimizedS3Search.js.map

dist/storage/adapters/s3CompatibleStorage.d.ts

@@ -0,0 +1,493 @@
+/**
+ * S3-Compatible Storage Adapter
+ * Uses the AWS S3 client to interact with S3-compatible storage services
+ * including Amazon S3, Cloudflare R2, and Google Cloud Storage
+ */
+import { GraphVerb, HNSWNoun, HNSWVerb, StatisticsData } from '../../coreTypes.js';
+import { BaseStorage } from '../baseStorage.js';
+import { OperationConfig } from '../../utils/operationUtils.js';
+type HNSWNode = HNSWNoun;
+type Edge = HNSWVerb;
+interface ChangeLogEntry {
+    timestamp: number;
+    operation: 'add' | 'update' | 'delete';
+    entityType: 'noun' | 'verb' | 'metadata';
+    entityId: string;
+    data?: any;
+    instanceId?: string;
+}
+export { S3CompatibleStorage as R2Storage };
+/**
+ * S3-compatible storage adapter for server environments
+ * Uses the AWS S3 client to interact with S3-compatible storage services
+ * including Amazon S3, Cloudflare R2, and Google Cloud Storage
+ *
+ * To use this adapter with Amazon S3, you need to provide:
+ * - region: AWS region (e.g., 'us-east-1')
+ * - credentials: AWS credentials (accessKeyId and secretAccessKey)
+ * - bucketName: S3 bucket name
+ *
+ * To use this adapter with Cloudflare R2, you need to provide:
+ * - accountId: Cloudflare account ID
+ * - accessKeyId: R2 access key ID
+ * - secretAccessKey: R2 secret access key
+ * - bucketName: R2 bucket name
+ *
+ * To use this adapter with Google Cloud Storage, you need to provide:
+ * - region: GCS region (e.g., 'us-central1')
+ * - credentials: GCS credentials (accessKeyId and secretAccessKey)
+ * - endpoint: GCS endpoint (e.g., 'https://storage.googleapis.com')
+ * - bucketName: GCS bucket name
+ */
+export declare class S3CompatibleStorage extends BaseStorage {
+    private s3Client;
+    private bucketName;
+    private serviceType;
+    private region;
+    private endpoint?;
+    private accountId?;
+    private accessKeyId;
+    private secretAccessKey;
+    private sessionToken?;
+    private nounPrefix;
+    private verbPrefix;
+    private metadataPrefix;
+    private verbMetadataPrefix;
+    private indexPrefix;
+    private systemPrefix;
+    private useDualWrite;
+    protected statisticsCache: StatisticsData | null;
+    private lockPrefix;
+    private activeLocks;
+    private changeLogPrefix;
+    private pendingOperations;
+    private maxConcurrentOperations;
+    private baseBatchSize;
+    private currentBatchSize;
+    private lastMemoryCheck;
+    private memoryCheckInterval;
+    private consecutiveErrors;
+    private lastErrorReset;
+    private socketManager;
+    private backpressure;
+    private nounWriteBuffer;
+    private verbWriteBuffer;
+    private requestCoalescer;
+    private highVolumeMode;
+    private lastVolumeCheck;
+    private volumeCheckInterval;
+    private forceHighVolumeMode;
+    private operationExecutors;
+    private nounCacheManager;
+    private verbCacheManager;
+    private logger;
+    /**
+     * Initialize the storage adapter
+     * @param options Configuration options for the S3-compatible storage
+     */
+    constructor(options: {
+        bucketName: string;
+        region?: string;
+        endpoint?: string;
+        accountId?: string;
+        accessKeyId: string;
+        secretAccessKey: string;
+        sessionToken?: string;
+        serviceType?: string;
+        operationConfig?: OperationConfig;
+        cacheConfig?: {
+            hotCacheMaxSize?: number;
+            hotCacheEvictionThreshold?: number;
+            warmCacheTTL?: number;
+        };
+        readOnly?: boolean;
+    });
+    /**
+     * Initialize the storage adapter
+     */
+    init(): Promise<void>;
+    /**
+     * Override base class method to detect S3-specific throttling errors
+     */
+    protected isThrottlingError(error: any): boolean;
+    /**
+     * Override to add S3-specific logging
+     */
+    handleThrottling(error: any, service?: string): Promise<void>;
+    /**
+     * Smart delay based on current throttling status
+     */
+    private smartDelay;
+    /**
+     * Auto-cleanup legacy /index folder during initialization
+     * This removes old index data that has been migrated to _system
+     */
+    private cleanupLegacyIndexFolder;
+    /**
+     * Initialize write buffers for high-volume scenarios
+     */
+    private initializeBuffers;
+    /**
+     * Initialize request coalescer
+     */
+    private initializeCoalescer;
+    /**
+     * Check if we should enable high-volume mode
+     */
+    private checkVolumeMode;
+    /**
+     * Bulk write nouns to S3
+     */
+    private bulkWriteNouns;
+    /**
+     * Bulk write verbs to S3
+     */
+    private bulkWriteVerbs;
+    /**
+     * Process coalesced batch of operations
+     */
+    private processCoalescedBatch;
+    /**
+     * Process bulk deletes
+     */
+    private processBulkDeletes;
+    /**
+     * Process bulk writes
+     */
+    private processBulkWrites;
+    /**
+     * Process bulk reads
+     */
+    private processBulkReads;
+    /**
+     * Dynamically adjust batch size based on memory pressure and error rates
+     */
+    private adjustBatchSize;
+    /**
+     * Apply backpressure when system is under load
+     */
+    private applyBackpressure;
+    /**
+     * Release backpressure after operation completes
+     */
+    private releaseBackpressure;
+    /**
+     * Get current batch size for operations
+     */
+    private getBatchSize;
+    /**
+     * Save a noun to storage (internal implementation)
+     */
+    protected saveNoun_internal(noun: HNSWNoun): Promise<void>;
+    /**
+     * Save a node to storage
+     */
+    protected saveNode(node: HNSWNode): Promise<void>;
+    /**
+     * Get a noun from storage (internal implementation)
+     */
+    protected getNoun_internal(id: string): Promise<HNSWNoun | null>;
+    /**
+     * Get a node from storage
+     */
+    protected getNode(id: string): Promise<HNSWNode | null>;
+    private nodeCache;
+    /**
+     * Get all nodes from storage
+     * @deprecated This method is deprecated and will be removed in a future version.
+     * It can cause memory issues with large datasets. Use getNodesWithPagination() instead.
+     */
+    protected getAllNodes(): Promise<HNSWNode[]>;
+    /**
+     * Get nodes with pagination
+     * @param options Pagination options
+     * @returns Promise that resolves to a paginated result of nodes
+     */
+    protected getNodesWithPagination(options?: {
+        limit?: number;
+        cursor?: string;
+        useCache?: boolean;
+    }): Promise<{
+        nodes: HNSWNode[];
+        hasMore: boolean;
+        nextCursor?: string;
+    }>;
+    /**
+     * Get nouns by noun type (internal implementation)
+     * @param nounType The noun type to filter by
+     * @returns Promise that resolves to an array of nouns of the specified noun type
+     */
+    protected getNounsByNounType_internal(nounType: string): Promise<HNSWNoun[]>;
+    /**
+     * Get nodes by noun type
+     * @param nounType The noun type to filter by
+     * @returns Promise that resolves to an array of nodes of the specified noun type
+     */
+    protected getNodesByNounType(nounType: string): Promise<HNSWNode[]>;
+    /**
+     * Delete a noun from storage (internal implementation)
+     */
+    protected deleteNoun_internal(id: string): Promise<void>;
+    /**
+     * Delete a node from storage
+     */
+    protected deleteNode(id: string): Promise<void>;
+    /**
+     * Save a verb to storage (internal implementation)
+     */
+    protected saveVerb_internal(verb: HNSWVerb): Promise<void>;
+    /**
+     * Save an edge to storage
+     */
+    protected saveEdge(edge: Edge): Promise<void>;
+    /**
+     * Get a verb from storage (internal implementation)
+     */
+    protected getVerb_internal(id: string): Promise<HNSWVerb | null>;
+    /**
+     * Get an edge from storage
+     */
+    protected getEdge(id: string): Promise<Edge | null>;
+    /**
+     * Get all edges from storage
+     * @deprecated This method is deprecated and will be removed in a future version.
+     * It can cause memory issues with large datasets. Use getEdgesWithPagination() instead.
+     */
+    protected getAllEdges(): Promise<Edge[]>;
+    /**
+     * Get edges with pagination
+     * @param options Pagination options
+     * @returns Promise that resolves to a paginated result of edges
+     */
+    protected getEdgesWithPagination(options?: {
+        limit?: number;
+        cursor?: string;
+        useCache?: boolean;
+        filter?: {
+            sourceId?: string;
+            targetId?: string;
+            type?: string;
+        };
+    }): Promise<{
+        edges: Edge[];
+        hasMore: boolean;
+        nextCursor?: string;
+    }>;
+    /**
+     * Filter an edge based on filter criteria
+     * @param edge The edge to filter
+     * @param filter The filter criteria
+     * @returns True if the edge matches the filter, false otherwise
+     */
+    private filterEdge;
+    /**
+     * Get verbs with pagination
+     * @param options Pagination options
+     * @returns Promise that resolves to a paginated result of verbs
+     */
+    getVerbsWithPagination(options?: {
+        limit?: number;
+        cursor?: string;
+        filter?: {
+            verbType?: string | string[];
+            sourceId?: string | string[];
+            targetId?: string | string[];
+            service?: string | string[];
+            metadata?: Record<string, any>;
+        };
+    }): Promise<{
+        items: GraphVerb[];
+        totalCount?: number;
+        hasMore: boolean;
+        nextCursor?: string;
+    }>;
+    /**
+     * Get verbs by source (internal implementation)
+     */
+    protected getVerbsBySource_internal(sourceId: string): Promise<GraphVerb[]>;
+    /**
+     * Get verbs by target (internal implementation)
+     */
+    protected getVerbsByTarget_internal(targetId: string): Promise<GraphVerb[]>;
+    /**
+     * Get verbs by type (internal implementation)
+     */
+    protected getVerbsByType_internal(type: string): Promise<GraphVerb[]>;
+    /**
+     * Delete a verb from storage (internal implementation)
+     */
+    protected deleteVerb_internal(id: string): Promise<void>;
+    /**
+     * Delete an edge from storage
+     */
+    protected deleteEdge(id: string): Promise<void>;
+    /**
+     * Save metadata to storage
+     */
+    saveMetadata(id: string, metadata: any): Promise<void>;
+    /**
+     * Save verb metadata to storage
+     */
+    saveVerbMetadata(id: string, metadata: any): Promise<void>;
+    /**
+     * Get verb metadata from storage
+     */
+    getVerbMetadata(id: string): Promise<any | null>;
+    /**
+     * Save noun metadata to storage
+     */
+    saveNounMetadata(id: string, metadata: any): Promise<void>;
+    /**
+     * Get multiple metadata objects in batches (CRITICAL: Prevents socket exhaustion)
+     * This is the solution to the metadata reading socket exhaustion during initialization
+     */
+    getMetadataBatch(ids: string[]): Promise<Map<string, any>>;
+    /**
+     * Get multiple verb metadata objects in batches (prevents socket exhaustion)
+     */
+    getVerbMetadataBatch(ids: string[]): Promise<Map<string, any>>;
+    /**
+     * Get noun metadata from storage
+     */
+    getNounMetadata(id: string): Promise<any | null>;
+    /**
+     * Get metadata from storage
+     */
+    getMetadata(id: string): Promise<any | null>;
+    /**
+     * Clear all data from storage
+     */
+    clear(): Promise<void>;
+    /**
+     * Get information about storage usage and capacity
+     * Optimized version that uses cached statistics instead of expensive full scans
+     */
+    getStorageStatus(): Promise<{
+        type: string;
+        used: number;
+        quota: number | null;
+        details?: Record<string, any>;
+    }>;
+    protected statisticsBatchUpdateTimerId: NodeJS.Timeout | null;
+    protected statisticsModified: boolean;
+    protected lastStatisticsFlushTime: number;
+    protected readonly MIN_FLUSH_INTERVAL_MS = 5000;
+    protected readonly MAX_FLUSH_DELAY_MS = 30000;
+    /**
+     * Get the statistics key for a specific date
+     * @param date The date to get the key for
+     * @returns The statistics key for the specified date
+     */
+    private getStatisticsKeyForDate;
+    /**
+     * Get the current statistics key
+     * @returns The current statistics key
+     */
+    private getCurrentStatisticsKey;
+    /**
+     * Get the legacy statistics key (DEPRECATED - /index folder is auto-cleaned)
+     * @returns The legacy statistics key
+     * @deprecated Legacy /index folder is automatically cleaned on initialization
+     */
+    private getLegacyStatisticsKey;
+    /**
+     * Schedule a batch update of statistics
+     */
+    protected scheduleBatchUpdate(): void;
+    /**
+     * Flush statistics to storage with distributed locking
+     */
+    protected flushStatistics(): Promise<void>;
+    /**
+     * Merge statistics from storage with local statistics
+     * @param storageStats Statistics from storage
+     * @param localStats Local statistics to merge
+     * @returns Merged statistics data
+     */
+    private mergeStatistics;
+    /**
+     * Save statistics data to storage
+     * @param statistics The statistics data to save
+     */
+    protected saveStatisticsData(statistics: StatisticsData): Promise<void>;
+    /**
+     * Get statistics data from storage
+     * @returns Promise that resolves to the statistics data or null if not found
+     */
+    protected getStatisticsData(): Promise<StatisticsData | null>;
+    /**
+     * Check if we should try yesterday's statistics file
+     * Only try within 2 hours of midnight to avoid unnecessary calls
+     */
+    private shouldTryYesterday;
+    /**
+     * Get yesterday's date
+     */
+    private getYesterday;
+    /**
+     * Try to get statistics from a specific key
+     * @param key The key to try to get statistics from
+     * @returns The statistics data or null if not found
+     */
+    private tryGetStatisticsFromKey;
+    /**
+     * Append an entry to the change log for efficient synchronization
+     * @param entry The change log entry to append
+     */
+    private appendToChangeLog;
+    /**
+     * Get changes from the change log since a specific timestamp
+     * @param sinceTimestamp Timestamp to get changes since
+     * @param maxEntries Maximum number of entries to return (default: 1000)
+     * @returns Array of change log entries
+     */
+    getChangesSince(sinceTimestamp: number, maxEntries?: number): Promise<ChangeLogEntry[]>;
+    /**
+     * Clean up old change log entries to prevent unlimited growth
+     * @param olderThanTimestamp Remove entries older than this timestamp
+     */
+    cleanupOldChangeLogs(olderThanTimestamp: number): Promise<void>;
+    /**
+     * Sample-based storage estimation as fallback when statistics unavailable
+     * Much faster than full scans - samples first 50 objects per prefix
+     */
+    private getSampleBasedStorageEstimate;
+    /**
+     * Acquire a distributed lock for coordinating operations across multiple instances
+     * @param lockKey The key to lock on
+     * @param ttl Time to live for the lock in milliseconds (default: 30 seconds)
+     * @returns Promise that resolves to true if lock was acquired, false otherwise
+     */
+    private acquireLock;
+    /**
+     * Release a distributed lock
+     * @param lockKey The key to unlock
+     * @param lockValue The value used when acquiring the lock (for verification)
+     * @returns Promise that resolves when lock is released
+     */
+    private releaseLock;
+    /**
+     * Clean up expired locks to prevent lock leakage
+     * This method should be called periodically
+     */
+    private cleanupExpiredLocks;
+    /**
+     * Get nouns with pagination support
+     * @param options Pagination options
+     * @returns Promise that resolves to a paginated result of nouns
+     */
+    getNounsWithPagination(options?: {
+        limit?: number;
+        cursor?: string;
+        filter?: {
+            nounType?: string | string[];
+            service?: string | string[];
+            metadata?: Record<string, any>;
+        };
+    }): Promise<{
+        items: HNSWNoun[];
+        totalCount?: number;
+        hasMore: boolean;
+        nextCursor?: string;
+    }>;
+}

dist/storage/backwardCompatibility.d.ts

@@ -0,0 +1,84 @@
+/**
+ * Backward Compatibility Layer for Storage Migration
+ *
+ * Handles the transition from 'index' to '_system' directory
+ * Ensures services running different versions can coexist
+ */
+import { StatisticsData } from '../coreTypes.js';
+export interface MigrationMetadata {
+    schemaVersion: number;
+    migrationStarted?: string;
+    migrationCompleted?: string;
+    lastUpdatedBy?: string;
+}
+/**
+ * Backward compatibility strategy for directory migration
+ */
+export declare class StorageCompatibilityLayer {
+    private migrationMetadata;
+    /**
+     * Determines the read strategy based on what's available
+     * @returns Priority-ordered list of directories to try
+     */
+    static getReadPriority(): string[];
+    /**
+     * Determines write strategy based on migration state
+     * @param migrationComplete Whether migration is complete
+     * @returns List of directories to write to
+     */
+    static getWriteTargets(migrationComplete?: boolean): string[];
+    /**
+     * Check if we should perform migration based on service coordination
+     * @param existingStats Statistics from storage
+     * @returns Whether to initiate migration
+     */
+    static shouldMigrate(existingStats: StatisticsData | null): boolean;
+    /**
+     * Creates migration metadata
+     */
+    static createMigrationMetadata(): MigrationMetadata;
+    /**
+     * Merge statistics from multiple locations (deduplication)
+     */
+    static mergeStatistics(primary: StatisticsData | null, fallback: StatisticsData | null): StatisticsData | null;
+    /**
+     * Determines if dual-write is needed based on environment
+     * @param storageType The type of storage being used
+     * @returns Whether to write to both old and new locations
+     */
+    static needsDualWrite(storageType: string): boolean;
+    /**
+     * Grace period for migration (30 days default)
+     * After this period, services can stop reading from old location
+     */
+    static getMigrationGracePeriodMs(): number;
+    /**
+     * Check if migration grace period has expired
+     */
+    static isGracePeriodExpired(migrationStarted: string): boolean;
+    /**
+     * Log migration events for monitoring
+     */
+    static logMigrationEvent(event: string, details?: any): void;
+}
+/**
+ * Storage paths helper for migration
+ */
+export declare class StoragePaths {
+    /**
+     * Get the statistics file path for a given directory
+     */
+    static getStatisticsPath(baseDir: string, filename?: string): string;
+    /**
+     * Get distributed config path
+     */
+    static getDistributedConfigPath(baseDir: string): string;
+    /**
+     * Check if a path is using the old structure
+     */
+    static isLegacyPath(path: string): boolean;
+    /**
+     * Convert legacy path to new structure
+     */
+    static modernizePath(path: string): string;
+}

dist/storage/backwardCompatibility.js

@@ -0,0 +1,141 @@
+/**
+ * Backward Compatibility Layer for Storage Migration
+ *
+ * Handles the transition from 'index' to '_system' directory
+ * Ensures services running different versions can coexist
+ */
+/**
+ * Backward compatibility strategy for directory migration
+ */
+export class StorageCompatibilityLayer {
+    constructor() {
+        this.migrationMetadata = null;
+    }
+    /**
+     * Determines the read strategy based on what's available
+     * @returns Priority-ordered list of directories to try
+     */
+    static getReadPriority() {
+        return ['_system', 'index']; // Try new location first, fallback to old
+    }
+    /**
+     * Determines write strategy based on migration state
+     * @param migrationComplete Whether migration is complete
+     * @returns List of directories to write to
+     */
+    static getWriteTargets(migrationComplete = false) {
+        if (migrationComplete) {
+            return ['_system']; // Only write to new location
+        }
+        // During migration, write to both for compatibility
+        return ['_system', 'index'];
+    }
+    /**
+     * Check if we should perform migration based on service coordination
+     * @param existingStats Statistics from storage
+     * @returns Whether to initiate migration
+     */
+    static shouldMigrate(existingStats) {
+        if (!existingStats)
+            return true; // No data yet, use new structure
+        // Check if we have migration metadata in stats
+        const migrationData = existingStats.migrationMetadata;
+        if (!migrationData)
+            return true; // No migration data, start migration
+        // Check schema version
+        if (migrationData.schemaVersion < 2)
+            return true;
+        // Already migrated
+        return false;
+    }
+    /**
+     * Creates migration metadata
+     */
+    static createMigrationMetadata() {
+        return {
+            schemaVersion: 2,
+            migrationStarted: new Date().toISOString(),
+            lastUpdatedBy: process.env.HOSTNAME || process.env.INSTANCE_ID || 'unknown'
+        };
+    }
+    /**
+     * Merge statistics from multiple locations (deduplication)
+     */
+    static mergeStatistics(primary, fallback) {
+        if (!primary && !fallback)
+            return null;
+        if (!fallback)
+            return primary;
+        if (!primary)
+            return fallback;
+        // Return the most recently updated
+        const primaryTime = new Date(primary.lastUpdated).getTime();
+        const fallbackTime = new Date(fallback.lastUpdated).getTime();
+        return primaryTime >= fallbackTime ? primary : fallback;
+    }
+    /**
+     * Determines if dual-write is needed based on environment
+     * @param storageType The type of storage being used
+     * @returns Whether to write to both old and new locations
+     */
+    static needsDualWrite(storageType) {
+        // Only need dual-write for shared storage systems
+        const sharedStorageTypes = ['s3', 'r2', 'gcs', 'filesystem'];
+        return sharedStorageTypes.includes(storageType.toLowerCase());
+    }
+    /**
+     * Grace period for migration (30 days default)
+     * After this period, services can stop reading from old location
+     */
+    static getMigrationGracePeriodMs() {
+        const days = parseInt(process.env.BRAINY_MIGRATION_GRACE_DAYS || '30', 10);
+        return days * 24 * 60 * 60 * 1000;
+    }
+    /**
+     * Check if migration grace period has expired
+     */
+    static isGracePeriodExpired(migrationStarted) {
+        const startTime = new Date(migrationStarted).getTime();
+        const now = Date.now();
+        const gracePeriod = this.getMigrationGracePeriodMs();
+        return (now - startTime) > gracePeriod;
+    }
+    /**
+     * Log migration events for monitoring
+     */
+    static logMigrationEvent(event, details) {
+        if (process.env.NODE_ENV !== 'test') {
+            console.log(`[Brainy Storage Migration] ${event}`, details || '');
+        }
+    }
+}
+/**
+ * Storage paths helper for migration
+ */
+export class StoragePaths {
+    /**
+     * Get the statistics file path for a given directory
+     */
+    static getStatisticsPath(baseDir, filename = 'statistics') {
+        return `${baseDir}/${filename}.json`;
+    }
+    /**
+     * Get distributed config path
+     */
+    static getDistributedConfigPath(baseDir) {
+        return `${baseDir}/distributed_config.json`;
+    }
+    /**
+     * Check if a path is using the old structure
+     */
+    static isLegacyPath(path) {
+        return path.includes('/index/') || path.endsWith('/index');
+    }
+    /**
+     * Convert legacy path to new structure
+     */
+    static modernizePath(path) {
+        return path.replace('/index/', '/_system/').replace('/index', '/_system');
+    }
+}
+//# sourceMappingURL=backwardCompatibility.js.map

dist/storage/backwardCompatibility.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"backwardCompatibility.js","sourceRoot":"","sources":["../../src/storage/backwardCompatibility.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAWH;;GAEG;AACH,MAAM,OAAO,yBAAyB;IAAtC;QACU,sBAAiB,GAA6B,IAAI,CAAA;IA8G5D,CAAC;IA5GC;;;OAGG;IACH,MAAM,CAAC,eAAe;QACpB,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,CAAA,CAAE,0CAA0C;IACzE,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,eAAe,CAAC,oBAA6B,KAAK;QACvD,IAAI,iBAAiB,EAAE,CAAC;YACtB,OAAO,CAAC,SAAS,CAAC,CAAA,CAAE,6BAA6B;QACnD,CAAC;QACD,oDAAoD;QACpD,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,CAAA;IAC7B,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,aAAa,CAAC,aAAoC;QACvD,IAAI,CAAC,aAAa;YAAE,OAAO,IAAI,CAAA,CAAE,iCAAiC;QAElE,+CAA+C;QAC/C,MAAM,aAAa,GAAI,aAAqB,CAAC,iBAAiB,CAAA;QAC9D,IAAI,CAAC,aAAa;YAAE,OAAO,IAAI,CAAA,CAAE,qCAAqC;QAEtE,uBAAuB;QACvB,IAAI,aAAa,CAAC,aAAa,GAAG,CAAC;YAAE,OAAO,IAAI,CAAA;QAEhD,mBAAmB;QACnB,OAAO,KAAK,CAAA;IACd,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,uBAAuB;QAC5B,OAAO;YACL,aAAa,EAAE,CAAC;YAChB,gBAAgB,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YAC1C,aAAa,EAAE,OAAO,CAAC,GAAG,CAAC,QAAQ,IAAI,OAAO,CAAC,GAAG,CAAC,WAAW,IAAI,SAAS;SAC5E,CAAA;IACH,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,eAAe,CACpB,OAA8B,EAC9B,QAA+B;QAE/B,IAAI,CAAC,OAAO,IAAI,CAAC,QAAQ;YAAE,OAAO,IAAI,CAAA;QACtC,IAAI,CAAC,QAAQ;YAAE,OAAO,OAAO,CAAA;QAC7B,IAAI,CAAC,OAAO;YAAE,OAAO,QAAQ,CAAA;QAE7B,mCAAmC;QACnC,MAAM,WAAW,GAAG,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,OAAO,EAAE,CAAA;QAC3D,MAAM,YAAY,GAAG,IAAI,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,OAAO,EAAE,CAAA;QAE7D,OAAO,WAAW,IAAI,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAA;IACzD,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,cAAc,CAAC,WAAmB;QACvC,kDAAkD;QAClD,MAAM,kBAAkB,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,YAAY,CAAC,CAAA;QAC5D,OAAO,kBAAkB,CAAC,QAAQ,CAAC,WAAW,CAAC,WAAW,EAAE,CAAC,CAAA;IAC/D,CAAC;IAED;;;OAGG;IACH,MAAM,CAAC,yBAAyB;QAC9B,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,2BAA2B,IAAI,IAAI,EAAE,EAAE,CAAC,CAAA;QAC1E,OAAO,IAAI,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAA;IACnC,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,oBAAoB,CAAC,gBAAwB;QAClD,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,gBAAgB,CAAC,CAAC,OAAO,EAAE,CAAA;QACtD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QACtB,MAAM,WAAW,GAAG,IAAI,CAAC,yBAAyB,EAAE,CAAA;QAEpD,OAAO,CAAC,GAAG,GAAG,SAAS,CAAC,GAAG,WAAW,CAAA;IACxC,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,iBAAiB,CAAC,KAAa,EAAE,OAAa;QACnD,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,MAAM,EAAE,CAAC;YACpC,OAAO,CAAC,GAAG,CAAC,8BAA8B,KAAK,EAAE,EAAE,OAAO,IAAI,EAAE,CAAC,CAAA;QACnE,CAAC;IACH,CAAC;CACF;AAED;;GAEG;AACH,MAAM,OAAO,YAAY;IACvB;;OAEG;IACH,MAAM,CAAC,iBAAiB,CAAC,OAAe,EAAE,WAAmB,YAAY;QACvE,OAAO,GAAG,OAAO,IAAI,QAAQ,OAAO,CAAA;IACtC,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,wBAAwB,CAAC,OAAe;QAC7C,OAAO,GAAG,OAAO,0BAA0B,CAAA;IAC7C,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,YAAY,CAAC,IAAY;QAC9B,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAA;IAC5D,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,aAAa,CAAC,IAAY;QAC/B,OAAO,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAA;IAC3E,CAAC;CACF"}

dist/storage/baseStorage.d.ts

@@ -0,0 +1,267 @@
+/**
+ * Base Storage Adapter
+ * Provides common functionality for all storage adapters
+ */
+import { GraphVerb, HNSWNoun, HNSWVerb, StatisticsData } from '../coreTypes.js';
+import { BaseStorageAdapter } from './adapters/baseStorageAdapter.js';
+export declare const ENTITIES_DIR = "entities";
+export declare const NOUNS_VECTOR_DIR = "entities/nouns/vectors";
+export declare const NOUNS_METADATA_DIR = "entities/nouns/metadata";
+export declare const VERBS_VECTOR_DIR = "entities/verbs/vectors";
+export declare const VERBS_METADATA_DIR = "entities/verbs/metadata";
+export declare const INDEXES_DIR = "indexes";
+export declare const METADATA_INDEX_DIR = "indexes/metadata";
+export declare const NOUNS_DIR = "nouns";
+export declare const VERBS_DIR = "verbs";
+export declare const METADATA_DIR = "metadata";
+export declare const NOUN_METADATA_DIR = "noun-metadata";
+export declare const VERB_METADATA_DIR = "verb-metadata";
+export declare const INDEX_DIR = "index";
+export declare const SYSTEM_DIR = "_system";
+export declare const STATISTICS_KEY = "statistics";
+export declare const STORAGE_SCHEMA_VERSION = 3;
+export declare const USE_ENTITY_BASED_STRUCTURE = true;
+/**
+ * Get the appropriate directory path based on configuration
+ */
+export declare function getDirectoryPath(entityType: 'noun' | 'verb', dataType: 'vector' | 'metadata'): string;
+/**
+ * Base storage adapter that implements common functionality
+ * This is an abstract class that should be extended by specific storage adapters
+ */
+export declare abstract class BaseStorage extends BaseStorageAdapter {
+    protected isInitialized: boolean;
+    protected readOnly: boolean;
+    /**
+     * Initialize the storage adapter
+     * This method should be implemented by each specific adapter
+     */
+    abstract init(): Promise<void>;
+    /**
+     * Ensure the storage adapter is initialized
+     */
+    protected ensureInitialized(): Promise<void>;
+    /**
+     * Save a noun to storage
+     */
+    saveNoun(noun: HNSWNoun): Promise<void>;
+    /**
+     * Get a noun from storage
+     */
+    getNoun(id: string): Promise<HNSWNoun | null>;
+    /**
+     * Get nouns by noun type
+     * @param nounType The noun type to filter by
+     * @returns Promise that resolves to an array of nouns of the specified noun type
+     */
+    getNounsByNounType(nounType: string): Promise<HNSWNoun[]>;
+    /**
+     * Delete a noun from storage
+     */
+    deleteNoun(id: string): Promise<void>;
+    /**
+     * Save a verb to storage
+     */
+    saveVerb(verb: GraphVerb): Promise<void>;
+    /**
+     * Get a verb from storage
+     */
+    getVerb(id: string): Promise<GraphVerb | null>;
+    /**
+     * Convert HNSWVerb to GraphVerb by combining with metadata
+     */
+    protected convertHNSWVerbToGraphVerb(hnswVerb: HNSWVerb): Promise<GraphVerb | null>;
+    /**
+     * Internal method for loading all verbs - used by performance optimizations
+     * @internal - Do not use directly, use getVerbs() with pagination instead
+     */
+    protected _loadAllVerbsForOptimization(): Promise<HNSWVerb[]>;
+    /**
+     * Get verbs by source
+     */
+    getVerbsBySource(sourceId: string): Promise<GraphVerb[]>;
+    /**
+     * Get verbs by target
+     */
+    getVerbsByTarget(targetId: string): Promise<GraphVerb[]>;
+    /**
+     * Get verbs by type
+     */
+    getVerbsByType(type: string): Promise<GraphVerb[]>;
+    /**
+     * Internal method for loading all nouns - used by performance optimizations
+     * @internal - Do not use directly, use getNouns() with pagination instead
+     */
+    protected _loadAllNounsForOptimization(): Promise<HNSWNoun[]>;
+    /**
+     * Get nouns with pagination and filtering
+     * @param options Pagination and filtering options
+     * @returns Promise that resolves to a paginated result of nouns
+     */
+    getNouns(options?: {
+        pagination?: {
+            offset?: number;
+            limit?: number;
+            cursor?: string;
+        };
+        filter?: {
+            nounType?: string | string[];
+            service?: string | string[];
+            metadata?: Record<string, any>;
+        };
+    }): Promise<{
+        items: HNSWNoun[];
+        totalCount?: number;
+        hasMore: boolean;
+        nextCursor?: string;
+    }>;
+    /**
+     * Get verbs with pagination and filtering
+     * @param options Pagination and filtering options
+     * @returns Promise that resolves to a paginated result of verbs
+     */
+    getVerbs(options?: {
+        pagination?: {
+            offset?: number;
+            limit?: number;
+            cursor?: string;
+        };
+        filter?: {
+            verbType?: string | string[];
+            sourceId?: string | string[];
+            targetId?: string | string[];
+            service?: string | string[];
+            metadata?: Record<string, any>;
+        };
+    }): Promise<{
+        items: GraphVerb[];
+        totalCount?: number;
+        hasMore: boolean;
+        nextCursor?: string;
+    }>;
+    /**
+     * Delete a verb from storage
+     */
+    deleteVerb(id: string): Promise<void>;
+    /**
+     * Clear all data from storage
+     * This method should be implemented by each specific adapter
+     */
+    abstract clear(): Promise<void>;
+    /**
+     * Get information about storage usage and capacity
+     * This method should be implemented by each specific adapter
+     */
+    abstract getStorageStatus(): Promise<{
+        type: string;
+        used: number;
+        quota: number | null;
+        details?: Record<string, any>;
+    }>;
+    /**
+     * Save metadata to storage
+     * This method should be implemented by each specific adapter
+     */
+    abstract saveMetadata(id: string, metadata: any): Promise<void>;
+    /**
+     * Get metadata from storage
+     * This method should be implemented by each specific adapter
+     */
+    abstract getMetadata(id: string): Promise<any | null>;
+    /**
+     * Save noun metadata to storage
+     * This method should be implemented by each specific adapter
+     */
+    abstract saveNounMetadata(id: string, metadata: any): Promise<void>;
+    /**
+     * Get noun metadata from storage
+     * This method should be implemented by each specific adapter
+     */
+    abstract getNounMetadata(id: string): Promise<any | null>;
+    /**
+     * Save verb metadata to storage
+     * This method should be implemented by each specific adapter
+     */
+    abstract saveVerbMetadata(id: string, metadata: any): Promise<void>;
+    /**
+     * Get verb metadata from storage
+     * This method should be implemented by each specific adapter
+     */
+    abstract getVerbMetadata(id: string): Promise<any | null>;
+    /**
+     * Save a noun to storage
+     * This method should be implemented by each specific adapter
+     */
+    protected abstract saveNoun_internal(noun: HNSWNoun): Promise<void>;
+    /**
+     * Get a noun from storage
+     * This method should be implemented by each specific adapter
+     */
+    protected abstract getNoun_internal(id: string): Promise<HNSWNoun | null>;
+    /**
+     * Get nouns by noun type
+     * This method should be implemented by each specific adapter
+     */
+    protected abstract getNounsByNounType_internal(nounType: string): Promise<HNSWNoun[]>;
+    /**
+     * Delete a noun from storage
+     * This method should be implemented by each specific adapter
+     */
+    protected abstract deleteNoun_internal(id: string): Promise<void>;
+    /**
+     * Save a verb to storage
+     * This method should be implemented by each specific adapter
+     */
+    protected abstract saveVerb_internal(verb: HNSWVerb): Promise<void>;
+    /**
+     * Get a verb from storage
+     * This method should be implemented by each specific adapter
+     */
+    protected abstract getVerb_internal(id: string): Promise<HNSWVerb | null>;
+    /**
+     * Get verbs by source
+     * This method should be implemented by each specific adapter
+     */
+    protected abstract getVerbsBySource_internal(sourceId: string): Promise<GraphVerb[]>;
+    /**
+     * Get verbs by target
+     * This method should be implemented by each specific adapter
+     */
+    protected abstract getVerbsByTarget_internal(targetId: string): Promise<GraphVerb[]>;
+    /**
+     * Get verbs by type
+     * This method should be implemented by each specific adapter
+     */
+    protected abstract getVerbsByType_internal(type: string): Promise<GraphVerb[]>;
+    /**
+     * Delete a verb from storage
+     * This method should be implemented by each specific adapter
+     */
+    protected abstract deleteVerb_internal(id: string): Promise<void>;
+    /**
+     * Helper method to convert a Map to a plain object for serialization
+     */
+    protected mapToObject<K extends string | number, V>(map: Map<K, V>, valueTransformer?: (value: V) => any): Record<string, any>;
+    /**
+     * Save statistics data to storage (public interface)
+     * @param statistics The statistics data to save
+     */
+    saveStatistics(statistics: StatisticsData): Promise<void>;
+    /**
+     * Get statistics data from storage (public interface)
+     * @returns Promise that resolves to the statistics data or null if not found
+     */
+    getStatistics(): Promise<StatisticsData | null>;
+    /**
+     * Save statistics data to storage
+     * This method should be implemented by each specific adapter
+     * @param statistics The statistics data to save
+     */
+    protected abstract saveStatisticsData(statistics: StatisticsData): Promise<void>;
+    /**
+     * Get statistics data from storage
+     * This method should be implemented by each specific adapter
+     * @returns Promise that resolves to the statistics data or null if not found
+     */
+    protected abstract getStatisticsData(): Promise<StatisticsData | null>;
+}

dist/storage/baseStorage.js

@@ -0,0 +1,516 @@
+/**
+ * Base Storage Adapter
+ * Provides common functionality for all storage adapters
+ */
+import { BaseStorageAdapter } from './adapters/baseStorageAdapter.js';
+// Common directory/prefix names
+// Option A: Entity-Based Directory Structure
+export const ENTITIES_DIR = 'entities';
+export const NOUNS_VECTOR_DIR = 'entities/nouns/vectors';
+export const NOUNS_METADATA_DIR = 'entities/nouns/metadata';
+export const VERBS_VECTOR_DIR = 'entities/verbs/vectors';
+export const VERBS_METADATA_DIR = 'entities/verbs/metadata';
+export const INDEXES_DIR = 'indexes';
+export const METADATA_INDEX_DIR = 'indexes/metadata';
+// Legacy paths - kept for backward compatibility during migration
+export const NOUNS_DIR = 'nouns'; // Legacy: now maps to entities/nouns/vectors
+export const VERBS_DIR = 'verbs'; // Legacy: now maps to entities/verbs/vectors
+export const METADATA_DIR = 'metadata'; // Legacy: now maps to entities/nouns/metadata
+export const NOUN_METADATA_DIR = 'noun-metadata'; // Legacy: now maps to entities/nouns/metadata
+export const VERB_METADATA_DIR = 'verb-metadata'; // Legacy: now maps to entities/verbs/metadata
+export const INDEX_DIR = 'index'; // Legacy - kept for backward compatibility
+export const SYSTEM_DIR = '_system'; // System config & metadata indexes
+export const STATISTICS_KEY = 'statistics';
+// Migration version to track compatibility
+export const STORAGE_SCHEMA_VERSION = 3; // v3: Entity-Based Directory Structure (Option A)
+// Configuration flag to enable new directory structure
+export const USE_ENTITY_BASED_STRUCTURE = true; // Set to true to use Option A structure
+/**
+ * Get the appropriate directory path based on configuration
+ */
+export function getDirectoryPath(entityType, dataType) {
+    if (USE_ENTITY_BASED_STRUCTURE) {
+        // Option A: Entity-Based Structure
+        if (entityType === 'noun') {
+            return dataType === 'vector' ? NOUNS_VECTOR_DIR : NOUNS_METADATA_DIR;
+        }
+        else {
+            return dataType === 'vector' ? VERBS_VECTOR_DIR : VERBS_METADATA_DIR;
+        }
+    }
+    else {
+        // Legacy structure
+        if (entityType === 'noun') {
+            return dataType === 'vector' ? NOUNS_DIR : METADATA_DIR;
+        }
+        else {
+            return dataType === 'vector' ? VERBS_DIR : VERB_METADATA_DIR;
+        }
+    }
+}
+/**
+ * Base storage adapter that implements common functionality
+ * This is an abstract class that should be extended by specific storage adapters
+ */
+export class BaseStorage extends BaseStorageAdapter {
+    constructor() {
+        super(...arguments);
+        this.isInitialized = false;
+        this.readOnly = false;
+    }
+    /**
+     * Ensure the storage adapter is initialized
+     */
+    async ensureInitialized() {
+        if (!this.isInitialized) {
+            await this.init();
+        }
+    }
+    /**
+     * Save a noun to storage
+     */
+    async saveNoun(noun) {
+        await this.ensureInitialized();
+        return this.saveNoun_internal(noun);
+    }
+    /**
+     * Get a noun from storage
+     */
+    async getNoun(id) {
+        await this.ensureInitialized();
+        return this.getNoun_internal(id);
+    }
+    /**
+     * Get nouns by noun type
+     * @param nounType The noun type to filter by
+     * @returns Promise that resolves to an array of nouns of the specified noun type
+     */
+    async getNounsByNounType(nounType) {
+        await this.ensureInitialized();
+        return this.getNounsByNounType_internal(nounType);
+    }
+    /**
+     * Delete a noun from storage
+     */
+    async deleteNoun(id) {
+        await this.ensureInitialized();
+        return this.deleteNoun_internal(id);
+    }
+    /**
+     * Save a verb to storage
+     */
+    async saveVerb(verb) {
+        await this.ensureInitialized();
+        // Extract the lightweight HNSWVerb data
+        const hnswVerb = {
+            id: verb.id,
+            vector: verb.vector,
+            connections: verb.connections || new Map()
+        };
+        // Extract and save the metadata separately
+        const metadata = {
+            sourceId: verb.sourceId || verb.source,
+            targetId: verb.targetId || verb.target,
+            source: verb.source || verb.sourceId,
+            target: verb.target || verb.targetId,
+            type: verb.type || verb.verb,
+            verb: verb.verb || verb.type,
+            weight: verb.weight,
+            metadata: verb.metadata,
+            data: verb.data,
+            createdAt: verb.createdAt,
+            updatedAt: verb.updatedAt,
+            createdBy: verb.createdBy,
+            embedding: verb.embedding
+        };
+        // Save both the HNSWVerb and metadata
+        await this.saveVerb_internal(hnswVerb);
+        await this.saveVerbMetadata(verb.id, metadata);
+    }
+    /**
+     * Get a verb from storage
+     */
+    async getVerb(id) {
+        await this.ensureInitialized();
+        const hnswVerb = await this.getVerb_internal(id);
+        if (!hnswVerb) {
+            return null;
+        }
+        return this.convertHNSWVerbToGraphVerb(hnswVerb);
+    }
+    /**
+     * Convert HNSWVerb to GraphVerb by combining with metadata
+     */
+    async convertHNSWVerbToGraphVerb(hnswVerb) {
+        try {
+            const metadata = await this.getVerbMetadata(hnswVerb.id);
+            if (!metadata) {
+                return null;
+            }
+            // Create default timestamp if not present
+            const defaultTimestamp = {
+                seconds: Math.floor(Date.now() / 1000),
+                nanoseconds: (Date.now() % 1000) * 1000000
+            };
+            // Create default createdBy if not present
+            const defaultCreatedBy = {
+                augmentation: 'unknown',
+                version: '1.0'
+            };
+            return {
+                id: hnswVerb.id,
+                vector: hnswVerb.vector,
+                sourceId: metadata.sourceId,
+                targetId: metadata.targetId,
+                source: metadata.source,
+                target: metadata.target,
+                verb: metadata.verb,
+                type: metadata.type,
+                weight: metadata.weight || 1.0,
+                metadata: metadata.metadata || {},
+                createdAt: metadata.createdAt || defaultTimestamp,
+                updatedAt: metadata.updatedAt || defaultTimestamp,
+                createdBy: metadata.createdBy || defaultCreatedBy,
+                data: metadata.data,
+                embedding: hnswVerb.vector
+            };
+        }
+        catch (error) {
+            console.error(`Failed to convert HNSWVerb to GraphVerb for ${hnswVerb.id}:`, error);
+            return null;
+        }
+    }
+    /**
+     * Internal method for loading all verbs - used by performance optimizations
+     * @internal - Do not use directly, use getVerbs() with pagination instead
+     */
+    async _loadAllVerbsForOptimization() {
+        await this.ensureInitialized();
+        // Only use this for internal optimizations when safe
+        const result = await this.getVerbs({
+            pagination: { limit: Number.MAX_SAFE_INTEGER }
+        });
+        // Convert GraphVerbs back to HNSWVerbs for internal use
+        const hnswVerbs = [];
+        for (const graphVerb of result.items) {
+            const hnswVerb = {
+                id: graphVerb.id,
+                vector: graphVerb.vector,
+                connections: new Map()
+            };
+            hnswVerbs.push(hnswVerb);
+        }
+        return hnswVerbs;
+    }
+    /**
+     * Get verbs by source
+     */
+    async getVerbsBySource(sourceId) {
+        await this.ensureInitialized();
+        // Use the paginated getVerbs method with source filter
+        const result = await this.getVerbs({
+            filter: { sourceId }
+        });
+        return result.items;
+    }
+    /**
+     * Get verbs by target
+     */
+    async getVerbsByTarget(targetId) {
+        await this.ensureInitialized();
+        // Use the paginated getVerbs method with target filter
+        const result = await this.getVerbs({
+            filter: { targetId }
+        });
+        return result.items;
+    }
+    /**
+     * Get verbs by type
+     */
+    async getVerbsByType(type) {
+        await this.ensureInitialized();
+        // Use the paginated getVerbs method with type filter
+        const result = await this.getVerbs({
+            filter: { verbType: type }
+        });
+        return result.items;
+    }
+    /**
+     * Internal method for loading all nouns - used by performance optimizations
+     * @internal - Do not use directly, use getNouns() with pagination instead
+     */
+    async _loadAllNounsForOptimization() {
+        await this.ensureInitialized();
+        // Only use this for internal optimizations when safe
+        const result = await this.getNouns({
+            pagination: { limit: Number.MAX_SAFE_INTEGER }
+        });
+        return result.items;
+    }
+    /**
+     * Get nouns with pagination and filtering
+     * @param options Pagination and filtering options
+     * @returns Promise that resolves to a paginated result of nouns
+     */
+    async getNouns(options) {
+        await this.ensureInitialized();
+        // Set default pagination values
+        const pagination = options?.pagination || {};
+        const limit = pagination.limit || 100;
+        const offset = pagination.offset || 0;
+        const cursor = pagination.cursor;
+        // Optimize for common filter cases to avoid loading all nouns
+        if (options?.filter) {
+            // If filtering by nounType only, use the optimized method
+            if (options.filter.nounType &&
+                !options.filter.service &&
+                !options.filter.metadata) {
+                const nounType = Array.isArray(options.filter.nounType)
+                    ? options.filter.nounType[0]
+                    : options.filter.nounType;
+                // Get nouns by type directly
+                const nounsByType = await this.getNounsByNounType_internal(nounType);
+                // Apply pagination
+                const paginatedNouns = nounsByType.slice(offset, offset + limit);
+                const hasMore = offset + limit < nounsByType.length;
+                // Set next cursor if there are more items
+                let nextCursor = undefined;
+                if (hasMore && paginatedNouns.length > 0) {
+                    const lastItem = paginatedNouns[paginatedNouns.length - 1];
+                    nextCursor = lastItem.id;
+                }
+                return {
+                    items: paginatedNouns,
+                    totalCount: nounsByType.length,
+                    hasMore,
+                    nextCursor
+                };
+            }
+        }
+        // For more complex filtering or no filtering, use a paginated approach
+        // that avoids loading all nouns into memory at once
+        try {
+            // First, try to get a count of total nouns (if the adapter supports it)
+            let totalCount = undefined;
+            try {
+                // This is an optional method that adapters may implement
+                if (typeof this.countNouns === 'function') {
+                    totalCount = await this.countNouns(options?.filter);
+                }
+            }
+            catch (countError) {
+                // Ignore errors from count method, it's optional
+                console.warn('Error getting noun count:', countError);
+            }
+            // Check if the adapter has a paginated method for getting nouns
+            if (typeof this.getNounsWithPagination === 'function') {
+                // Use the adapter's paginated method
+                const result = await this.getNounsWithPagination({
+                    limit,
+                    cursor,
+                    filter: options?.filter
+                });
+                // Apply offset if needed (some adapters might not support offset)
+                const items = result.items.slice(offset);
+                return {
+                    items,
+                    totalCount: result.totalCount || totalCount,
+                    hasMore: result.hasMore,
+                    nextCursor: result.nextCursor
+                };
+            }
+            // Storage adapter does not support pagination
+            console.error('Storage adapter does not support pagination. The deprecated getAllNouns_internal() method has been removed. Please implement getNounsWithPagination() in your storage adapter.');
+            return {
+                items: [],
+                totalCount: 0,
+                hasMore: false
+            };
+        }
+        catch (error) {
+            console.error('Error getting nouns with pagination:', error);
+            return {
+                items: [],
+                totalCount: 0,
+                hasMore: false
+            };
+        }
+    }
+    /**
+     * Get verbs with pagination and filtering
+     * @param options Pagination and filtering options
+     * @returns Promise that resolves to a paginated result of verbs
+     */
+    async getVerbs(options) {
+        await this.ensureInitialized();
+        // Set default pagination values
+        const pagination = options?.pagination || {};
+        const limit = pagination.limit || 100;
+        const offset = pagination.offset || 0;
+        const cursor = pagination.cursor;
+        // Optimize for common filter cases to avoid loading all verbs
+        if (options?.filter) {
+            // If filtering by sourceId only, use the optimized method
+            if (options.filter.sourceId &&
+                !options.filter.verbType &&
+                !options.filter.targetId &&
+                !options.filter.service &&
+                !options.filter.metadata) {
+                const sourceId = Array.isArray(options.filter.sourceId)
+                    ? options.filter.sourceId[0]
+                    : options.filter.sourceId;
+                // Get verbs by source directly
+                const verbsBySource = await this.getVerbsBySource_internal(sourceId);
+                // Apply pagination
+                const paginatedVerbs = verbsBySource.slice(offset, offset + limit);
+                const hasMore = offset + limit < verbsBySource.length;
+                // Set next cursor if there are more items
+                let nextCursor = undefined;
+                if (hasMore && paginatedVerbs.length > 0) {
+                    const lastItem = paginatedVerbs[paginatedVerbs.length - 1];
+                    nextCursor = lastItem.id;
+                }
+                return {
+                    items: paginatedVerbs,
+                    totalCount: verbsBySource.length,
+                    hasMore,
+                    nextCursor
+                };
+            }
+            // If filtering by targetId only, use the optimized method
+            if (options.filter.targetId &&
+                !options.filter.verbType &&
+                !options.filter.sourceId &&
+                !options.filter.service &&
+                !options.filter.metadata) {
+                const targetId = Array.isArray(options.filter.targetId)
+                    ? options.filter.targetId[0]
+                    : options.filter.targetId;
+                // Get verbs by target directly
+                const verbsByTarget = await this.getVerbsByTarget_internal(targetId);
+                // Apply pagination
+                const paginatedVerbs = verbsByTarget.slice(offset, offset + limit);
+                const hasMore = offset + limit < verbsByTarget.length;
+                // Set next cursor if there are more items
+                let nextCursor = undefined;
+                if (hasMore && paginatedVerbs.length > 0) {
+                    const lastItem = paginatedVerbs[paginatedVerbs.length - 1];
+                    nextCursor = lastItem.id;
+                }
+                return {
+                    items: paginatedVerbs,
+                    totalCount: verbsByTarget.length,
+                    hasMore,
+                    nextCursor
+                };
+            }
+            // If filtering by verbType only, use the optimized method
+            if (options.filter.verbType &&
+                !options.filter.sourceId &&
+                !options.filter.targetId &&
+                !options.filter.service &&
+                !options.filter.metadata) {
+                const verbType = Array.isArray(options.filter.verbType)
+                    ? options.filter.verbType[0]
+                    : options.filter.verbType;
+                // Get verbs by type directly
+                const verbsByType = await this.getVerbsByType_internal(verbType);
+                // Apply pagination
+                const paginatedVerbs = verbsByType.slice(offset, offset + limit);
+                const hasMore = offset + limit < verbsByType.length;
+                // Set next cursor if there are more items
+                let nextCursor = undefined;
+                if (hasMore && paginatedVerbs.length > 0) {
+                    const lastItem = paginatedVerbs[paginatedVerbs.length - 1];
+                    nextCursor = lastItem.id;
+                }
+                return {
+                    items: paginatedVerbs,
+                    totalCount: verbsByType.length,
+                    hasMore,
+                    nextCursor
+                };
+            }
+        }
+        // For more complex filtering or no filtering, use a paginated approach
+        // that avoids loading all verbs into memory at once
+        try {
+            // First, try to get a count of total verbs (if the adapter supports it)
+            let totalCount = undefined;
+            try {
+                // This is an optional method that adapters may implement
+                if (typeof this.countVerbs === 'function') {
+                    totalCount = await this.countVerbs(options?.filter);
+                }
+            }
+            catch (countError) {
+                // Ignore errors from count method, it's optional
+                console.warn('Error getting verb count:', countError);
+            }
+            // Check if the adapter has a paginated method for getting verbs
+            if (typeof this.getVerbsWithPagination === 'function') {
+                // Use the adapter's paginated method
+                const result = await this.getVerbsWithPagination({
+                    limit,
+                    cursor,
+                    filter: options?.filter
+                });
+                // Apply offset if needed (some adapters might not support offset)
+                const items = result.items.slice(offset);
+                return {
+                    items,
+                    totalCount: result.totalCount || totalCount,
+                    hasMore: result.hasMore,
+                    nextCursor: result.nextCursor
+                };
+            }
+            // Storage adapter does not support pagination
+            console.error('Storage adapter does not support pagination. The deprecated getAllVerbs_internal() method has been removed. Please implement getVerbsWithPagination() in your storage adapter.');
+            return {
+                items: [],
+                totalCount: 0,
+                hasMore: false
+            };
+        }
+        catch (error) {
+            console.error('Error getting verbs with pagination:', error);
+            return {
+                items: [],
+                totalCount: 0,
+                hasMore: false
+            };
+        }
+    }
+    /**
+     * Delete a verb from storage
+     */
+    async deleteVerb(id) {
+        await this.ensureInitialized();
+        return this.deleteVerb_internal(id);
+    }
+    /**
+     * Helper method to convert a Map to a plain object for serialization
+     */
+    mapToObject(map, valueTransformer = (v) => v) {
+        const obj = {};
+        for (const [key, value] of map.entries()) {
+            obj[key.toString()] = valueTransformer(value);
+        }
+        return obj;
+    }
+    /**
+     * Save statistics data to storage (public interface)
+     * @param statistics The statistics data to save
+     */
+    async saveStatistics(statistics) {
+        return this.saveStatisticsData(statistics);
+    }
+    /**
+     * Get statistics data from storage (public interface)
+     * @returns Promise that resolves to the statistics data or null if not found
+     */
+    async getStatistics() {
+        return this.getStatisticsData();
+    }
+}
+//# sourceMappingURL=baseStorage.js.map

dist/storage/cacheManager.d.ts

@@ -0,0 +1,331 @@
+/**
+ * Multi-level Cache Manager
+ *
+ * Implements a three-level caching strategy:
+ * - Level 1: Hot cache (most accessed nodes) - RAM (automatically detecting and adjusting in each environment)
+ * - Level 2: Warm cache (recent nodes) - OPFS, Filesystem or S3 depending on environment
+ * - Level 3: Cold storage (all nodes) - OPFS, Filesystem or S3 depending on environment
+ */
+import { HNSWNoun, GraphVerb, HNSWVerb } from '../coreTypes.js';
+declare global {
+    interface Navigator {
+        deviceMemory?: number;
+    }
+    interface WorkerGlobalScope {
+        storage?: {
+            getDirectory?: () => Promise<any>;
+            [key: string]: any;
+        };
+    }
+}
+type HNSWNode = HNSWNoun;
+type Edge = GraphVerb;
+interface CacheStats {
+    hits: number;
+    misses: number;
+    evictions: number;
+    size: number;
+    maxSize: number;
+    hotCacheSize: number;
+    warmCacheSize: number;
+    hotCacheHits: number;
+    hotCacheMisses: number;
+    warmCacheHits: number;
+    warmCacheMisses: number;
+}
+/**
+ * Multi-level cache manager for efficient data access
+ */
+export declare class CacheManager<T extends HNSWNode | Edge | HNSWVerb> {
+    private hotCache;
+    private stats;
+    private environment;
+    private warmStorageType;
+    private coldStorageType;
+    private hotCacheMaxSize;
+    private hotCacheEvictionThreshold;
+    private warmCacheTTL;
+    private batchSize;
+    private autoTune;
+    private lastAutoTuneTime;
+    private autoTuneInterval;
+    private storageStatistics;
+    private warmStorage;
+    private coldStorage;
+    private options;
+    /**
+     * Initialize the cache manager
+     * @param options Configuration options
+     */
+    constructor(options?: {
+        hotCacheMaxSize?: number;
+        hotCacheEvictionThreshold?: number;
+        warmCacheTTL?: number;
+        batchSize?: number;
+        autoTune?: boolean;
+        warmStorage?: any;
+        coldStorage?: any;
+        readOnly?: boolean;
+        environmentConfig?: {
+            node?: {
+                hotCacheMaxSize?: number;
+                hotCacheEvictionThreshold?: number;
+                warmCacheTTL?: number;
+                batchSize?: number;
+            };
+            browser?: {
+                hotCacheMaxSize?: number;
+                hotCacheEvictionThreshold?: number;
+                warmCacheTTL?: number;
+                batchSize?: number;
+            };
+            worker?: {
+                hotCacheMaxSize?: number;
+                hotCacheEvictionThreshold?: number;
+                warmCacheTTL?: number;
+                batchSize?: number;
+            };
+            [key: string]: {
+                hotCacheMaxSize?: number;
+                hotCacheEvictionThreshold?: number;
+                warmCacheTTL?: number;
+                batchSize?: number;
+            } | undefined;
+        };
+    });
+    /**
+     * Detect the current environment
+     */
+    private detectEnvironment;
+    /**
+     * Detect the optimal cache size based on available memory and operating mode
+     *
+     * Enhanced to better handle large datasets in S3 or other storage:
+     * - Increases cache size for read-only mode
+     * - Adjusts based on total dataset size when available
+     * - Provides more aggressive caching for large datasets
+     * - Optimizes memory usage based on environment
+     */
+    private detectOptimalCacheSize;
+    /**
+     * Async version of detectOptimalCacheSize that uses dynamic imports
+     * to access system information in Node.js environments
+     *
+     * This method provides more accurate memory detection by using
+     * the OS module's dynamic import in Node.js environments
+     */
+    private detectOptimalCacheSizeAsync;
+    /**
+     * Detects available memory across different environments
+     *
+     * This method uses different techniques to detect memory in:
+     * - Node.js: Uses the OS module with dynamic import
+     * - Browser: Uses performance.memory or navigator.deviceMemory
+     * - Worker: Uses performance.memory if available
+     *
+     * @returns An object with totalMemory and freeMemory in bytes, or null if detection fails
+     */
+    private detectAvailableMemory;
+    /**
+     * Tune cache parameters based on statistics and environment
+     * This method is called periodically if auto-tuning is enabled
+     *
+     * The auto-tuning process:
+     * 1. Retrieves storage statistics if available
+     * 2. Tunes each parameter based on statistics and environment
+     * 3. Logs the tuned parameters if debug is enabled
+     *
+     * Auto-tuning helps optimize cache performance by adapting to:
+     * - The current environment (Node.js, browser, worker)
+     * - Available system resources (memory, CPU)
+     * - Usage patterns (read-heavy vs. write-heavy workloads)
+     * - Cache efficiency (hit/miss ratios)
+     */
+    private tuneParameters;
+    /**
+     * Tune hot cache size based on statistics, environment, and operating mode
+     *
+     * The hot cache size is tuned based on:
+     * 1. Available memory in the current environment
+     * 2. Total number of nodes and edges in the system
+     * 3. Cache hit/miss ratio
+     * 4. Operating mode (read-only vs. read-write)
+     * 5. Storage type (S3, filesystem, memory)
+     *
+     * Enhanced algorithm:
+     * - Start with a size based on available memory and operating mode
+     * - For large datasets in S3 or other remote storage, use more aggressive caching
+     * - Adjust based on access patterns (read-heavy vs. write-heavy)
+     * - For read-only mode, prioritize cache size over eviction speed
+     * - Dynamically adjust based on hit/miss ratio and query patterns
+     */
+    private tuneHotCacheSize;
+    /**
+     * Tune eviction threshold based on statistics
+     *
+     * The eviction threshold determines when items start being evicted from the hot cache.
+     * It is tuned based on:
+     * 1. Cache hit/miss ratio
+     * 2. Operation patterns (read-heavy vs. write-heavy workloads)
+     * 3. Memory pressure and available resources
+     *
+     * Algorithm:
+     * - Start with a default threshold of 0.8 (80% of max size)
+     * - For high hit ratios, increase the threshold to keep more items in cache
+     * - For low hit ratios, decrease the threshold to evict items more aggressively
+     * - For read-heavy workloads, use a higher threshold
+     * - For write-heavy workloads, use a lower threshold
+     * - Under memory pressure, use a lower threshold to conserve resources
+     *
+     * @param cacheStats Optional cache statistics for more adaptive tuning
+     */
+    private tuneEvictionThreshold;
+    /**
+     * Tune warm cache TTL based on statistics
+     *
+     * The warm cache TTL determines how long items remain in the warm cache.
+     * It is tuned based on:
+     * 1. Update frequency from operation statistics
+     * 2. Warm cache hit/miss ratio
+     * 3. Access patterns and frequency
+     * 4. Available storage resources
+     *
+     * Algorithm:
+     * - Start with a default TTL of 24 hours
+     * - For frequently updated data, use a shorter TTL
+     * - For rarely updated data, use a longer TTL
+     * - For frequently accessed data, use a longer TTL
+     * - For rarely accessed data, use a shorter TTL
+     * - Under storage pressure, use a shorter TTL
+     *
+     * @param cacheStats Optional cache statistics for more adaptive tuning
+     */
+    private tuneWarmCacheTTL;
+    /**
+     * Tune batch size based on environment, statistics, and operating mode
+     *
+     * The batch size determines how many items are processed in a single batch
+     * for operations like prefetching. It is tuned based on:
+     * 1. Current environment (Node.js, browser, worker)
+     * 2. Available memory
+     * 3. Operation patterns
+     * 4. Cache hit/miss ratio
+     * 5. Operating mode (read-only vs. read-write)
+     * 6. Storage type (S3, filesystem, memory)
+     * 7. Dataset size
+     * 8. Cache efficiency and access patterns
+     *
+     * Enhanced algorithm:
+     * - Start with a default based on the environment
+     * - For large datasets in S3 or other remote storage, use larger batches
+     * - For read-only mode, use larger batches to improve throughput
+     * - Dynamically adjust based on network latency and throughput
+     * - Balance between memory usage and performance
+     * - Adapt to cache hit/miss patterns
+     *
+     * @param cacheStats Optional cache statistics for more adaptive tuning
+     */
+    private tuneBatchSize;
+    /**
+     * Detect the appropriate warm storage type based on environment
+     */
+    private detectWarmStorageType;
+    /**
+     * Detect the appropriate cold storage type based on environment
+     */
+    private detectColdStorageType;
+    /**
+     * Initialize warm storage adapter
+     */
+    private initializeWarmStorage;
+    /**
+     * Initialize cold storage adapter
+     */
+    private initializeColdStorage;
+    /**
+     * Get an item from cache, trying each level in order
+     * @param id The item ID
+     * @returns The cached item or null if not found
+     */
+    get(id: string): Promise<T | null>;
+    /**
+     * Get an item from warm cache
+     * @param id The item ID
+     * @returns The cached item or null if not found
+     */
+    private getFromWarmCache;
+    /**
+     * Get an item from cold storage
+     * @param id The item ID
+     * @returns The item or null if not found
+     */
+    private getFromColdStorage;
+    /**
+     * Add an item to hot cache
+     * @param id The item ID
+     * @param item The item to cache
+     */
+    private addToHotCache;
+    /**
+     * Add an item to warm cache
+     * @param id The item ID
+     * @param item The item to cache
+     */
+    private addToWarmCache;
+    /**
+     * Evict items from hot cache based on LRU policy
+     */
+    private evictFromHotCache;
+    /**
+     * Set an item in all cache levels
+     * @param id The item ID
+     * @param item The item to cache
+     */
+    set(id: string, item: T): Promise<void>;
+    /**
+     * Delete an item from all cache levels
+     * @param id The item ID to delete
+     */
+    delete(id: string): Promise<void>;
+    /**
+     * Clear all cache levels
+     */
+    clear(): Promise<void>;
+    /**
+     * Get cache statistics
+     * @returns Cache statistics
+     */
+    getStats(): CacheStats;
+    /**
+     * Prefetch items based on ID patterns or relationships
+     * @param ids Array of IDs to prefetch
+     */
+    prefetch(ids: string[]): Promise<void>;
+    /**
+     * Check if it's time to tune parameters and do so if needed
+     * This is called before operations that might benefit from tuned parameters
+     *
+     * This method serves as a checkpoint for auto-tuning, ensuring that:
+     * 1. Parameters are tuned periodically based on the auto-tune interval
+     * 2. Tuning happens before critical operations that would benefit from optimized parameters
+     * 3. Tuning doesn't happen too frequently, which could impact performance
+     *
+     * By calling this method before get(), getMany(), and prefetch() operations,
+     * we ensure that the cache parameters are optimized for the current workload
+     * without adding unnecessary overhead to every operation.
+     */
+    private checkAndTuneParameters;
+    /**
+     * Get multiple items at once, optimizing for batch retrieval
+     * @param ids Array of IDs to get
+     * @returns Map of ID to item
+     */
+    getMany(ids: string[]): Promise<Map<string, T>>;
+    /**
+     * Set the storage adapters for warm and cold caches
+     * @param warmStorage Warm cache storage adapter
+     * @param coldStorage Cold storage adapter
+     */
+    setStorageAdapters(warmStorage: any, coldStorage: any): void;
+}
+export {};

dist/storage/enhancedCacheManager.d.ts

@@ -0,0 +1,141 @@
+/**
+ * Enhanced Multi-Level Cache Manager with Predictive Prefetching
+ * Optimized for HNSW search patterns and large-scale vector operations
+ */
+import { HNSWNoun, HNSWVerb } from '../coreTypes.js';
+import { BatchS3Operations } from './adapters/batchS3Operations.js';
+declare enum PrefetchStrategy {
+    GRAPH_CONNECTIVITY = "connectivity",
+    VECTOR_SIMILARITY = "similarity",
+    ACCESS_PATTERN = "pattern",
+    HYBRID = "hybrid"
+}
+interface EnhancedCacheConfig {
+    hotCacheMaxSize?: number;
+    hotCacheEvictionThreshold?: number;
+    warmCacheMaxSize?: number;
+    warmCacheTTL?: number;
+    prefetchEnabled?: boolean;
+    prefetchStrategy?: PrefetchStrategy;
+    prefetchBatchSize?: number;
+    predictionLookahead?: number;
+    similarityThreshold?: number;
+    maxSimilarityDistance?: number;
+    backgroundOptimization?: boolean;
+    statisticsCollection?: boolean;
+}
+/**
+ * Enhanced cache manager with intelligent prefetching for HNSW operations
+ * Provides multi-level caching optimized for vector search workloads
+ */
+export declare class EnhancedCacheManager<T extends HNSWNoun | HNSWVerb> {
+    private hotCache;
+    private warmCache;
+    private prefetchQueue;
+    private accessPatterns;
+    private vectorIndex;
+    private config;
+    private batchOperations?;
+    private storageAdapter?;
+    private prefetchInProgress;
+    private stats;
+    constructor(config?: EnhancedCacheConfig);
+    /**
+     * Set storage adapters for warm/cold storage operations
+     */
+    setStorageAdapters(storageAdapter: any, batchOperations?: BatchS3Operations): void;
+    /**
+     * Get item with intelligent prefetching
+     */
+    get(id: string): Promise<T | null>;
+    /**
+     * Get multiple items efficiently with batch operations
+     */
+    getMany(ids: string[]): Promise<Map<string, T>>;
+    /**
+     * Set item in cache with metadata
+     */
+    set(id: string, item: T): Promise<void>;
+    /**
+     * Intelligent prefetch based on access patterns and graph structure
+     */
+    private schedulePrefetch;
+    /**
+     * Predict next nodes based on graph connectivity
+     */
+    private predictByConnectivity;
+    /**
+     * Predict next nodes based on vector similarity
+     */
+    private predictBySimilarity;
+    /**
+     * Predict based on historical access patterns
+     */
+    private predictByAccessPattern;
+    /**
+     * Hybrid prediction combining multiple strategies
+     */
+    private hybridPrediction;
+    /**
+     * Execute prefetch operation in background
+     */
+    private executePrefetch;
+    /**
+     * Load item from storage adapter
+     */
+    private loadFromStorage;
+    /**
+     * Promote frequently accessed item to hot cache
+     */
+    private promoteToHotCache;
+    /**
+     * Evict least recently used items from hot cache
+     */
+    private evictFromHotCache;
+    /**
+     * Evict expired items from warm cache
+     */
+    private evictFromWarmCache;
+    /**
+     * Record access pattern for prediction
+     */
+    private recordAccess;
+    /**
+     * Extract connected node IDs from HNSW item
+     */
+    private extractConnectedNodes;
+    /**
+     * Check if cache entry is expired
+     */
+    private isExpired;
+    /**
+     * Calculate cosine similarity between vectors
+     */
+    private cosineSimilarity;
+    /**
+     * Calculate pattern similarity between access patterns
+     */
+    private patternSimilarity;
+    /**
+     * Start background optimization process
+     */
+    private startBackgroundOptimization;
+    /**
+     * Run background optimization tasks
+     */
+    private runBackgroundOptimization;
+    /**
+     * Get cache statistics
+     */
+    getStats(): typeof this.stats & {
+        hotCacheSize: number;
+        warmCacheSize: number;
+        prefetchQueueSize: number;
+        accessPatternsTracked: number;
+    };
+    /**
+     * Clear all caches
+     */
+    clear(): void;
+}
+export {};

dist/storage/enhancedCacheManager.js

@@ -0,0 +1,520 @@
+/**
+ * Enhanced Multi-Level Cache Manager with Predictive Prefetching
+ * Optimized for HNSW search patterns and large-scale vector operations
+ */
+// Prefetch prediction strategies
+var PrefetchStrategy;
+(function (PrefetchStrategy) {
+    PrefetchStrategy["GRAPH_CONNECTIVITY"] = "connectivity";
+    PrefetchStrategy["VECTOR_SIMILARITY"] = "similarity";
+    PrefetchStrategy["ACCESS_PATTERN"] = "pattern";
+    PrefetchStrategy["HYBRID"] = "hybrid";
+})(PrefetchStrategy || (PrefetchStrategy = {}));
+/**
+ * Enhanced cache manager with intelligent prefetching for HNSW operations
+ * Provides multi-level caching optimized for vector search workloads
+ */
+export class EnhancedCacheManager {
+    constructor(config = {}) {
+        this.hotCache = new Map();
+        this.warmCache = new Map();
+        this.prefetchQueue = new Set();
+        this.accessPatterns = new Map(); // Track access times
+        this.vectorIndex = new Map(); // For similarity calculations
+        this.prefetchInProgress = false;
+        // Statistics and monitoring
+        this.stats = {
+            hotCacheHits: 0,
+            hotCacheMisses: 0,
+            warmCacheHits: 0,
+            warmCacheMisses: 0,
+            prefetchHits: 0,
+            prefetchMisses: 0,
+            totalPrefetched: 0,
+            predictionAccuracy: 0,
+            backgroundOptimizations: 0
+        };
+        this.config = {
+            hotCacheMaxSize: 1000,
+            hotCacheEvictionThreshold: 0.8,
+            warmCacheMaxSize: 10000,
+            warmCacheTTL: 300000, // 5 minutes
+            prefetchEnabled: true,
+            prefetchStrategy: PrefetchStrategy.HYBRID,
+            prefetchBatchSize: 50,
+            predictionLookahead: 3,
+            similarityThreshold: 0.8,
+            maxSimilarityDistance: 2.0,
+            backgroundOptimization: true,
+            statisticsCollection: true,
+            ...config
+        };
+        // Start background optimization if enabled
+        if (this.config.backgroundOptimization) {
+            this.startBackgroundOptimization();
+        }
+    }
+    /**
+     * Set storage adapters for warm/cold storage operations
+     */
+    setStorageAdapters(storageAdapter, batchOperations) {
+        this.storageAdapter = storageAdapter;
+        this.batchOperations = batchOperations;
+    }
+    /**
+     * Get item with intelligent prefetching
+     */
+    async get(id) {
+        const startTime = Date.now();
+        // Update access pattern
+        this.recordAccess(id, startTime);
+        // Check hot cache first
+        let entry = this.hotCache.get(id);
+        if (entry && !this.isExpired(entry)) {
+            entry.lastAccessed = startTime;
+            entry.accessCount++;
+            this.stats.hotCacheHits++;
+            // Trigger predictive prefetch
+            if (this.config.prefetchEnabled) {
+                this.schedulePrefetch(id, entry.data);
+            }
+            return entry.data;
+        }
+        this.stats.hotCacheMisses++;
+        // Check warm cache
+        entry = this.warmCache.get(id);
+        if (entry && !this.isExpired(entry)) {
+            entry.lastAccessed = startTime;
+            entry.accessCount++;
+            this.stats.warmCacheHits++;
+            // Promote to hot cache if frequently accessed
+            if (entry.accessCount > 3) {
+                this.promoteToHotCache(id, entry);
+            }
+            return entry.data;
+        }
+        this.stats.warmCacheMisses++;
+        // Load from storage
+        const item = await this.loadFromStorage(id);
+        if (item) {
+            // Cache the item
+            await this.set(id, item);
+            // Trigger predictive prefetch
+            if (this.config.prefetchEnabled) {
+                this.schedulePrefetch(id, item);
+            }
+        }
+        return item;
+    }
+    /**
+     * Get multiple items efficiently with batch operations
+     */
+    async getMany(ids) {
+        const result = new Map();
+        const uncachedIds = [];
+        // Check caches first
+        for (const id of ids) {
+            const cached = await this.get(id);
+            if (cached) {
+                result.set(id, cached);
+            }
+            else {
+                uncachedIds.push(id);
+            }
+        }
+        // Batch load uncached items
+        if (uncachedIds.length > 0 && this.batchOperations) {
+            const batchResult = await this.batchOperations.batchGetNodes(uncachedIds);
+            // Cache loaded items
+            for (const [id, item] of batchResult.items) {
+                await this.set(id, item);
+                result.set(id, item);
+            }
+        }
+        return result;
+    }
+    /**
+     * Set item in cache with metadata
+     */
+    async set(id, item) {
+        const now = Date.now();
+        const entry = {
+            data: item,
+            lastAccessed: now,
+            accessCount: 1,
+            expiresAt: now + this.config.warmCacheTTL,
+            connectedNodes: this.extractConnectedNodes(item),
+            predictionScore: 0
+        };
+        // Store vector for similarity calculations
+        if ('vector' in item && item.vector) {
+            this.vectorIndex.set(id, item.vector);
+            entry.vectorSimilarity = 0;
+        }
+        // Add to warm cache initially
+        this.warmCache.set(id, entry);
+        // Clean up if needed
+        if (this.warmCache.size > this.config.warmCacheMaxSize) {
+            this.evictFromWarmCache();
+        }
+        // Update statistics
+        this.stats.warmCacheHits++; // Count as a potential future hit
+    }
+    /**
+     * Intelligent prefetch based on access patterns and graph structure
+     */
+    async schedulePrefetch(currentId, currentItem) {
+        if (this.prefetchInProgress || !this.config.prefetchEnabled) {
+            return;
+        }
+        // Use different strategies based on configuration
+        let candidateIds = [];
+        switch (this.config.prefetchStrategy) {
+            case PrefetchStrategy.GRAPH_CONNECTIVITY:
+                candidateIds = this.predictByConnectivity(currentId, currentItem);
+                break;
+            case PrefetchStrategy.VECTOR_SIMILARITY:
+                candidateIds = await this.predictBySimilarity(currentId, currentItem);
+                break;
+            case PrefetchStrategy.ACCESS_PATTERN:
+                candidateIds = this.predictByAccessPattern(currentId);
+                break;
+            case PrefetchStrategy.HYBRID:
+                candidateIds = await this.hybridPrediction(currentId, currentItem);
+                break;
+        }
+        // Filter out already cached items
+        const uncachedIds = candidateIds.filter(id => !this.hotCache.has(id) && !this.warmCache.has(id)).slice(0, this.config.prefetchBatchSize);
+        if (uncachedIds.length > 0) {
+            this.executePrefetch(uncachedIds);
+        }
+    }
+    /**
+     * Predict next nodes based on graph connectivity
+     */
+    predictByConnectivity(currentId, currentItem) {
+        const candidates = [];
+        if ('connections' in currentItem && currentItem.connections) {
+            const connections = currentItem.connections;
+            // Add immediate neighbors with higher priority for lower levels
+            for (const [level, nodeIds] of connections.entries()) {
+                const priority = Math.max(1, 5 - level); // Higher priority for level 0
+                for (const nodeId of nodeIds) {
+                    // Add based on priority
+                    for (let i = 0; i < priority; i++) {
+                        candidates.push(nodeId);
+                    }
+                }
+            }
+        }
+        // Shuffle and deduplicate
+        const shuffled = candidates.sort(() => Math.random() - 0.5);
+        return [...new Set(shuffled)];
+    }
+    /**
+     * Predict next nodes based on vector similarity
+     */
+    async predictBySimilarity(currentId, currentItem) {
+        if (!('vector' in currentItem) || !currentItem.vector) {
+            return [];
+        }
+        const currentVector = currentItem.vector;
+        const similarities = [];
+        // Calculate similarities with vectors in cache
+        for (const [id, vector] of this.vectorIndex.entries()) {
+            if (id === currentId)
+                continue;
+            const similarity = this.cosineSimilarity(currentVector, vector);
+            if (similarity > this.config.similarityThreshold) {
+                similarities.push([id, similarity]);
+            }
+        }
+        // Sort by similarity and return top candidates
+        similarities.sort((a, b) => b[1] - a[1]);
+        return similarities.slice(0, this.config.prefetchBatchSize).map(([id]) => id);
+    }
+    /**
+     * Predict based on historical access patterns
+     */
+    predictByAccessPattern(currentId) {
+        const currentPattern = this.accessPatterns.get(currentId);
+        if (!currentPattern || currentPattern.length < 2) {
+            return [];
+        }
+        // Find similar access patterns
+        const candidates = [];
+        for (const [id, pattern] of this.accessPatterns.entries()) {
+            if (id === currentId || pattern.length < 2)
+                continue;
+            const similarity = this.patternSimilarity(currentPattern, pattern);
+            if (similarity > 0.5) {
+                candidates.push([id, similarity]);
+            }
+        }
+        candidates.sort((a, b) => b[1] - a[1]);
+        return candidates.slice(0, this.config.prefetchBatchSize).map(([id]) => id);
+    }
+    /**
+     * Hybrid prediction combining multiple strategies
+     */
+    async hybridPrediction(currentId, currentItem) {
+        const connectivityCandidates = this.predictByConnectivity(currentId, currentItem);
+        const similarityCandidates = await this.predictBySimilarity(currentId, currentItem);
+        const patternCandidates = this.predictByAccessPattern(currentId);
+        // Weighted combination
+        const candidateScores = new Map();
+        // Connectivity gets highest weight (40%)
+        connectivityCandidates.forEach((id, index) => {
+            const score = (connectivityCandidates.length - index) / connectivityCandidates.length * 0.4;
+            candidateScores.set(id, (candidateScores.get(id) || 0) + score);
+        });
+        // Similarity gets medium weight (35%)
+        similarityCandidates.forEach((id, index) => {
+            const score = (similarityCandidates.length - index) / similarityCandidates.length * 0.35;
+            candidateScores.set(id, (candidateScores.get(id) || 0) + score);
+        });
+        // Pattern gets lower weight (25%)
+        patternCandidates.forEach((id, index) => {
+            const score = (patternCandidates.length - index) / patternCandidates.length * 0.25;
+            candidateScores.set(id, (candidateScores.get(id) || 0) + score);
+        });
+        // Sort by combined score
+        const sortedCandidates = Array.from(candidateScores.entries())
+            .sort((a, b) => b[1] - a[1])
+            .map(([id]) => id);
+        return sortedCandidates.slice(0, this.config.prefetchBatchSize);
+    }
+    /**
+     * Execute prefetch operation in background
+     */
+    async executePrefetch(ids) {
+        if (this.prefetchInProgress || !this.batchOperations) {
+            return;
+        }
+        this.prefetchInProgress = true;
+        try {
+            const batchResult = await this.batchOperations.batchGetNodes(ids);
+            // Cache prefetched items
+            for (const [id, item] of batchResult.items) {
+                const entry = {
+                    data: item,
+                    lastAccessed: Date.now(),
+                    accessCount: 0, // Prefetched items start with 0 access count
+                    expiresAt: Date.now() + this.config.warmCacheTTL,
+                    connectedNodes: this.extractConnectedNodes(item),
+                    predictionScore: 1 // Mark as prefetched
+                };
+                this.warmCache.set(id, entry);
+            }
+            this.stats.totalPrefetched += batchResult.items.size;
+        }
+        catch (error) {
+            console.warn('Prefetch operation failed:', error);
+        }
+        finally {
+            this.prefetchInProgress = false;
+        }
+    }
+    /**
+     * Load item from storage adapter
+     */
+    async loadFromStorage(id) {
+        if (!this.storageAdapter) {
+            return null;
+        }
+        try {
+            return await this.storageAdapter.get(id);
+        }
+        catch (error) {
+            console.warn(`Failed to load ${id} from storage:`, error);
+            return null;
+        }
+    }
+    /**
+     * Promote frequently accessed item to hot cache
+     */
+    promoteToHotCache(id, entry) {
+        // Remove from warm cache
+        this.warmCache.delete(id);
+        // Add to hot cache
+        this.hotCache.set(id, entry);
+        // Evict if necessary
+        if (this.hotCache.size > this.config.hotCacheMaxSize) {
+            this.evictFromHotCache();
+        }
+    }
+    /**
+     * Evict least recently used items from hot cache
+     */
+    evictFromHotCache() {
+        const threshold = Math.floor(this.config.hotCacheMaxSize * this.config.hotCacheEvictionThreshold);
+        if (this.hotCache.size <= threshold) {
+            return;
+        }
+        // Sort by last accessed time and access count
+        const entries = Array.from(this.hotCache.entries())
+            .sort((a, b) => {
+            const scoreA = a[1].accessCount * 0.7 + (Date.now() - a[1].lastAccessed) * -0.3;
+            const scoreB = b[1].accessCount * 0.7 + (Date.now() - b[1].lastAccessed) * -0.3;
+            return scoreA - scoreB;
+        });
+        // Remove least valuable entries
+        const toRemove = entries.slice(0, this.hotCache.size - threshold);
+        for (const [id] of toRemove) {
+            this.hotCache.delete(id);
+        }
+    }
+    /**
+     * Evict expired items from warm cache
+     */
+    evictFromWarmCache() {
+        const now = Date.now();
+        const toRemove = [];
+        for (const [id, entry] of this.warmCache.entries()) {
+            if (this.isExpired(entry)) {
+                toRemove.push(id);
+            }
+        }
+        // Remove expired items
+        for (const id of toRemove) {
+            this.warmCache.delete(id);
+            this.vectorIndex.delete(id);
+        }
+        // If still over limit, remove LRU items
+        if (this.warmCache.size > this.config.warmCacheMaxSize) {
+            const entries = Array.from(this.warmCache.entries())
+                .sort((a, b) => a[1].lastAccessed - b[1].lastAccessed);
+            const excess = this.warmCache.size - this.config.warmCacheMaxSize;
+            for (let i = 0; i < excess; i++) {
+                const [id] = entries[i];
+                this.warmCache.delete(id);
+                this.vectorIndex.delete(id);
+            }
+        }
+    }
+    /**
+     * Record access pattern for prediction
+     */
+    recordAccess(id, timestamp) {
+        if (!this.config.statisticsCollection) {
+            return;
+        }
+        let pattern = this.accessPatterns.get(id);
+        if (!pattern) {
+            pattern = [];
+            this.accessPatterns.set(id, pattern);
+        }
+        pattern.push(timestamp);
+        // Keep only recent accesses (last 10)
+        if (pattern.length > 10) {
+            pattern.shift();
+        }
+    }
+    /**
+     * Extract connected node IDs from HNSW item
+     */
+    extractConnectedNodes(item) {
+        const connected = new Set();
+        if ('connections' in item && item.connections) {
+            const connections = item.connections;
+            for (const nodeIds of connections.values()) {
+                nodeIds.forEach(id => connected.add(id));
+            }
+        }
+        return connected;
+    }
+    /**
+     * Check if cache entry is expired
+     */
+    isExpired(entry) {
+        return entry.expiresAt !== null && Date.now() > entry.expiresAt;
+    }
+    /**
+     * Calculate cosine similarity between vectors
+     */
+    cosineSimilarity(a, b) {
+        if (a.length !== b.length)
+            return 0;
+        let dotProduct = 0;
+        let normA = 0;
+        let normB = 0;
+        for (let i = 0; i < a.length; i++) {
+            dotProduct += a[i] * b[i];
+            normA += a[i] * a[i];
+            normB += b[i] * b[i];
+        }
+        const magnitude = Math.sqrt(normA) * Math.sqrt(normB);
+        return magnitude === 0 ? 0 : dotProduct / magnitude;
+    }
+    /**
+     * Calculate pattern similarity between access patterns
+     */
+    patternSimilarity(pattern1, pattern2) {
+        const minLength = Math.min(pattern1.length, pattern2.length);
+        if (minLength < 2)
+            return 0;
+        // Calculate intervals between accesses
+        const intervals1 = pattern1.slice(1).map((t, i) => t - pattern1[i]);
+        const intervals2 = pattern2.slice(1).map((t, i) => t - pattern2[i]);
+        // Compare interval patterns
+        let similarity = 0;
+        const compareLength = Math.min(intervals1.length, intervals2.length);
+        for (let i = 0; i < compareLength; i++) {
+            const diff = Math.abs(intervals1[i] - intervals2[i]);
+            const maxInterval = Math.max(intervals1[i], intervals2[i]);
+            similarity += maxInterval === 0 ? 1 : 1 - (diff / maxInterval);
+        }
+        return compareLength === 0 ? 0 : similarity / compareLength;
+    }
+    /**
+     * Start background optimization process
+     */
+    startBackgroundOptimization() {
+        setInterval(() => {
+            this.runBackgroundOptimization();
+        }, 60000); // Run every minute
+    }
+    /**
+     * Run background optimization tasks
+     */
+    runBackgroundOptimization() {
+        // Clean up expired entries
+        this.evictFromWarmCache();
+        this.evictFromHotCache();
+        // Clean up old access patterns
+        const cutoff = Date.now() - 3600000; // 1 hour
+        for (const [id, pattern] of this.accessPatterns.entries()) {
+            const recentAccesses = pattern.filter(t => t > cutoff);
+            if (recentAccesses.length === 0) {
+                this.accessPatterns.delete(id);
+            }
+            else {
+                this.accessPatterns.set(id, recentAccesses);
+            }
+        }
+        this.stats.backgroundOptimizations++;
+    }
+    /**
+     * Get cache statistics
+     */
+    getStats() {
+        return {
+            ...this.stats,
+            hotCacheSize: this.hotCache.size,
+            warmCacheSize: this.warmCache.size,
+            prefetchQueueSize: this.prefetchQueue.size,
+            accessPatternsTracked: this.accessPatterns.size
+        };
+    }
+    /**
+     * Clear all caches
+     */
+    clear() {
+        this.hotCache.clear();
+        this.warmCache.clear();
+        this.prefetchQueue.clear();
+        this.accessPatterns.clear();
+        this.vectorIndex.clear();
+    }
+}
+//# sourceMappingURL=enhancedCacheManager.js.map

dist/storage/readOnlyOptimizations.d.ts

@@ -0,0 +1,133 @@
+/**
+ * Read-Only Storage Optimizations for Production Deployments
+ * Implements compression, memory-mapping, and pre-built index segments
+ */
+import { HNSWNoun, Vector } from '../coreTypes.js';
+declare enum CompressionType {
+    NONE = "none",
+    GZIP = "gzip",
+    BROTLI = "brotli",
+    QUANTIZATION = "quantization",
+    HYBRID = "hybrid"
+}
+declare enum QuantizationType {
+    SCALAR = "scalar",// 8-bit scalar quantization
+    PRODUCT = "product",// Product quantization
+    BINARY = "binary"
+}
+interface CompressionConfig {
+    vectorCompression: CompressionType;
+    metadataCompression: CompressionType;
+    quantizationType?: QuantizationType;
+    quantizationBits?: number;
+    compressionLevel?: number;
+}
+interface ReadOnlyConfig {
+    prebuiltIndexPath?: string;
+    memoryMapped?: boolean;
+    compression: CompressionConfig;
+    segmentSize?: number;
+    prefetchSegments?: number;
+    cacheIndexInMemory?: boolean;
+}
+interface IndexSegment {
+    id: string;
+    nodeCount: number;
+    vectorDimension: number;
+    compression: CompressionType;
+    s3Key?: string;
+    localPath?: string;
+    loadedInMemory: boolean;
+    lastAccessed: number;
+}
+/**
+ * Read-only storage optimizations for high-performance production deployments
+ */
+export declare class ReadOnlyOptimizations {
+    private config;
+    private segments;
+    private compressionStats;
+    private quantizationCodebooks;
+    private memoryMappedBuffers;
+    constructor(config?: Partial<ReadOnlyConfig>);
+    /**
+     * Compress vector data using specified compression method
+     */
+    compressVector(vector: Vector, segmentId: string): Promise<ArrayBuffer>;
+    /**
+     * Decompress vector data
+     */
+    decompressVector(compressedData: ArrayBuffer, segmentId: string, originalDimension: number): Promise<Vector>;
+    /**
+     * Scalar quantization of vectors to 8-bit integers
+     */
+    private quantizeVector;
+    /**
+     * Dequantize 8-bit vectors back to float32
+     */
+    private dequantizeVector;
+    /**
+     * GZIP compression using browser/Node.js APIs
+     */
+    private gzipCompress;
+    /**
+     * GZIP decompression
+     */
+    private gzipDecompress;
+    /**
+     * Brotli compression (placeholder - similar to GZIP)
+     */
+    private brotliCompress;
+    /**
+     * Brotli decompression (placeholder)
+     */
+    private brotliDecompress;
+    /**
+     * Create prebuilt index segments for faster loading
+     */
+    createPrebuiltSegments(nodes: HNSWNoun[], outputPath: string): Promise<IndexSegment[]>;
+    /**
+     * Compress an entire segment of nodes
+     */
+    private compressSegment;
+    /**
+     * Load a segment from storage with caching
+     */
+    loadSegment(segmentId: string): Promise<HNSWNoun[]>;
+    /**
+     * Load segment data from storage
+     */
+    private loadSegmentFromStorage;
+    /**
+     * Deserialize and decompress segment data
+     */
+    private deserializeSegment;
+    /**
+     * Serialize connections Map for storage
+     */
+    private serializeConnections;
+    /**
+     * Deserialize connections from storage format
+     */
+    private deserializeConnections;
+    /**
+     * Prefetch segments based on access patterns
+     */
+    prefetchSegments(currentSegmentId: string): Promise<void>;
+    /**
+     * Update compression statistics
+     */
+    private updateCompressionRatio;
+    /**
+     * Get compression statistics
+     */
+    getCompressionStats(): typeof this.compressionStats & {
+        segmentCount: number;
+        memoryUsage: number;
+    };
+    /**
+     * Cleanup memory-mapped buffers
+     */
+    cleanup(): void;
+}
+export {};

dist/storage/readOnlyOptimizations.js

@@ -0,0 +1,425 @@
+/**
+ * Read-Only Storage Optimizations for Production Deployments
+ * Implements compression, memory-mapping, and pre-built index segments
+ */
+// Compression types supported
+var CompressionType;
+(function (CompressionType) {
+    CompressionType["NONE"] = "none";
+    CompressionType["GZIP"] = "gzip";
+    CompressionType["BROTLI"] = "brotli";
+    CompressionType["QUANTIZATION"] = "quantization";
+    CompressionType["HYBRID"] = "hybrid";
+})(CompressionType || (CompressionType = {}));
+// Vector quantization methods
+var QuantizationType;
+(function (QuantizationType) {
+    QuantizationType["SCALAR"] = "scalar";
+    QuantizationType["PRODUCT"] = "product";
+    QuantizationType["BINARY"] = "binary"; // Binary quantization
+})(QuantizationType || (QuantizationType = {}));
+/**
+ * Read-only storage optimizations for high-performance production deployments
+ */
+export class ReadOnlyOptimizations {
+    constructor(config = {}) {
+        this.segments = new Map();
+        this.compressionStats = {
+            originalSize: 0,
+            compressedSize: 0,
+            compressionRatio: 0,
+            decompressionTime: 0
+        };
+        // Quantization codebooks for vector compression
+        this.quantizationCodebooks = new Map();
+        // Memory-mapped buffers for large datasets
+        this.memoryMappedBuffers = new Map();
+        this.config = {
+            prebuiltIndexPath: '',
+            memoryMapped: true,
+            compression: {
+                vectorCompression: CompressionType.QUANTIZATION,
+                metadataCompression: CompressionType.GZIP,
+                quantizationType: QuantizationType.SCALAR,
+                quantizationBits: 8,
+                compressionLevel: 6
+            },
+            segmentSize: 10000, // 10k nodes per segment
+            prefetchSegments: 3,
+            cacheIndexInMemory: false,
+            ...config
+        };
+        if (config.compression) {
+            this.config.compression = { ...this.config.compression, ...config.compression };
+        }
+    }
+    /**
+     * Compress vector data using specified compression method
+     */
+    async compressVector(vector, segmentId) {
+        const startTime = Date.now();
+        let compressedData;
+        switch (this.config.compression.vectorCompression) {
+            case CompressionType.QUANTIZATION:
+                compressedData = await this.quantizeVector(vector, segmentId);
+                break;
+            case CompressionType.GZIP:
+                const gzipBuffer = new Float32Array(vector).buffer;
+                compressedData = await this.gzipCompress(gzipBuffer.slice(0));
+                break;
+            case CompressionType.BROTLI:
+                const brotliBuffer = new Float32Array(vector).buffer;
+                compressedData = await this.brotliCompress(brotliBuffer.slice(0));
+                break;
+            case CompressionType.HYBRID:
+                // First quantize, then compress
+                const quantized = await this.quantizeVector(vector, segmentId);
+                compressedData = await this.gzipCompress(quantized);
+                break;
+            default:
+                const defaultBuffer = new Float32Array(vector).buffer;
+                compressedData = defaultBuffer.slice(0);
+                break;
+        }
+        // Update compression statistics
+        const originalSize = vector.length * 4; // 4 bytes per float32
+        this.compressionStats.originalSize += originalSize;
+        this.compressionStats.compressedSize += compressedData.byteLength;
+        this.compressionStats.decompressionTime += Date.now() - startTime;
+        this.updateCompressionRatio();
+        return compressedData;
+    }
+    /**
+     * Decompress vector data
+     */
+    async decompressVector(compressedData, segmentId, originalDimension) {
+        switch (this.config.compression.vectorCompression) {
+            case CompressionType.QUANTIZATION:
+                return this.dequantizeVector(compressedData, segmentId, originalDimension);
+            case CompressionType.GZIP:
+                const gzipDecompressed = await this.gzipDecompress(compressedData);
+                return Array.from(new Float32Array(gzipDecompressed));
+            case CompressionType.BROTLI:
+                const brotliDecompressed = await this.brotliDecompress(compressedData);
+                return Array.from(new Float32Array(brotliDecompressed));
+            case CompressionType.HYBRID:
+                const gzipStage = await this.gzipDecompress(compressedData);
+                return this.dequantizeVector(gzipStage, segmentId, originalDimension);
+            default:
+                return Array.from(new Float32Array(compressedData));
+        }
+    }
+    /**
+     * Scalar quantization of vectors to 8-bit integers
+     */
+    async quantizeVector(vector, segmentId) {
+        let codebook = this.quantizationCodebooks.get(segmentId);
+        if (!codebook) {
+            // Create codebook (min/max values for scaling)
+            const min = Math.min(...vector);
+            const max = Math.max(...vector);
+            codebook = new Float32Array([min, max]);
+            this.quantizationCodebooks.set(segmentId, codebook);
+        }
+        const [min, max] = codebook;
+        const scale = (max - min) / 255; // 8-bit quantization
+        const quantized = new Uint8Array(vector.length);
+        for (let i = 0; i < vector.length; i++) {
+            quantized[i] = Math.round((vector[i] - min) / scale);
+        }
+        // Store codebook with quantized data
+        const result = new ArrayBuffer(quantized.byteLength + codebook.byteLength);
+        const resultView = new Uint8Array(result);
+        // First 8 bytes: codebook (min, max as float32)
+        resultView.set(new Uint8Array(codebook.buffer), 0);
+        // Remaining bytes: quantized vector
+        resultView.set(quantized, codebook.byteLength);
+        return result;
+    }
+    /**
+     * Dequantize 8-bit vectors back to float32
+     */
+    dequantizeVector(quantizedData, segmentId, dimension) {
+        const dataView = new Uint8Array(quantizedData);
+        // Extract codebook (first 8 bytes)
+        const codebookBytes = dataView.slice(0, 8);
+        const codebook = new Float32Array(codebookBytes.buffer);
+        const [min, max] = codebook;
+        // Extract quantized vector
+        const quantized = dataView.slice(8);
+        const scale = (max - min) / 255;
+        const result = [];
+        for (let i = 0; i < dimension; i++) {
+            result[i] = min + quantized[i] * scale;
+        }
+        return result;
+    }
+    /**
+     * GZIP compression using browser/Node.js APIs
+     */
+    async gzipCompress(data) {
+        if (typeof CompressionStream !== 'undefined') {
+            // Browser environment
+            const stream = new CompressionStream('gzip');
+            const writer = stream.writable.getWriter();
+            const reader = stream.readable.getReader();
+            writer.write(new Uint8Array(data));
+            writer.close();
+            const chunks = [];
+            let result = await reader.read();
+            while (!result.done) {
+                chunks.push(result.value);
+                result = await reader.read();
+            }
+            // Combine chunks
+            const totalLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
+            const combined = new Uint8Array(totalLength);
+            let offset = 0;
+            for (const chunk of chunks) {
+                combined.set(chunk, offset);
+                offset += chunk.length;
+            }
+            return combined.buffer;
+        }
+        else {
+            // Node.js environment - would use zlib
+            console.warn('GZIP compression not available, returning original data');
+            return data;
+        }
+    }
+    /**
+     * GZIP decompression
+     */
+    async gzipDecompress(compressedData) {
+        if (typeof DecompressionStream !== 'undefined') {
+            // Browser environment
+            const stream = new DecompressionStream('gzip');
+            const writer = stream.writable.getWriter();
+            const reader = stream.readable.getReader();
+            writer.write(new Uint8Array(compressedData));
+            writer.close();
+            const chunks = [];
+            let result = await reader.read();
+            while (!result.done) {
+                chunks.push(result.value);
+                result = await reader.read();
+            }
+            // Combine chunks
+            const totalLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
+            const combined = new Uint8Array(totalLength);
+            let offset = 0;
+            for (const chunk of chunks) {
+                combined.set(chunk, offset);
+                offset += chunk.length;
+            }
+            return combined.buffer;
+        }
+        else {
+            console.warn('GZIP decompression not available, returning original data');
+            return compressedData;
+        }
+    }
+    /**
+     * Brotli compression (placeholder - similar to GZIP)
+     */
+    async brotliCompress(data) {
+        // Would implement Brotli compression here
+        console.warn('Brotli compression not implemented, falling back to GZIP');
+        return this.gzipCompress(data);
+    }
+    /**
+     * Brotli decompression (placeholder)
+     */
+    async brotliDecompress(compressedData) {
+        console.warn('Brotli decompression not implemented, falling back to GZIP');
+        return this.gzipDecompress(compressedData);
+    }
+    /**
+     * Create prebuilt index segments for faster loading
+     */
+    async createPrebuiltSegments(nodes, outputPath) {
+        const segments = [];
+        const segmentSize = this.config.segmentSize;
+        console.log(`Creating ${Math.ceil(nodes.length / segmentSize)} prebuilt segments`);
+        for (let i = 0; i < nodes.length; i += segmentSize) {
+            const segmentNodes = nodes.slice(i, i + segmentSize);
+            const segmentId = `segment_${Math.floor(i / segmentSize)}`;
+            const segment = {
+                id: segmentId,
+                nodeCount: segmentNodes.length,
+                vectorDimension: segmentNodes[0]?.vector.length || 0,
+                compression: this.config.compression.vectorCompression,
+                localPath: `${outputPath}/${segmentId}.dat`,
+                loadedInMemory: false,
+                lastAccessed: 0
+            };
+            // Compress and serialize segment data
+            const compressedData = await this.compressSegment(segmentNodes);
+            // In a real implementation, you would write this to disk/S3
+            console.log(`Created segment ${segmentId} with ${compressedData.byteLength} bytes`);
+            segments.push(segment);
+            this.segments.set(segmentId, segment);
+        }
+        return segments;
+    }
+    /**
+     * Compress an entire segment of nodes
+     */
+    async compressSegment(nodes) {
+        const serialized = JSON.stringify(nodes.map(node => ({
+            id: node.id,
+            vector: node.vector,
+            connections: this.serializeConnections(node.connections)
+        })));
+        const encoder = new TextEncoder();
+        const data = encoder.encode(serialized);
+        // Apply metadata compression
+        switch (this.config.compression.metadataCompression) {
+            case CompressionType.GZIP:
+                return this.gzipCompress(data.buffer.slice(0));
+            case CompressionType.BROTLI:
+                return this.brotliCompress(data.buffer.slice(0));
+            default:
+                return data.buffer.slice(0);
+        }
+    }
+    /**
+     * Load a segment from storage with caching
+     */
+    async loadSegment(segmentId) {
+        const segment = this.segments.get(segmentId);
+        if (!segment) {
+            throw new Error(`Segment ${segmentId} not found`);
+        }
+        segment.lastAccessed = Date.now();
+        // Check if segment is already loaded in memory
+        if (segment.loadedInMemory && this.memoryMappedBuffers.has(segmentId)) {
+            return this.deserializeSegment(this.memoryMappedBuffers.get(segmentId));
+        }
+        // Load from storage (S3, disk, etc.)
+        const compressedData = await this.loadSegmentFromStorage(segment);
+        // Cache in memory if configured
+        if (this.config.cacheIndexInMemory) {
+            this.memoryMappedBuffers.set(segmentId, compressedData);
+            segment.loadedInMemory = true;
+        }
+        return this.deserializeSegment(compressedData);
+    }
+    /**
+     * Load segment data from storage
+     */
+    async loadSegmentFromStorage(segment) {
+        // This would integrate with your S3 storage adapter
+        // For now, return a placeholder
+        console.log(`Loading segment ${segment.id} from storage`);
+        return new ArrayBuffer(0);
+    }
+    /**
+     * Deserialize and decompress segment data
+     */
+    async deserializeSegment(compressedData) {
+        // Decompress metadata
+        let decompressed;
+        switch (this.config.compression.metadataCompression) {
+            case CompressionType.GZIP:
+                decompressed = await this.gzipDecompress(compressedData);
+                break;
+            case CompressionType.BROTLI:
+                decompressed = await this.brotliDecompress(compressedData);
+                break;
+            default:
+                decompressed = compressedData;
+                break;
+        }
+        // Parse JSON
+        const decoder = new TextDecoder();
+        const jsonStr = decoder.decode(decompressed);
+        const parsed = JSON.parse(jsonStr);
+        // Reconstruct HNSWNoun objects
+        return parsed.map((item) => ({
+            id: item.id,
+            vector: item.vector,
+            connections: this.deserializeConnections(item.connections)
+        }));
+    }
+    /**
+     * Serialize connections Map for storage
+     */
+    serializeConnections(connections) {
+        const result = {};
+        for (const [level, nodeIds] of connections.entries()) {
+            result[level.toString()] = Array.from(nodeIds);
+        }
+        return result;
+    }
+    /**
+     * Deserialize connections from storage format
+     */
+    deserializeConnections(serialized) {
+        const result = new Map();
+        for (const [levelStr, nodeIds] of Object.entries(serialized)) {
+            result.set(parseInt(levelStr), new Set(nodeIds));
+        }
+        return result;
+    }
+    /**
+     * Prefetch segments based on access patterns
+     */
+    async prefetchSegments(currentSegmentId) {
+        const segment = this.segments.get(currentSegmentId);
+        if (!segment)
+            return;
+        // Simple prefetching strategy - load adjacent segments
+        const segmentNumber = parseInt(currentSegmentId.split('_')[1]);
+        const toPrefetch = [];
+        for (let i = 1; i <= this.config.prefetchSegments; i++) {
+            const nextId = `segment_${segmentNumber + i}`;
+            const prevId = `segment_${segmentNumber - i}`;
+            if (this.segments.has(nextId) && !this.memoryMappedBuffers.has(nextId)) {
+                toPrefetch.push(nextId);
+            }
+            if (this.segments.has(prevId) && !this.memoryMappedBuffers.has(prevId)) {
+                toPrefetch.push(prevId);
+            }
+        }
+        // Prefetch in background
+        for (const segmentId of toPrefetch) {
+            this.loadSegment(segmentId).catch(error => {
+                console.warn(`Failed to prefetch segment ${segmentId}:`, error);
+            });
+        }
+    }
+    /**
+     * Update compression statistics
+     */
+    updateCompressionRatio() {
+        if (this.compressionStats.originalSize > 0) {
+            this.compressionStats.compressionRatio =
+                this.compressionStats.compressedSize / this.compressionStats.originalSize;
+        }
+    }
+    /**
+     * Get compression statistics
+     */
+    getCompressionStats() {
+        const memoryUsage = Array.from(this.memoryMappedBuffers.values())
+            .reduce((sum, buffer) => sum + buffer.byteLength, 0);
+        return {
+            ...this.compressionStats,
+            segmentCount: this.segments.size,
+            memoryUsage
+        };
+    }
+    /**
+     * Cleanup memory-mapped buffers
+     */
+    cleanup() {
+        this.memoryMappedBuffers.clear();
+        this.quantizationCodebooks.clear();
+        // Mark all segments as not loaded
+        for (const segment of this.segments.values()) {
+            segment.loadedInMemory = false;
+        }
+    }
+}
+//# sourceMappingURL=readOnlyOptimizations.js.map

dist/storage/storageFactory.d.ts

@@ -0,0 +1,199 @@
+/**
+ * Storage Factory
+ * Creates the appropriate storage adapter based on the environment and configuration
+ */
+import { StorageAdapter } from '../coreTypes.js';
+import { MemoryStorage } from './adapters/memoryStorage.js';
+import { OPFSStorage } from './adapters/opfsStorage.js';
+import { S3CompatibleStorage, R2Storage } from './adapters/s3CompatibleStorage.js';
+import { OperationConfig } from '../utils/operationUtils.js';
+/**
+ * Options for creating a storage adapter
+ */
+export interface StorageOptions {
+    /**
+     * The type of storage to use
+     * - 'auto': Automatically select the best storage adapter based on the environment
+     * - 'memory': Use in-memory storage
+     * - 'opfs': Use Origin Private File System storage (browser only)
+     * - 'filesystem': Use file system storage (Node.js only)
+     * - 's3': Use Amazon S3 storage
+     * - 'r2': Use Cloudflare R2 storage
+     * - 'gcs': Use Google Cloud Storage
+     */
+    type?: 'auto' | 'memory' | 'opfs' | 'filesystem' | 's3' | 'r2' | 'gcs';
+    /**
+     * Force the use of memory storage even if other storage types are available
+     */
+    forceMemoryStorage?: boolean;
+    /**
+     * Force the use of file system storage even if other storage types are available
+     */
+    forceFileSystemStorage?: boolean;
+    /**
+     * Request persistent storage permission from the user (browser only)
+     */
+    requestPersistentStorage?: boolean;
+    /**
+     * Root directory for file system storage (Node.js only)
+     */
+    rootDirectory?: string;
+    /**
+     * Configuration for Amazon S3 storage
+     */
+    s3Storage?: {
+        /**
+         * S3 bucket name
+         */
+        bucketName: string;
+        /**
+         * AWS region (e.g., 'us-east-1')
+         */
+        region?: string;
+        /**
+         * AWS access key ID
+         */
+        accessKeyId: string;
+        /**
+         * AWS secret access key
+         */
+        secretAccessKey: string;
+        /**
+         * AWS session token (optional)
+         */
+        sessionToken?: string;
+    };
+    /**
+     * Configuration for Cloudflare R2 storage
+     */
+    r2Storage?: {
+        /**
+         * R2 bucket name
+         */
+        bucketName: string;
+        /**
+         * Cloudflare account ID
+         */
+        accountId: string;
+        /**
+         * R2 access key ID
+         */
+        accessKeyId: string;
+        /**
+         * R2 secret access key
+         */
+        secretAccessKey: string;
+    };
+    /**
+     * Configuration for Google Cloud Storage
+     */
+    gcsStorage?: {
+        /**
+         * GCS bucket name
+         */
+        bucketName: string;
+        /**
+         * GCS region (e.g., 'us-central1')
+         */
+        region?: string;
+        /**
+         * GCS access key ID
+         */
+        accessKeyId: string;
+        /**
+         * GCS secret access key
+         */
+        secretAccessKey: string;
+        /**
+         * GCS endpoint (e.g., 'https://storage.googleapis.com')
+         */
+        endpoint?: string;
+    };
+    /**
+     * Configuration for custom S3-compatible storage
+     */
+    customS3Storage?: {
+        /**
+         * S3-compatible bucket name
+         */
+        bucketName: string;
+        /**
+         * S3-compatible region
+         */
+        region?: string;
+        /**
+         * S3-compatible endpoint URL
+         */
+        endpoint: string;
+        /**
+         * S3-compatible access key ID
+         */
+        accessKeyId: string;
+        /**
+         * S3-compatible secret access key
+         */
+        secretAccessKey: string;
+        /**
+         * S3-compatible service type (for logging and error messages)
+         */
+        serviceType?: string;
+    };
+    /**
+     * Operation configuration for timeout and retry behavior
+     */
+    operationConfig?: OperationConfig;
+    /**
+     * Cache configuration for optimizing data access
+     * Particularly important for S3 and other remote storage
+     */
+    cacheConfig?: {
+        /**
+         * Maximum size of the hot cache (most frequently accessed items)
+         * For large datasets, consider values between 5000-50000 depending on available memory
+         */
+        hotCacheMaxSize?: number;
+        /**
+         * Threshold at which to start evicting items from the hot cache
+         * Expressed as a fraction of hotCacheMaxSize (0.0 to 1.0)
+         * Default: 0.8 (start evicting when cache is 80% full)
+         */
+        hotCacheEvictionThreshold?: number;
+        /**
+         * Time-to-live for items in the warm cache in milliseconds
+         * Default: 3600000 (1 hour)
+         */
+        warmCacheTTL?: number;
+        /**
+         * Batch size for operations like prefetching
+         * Larger values improve throughput but use more memory
+         */
+        batchSize?: number;
+        /**
+         * Whether to enable auto-tuning of cache parameters
+         * When true, the system will automatically adjust cache sizes based on usage patterns
+         * Default: true
+         */
+        autoTune?: boolean;
+        /**
+         * The interval (in milliseconds) at which to auto-tune cache parameters
+         * Only applies when autoTune is true
+         * Default: 60000 (1 minute)
+         */
+        autoTuneInterval?: number;
+        /**
+         * Whether the storage is in read-only mode
+         * This affects cache sizing and prefetching strategies
+         */
+        readOnly?: boolean;
+    };
+}
+/**
+ * Create a storage adapter based on the environment and configuration
+ * @param options Options for creating the storage adapter
+ * @returns Promise that resolves to a storage adapter
+ */
+export declare function createStorage(options?: StorageOptions): Promise<StorageAdapter>;
+/**
+ * Export storage adapters
+ */
+export { MemoryStorage, OPFSStorage, S3CompatibleStorage, R2Storage };

dist/storage/storageFactory.js

@@ -0,0 +1,227 @@
+/**
+ * Storage Factory
+ * Creates the appropriate storage adapter based on the environment and configuration
+ */
+import { MemoryStorage } from './adapters/memoryStorage.js';
+import { OPFSStorage } from './adapters/opfsStorage.js';
+import { S3CompatibleStorage, R2Storage } from './adapters/s3CompatibleStorage.js';
+// FileSystemStorage is dynamically imported to avoid issues in browser environments
+import { isBrowser } from '../utils/environment.js';
+/**
+ * Create a storage adapter based on the environment and configuration
+ * @param options Options for creating the storage adapter
+ * @returns Promise that resolves to a storage adapter
+ */
+export async function createStorage(options = {}) {
+    // If memory storage is forced, use it regardless of other options
+    if (options.forceMemoryStorage) {
+        console.log('Using memory storage (forced)');
+        return new MemoryStorage();
+    }
+    // If file system storage is forced, use it regardless of other options
+    if (options.forceFileSystemStorage) {
+        if (isBrowser()) {
+            console.warn('FileSystemStorage is not available in browser environments, falling back to memory storage');
+            return new MemoryStorage();
+        }
+        console.log('Using file system storage (forced)');
+        try {
+            const { FileSystemStorage } = await import('./adapters/fileSystemStorage.js');
+            return new FileSystemStorage(options.rootDirectory || './brainy-data');
+        }
+        catch (error) {
+            console.warn('Failed to load FileSystemStorage, falling back to memory storage:', error);
+            return new MemoryStorage();
+        }
+    }
+    // If a specific storage type is specified, use it
+    if (options.type && options.type !== 'auto') {
+        switch (options.type) {
+            case 'memory':
+                console.log('Using memory storage');
+                return new MemoryStorage();
+            case 'opfs': {
+                // Check if OPFS is available
+                const opfsStorage = new OPFSStorage();
+                if (opfsStorage.isOPFSAvailable()) {
+                    console.log('Using OPFS storage');
+                    await opfsStorage.init();
+                    // Request persistent storage if specified
+                    if (options.requestPersistentStorage) {
+                        const isPersistent = await opfsStorage.requestPersistentStorage();
+                        console.log(`Persistent storage ${isPersistent ? 'granted' : 'denied'}`);
+                    }
+                    return opfsStorage;
+                }
+                else {
+                    console.warn('OPFS storage is not available, falling back to memory storage');
+                    return new MemoryStorage();
+                }
+            }
+            case 'filesystem': {
+                if (isBrowser()) {
+                    console.warn('FileSystemStorage is not available in browser environments, falling back to memory storage');
+                    return new MemoryStorage();
+                }
+                console.log('Using file system storage');
+                try {
+                    const { FileSystemStorage } = await import('./adapters/fileSystemStorage.js');
+                    return new FileSystemStorage(options.rootDirectory || './brainy-data');
+                }
+                catch (error) {
+                    console.warn('Failed to load FileSystemStorage, falling back to memory storage:', error);
+                    return new MemoryStorage();
+                }
+            }
+            case 's3':
+                if (options.s3Storage) {
+                    console.log('Using Amazon S3 storage');
+                    return new S3CompatibleStorage({
+                        bucketName: options.s3Storage.bucketName,
+                        region: options.s3Storage.region,
+                        accessKeyId: options.s3Storage.accessKeyId,
+                        secretAccessKey: options.s3Storage.secretAccessKey,
+                        sessionToken: options.s3Storage.sessionToken,
+                        serviceType: 's3',
+                        operationConfig: options.operationConfig,
+                        cacheConfig: options.cacheConfig
+                    });
+                }
+                else {
+                    console.warn('S3 storage configuration is missing, falling back to memory storage');
+                    return new MemoryStorage();
+                }
+            case 'r2':
+                if (options.r2Storage) {
+                    console.log('Using Cloudflare R2 storage');
+                    return new R2Storage({
+                        bucketName: options.r2Storage.bucketName,
+                        accountId: options.r2Storage.accountId,
+                        accessKeyId: options.r2Storage.accessKeyId,
+                        secretAccessKey: options.r2Storage.secretAccessKey,
+                        serviceType: 'r2',
+                        cacheConfig: options.cacheConfig
+                    });
+                }
+                else {
+                    console.warn('R2 storage configuration is missing, falling back to memory storage');
+                    return new MemoryStorage();
+                }
+            case 'gcs':
+                if (options.gcsStorage) {
+                    console.log('Using Google Cloud Storage');
+                    return new S3CompatibleStorage({
+                        bucketName: options.gcsStorage.bucketName,
+                        region: options.gcsStorage.region,
+                        endpoint: options.gcsStorage.endpoint || 'https://storage.googleapis.com',
+                        accessKeyId: options.gcsStorage.accessKeyId,
+                        secretAccessKey: options.gcsStorage.secretAccessKey,
+                        serviceType: 'gcs',
+                        cacheConfig: options.cacheConfig
+                    });
+                }
+                else {
+                    console.warn('GCS storage configuration is missing, falling back to memory storage');
+                    return new MemoryStorage();
+                }
+            default:
+                console.warn(`Unknown storage type: ${options.type}, falling back to memory storage`);
+                return new MemoryStorage();
+        }
+    }
+    // If custom S3-compatible storage is specified, use it
+    if (options.customS3Storage) {
+        console.log(`Using custom S3-compatible storage: ${options.customS3Storage.serviceType || 'custom'}`);
+        return new S3CompatibleStorage({
+            bucketName: options.customS3Storage.bucketName,
+            region: options.customS3Storage.region,
+            endpoint: options.customS3Storage.endpoint,
+            accessKeyId: options.customS3Storage.accessKeyId,
+            secretAccessKey: options.customS3Storage.secretAccessKey,
+            serviceType: options.customS3Storage.serviceType || 'custom',
+            cacheConfig: options.cacheConfig
+        });
+    }
+    // If R2 storage is specified, use it
+    if (options.r2Storage) {
+        console.log('Using Cloudflare R2 storage');
+        return new R2Storage({
+            bucketName: options.r2Storage.bucketName,
+            accountId: options.r2Storage.accountId,
+            accessKeyId: options.r2Storage.accessKeyId,
+            secretAccessKey: options.r2Storage.secretAccessKey,
+            serviceType: 'r2',
+            cacheConfig: options.cacheConfig
+        });
+    }
+    // If S3 storage is specified, use it
+    if (options.s3Storage) {
+        console.log('Using Amazon S3 storage');
+        return new S3CompatibleStorage({
+            bucketName: options.s3Storage.bucketName,
+            region: options.s3Storage.region,
+            accessKeyId: options.s3Storage.accessKeyId,
+            secretAccessKey: options.s3Storage.secretAccessKey,
+            sessionToken: options.s3Storage.sessionToken,
+            serviceType: 's3',
+            cacheConfig: options.cacheConfig
+        });
+    }
+    // If GCS storage is specified, use it
+    if (options.gcsStorage) {
+        console.log('Using Google Cloud Storage');
+        return new S3CompatibleStorage({
+            bucketName: options.gcsStorage.bucketName,
+            region: options.gcsStorage.region,
+            endpoint: options.gcsStorage.endpoint || 'https://storage.googleapis.com',
+            accessKeyId: options.gcsStorage.accessKeyId,
+            secretAccessKey: options.gcsStorage.secretAccessKey,
+            serviceType: 'gcs',
+            cacheConfig: options.cacheConfig
+        });
+    }
+    // Auto-detect the best storage adapter based on the environment
+    // First, try OPFS (browser only)
+    const opfsStorage = new OPFSStorage();
+    if (opfsStorage.isOPFSAvailable()) {
+        console.log('Using OPFS storage (auto-detected)');
+        await opfsStorage.init();
+        // Request persistent storage if specified
+        if (options.requestPersistentStorage) {
+            const isPersistent = await opfsStorage.requestPersistentStorage();
+            console.log(`Persistent storage ${isPersistent ? 'granted' : 'denied'}`);
+        }
+        return opfsStorage;
+    }
+    // Next, try file system storage (Node.js only)
+    try {
+        // Check if we're in a Node.js environment
+        if (typeof process !== 'undefined' &&
+            process.versions &&
+            process.versions.node) {
+            console.log('Using file system storage (auto-detected)');
+            try {
+                const { FileSystemStorage } = await import('./adapters/fileSystemStorage.js');
+                return new FileSystemStorage(options.rootDirectory || './brainy-data');
+            }
+            catch (fsError) {
+                console.warn('Failed to load FileSystemStorage, falling back to memory storage:', fsError);
+            }
+        }
+    }
+    catch (error) {
+        // Not in a Node.js environment or file system is not available
+        console.warn('Not in a Node.js environment:', error);
+    }
+    // Finally, fall back to memory storage
+    console.log('Using memory storage (auto-detected)');
+    return new MemoryStorage();
+}
+/**
+ * Export storage adapters
+ */
+export { MemoryStorage, OPFSStorage, S3CompatibleStorage, R2Storage };
+// Export FileSystemStorage conditionally
+// NOTE: FileSystemStorage is now only imported dynamically to avoid fs imports in browser builds
+// export { FileSystemStorage } from './adapters/fileSystemStorage.js'
+//# sourceMappingURL=storageFactory.js.map