chore: recovery checkpoint - v3.0 API successfully recovered
CRITICAL CHECKPOINT - DO NOT PUSH TO GITHUB Recovery Status: - Successfully recovered brainy.ts from compiled JavaScript - All core v3.0 API methods functional (add, get, update, delete, relate, find, etc.) - Neural subsystem intact (562KB embedded patterns, NLP working) - Augmentation pipeline operational (20+ augmentations) - HNSW clustering system complete - Triple Intelligence compiled (needs constructor fix) - Test suite validates functionality Changes preserved: - 898 files with changes from last 3 days - 144,475 insertions - All augmentation improvements - All test coverage enhancements - Complete v3.0 feature set This is a LOCAL checkpoint only - contains recovered work after corruption incident. Created backup in .backups/brainy-full-20250910-151314.tar.gz Branch: recovery-checkpoint-20250910-151433 Date: Wed Sep 10 03:18:04 PM PDT 2025
This commit is contained in:
parent
f65455fb22
commit
8ff382ca3b
895 changed files with 143654 additions and 28268 deletions
256
.recovery-workspace/dist-backup-20250910-141917/storage/adapters/baseStorageAdapter.d.ts
vendored
Normal file
256
.recovery-workspace/dist-backup-20250910-141917/storage/adapters/baseStorageAdapter.d.ts
vendored
Normal file
|
|
@ -0,0 +1,256 @@
|
|||
/**
|
||||
* Base Storage Adapter
|
||||
* Provides common functionality for all storage adapters, including statistics tracking
|
||||
*/
|
||||
import { StatisticsData, StorageAdapter } from '../../coreTypes.js';
|
||||
/**
|
||||
* Base class for storage adapters that implements statistics tracking
|
||||
*/
|
||||
export declare abstract class BaseStorageAdapter implements StorageAdapter {
|
||||
abstract init(): Promise<void>;
|
||||
abstract saveNoun(noun: any): Promise<void>;
|
||||
abstract getNoun(id: string): Promise<any | null>;
|
||||
abstract getNounsByNounType(nounType: string): Promise<any[]>;
|
||||
abstract deleteNoun(id: string): Promise<void>;
|
||||
abstract saveVerb(verb: any): Promise<void>;
|
||||
abstract getVerb(id: string): Promise<any | null>;
|
||||
abstract getVerbsBySource(sourceId: string): Promise<any[]>;
|
||||
abstract getVerbsByTarget(targetId: string): Promise<any[]>;
|
||||
abstract getVerbsByType(type: string): Promise<any[]>;
|
||||
abstract deleteVerb(id: string): Promise<void>;
|
||||
abstract saveMetadata(id: string, metadata: any): Promise<void>;
|
||||
abstract getMetadata(id: string): Promise<any | null>;
|
||||
abstract saveVerbMetadata(id: string, metadata: any): Promise<void>;
|
||||
abstract getVerbMetadata(id: string): Promise<any | null>;
|
||||
abstract clear(): Promise<void>;
|
||||
abstract getStorageStatus(): Promise<{
|
||||
type: string;
|
||||
used: number;
|
||||
quota: number | null;
|
||||
details?: Record<string, any>;
|
||||
}>;
|
||||
/**
|
||||
* Get nouns with pagination and filtering
|
||||
* @param options Pagination and filtering options
|
||||
* @returns Promise that resolves to a paginated result of nouns
|
||||
*/
|
||||
abstract getNouns(options?: {
|
||||
pagination?: {
|
||||
offset?: number;
|
||||
limit?: number;
|
||||
cursor?: string;
|
||||
};
|
||||
filter?: {
|
||||
nounType?: string | string[];
|
||||
service?: string | string[];
|
||||
metadata?: Record<string, any>;
|
||||
};
|
||||
}): Promise<{
|
||||
items: any[];
|
||||
totalCount?: number;
|
||||
hasMore: boolean;
|
||||
nextCursor?: string;
|
||||
}>;
|
||||
/**
|
||||
* Get verbs with pagination and filtering
|
||||
* @param options Pagination and filtering options
|
||||
* @returns Promise that resolves to a paginated result of verbs
|
||||
*/
|
||||
abstract getVerbs(options?: {
|
||||
pagination?: {
|
||||
offset?: number;
|
||||
limit?: number;
|
||||
cursor?: string;
|
||||
};
|
||||
filter?: {
|
||||
verbType?: string | string[];
|
||||
sourceId?: string | string[];
|
||||
targetId?: string | string[];
|
||||
service?: string | string[];
|
||||
metadata?: Record<string, any>;
|
||||
};
|
||||
}): Promise<{
|
||||
items: any[];
|
||||
totalCount?: number;
|
||||
hasMore: boolean;
|
||||
nextCursor?: string;
|
||||
}>;
|
||||
/**
|
||||
* Get nouns with pagination (internal implementation)
|
||||
* This method should be implemented by storage adapters to support efficient pagination
|
||||
* @param options Pagination options
|
||||
* @returns Promise that resolves to a paginated result of nouns
|
||||
*/
|
||||
getNounsWithPagination?(options: {
|
||||
limit?: number;
|
||||
cursor?: string;
|
||||
filter?: {
|
||||
nounType?: string | string[];
|
||||
service?: string | string[];
|
||||
metadata?: Record<string, any>;
|
||||
};
|
||||
}): Promise<{
|
||||
items: any[];
|
||||
totalCount?: number;
|
||||
hasMore: boolean;
|
||||
nextCursor?: string;
|
||||
}>;
|
||||
/**
|
||||
* Get verbs with pagination (internal implementation)
|
||||
* This method should be implemented by storage adapters to support efficient pagination
|
||||
* @param options Pagination options
|
||||
* @returns Promise that resolves to a paginated result of verbs
|
||||
*/
|
||||
getVerbsWithPagination?(options: {
|
||||
limit?: number;
|
||||
cursor?: string;
|
||||
filter?: {
|
||||
verbType?: string | string[];
|
||||
sourceId?: string | string[];
|
||||
targetId?: string | string[];
|
||||
service?: string | string[];
|
||||
metadata?: Record<string, any>;
|
||||
};
|
||||
}): Promise<{
|
||||
items: any[];
|
||||
totalCount?: number;
|
||||
hasMore: boolean;
|
||||
nextCursor?: string;
|
||||
}>;
|
||||
protected statisticsCache: StatisticsData | null;
|
||||
protected statisticsBatchUpdateTimerId: NodeJS.Timeout | null;
|
||||
protected statisticsModified: boolean;
|
||||
protected lastStatisticsFlushTime: number;
|
||||
protected readonly MIN_FLUSH_INTERVAL_MS = 5000;
|
||||
protected readonly MAX_FLUSH_DELAY_MS = 30000;
|
||||
protected throttlingDetected: boolean;
|
||||
protected throttlingBackoffMs: number;
|
||||
protected maxBackoffMs: number;
|
||||
protected consecutiveThrottleEvents: number;
|
||||
protected lastThrottleTime: number;
|
||||
protected totalThrottleEvents: number;
|
||||
protected throttleEventsByHour: number[];
|
||||
protected throttleReasons: Record<string, number>;
|
||||
protected lastThrottleHourIndex: number;
|
||||
protected delayedOperations: number;
|
||||
protected retriedOperations: number;
|
||||
protected failedDueToThrottling: number;
|
||||
protected totalDelayMs: number;
|
||||
protected serviceThrottling: Map<string, {
|
||||
throttleCount: number;
|
||||
lastThrottle: number;
|
||||
status: 'normal' | 'throttled' | 'recovering';
|
||||
}>;
|
||||
protected abstract saveStatisticsData(statistics: StatisticsData): Promise<void>;
|
||||
protected abstract getStatisticsData(): Promise<StatisticsData | null>;
|
||||
/**
|
||||
* Save statistics data
|
||||
* @param statistics The statistics data to save
|
||||
*/
|
||||
saveStatistics(statistics: StatisticsData): Promise<void>;
|
||||
/**
|
||||
* Get statistics data
|
||||
* @returns Promise that resolves to the statistics data
|
||||
*/
|
||||
getStatistics(): Promise<StatisticsData | null>;
|
||||
/**
|
||||
* Schedule a batch update of statistics
|
||||
*/
|
||||
protected scheduleBatchUpdate(): void;
|
||||
/**
|
||||
* Flush statistics to storage
|
||||
*/
|
||||
protected flushStatistics(): Promise<void>;
|
||||
/**
|
||||
* Increment a statistic counter
|
||||
* @param type The type of statistic to increment ('noun', 'verb', 'metadata')
|
||||
* @param service The service that inserted the data
|
||||
* @param amount The amount to increment by (default: 1)
|
||||
*/
|
||||
incrementStatistic(type: 'noun' | 'verb' | 'metadata', service: string, amount?: number): Promise<void>;
|
||||
/**
|
||||
* Track service activity (first/last activity, operation counts)
|
||||
* @param service The service name
|
||||
* @param operation The operation type
|
||||
*/
|
||||
protected trackServiceActivity(service: string, operation: 'add' | 'update' | 'delete'): void;
|
||||
/**
|
||||
* Decrement a statistic counter
|
||||
* @param type The type of statistic to decrement ('noun', 'verb', 'metadata')
|
||||
* @param service The service that inserted the data
|
||||
* @param amount The amount to decrement by (default: 1)
|
||||
*/
|
||||
decrementStatistic(type: 'noun' | 'verb' | 'metadata', service: string, amount?: number): Promise<void>;
|
||||
/**
|
||||
* Update the HNSW index size statistic
|
||||
* @param size The new size of the HNSW index
|
||||
*/
|
||||
updateHnswIndexSize(size: number): Promise<void>;
|
||||
/**
|
||||
* Force an immediate flush of statistics to storage
|
||||
* This ensures that any pending statistics updates are written to persistent storage
|
||||
*/
|
||||
flushStatisticsToStorage(): Promise<void>;
|
||||
/**
|
||||
* Track field names from a JSON document
|
||||
* @param jsonDocument The JSON document to extract field names from
|
||||
* @param service The service that inserted the data
|
||||
*/
|
||||
trackFieldNames(jsonDocument: any, service: string): Promise<void>;
|
||||
/**
|
||||
* Get available field names by service
|
||||
* @returns Record of field names by service
|
||||
*/
|
||||
getAvailableFieldNames(): Promise<Record<string, string[]>>;
|
||||
/**
|
||||
* Get standard field mappings
|
||||
* @returns Record of standard field mappings
|
||||
*/
|
||||
getStandardFieldMappings(): Promise<Record<string, Record<string, string[]>>>;
|
||||
/**
|
||||
* Create default statistics data
|
||||
* @returns Default statistics data
|
||||
*/
|
||||
protected createDefaultStatistics(): StatisticsData;
|
||||
/**
|
||||
* Detect if an error is a throttling error
|
||||
* Override this method in specific adapters for custom detection
|
||||
*/
|
||||
protected isThrottlingError(error: any): boolean;
|
||||
/**
|
||||
* Track a throttling event
|
||||
* @param error The error that caused throttling
|
||||
* @param service Optional service that was throttled
|
||||
*/
|
||||
protected trackThrottlingEvent(error: any, service?: string): void;
|
||||
/**
|
||||
* Get the reason for throttling from an error
|
||||
*/
|
||||
protected getThrottleReason(error: any): string;
|
||||
/**
|
||||
* Clear throttling state after successful operations
|
||||
*/
|
||||
protected clearThrottlingState(): void;
|
||||
/**
|
||||
* Handle throttling by implementing exponential backoff
|
||||
* @param error The error that triggered throttling
|
||||
* @param service Optional service that was throttled
|
||||
*/
|
||||
handleThrottling(error: any, service?: string): Promise<void>;
|
||||
/**
|
||||
* Track a retried operation
|
||||
*/
|
||||
protected trackRetriedOperation(): void;
|
||||
/**
|
||||
* Track an operation that failed due to throttling
|
||||
*/
|
||||
protected trackFailedDueToThrottling(): void;
|
||||
/**
|
||||
* Get current throttling metrics
|
||||
*/
|
||||
protected getThrottlingMetrics(): StatisticsData['throttlingMetrics'];
|
||||
/**
|
||||
* Include throttling metrics in statistics
|
||||
*/
|
||||
getStatisticsWithThrottling(): Promise<StatisticsData | null>;
|
||||
}
|
||||
|
|
@ -0,0 +1,613 @@
|
|||
/**
|
||||
* Base Storage Adapter
|
||||
* Provides common functionality for all storage adapters, including statistics tracking
|
||||
*/
|
||||
import { extractFieldNamesFromJson, mapToStandardField } from '../../utils/fieldNameTracking.js';
|
||||
/**
|
||||
* Base class for storage adapters that implements statistics tracking
|
||||
*/
|
||||
export class BaseStorageAdapter {
|
||||
constructor() {
|
||||
// Statistics cache
|
||||
this.statisticsCache = null;
|
||||
// Batch update timer ID
|
||||
this.statisticsBatchUpdateTimerId = null;
|
||||
// Flag to indicate if statistics have been modified since last save
|
||||
this.statisticsModified = false;
|
||||
// Time of last statistics flush to storage
|
||||
this.lastStatisticsFlushTime = 0;
|
||||
// Minimum time between statistics flushes (5 seconds)
|
||||
this.MIN_FLUSH_INTERVAL_MS = 5000;
|
||||
// Maximum time to wait before flushing statistics (30 seconds)
|
||||
this.MAX_FLUSH_DELAY_MS = 30000;
|
||||
// Throttling tracking properties
|
||||
this.throttlingDetected = false;
|
||||
this.throttlingBackoffMs = 1000; // Start with 1 second
|
||||
this.maxBackoffMs = 30000; // Max 30 seconds
|
||||
this.consecutiveThrottleEvents = 0;
|
||||
this.lastThrottleTime = 0;
|
||||
this.totalThrottleEvents = 0;
|
||||
this.throttleEventsByHour = new Array(24).fill(0);
|
||||
this.throttleReasons = {};
|
||||
this.lastThrottleHourIndex = -1;
|
||||
// Operation impact tracking
|
||||
this.delayedOperations = 0;
|
||||
this.retriedOperations = 0;
|
||||
this.failedDueToThrottling = 0;
|
||||
this.totalDelayMs = 0;
|
||||
// Service-level throttling
|
||||
this.serviceThrottling = new Map();
|
||||
}
|
||||
/**
|
||||
* Save statistics data
|
||||
* @param statistics The statistics data to save
|
||||
*/
|
||||
async saveStatistics(statistics) {
|
||||
// Update the cache with a deep copy to avoid reference issues
|
||||
this.statisticsCache = {
|
||||
nounCount: { ...statistics.nounCount },
|
||||
verbCount: { ...statistics.verbCount },
|
||||
metadataCount: { ...statistics.metadataCount },
|
||||
hnswIndexSize: statistics.hnswIndexSize,
|
||||
lastUpdated: statistics.lastUpdated,
|
||||
// Include serviceActivity if present
|
||||
...(statistics.serviceActivity && {
|
||||
serviceActivity: Object.fromEntries(Object.entries(statistics.serviceActivity).map(([k, v]) => [k, { ...v }]))
|
||||
}),
|
||||
// Include services if present
|
||||
...(statistics.services && {
|
||||
services: statistics.services.map(s => ({ ...s }))
|
||||
})
|
||||
};
|
||||
// Schedule a batch update instead of saving immediately
|
||||
this.scheduleBatchUpdate();
|
||||
}
|
||||
/**
|
||||
* Get statistics data
|
||||
* @returns Promise that resolves to the statistics data
|
||||
*/
|
||||
async getStatistics() {
|
||||
// If we have cached statistics, return a deep copy
|
||||
if (this.statisticsCache) {
|
||||
return {
|
||||
nounCount: { ...this.statisticsCache.nounCount },
|
||||
verbCount: { ...this.statisticsCache.verbCount },
|
||||
metadataCount: { ...this.statisticsCache.metadataCount },
|
||||
hnswIndexSize: this.statisticsCache.hnswIndexSize,
|
||||
lastUpdated: this.statisticsCache.lastUpdated
|
||||
};
|
||||
}
|
||||
// Otherwise, get from storage
|
||||
const statistics = await this.getStatisticsData();
|
||||
// If we found statistics, update the cache
|
||||
if (statistics) {
|
||||
// Update the cache with a deep copy
|
||||
this.statisticsCache = {
|
||||
nounCount: { ...statistics.nounCount },
|
||||
verbCount: { ...statistics.verbCount },
|
||||
metadataCount: { ...statistics.metadataCount },
|
||||
hnswIndexSize: statistics.hnswIndexSize,
|
||||
lastUpdated: statistics.lastUpdated
|
||||
};
|
||||
}
|
||||
return statistics;
|
||||
}
|
||||
/**
|
||||
* Schedule a batch update of statistics
|
||||
*/
|
||||
scheduleBatchUpdate() {
|
||||
// Mark statistics as modified
|
||||
this.statisticsModified = true;
|
||||
// If a timer is already set, don't set another one
|
||||
if (this.statisticsBatchUpdateTimerId !== null) {
|
||||
return;
|
||||
}
|
||||
// Calculate time since last flush
|
||||
const now = Date.now();
|
||||
const timeSinceLastFlush = now - this.lastStatisticsFlushTime;
|
||||
// If we've recently flushed, wait longer before the next flush
|
||||
const delayMs = timeSinceLastFlush < this.MIN_FLUSH_INTERVAL_MS
|
||||
? this.MAX_FLUSH_DELAY_MS
|
||||
: this.MIN_FLUSH_INTERVAL_MS;
|
||||
// Schedule the batch update
|
||||
this.statisticsBatchUpdateTimerId = setTimeout(() => {
|
||||
this.flushStatistics();
|
||||
}, delayMs);
|
||||
}
|
||||
/**
|
||||
* Flush statistics to storage
|
||||
*/
|
||||
async flushStatistics() {
|
||||
// Clear the timer
|
||||
if (this.statisticsBatchUpdateTimerId !== null) {
|
||||
clearTimeout(this.statisticsBatchUpdateTimerId);
|
||||
this.statisticsBatchUpdateTimerId = null;
|
||||
}
|
||||
// If statistics haven't been modified, no need to flush
|
||||
if (!this.statisticsModified || !this.statisticsCache) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
// Save the statistics to storage
|
||||
await this.saveStatisticsData(this.statisticsCache);
|
||||
// Update the last flush time
|
||||
this.lastStatisticsFlushTime = Date.now();
|
||||
// Reset the modified flag
|
||||
this.statisticsModified = false;
|
||||
}
|
||||
catch (error) {
|
||||
console.error('Failed to flush statistics data:', error);
|
||||
// Mark as still modified so we'll try again later
|
||||
this.statisticsModified = true;
|
||||
// Don't throw the error to avoid disrupting the application
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Increment a statistic counter
|
||||
* @param type The type of statistic to increment ('noun', 'verb', 'metadata')
|
||||
* @param service The service that inserted the data
|
||||
* @param amount The amount to increment by (default: 1)
|
||||
*/
|
||||
async incrementStatistic(type, service, amount = 1) {
|
||||
// Get current statistics from cache or storage
|
||||
let statistics = this.statisticsCache;
|
||||
if (!statistics) {
|
||||
statistics = await this.getStatisticsData();
|
||||
if (!statistics) {
|
||||
statistics = this.createDefaultStatistics();
|
||||
}
|
||||
// Update the cache
|
||||
this.statisticsCache = {
|
||||
nounCount: { ...statistics.nounCount },
|
||||
verbCount: { ...statistics.verbCount },
|
||||
metadataCount: { ...statistics.metadataCount },
|
||||
hnswIndexSize: statistics.hnswIndexSize,
|
||||
lastUpdated: statistics.lastUpdated,
|
||||
// Include serviceActivity if present
|
||||
...(statistics.serviceActivity && {
|
||||
serviceActivity: Object.fromEntries(Object.entries(statistics.serviceActivity).map(([k, v]) => [k, { ...v }]))
|
||||
}),
|
||||
// Include services if present
|
||||
...(statistics.services && {
|
||||
services: statistics.services.map(s => ({ ...s }))
|
||||
})
|
||||
};
|
||||
}
|
||||
// Increment the appropriate counter
|
||||
const counterMap = {
|
||||
noun: this.statisticsCache.nounCount,
|
||||
verb: this.statisticsCache.verbCount,
|
||||
metadata: this.statisticsCache.metadataCount
|
||||
};
|
||||
const counter = counterMap[type];
|
||||
counter[service] = (counter[service] || 0) + amount;
|
||||
// Track service activity
|
||||
this.trackServiceActivity(service, 'add');
|
||||
// Update timestamp
|
||||
this.statisticsCache.lastUpdated = new Date().toISOString();
|
||||
// Schedule a batch update instead of saving immediately
|
||||
this.scheduleBatchUpdate();
|
||||
}
|
||||
/**
|
||||
* Track service activity (first/last activity, operation counts)
|
||||
* @param service The service name
|
||||
* @param operation The operation type
|
||||
*/
|
||||
trackServiceActivity(service, operation) {
|
||||
if (!this.statisticsCache) {
|
||||
return;
|
||||
}
|
||||
// Initialize serviceActivity if it doesn't exist
|
||||
if (!this.statisticsCache.serviceActivity) {
|
||||
this.statisticsCache.serviceActivity = {};
|
||||
}
|
||||
const now = new Date().toISOString();
|
||||
const activity = this.statisticsCache.serviceActivity[service];
|
||||
if (!activity) {
|
||||
// First activity for this service
|
||||
this.statisticsCache.serviceActivity[service] = {
|
||||
firstActivity: now,
|
||||
lastActivity: now,
|
||||
totalOperations: 1
|
||||
};
|
||||
}
|
||||
else {
|
||||
// Update existing activity
|
||||
activity.lastActivity = now;
|
||||
activity.totalOperations++;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Decrement a statistic counter
|
||||
* @param type The type of statistic to decrement ('noun', 'verb', 'metadata')
|
||||
* @param service The service that inserted the data
|
||||
* @param amount The amount to decrement by (default: 1)
|
||||
*/
|
||||
async decrementStatistic(type, service, amount = 1) {
|
||||
// Get current statistics from cache or storage
|
||||
let statistics = this.statisticsCache;
|
||||
if (!statistics) {
|
||||
statistics = await this.getStatisticsData();
|
||||
if (!statistics) {
|
||||
statistics = this.createDefaultStatistics();
|
||||
}
|
||||
// Update the cache
|
||||
this.statisticsCache = {
|
||||
nounCount: { ...statistics.nounCount },
|
||||
verbCount: { ...statistics.verbCount },
|
||||
metadataCount: { ...statistics.metadataCount },
|
||||
hnswIndexSize: statistics.hnswIndexSize,
|
||||
lastUpdated: statistics.lastUpdated,
|
||||
// Include serviceActivity if present
|
||||
...(statistics.serviceActivity && {
|
||||
serviceActivity: Object.fromEntries(Object.entries(statistics.serviceActivity).map(([k, v]) => [k, { ...v }]))
|
||||
}),
|
||||
// Include services if present
|
||||
...(statistics.services && {
|
||||
services: statistics.services.map(s => ({ ...s }))
|
||||
})
|
||||
};
|
||||
}
|
||||
// Decrement the appropriate counter
|
||||
const counterMap = {
|
||||
noun: this.statisticsCache.nounCount,
|
||||
verb: this.statisticsCache.verbCount,
|
||||
metadata: this.statisticsCache.metadataCount
|
||||
};
|
||||
const counter = counterMap[type];
|
||||
counter[service] = Math.max(0, (counter[service] || 0) - amount);
|
||||
// Track service activity
|
||||
this.trackServiceActivity(service, 'delete');
|
||||
// Update timestamp
|
||||
this.statisticsCache.lastUpdated = new Date().toISOString();
|
||||
// Schedule a batch update instead of saving immediately
|
||||
this.scheduleBatchUpdate();
|
||||
}
|
||||
/**
|
||||
* Update the HNSW index size statistic
|
||||
* @param size The new size of the HNSW index
|
||||
*/
|
||||
async updateHnswIndexSize(size) {
|
||||
// Get current statistics from cache or storage
|
||||
let statistics = this.statisticsCache;
|
||||
if (!statistics) {
|
||||
statistics = await this.getStatisticsData();
|
||||
if (!statistics) {
|
||||
statistics = this.createDefaultStatistics();
|
||||
}
|
||||
// Update the cache
|
||||
this.statisticsCache = {
|
||||
nounCount: { ...statistics.nounCount },
|
||||
verbCount: { ...statistics.verbCount },
|
||||
metadataCount: { ...statistics.metadataCount },
|
||||
hnswIndexSize: statistics.hnswIndexSize,
|
||||
lastUpdated: statistics.lastUpdated,
|
||||
// Include serviceActivity if present
|
||||
...(statistics.serviceActivity && {
|
||||
serviceActivity: Object.fromEntries(Object.entries(statistics.serviceActivity).map(([k, v]) => [k, { ...v }]))
|
||||
}),
|
||||
// Include services if present
|
||||
...(statistics.services && {
|
||||
services: statistics.services.map(s => ({ ...s }))
|
||||
})
|
||||
};
|
||||
}
|
||||
// Update HNSW index size
|
||||
this.statisticsCache.hnswIndexSize = size;
|
||||
// Update timestamp
|
||||
this.statisticsCache.lastUpdated = new Date().toISOString();
|
||||
// Schedule a batch update instead of saving immediately
|
||||
this.scheduleBatchUpdate();
|
||||
}
|
||||
/**
|
||||
* Force an immediate flush of statistics to storage
|
||||
* This ensures that any pending statistics updates are written to persistent storage
|
||||
*/
|
||||
async flushStatisticsToStorage() {
|
||||
// If there are no statistics in cache or they haven't been modified, nothing to flush
|
||||
if (!this.statisticsCache || !this.statisticsModified) {
|
||||
return;
|
||||
}
|
||||
// Call the protected flushStatistics method to immediately write to storage
|
||||
await this.flushStatistics();
|
||||
}
|
||||
/**
|
||||
* Track field names from a JSON document
|
||||
* @param jsonDocument The JSON document to extract field names from
|
||||
* @param service The service that inserted the data
|
||||
*/
|
||||
async trackFieldNames(jsonDocument, service) {
|
||||
// Skip if not a JSON object
|
||||
if (typeof jsonDocument !== 'object' || jsonDocument === null || Array.isArray(jsonDocument)) {
|
||||
return;
|
||||
}
|
||||
// Get current statistics from cache or storage
|
||||
let statistics = this.statisticsCache;
|
||||
if (!statistics) {
|
||||
statistics = await this.getStatisticsData();
|
||||
if (!statistics) {
|
||||
statistics = this.createDefaultStatistics();
|
||||
}
|
||||
// Update the cache
|
||||
this.statisticsCache = {
|
||||
...statistics,
|
||||
nounCount: { ...statistics.nounCount },
|
||||
verbCount: { ...statistics.verbCount },
|
||||
metadataCount: { ...statistics.metadataCount },
|
||||
fieldNames: { ...statistics.fieldNames },
|
||||
standardFieldMappings: { ...statistics.standardFieldMappings }
|
||||
};
|
||||
}
|
||||
// Ensure fieldNames exists
|
||||
if (!this.statisticsCache.fieldNames) {
|
||||
this.statisticsCache.fieldNames = {};
|
||||
}
|
||||
// Ensure standardFieldMappings exists
|
||||
if (!this.statisticsCache.standardFieldMappings) {
|
||||
this.statisticsCache.standardFieldMappings = {};
|
||||
}
|
||||
// Extract field names from the JSON document
|
||||
const fieldNames = extractFieldNamesFromJson(jsonDocument);
|
||||
// Initialize service entry if it doesn't exist
|
||||
if (!this.statisticsCache.fieldNames[service]) {
|
||||
this.statisticsCache.fieldNames[service] = [];
|
||||
}
|
||||
// Add new field names to the service's list
|
||||
for (const fieldName of fieldNames) {
|
||||
if (!this.statisticsCache.fieldNames[service].includes(fieldName)) {
|
||||
this.statisticsCache.fieldNames[service].push(fieldName);
|
||||
}
|
||||
// Map to standard field if possible
|
||||
const standardField = mapToStandardField(fieldName);
|
||||
if (standardField) {
|
||||
// Initialize standard field entry if it doesn't exist
|
||||
if (!this.statisticsCache.standardFieldMappings[standardField]) {
|
||||
this.statisticsCache.standardFieldMappings[standardField] = {};
|
||||
}
|
||||
// Initialize service entry if it doesn't exist
|
||||
if (!this.statisticsCache.standardFieldMappings[standardField][service]) {
|
||||
this.statisticsCache.standardFieldMappings[standardField][service] = [];
|
||||
}
|
||||
// Add field name to standard field mapping if not already there
|
||||
if (!this.statisticsCache.standardFieldMappings[standardField][service].includes(fieldName)) {
|
||||
this.statisticsCache.standardFieldMappings[standardField][service].push(fieldName);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Update timestamp
|
||||
this.statisticsCache.lastUpdated = new Date().toISOString();
|
||||
// Schedule a batch update
|
||||
this.statisticsModified = true;
|
||||
this.scheduleBatchUpdate();
|
||||
}
|
||||
/**
|
||||
* Get available field names by service
|
||||
* @returns Record of field names by service
|
||||
*/
|
||||
async getAvailableFieldNames() {
|
||||
// Get current statistics from cache or storage
|
||||
let statistics = this.statisticsCache;
|
||||
if (!statistics) {
|
||||
statistics = await this.getStatisticsData();
|
||||
if (!statistics) {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
// Return field names by service
|
||||
return statistics.fieldNames || {};
|
||||
}
|
||||
/**
|
||||
* Get standard field mappings
|
||||
* @returns Record of standard field mappings
|
||||
*/
|
||||
async getStandardFieldMappings() {
|
||||
// Get current statistics from cache or storage
|
||||
let statistics = this.statisticsCache;
|
||||
if (!statistics) {
|
||||
statistics = await this.getStatisticsData();
|
||||
if (!statistics) {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
// Return standard field mappings
|
||||
return statistics.standardFieldMappings || {};
|
||||
}
|
||||
/**
|
||||
* Create default statistics data
|
||||
* @returns Default statistics data
|
||||
*/
|
||||
createDefaultStatistics() {
|
||||
return {
|
||||
nounCount: {},
|
||||
verbCount: {},
|
||||
metadataCount: {},
|
||||
hnswIndexSize: 0,
|
||||
fieldNames: {},
|
||||
standardFieldMappings: {},
|
||||
lastUpdated: new Date().toISOString()
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Detect if an error is a throttling error
|
||||
* Override this method in specific adapters for custom detection
|
||||
*/
|
||||
isThrottlingError(error) {
|
||||
const statusCode = error.$metadata?.httpStatusCode || error.statusCode || error.code;
|
||||
const message = error.message?.toLowerCase() || '';
|
||||
return (statusCode === 429 || // Too Many Requests
|
||||
statusCode === 503 || // Service Unavailable / Slow Down
|
||||
statusCode === 'ECONNRESET' || // Connection reset
|
||||
statusCode === 'ETIMEDOUT' || // Timeout
|
||||
message.includes('throttl') ||
|
||||
message.includes('slow down') ||
|
||||
message.includes('rate limit') ||
|
||||
message.includes('too many requests') ||
|
||||
message.includes('quota exceeded'));
|
||||
}
|
||||
/**
|
||||
* Track a throttling event
|
||||
* @param error The error that caused throttling
|
||||
* @param service Optional service that was throttled
|
||||
*/
|
||||
trackThrottlingEvent(error, service) {
|
||||
this.throttlingDetected = true;
|
||||
this.consecutiveThrottleEvents++;
|
||||
this.lastThrottleTime = Date.now();
|
||||
this.totalThrottleEvents++;
|
||||
// Track by hour
|
||||
const hourIndex = new Date().getHours();
|
||||
if (hourIndex !== this.lastThrottleHourIndex) {
|
||||
// Reset hour tracking if we've moved to a new hour
|
||||
this.throttleEventsByHour = new Array(24).fill(0);
|
||||
this.lastThrottleHourIndex = hourIndex;
|
||||
}
|
||||
this.throttleEventsByHour[hourIndex]++;
|
||||
// Track throttle reason
|
||||
const reason = this.getThrottleReason(error);
|
||||
this.throttleReasons[reason] = (this.throttleReasons[reason] || 0) + 1;
|
||||
// Track service-level throttling
|
||||
if (service) {
|
||||
const serviceInfo = this.serviceThrottling.get(service) || {
|
||||
throttleCount: 0,
|
||||
lastThrottle: 0,
|
||||
status: 'normal'
|
||||
};
|
||||
serviceInfo.throttleCount++;
|
||||
serviceInfo.lastThrottle = Date.now();
|
||||
serviceInfo.status = 'throttled';
|
||||
this.serviceThrottling.set(service, serviceInfo);
|
||||
}
|
||||
// Exponential backoff
|
||||
this.throttlingBackoffMs = Math.min(this.throttlingBackoffMs * 2, this.maxBackoffMs);
|
||||
}
|
||||
/**
|
||||
* Get the reason for throttling from an error
|
||||
*/
|
||||
getThrottleReason(error) {
|
||||
const statusCode = error.$metadata?.httpStatusCode || error.statusCode || error.code;
|
||||
if (statusCode === 429)
|
||||
return '429_TooManyRequests';
|
||||
if (statusCode === 503)
|
||||
return '503_ServiceUnavailable';
|
||||
if (statusCode === 'ECONNRESET')
|
||||
return 'ConnectionReset';
|
||||
if (statusCode === 'ETIMEDOUT')
|
||||
return 'Timeout';
|
||||
const message = error.message?.toLowerCase() || '';
|
||||
if (message.includes('throttl'))
|
||||
return 'Throttled';
|
||||
if (message.includes('slow down'))
|
||||
return 'SlowDown';
|
||||
if (message.includes('rate limit'))
|
||||
return 'RateLimit';
|
||||
if (message.includes('quota exceeded'))
|
||||
return 'QuotaExceeded';
|
||||
return 'Unknown';
|
||||
}
|
||||
/**
|
||||
* Clear throttling state after successful operations
|
||||
*/
|
||||
clearThrottlingState() {
|
||||
if (this.consecutiveThrottleEvents > 0) {
|
||||
this.consecutiveThrottleEvents = 0;
|
||||
this.throttlingBackoffMs = 1000; // Reset to initial backoff
|
||||
if (this.throttlingDetected) {
|
||||
this.throttlingDetected = false;
|
||||
// Update service statuses
|
||||
for (const [service, info] of this.serviceThrottling) {
|
||||
if (info.status === 'throttled') {
|
||||
info.status = 'recovering';
|
||||
}
|
||||
else if (info.status === 'recovering') {
|
||||
const timeSinceThrottle = Date.now() - info.lastThrottle;
|
||||
if (timeSinceThrottle > 60000) { // 1 minute recovery period
|
||||
info.status = 'normal';
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Handle throttling by implementing exponential backoff
|
||||
* @param error The error that triggered throttling
|
||||
* @param service Optional service that was throttled
|
||||
*/
|
||||
async handleThrottling(error, service) {
|
||||
if (this.isThrottlingError(error)) {
|
||||
this.trackThrottlingEvent(error, service);
|
||||
// Add delay for retry
|
||||
const delayMs = this.throttlingBackoffMs;
|
||||
this.totalDelayMs += delayMs;
|
||||
this.delayedOperations++;
|
||||
await new Promise(resolve => setTimeout(resolve, delayMs));
|
||||
}
|
||||
else {
|
||||
// Clear throttling state on non-throttling errors
|
||||
this.clearThrottlingState();
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Track a retried operation
|
||||
*/
|
||||
trackRetriedOperation() {
|
||||
this.retriedOperations++;
|
||||
}
|
||||
/**
|
||||
* Track an operation that failed due to throttling
|
||||
*/
|
||||
trackFailedDueToThrottling() {
|
||||
this.failedDueToThrottling++;
|
||||
}
|
||||
/**
|
||||
* Get current throttling metrics
|
||||
*/
|
||||
getThrottlingMetrics() {
|
||||
const averageDelayMs = this.delayedOperations > 0
|
||||
? this.totalDelayMs / this.delayedOperations
|
||||
: 0;
|
||||
// Convert service throttling map to record
|
||||
const serviceThrottlingRecord = {};
|
||||
for (const [service, info] of this.serviceThrottling) {
|
||||
serviceThrottlingRecord[service] = {
|
||||
throttleCount: info.throttleCount,
|
||||
lastThrottle: new Date(info.lastThrottle).toISOString(),
|
||||
status: info.status
|
||||
};
|
||||
}
|
||||
return {
|
||||
storage: {
|
||||
currentlyThrottled: this.throttlingDetected,
|
||||
lastThrottleTime: this.lastThrottleTime > 0
|
||||
? new Date(this.lastThrottleTime).toISOString()
|
||||
: undefined,
|
||||
consecutiveThrottleEvents: this.consecutiveThrottleEvents,
|
||||
currentBackoffMs: this.throttlingBackoffMs,
|
||||
totalThrottleEvents: this.totalThrottleEvents,
|
||||
throttleEventsByHour: [...this.throttleEventsByHour],
|
||||
throttleReasons: { ...this.throttleReasons }
|
||||
},
|
||||
operationImpact: {
|
||||
delayedOperations: this.delayedOperations,
|
||||
retriedOperations: this.retriedOperations,
|
||||
failedDueToThrottling: this.failedDueToThrottling,
|
||||
averageDelayMs,
|
||||
totalDelayMs: this.totalDelayMs
|
||||
},
|
||||
serviceThrottling: Object.keys(serviceThrottlingRecord).length > 0
|
||||
? serviceThrottlingRecord
|
||||
: undefined
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Include throttling metrics in statistics
|
||||
*/
|
||||
async getStatisticsWithThrottling() {
|
||||
const stats = await this.getStatistics();
|
||||
if (stats) {
|
||||
stats.throttlingMetrics = this.getThrottlingMetrics();
|
||||
}
|
||||
return stats;
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=baseStorageAdapter.js.map
|
||||
File diff suppressed because one or more lines are too long
71
.recovery-workspace/dist-backup-20250910-141917/storage/adapters/batchS3Operations.d.ts
vendored
Normal file
71
.recovery-workspace/dist-backup-20250910-141917/storage/adapters/batchS3Operations.d.ts
vendored
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
/**
|
||||
* Enhanced Batch S3 Operations for High-Performance Vector Retrieval
|
||||
* Implements optimized batch operations to reduce S3 API calls and latency
|
||||
*/
|
||||
import { HNSWNoun } from '../../coreTypes.js';
|
||||
type S3Client = any;
|
||||
export interface BatchRetrievalOptions {
|
||||
maxConcurrency?: number;
|
||||
prefetchSize?: number;
|
||||
useS3Select?: boolean;
|
||||
compressionEnabled?: boolean;
|
||||
}
|
||||
export interface BatchResult<T> {
|
||||
items: Map<string, T>;
|
||||
errors: Map<string, Error>;
|
||||
statistics: {
|
||||
totalRequested: number;
|
||||
totalRetrieved: number;
|
||||
totalErrors: number;
|
||||
duration: number;
|
||||
apiCalls: number;
|
||||
};
|
||||
}
|
||||
/**
|
||||
* High-performance batch operations for S3-compatible storage
|
||||
* Optimizes retrieval patterns for HNSW search operations
|
||||
*/
|
||||
export declare class BatchS3Operations {
|
||||
private s3Client;
|
||||
private bucketName;
|
||||
private options;
|
||||
constructor(s3Client: S3Client, bucketName: string, options?: BatchRetrievalOptions);
|
||||
/**
|
||||
* Batch retrieve HNSW nodes with intelligent prefetching
|
||||
*/
|
||||
batchGetNodes(nodeIds: string[], prefix?: string): Promise<BatchResult<HNSWNoun>>;
|
||||
/**
|
||||
* Parallel GetObject operations for small batches
|
||||
*/
|
||||
private parallelGetObjects;
|
||||
/**
|
||||
* Chunked parallel retrieval with intelligent batching
|
||||
*/
|
||||
private chunkedParallelGet;
|
||||
/**
|
||||
* List-based batch retrieval for large datasets
|
||||
* Uses S3 ListObjects to reduce API calls
|
||||
*/
|
||||
private listBasedBatchGet;
|
||||
/**
|
||||
* Intelligent prefetch based on HNSW graph connectivity
|
||||
*/
|
||||
prefetchConnectedNodes(currentNodeIds: string[], connectionMap: Map<string, Set<string>>, prefix?: string): Promise<BatchResult<HNSWNoun>>;
|
||||
/**
|
||||
* S3 Select-based retrieval for filtered queries
|
||||
*/
|
||||
selectiveRetrieve(prefix: string, filter: {
|
||||
vectorDimension?: number;
|
||||
metadataKey?: string;
|
||||
metadataValue?: any;
|
||||
}): Promise<BatchResult<HNSWNoun>>;
|
||||
/**
|
||||
* Parse stored object from JSON string
|
||||
*/
|
||||
private parseStoredObject;
|
||||
/**
|
||||
* Utility function to chunk arrays
|
||||
*/
|
||||
private chunkArray;
|
||||
}
|
||||
export {};
|
||||
|
|
@ -0,0 +1,287 @@
|
|||
/**
|
||||
* Enhanced Batch S3 Operations for High-Performance Vector Retrieval
|
||||
* Implements optimized batch operations to reduce S3 API calls and latency
|
||||
*/
|
||||
/**
|
||||
* High-performance batch operations for S3-compatible storage
|
||||
* Optimizes retrieval patterns for HNSW search operations
|
||||
*/
|
||||
export class BatchS3Operations {
|
||||
constructor(s3Client, bucketName, options = {}) {
|
||||
this.s3Client = s3Client;
|
||||
this.bucketName = bucketName;
|
||||
this.options = {
|
||||
maxConcurrency: 50, // AWS S3 rate limit friendly
|
||||
prefetchSize: 100,
|
||||
useS3Select: false,
|
||||
compressionEnabled: false,
|
||||
...options
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Batch retrieve HNSW nodes with intelligent prefetching
|
||||
*/
|
||||
async batchGetNodes(nodeIds, prefix = 'nodes/') {
|
||||
const startTime = Date.now();
|
||||
const result = {
|
||||
items: new Map(),
|
||||
errors: new Map(),
|
||||
statistics: {
|
||||
totalRequested: nodeIds.length,
|
||||
totalRetrieved: 0,
|
||||
totalErrors: 0,
|
||||
duration: 0,
|
||||
apiCalls: 0
|
||||
}
|
||||
};
|
||||
if (nodeIds.length === 0) {
|
||||
result.statistics.duration = Date.now() - startTime;
|
||||
return result;
|
||||
}
|
||||
// Use different strategies based on request size
|
||||
if (nodeIds.length <= 10) {
|
||||
// Small batch - use parallel GetObject
|
||||
await this.parallelGetObjects(nodeIds, prefix, result);
|
||||
}
|
||||
else if (nodeIds.length <= 1000) {
|
||||
// Medium batch - use chunked parallel with prefetching
|
||||
await this.chunkedParallelGet(nodeIds, prefix, result);
|
||||
}
|
||||
else {
|
||||
// Large batch - use S3 list-based approach with filtering
|
||||
await this.listBasedBatchGet(nodeIds, prefix, result);
|
||||
}
|
||||
result.statistics.duration = Date.now() - startTime;
|
||||
return result;
|
||||
}
|
||||
/**
|
||||
* Parallel GetObject operations for small batches
|
||||
*/
|
||||
async parallelGetObjects(ids, prefix, result) {
|
||||
const { GetObjectCommand } = await import('@aws-sdk/client-s3');
|
||||
const semaphore = new Semaphore(this.options.maxConcurrency);
|
||||
const promises = ids.map(async (id) => {
|
||||
await semaphore.acquire();
|
||||
try {
|
||||
result.statistics.apiCalls++;
|
||||
const response = await this.s3Client.send(new GetObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: `${prefix}${id}.json`
|
||||
}));
|
||||
if (response.Body) {
|
||||
const content = await response.Body.transformToString();
|
||||
const item = this.parseStoredObject(content);
|
||||
if (item) {
|
||||
result.items.set(id, item);
|
||||
result.statistics.totalRetrieved++;
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
result.errors.set(id, error);
|
||||
result.statistics.totalErrors++;
|
||||
}
|
||||
finally {
|
||||
semaphore.release();
|
||||
}
|
||||
});
|
||||
await Promise.all(promises);
|
||||
}
|
||||
/**
|
||||
* Chunked parallel retrieval with intelligent batching
|
||||
*/
|
||||
async chunkedParallelGet(ids, prefix, result) {
|
||||
const chunkSize = Math.min(50, Math.ceil(ids.length / 10));
|
||||
const chunks = this.chunkArray(ids, chunkSize);
|
||||
// Process chunks with controlled concurrency
|
||||
const semaphore = new Semaphore(Math.min(5, chunks.length));
|
||||
const chunkPromises = chunks.map(async (chunk) => {
|
||||
await semaphore.acquire();
|
||||
try {
|
||||
await this.parallelGetObjects(chunk, prefix, result);
|
||||
}
|
||||
finally {
|
||||
semaphore.release();
|
||||
}
|
||||
});
|
||||
await Promise.all(chunkPromises);
|
||||
}
|
||||
/**
|
||||
* List-based batch retrieval for large datasets
|
||||
* Uses S3 ListObjects to reduce API calls
|
||||
*/
|
||||
async listBasedBatchGet(ids, prefix, result) {
|
||||
const { ListObjectsV2Command, GetObjectCommand } = await import('@aws-sdk/client-s3');
|
||||
// Create a set for O(1) lookup
|
||||
const idSet = new Set(ids);
|
||||
// List objects with the prefix
|
||||
let continuationToken;
|
||||
const maxKeys = 1000;
|
||||
do {
|
||||
result.statistics.apiCalls++;
|
||||
const listResponse = await this.s3Client.send(new ListObjectsV2Command({
|
||||
Bucket: this.bucketName,
|
||||
Prefix: prefix,
|
||||
MaxKeys: maxKeys,
|
||||
ContinuationToken: continuationToken
|
||||
}));
|
||||
if (listResponse.Contents) {
|
||||
// Filter objects that match our requested IDs
|
||||
const matchingObjects = listResponse.Contents.filter((obj) => {
|
||||
if (!obj.Key)
|
||||
return false;
|
||||
const id = obj.Key.replace(prefix, '').replace('.json', '');
|
||||
return idSet.has(id);
|
||||
});
|
||||
// Batch retrieve matching objects
|
||||
const semaphore = new Semaphore(this.options.maxConcurrency);
|
||||
const retrievalPromises = matchingObjects.map(async (obj) => {
|
||||
if (!obj.Key)
|
||||
return;
|
||||
await semaphore.acquire();
|
||||
try {
|
||||
result.statistics.apiCalls++;
|
||||
const response = await this.s3Client.send(new GetObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: obj.Key
|
||||
}));
|
||||
if (response.Body) {
|
||||
const content = await response.Body.transformToString();
|
||||
const item = this.parseStoredObject(content);
|
||||
if (item) {
|
||||
const id = obj.Key.replace(prefix, '').replace('.json', '');
|
||||
result.items.set(id, item);
|
||||
result.statistics.totalRetrieved++;
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
const id = obj.Key.replace(prefix, '').replace('.json', '');
|
||||
result.errors.set(id, error);
|
||||
result.statistics.totalErrors++;
|
||||
}
|
||||
finally {
|
||||
semaphore.release();
|
||||
}
|
||||
});
|
||||
await Promise.all(retrievalPromises);
|
||||
}
|
||||
continuationToken = listResponse.NextContinuationToken;
|
||||
} while (continuationToken && result.items.size < ids.length);
|
||||
}
|
||||
/**
|
||||
* Intelligent prefetch based on HNSW graph connectivity
|
||||
*/
|
||||
async prefetchConnectedNodes(currentNodeIds, connectionMap, prefix = 'nodes/') {
|
||||
// Analyze connection patterns to predict next nodes
|
||||
const predictedNodes = new Set();
|
||||
for (const nodeId of currentNodeIds) {
|
||||
const connections = connectionMap.get(nodeId);
|
||||
if (connections) {
|
||||
// Add immediate neighbors
|
||||
connections.forEach(connId => predictedNodes.add(connId));
|
||||
// Add second-degree neighbors (limited)
|
||||
let count = 0;
|
||||
for (const connId of connections) {
|
||||
if (count >= 5)
|
||||
break; // Limit prefetch scope
|
||||
const secondDegree = connectionMap.get(connId);
|
||||
if (secondDegree) {
|
||||
secondDegree.forEach(id => {
|
||||
if (count < 20) {
|
||||
predictedNodes.add(id);
|
||||
count++;
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Remove nodes we already have
|
||||
const nodesToPrefetch = Array.from(predictedNodes).filter(id => !currentNodeIds.includes(id));
|
||||
return this.batchGetNodes(nodesToPrefetch.slice(0, this.options.prefetchSize), prefix);
|
||||
}
|
||||
/**
|
||||
* S3 Select-based retrieval for filtered queries
|
||||
*/
|
||||
async selectiveRetrieve(prefix, filter) {
|
||||
// This would use S3 Select to filter objects server-side
|
||||
// Reducing data transfer for large-scale operations
|
||||
const startTime = Date.now();
|
||||
const result = {
|
||||
items: new Map(),
|
||||
errors: new Map(),
|
||||
statistics: {
|
||||
totalRequested: 0,
|
||||
totalRetrieved: 0,
|
||||
totalErrors: 0,
|
||||
duration: 0,
|
||||
apiCalls: 0
|
||||
}
|
||||
};
|
||||
// S3 Select implementation would go here
|
||||
// For now, fall back to list-based approach
|
||||
console.warn('S3 Select not implemented, falling back to list-based retrieval');
|
||||
result.statistics.duration = Date.now() - startTime;
|
||||
return result;
|
||||
}
|
||||
/**
|
||||
* Parse stored object from JSON string
|
||||
*/
|
||||
parseStoredObject(content) {
|
||||
try {
|
||||
const parsed = JSON.parse(content);
|
||||
// Reconstruct HNSW node structure
|
||||
if (parsed.connections && typeof parsed.connections === 'object') {
|
||||
const connections = new Map();
|
||||
for (const [level, nodeIds] of Object.entries(parsed.connections)) {
|
||||
connections.set(Number(level), new Set(nodeIds));
|
||||
}
|
||||
parsed.connections = connections;
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
catch (error) {
|
||||
console.error('Failed to parse stored object:', error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Utility function to chunk arrays
|
||||
*/
|
||||
chunkArray(array, chunkSize) {
|
||||
const chunks = [];
|
||||
for (let i = 0; i < array.length; i += chunkSize) {
|
||||
chunks.push(array.slice(i, i + chunkSize));
|
||||
}
|
||||
return chunks;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Simple semaphore implementation for concurrency control
|
||||
*/
|
||||
class Semaphore {
|
||||
constructor(permits) {
|
||||
this.waiting = [];
|
||||
this.permits = permits;
|
||||
}
|
||||
async acquire() {
|
||||
if (this.permits > 0) {
|
||||
this.permits--;
|
||||
return Promise.resolve();
|
||||
}
|
||||
return new Promise((resolve) => {
|
||||
this.waiting.push(resolve);
|
||||
});
|
||||
}
|
||||
release() {
|
||||
if (this.waiting.length > 0) {
|
||||
const resolve = this.waiting.shift();
|
||||
resolve();
|
||||
}
|
||||
else {
|
||||
this.permits++;
|
||||
}
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=batchS3Operations.js.map
|
||||
File diff suppressed because one or more lines are too long
255
.recovery-workspace/dist-backup-20250910-141917/storage/adapters/fileSystemStorage.d.ts
vendored
Normal file
255
.recovery-workspace/dist-backup-20250910-141917/storage/adapters/fileSystemStorage.d.ts
vendored
Normal file
|
|
@ -0,0 +1,255 @@
|
|||
/**
|
||||
* File System Storage Adapter
|
||||
* File system storage adapter for Node.js environments
|
||||
*/
|
||||
import { GraphVerb, HNSWNoun, HNSWVerb, StatisticsData } from '../../coreTypes.js';
|
||||
import { BaseStorage } from '../baseStorage.js';
|
||||
type HNSWNode = HNSWNoun;
|
||||
type Edge = HNSWVerb;
|
||||
/**
|
||||
* File system storage adapter for Node.js environments
|
||||
* Uses the file system to store data in the specified directory structure
|
||||
*/
|
||||
export declare class FileSystemStorage extends BaseStorage {
|
||||
private rootDir;
|
||||
private nounsDir;
|
||||
private verbsDir;
|
||||
private metadataDir;
|
||||
private nounMetadataDir;
|
||||
private verbMetadataDir;
|
||||
private indexDir;
|
||||
private systemDir;
|
||||
private lockDir;
|
||||
private useDualWrite;
|
||||
private activeLocks;
|
||||
/**
|
||||
* Initialize the storage adapter
|
||||
* @param rootDirectory The root directory for storage
|
||||
*/
|
||||
constructor(rootDirectory: string);
|
||||
/**
|
||||
* Initialize the storage adapter
|
||||
*/
|
||||
init(): Promise<void>;
|
||||
/**
|
||||
* Check if a directory exists
|
||||
*/
|
||||
private directoryExists;
|
||||
/**
|
||||
* Ensure a directory exists, creating it if necessary
|
||||
*/
|
||||
private ensureDirectoryExists;
|
||||
/**
|
||||
* Save a node to storage
|
||||
*/
|
||||
protected saveNode(node: HNSWNode): Promise<void>;
|
||||
/**
|
||||
* Get a node from storage
|
||||
*/
|
||||
protected getNode(id: string): Promise<HNSWNode | null>;
|
||||
/**
|
||||
* Get all nodes from storage
|
||||
*/
|
||||
protected getAllNodes(): Promise<HNSWNode[]>;
|
||||
/**
|
||||
* Get nodes by noun type
|
||||
* @param nounType The noun type to filter by
|
||||
* @returns Promise that resolves to an array of nodes of the specified noun type
|
||||
*/
|
||||
protected getNodesByNounType(nounType: string): Promise<HNSWNode[]>;
|
||||
/**
|
||||
* Delete a node from storage
|
||||
*/
|
||||
protected deleteNode(id: string): Promise<void>;
|
||||
/**
|
||||
* Save an edge to storage
|
||||
*/
|
||||
protected saveEdge(edge: Edge): Promise<void>;
|
||||
/**
|
||||
* Get an edge from storage
|
||||
*/
|
||||
protected getEdge(id: string): Promise<Edge | null>;
|
||||
/**
|
||||
* Get all edges from storage
|
||||
*/
|
||||
protected getAllEdges(): Promise<Edge[]>;
|
||||
/**
|
||||
* Get edges by source
|
||||
*/
|
||||
protected getEdgesBySource(sourceId: string): Promise<Edge[]>;
|
||||
/**
|
||||
* Get edges by target
|
||||
*/
|
||||
protected getEdgesByTarget(targetId: string): Promise<Edge[]>;
|
||||
/**
|
||||
* Get edges by type
|
||||
*/
|
||||
protected getEdgesByType(type: string): Promise<Edge[]>;
|
||||
/**
|
||||
* Delete an edge from storage
|
||||
*/
|
||||
protected deleteEdge(id: string): Promise<void>;
|
||||
/**
|
||||
* Save metadata to storage
|
||||
*/
|
||||
saveMetadata(id: string, metadata: any): Promise<void>;
|
||||
/**
|
||||
* Get metadata from storage
|
||||
*/
|
||||
getMetadata(id: string): Promise<any | null>;
|
||||
/**
|
||||
* Get multiple metadata objects in batches (CRITICAL: Prevents socket exhaustion)
|
||||
* FileSystem implementation uses controlled concurrency to prevent too many file reads
|
||||
*/
|
||||
getMetadataBatch(ids: string[]): Promise<Map<string, any>>;
|
||||
/**
|
||||
* Save noun metadata to storage
|
||||
*/
|
||||
protected saveNounMetadata_internal(id: string, metadata: any): Promise<void>;
|
||||
/**
|
||||
* Get noun metadata from storage
|
||||
*/
|
||||
getNounMetadata(id: string): Promise<any | null>;
|
||||
/**
|
||||
* Save verb metadata to storage
|
||||
*/
|
||||
protected saveVerbMetadata_internal(id: string, metadata: any): Promise<void>;
|
||||
/**
|
||||
* Get verb metadata from storage
|
||||
*/
|
||||
getVerbMetadata(id: string): Promise<any | null>;
|
||||
/**
|
||||
* Get nouns with pagination support
|
||||
* @param options Pagination options
|
||||
*/
|
||||
getNounsWithPagination(options?: {
|
||||
limit?: number;
|
||||
cursor?: string;
|
||||
filter?: any;
|
||||
}): Promise<{
|
||||
items: HNSWNoun[];
|
||||
totalCount: number;
|
||||
hasMore: boolean;
|
||||
nextCursor?: string;
|
||||
}>;
|
||||
/**
|
||||
* Clear all data from storage
|
||||
*/
|
||||
clear(): Promise<void>;
|
||||
/**
|
||||
* Enhanced clear operation with safety mechanisms and performance optimizations
|
||||
* Provides progress tracking, backup options, and instance name confirmation
|
||||
*/
|
||||
clearEnhanced(options?: import('../enhancedClearOperations.js').ClearOptions): Promise<import('../enhancedClearOperations.js').ClearResult>;
|
||||
/**
|
||||
* Get information about storage usage and capacity
|
||||
*/
|
||||
getStorageStatus(): Promise<{
|
||||
type: string;
|
||||
used: number;
|
||||
quota: number | null;
|
||||
details?: Record<string, any>;
|
||||
}>;
|
||||
/**
|
||||
* Implementation of abstract methods from BaseStorage
|
||||
*/
|
||||
/**
|
||||
* Save a noun to storage
|
||||
*/
|
||||
protected saveNoun_internal(noun: HNSWNoun): Promise<void>;
|
||||
/**
|
||||
* Get a noun from storage
|
||||
*/
|
||||
protected getNoun_internal(id: string): Promise<HNSWNoun | null>;
|
||||
/**
|
||||
* Get nouns by noun type
|
||||
*/
|
||||
protected getNounsByNounType_internal(nounType: string): Promise<HNSWNoun[]>;
|
||||
/**
|
||||
* Delete a noun from storage
|
||||
*/
|
||||
protected deleteNoun_internal(id: string): Promise<void>;
|
||||
/**
|
||||
* Save a verb to storage
|
||||
*/
|
||||
protected saveVerb_internal(verb: HNSWVerb): Promise<void>;
|
||||
/**
|
||||
* Get a verb from storage
|
||||
*/
|
||||
protected getVerb_internal(id: string): Promise<HNSWVerb | null>;
|
||||
/**
|
||||
* Get verbs by source
|
||||
*/
|
||||
protected getVerbsBySource_internal(sourceId: string): Promise<GraphVerb[]>;
|
||||
/**
|
||||
* Get verbs by target
|
||||
*/
|
||||
protected getVerbsByTarget_internal(targetId: string): Promise<GraphVerb[]>;
|
||||
/**
|
||||
* Get verbs by type
|
||||
*/
|
||||
protected getVerbsByType_internal(type: string): Promise<GraphVerb[]>;
|
||||
/**
|
||||
* Get verbs with pagination
|
||||
* This method reads verb files from the filesystem and returns them with pagination
|
||||
*/
|
||||
getVerbsWithPagination(options?: {
|
||||
limit?: number;
|
||||
cursor?: string;
|
||||
filter?: {
|
||||
verbType?: string | string[];
|
||||
sourceId?: string | string[];
|
||||
targetId?: string | string[];
|
||||
service?: string | string[];
|
||||
metadata?: Record<string, any>;
|
||||
};
|
||||
}): Promise<{
|
||||
items: GraphVerb[];
|
||||
totalCount?: number;
|
||||
hasMore: boolean;
|
||||
nextCursor?: string;
|
||||
}>;
|
||||
/**
|
||||
* Delete a verb from storage
|
||||
*/
|
||||
protected deleteVerb_internal(id: string): Promise<void>;
|
||||
/**
|
||||
* Acquire a file-based lock for coordinating operations across multiple processes
|
||||
* @param lockKey The key to lock on
|
||||
* @param ttl Time to live for the lock in milliseconds (default: 30 seconds)
|
||||
* @returns Promise that resolves to true if lock was acquired, false otherwise
|
||||
*/
|
||||
private acquireLock;
|
||||
/**
|
||||
* Release a file-based lock
|
||||
* @param lockKey The key to unlock
|
||||
* @param lockValue The value used when acquiring the lock (for verification)
|
||||
* @returns Promise that resolves when lock is released
|
||||
*/
|
||||
private releaseLock;
|
||||
/**
|
||||
* Clean up expired lock files
|
||||
*/
|
||||
private cleanupExpiredLocks;
|
||||
/**
|
||||
* Save statistics data to storage with file-based locking
|
||||
*/
|
||||
protected saveStatisticsData(statistics: StatisticsData): Promise<void>;
|
||||
/**
|
||||
* Get statistics data from storage
|
||||
*/
|
||||
protected getStatisticsData(): Promise<StatisticsData | null>;
|
||||
/**
|
||||
* Save statistics with backward compatibility (dual write)
|
||||
*/
|
||||
private saveStatisticsWithBackwardCompat;
|
||||
/**
|
||||
* Get statistics with backward compatibility (dual read)
|
||||
*/
|
||||
private getStatisticsWithBackwardCompat;
|
||||
/**
|
||||
* Merge statistics from multiple sources
|
||||
*/
|
||||
private mergeStatistics;
|
||||
}
|
||||
export {};
|
||||
File diff suppressed because it is too large
Load diff
File diff suppressed because one or more lines are too long
172
.recovery-workspace/dist-backup-20250910-141917/storage/adapters/memoryStorage.d.ts
vendored
Normal file
172
.recovery-workspace/dist-backup-20250910-141917/storage/adapters/memoryStorage.d.ts
vendored
Normal file
|
|
@ -0,0 +1,172 @@
|
|||
/**
|
||||
* Memory Storage Adapter
|
||||
* In-memory storage adapter for environments where persistent storage is not available or needed
|
||||
*/
|
||||
import { GraphVerb, HNSWNoun, HNSWVerb, StatisticsData } from '../../coreTypes.js';
|
||||
import { BaseStorage } from '../baseStorage.js';
|
||||
import { PaginatedResult } from '../../types/paginationTypes.js';
|
||||
/**
|
||||
* In-memory storage adapter
|
||||
* Uses Maps to store data in memory
|
||||
*/
|
||||
export declare class MemoryStorage extends BaseStorage {
|
||||
private nouns;
|
||||
private verbs;
|
||||
private metadata;
|
||||
private nounMetadata;
|
||||
private verbMetadata;
|
||||
private statistics;
|
||||
constructor();
|
||||
/**
|
||||
* Initialize the storage adapter
|
||||
* Nothing to initialize for in-memory storage
|
||||
*/
|
||||
init(): Promise<void>;
|
||||
/**
|
||||
* Save a noun to storage
|
||||
*/
|
||||
protected saveNoun_internal(noun: HNSWNoun): Promise<void>;
|
||||
/**
|
||||
* Get a noun from storage
|
||||
*/
|
||||
protected getNoun_internal(id: string): Promise<HNSWNoun | null>;
|
||||
/**
|
||||
* Get nouns with pagination and filtering
|
||||
* @param options Pagination and filtering options
|
||||
* @returns Promise that resolves to a paginated result of nouns
|
||||
*/
|
||||
getNouns(options?: {
|
||||
pagination?: {
|
||||
offset?: number;
|
||||
limit?: number;
|
||||
cursor?: string;
|
||||
};
|
||||
filter?: {
|
||||
nounType?: string | string[];
|
||||
service?: string | string[];
|
||||
metadata?: Record<string, any>;
|
||||
};
|
||||
}): Promise<PaginatedResult<HNSWNoun>>;
|
||||
/**
|
||||
* Get nouns with pagination - simplified interface for compatibility
|
||||
*/
|
||||
getNounsWithPagination(options?: {
|
||||
limit?: number;
|
||||
cursor?: string;
|
||||
filter?: any;
|
||||
}): Promise<{
|
||||
items: HNSWNoun[];
|
||||
totalCount: number;
|
||||
hasMore: boolean;
|
||||
nextCursor?: string;
|
||||
}>;
|
||||
/**
|
||||
* Get nouns by noun type
|
||||
* @param nounType The noun type to filter by
|
||||
* @returns Promise that resolves to an array of nouns of the specified noun type
|
||||
* @deprecated Use getNouns() with filter.nounType instead
|
||||
*/
|
||||
protected getNounsByNounType_internal(nounType: string): Promise<HNSWNoun[]>;
|
||||
/**
|
||||
* Delete a noun from storage
|
||||
*/
|
||||
protected deleteNoun_internal(id: string): Promise<void>;
|
||||
/**
|
||||
* Save a verb to storage
|
||||
*/
|
||||
protected saveVerb_internal(verb: HNSWVerb): Promise<void>;
|
||||
/**
|
||||
* Get a verb from storage
|
||||
*/
|
||||
protected getVerb_internal(id: string): Promise<HNSWVerb | null>;
|
||||
/**
|
||||
* Get verbs with pagination and filtering
|
||||
* @param options Pagination and filtering options
|
||||
* @returns Promise that resolves to a paginated result of verbs
|
||||
*/
|
||||
getVerbs(options?: {
|
||||
pagination?: {
|
||||
offset?: number;
|
||||
limit?: number;
|
||||
cursor?: string;
|
||||
};
|
||||
filter?: {
|
||||
verbType?: string | string[];
|
||||
sourceId?: string | string[];
|
||||
targetId?: string | string[];
|
||||
service?: string | string[];
|
||||
metadata?: Record<string, any>;
|
||||
};
|
||||
}): Promise<PaginatedResult<GraphVerb>>;
|
||||
/**
|
||||
* Get verbs by source
|
||||
* @deprecated Use getVerbs() with filter.sourceId instead
|
||||
*/
|
||||
protected getVerbsBySource_internal(sourceId: string): Promise<GraphVerb[]>;
|
||||
/**
|
||||
* Get verbs by target
|
||||
* @deprecated Use getVerbs() with filter.targetId instead
|
||||
*/
|
||||
protected getVerbsByTarget_internal(targetId: string): Promise<GraphVerb[]>;
|
||||
/**
|
||||
* Get verbs by type
|
||||
* @deprecated Use getVerbs() with filter.verbType instead
|
||||
*/
|
||||
protected getVerbsByType_internal(type: string): Promise<GraphVerb[]>;
|
||||
/**
|
||||
* Delete a verb from storage
|
||||
*/
|
||||
protected deleteVerb_internal(id: string): Promise<void>;
|
||||
/**
|
||||
* Save metadata to storage
|
||||
*/
|
||||
saveMetadata(id: string, metadata: any): Promise<void>;
|
||||
/**
|
||||
* Get metadata from storage
|
||||
*/
|
||||
getMetadata(id: string): Promise<any | null>;
|
||||
/**
|
||||
* Get multiple metadata objects in batches (CRITICAL: Prevents socket exhaustion)
|
||||
* Memory storage implementation is simple since all data is already in memory
|
||||
*/
|
||||
getMetadataBatch(ids: string[]): Promise<Map<string, any>>;
|
||||
/**
|
||||
* Save noun metadata to storage (internal implementation)
|
||||
*/
|
||||
protected saveNounMetadata_internal(id: string, metadata: any): Promise<void>;
|
||||
/**
|
||||
* Get noun metadata from storage
|
||||
*/
|
||||
getNounMetadata(id: string): Promise<any | null>;
|
||||
/**
|
||||
* Save verb metadata to storage (internal implementation)
|
||||
*/
|
||||
protected saveVerbMetadata_internal(id: string, metadata: any): Promise<void>;
|
||||
/**
|
||||
* Get verb metadata from storage
|
||||
*/
|
||||
getVerbMetadata(id: string): Promise<any | null>;
|
||||
/**
|
||||
* Clear all data from storage
|
||||
*/
|
||||
clear(): Promise<void>;
|
||||
/**
|
||||
* Get information about storage usage and capacity
|
||||
*/
|
||||
getStorageStatus(): Promise<{
|
||||
type: string;
|
||||
used: number;
|
||||
quota: number | null;
|
||||
details?: Record<string, any>;
|
||||
}>;
|
||||
/**
|
||||
* Save statistics data to storage
|
||||
* @param statistics The statistics data to save
|
||||
*/
|
||||
protected saveStatisticsData(statistics: StatisticsData): Promise<void>;
|
||||
/**
|
||||
* Get statistics data from storage
|
||||
* @returns Promise that resolves to the statistics data or null if not found
|
||||
*/
|
||||
protected getStatisticsData(): Promise<StatisticsData | null>;
|
||||
}
|
||||
|
|
@ -0,0 +1,551 @@
|
|||
/**
|
||||
* Memory Storage Adapter
|
||||
* In-memory storage adapter for environments where persistent storage is not available or needed
|
||||
*/
|
||||
import { BaseStorage } from '../baseStorage.js';
|
||||
// No type aliases needed - using the original types directly
|
||||
/**
|
||||
* In-memory storage adapter
|
||||
* Uses Maps to store data in memory
|
||||
*/
|
||||
export class MemoryStorage extends BaseStorage {
|
||||
constructor() {
|
||||
super();
|
||||
// Single map of noun ID to noun
|
||||
this.nouns = new Map();
|
||||
this.verbs = new Map();
|
||||
this.metadata = new Map();
|
||||
this.nounMetadata = new Map();
|
||||
this.verbMetadata = new Map();
|
||||
this.statistics = null;
|
||||
}
|
||||
/**
|
||||
* Initialize the storage adapter
|
||||
* Nothing to initialize for in-memory storage
|
||||
*/
|
||||
async init() {
|
||||
this.isInitialized = true;
|
||||
}
|
||||
/**
|
||||
* Save a noun to storage
|
||||
*/
|
||||
async saveNoun_internal(noun) {
|
||||
// Create a deep copy to avoid reference issues
|
||||
const nounCopy = {
|
||||
id: noun.id,
|
||||
vector: [...noun.vector],
|
||||
connections: new Map(),
|
||||
level: noun.level || 0,
|
||||
metadata: noun.metadata
|
||||
};
|
||||
// Copy connections
|
||||
for (const [level, connections] of noun.connections.entries()) {
|
||||
nounCopy.connections.set(level, new Set(connections));
|
||||
}
|
||||
// Save the noun directly in the nouns map
|
||||
this.nouns.set(noun.id, nounCopy);
|
||||
}
|
||||
/**
|
||||
* Get a noun from storage
|
||||
*/
|
||||
async getNoun_internal(id) {
|
||||
// Get the noun directly from the nouns map
|
||||
const noun = this.nouns.get(id);
|
||||
// If not found, return null
|
||||
if (!noun) {
|
||||
return null;
|
||||
}
|
||||
// Return a deep copy to avoid reference issues
|
||||
const nounCopy = {
|
||||
id: noun.id,
|
||||
vector: [...noun.vector],
|
||||
connections: new Map(),
|
||||
level: noun.level || 0,
|
||||
metadata: noun.metadata
|
||||
};
|
||||
// Copy connections
|
||||
for (const [level, connections] of noun.connections.entries()) {
|
||||
nounCopy.connections.set(level, new Set(connections));
|
||||
}
|
||||
return nounCopy;
|
||||
}
|
||||
/**
|
||||
* Get nouns with pagination and filtering
|
||||
* @param options Pagination and filtering options
|
||||
* @returns Promise that resolves to a paginated result of nouns
|
||||
*/
|
||||
async getNouns(options = {}) {
|
||||
const pagination = options.pagination || {};
|
||||
const filter = options.filter || {};
|
||||
// Default values
|
||||
const offset = pagination.offset || 0;
|
||||
const limit = pagination.limit || 100;
|
||||
// Convert string types to arrays for consistent handling
|
||||
const nounTypes = filter.nounType
|
||||
? Array.isArray(filter.nounType) ? filter.nounType : [filter.nounType]
|
||||
: undefined;
|
||||
const services = filter.service
|
||||
? Array.isArray(filter.service) ? filter.service : [filter.service]
|
||||
: undefined;
|
||||
// First, collect all noun IDs that match the filter criteria
|
||||
const matchingIds = [];
|
||||
// Iterate through all nouns to find matches
|
||||
for (const [nounId, noun] of this.nouns.entries()) {
|
||||
// Get the metadata to check filters
|
||||
const metadata = await this.getMetadata(nounId);
|
||||
if (!metadata)
|
||||
continue;
|
||||
// Filter by noun type if specified
|
||||
if (nounTypes && !nounTypes.includes(metadata.noun)) {
|
||||
continue;
|
||||
}
|
||||
// Filter by service if specified
|
||||
if (services && metadata.service && !services.includes(metadata.service)) {
|
||||
continue;
|
||||
}
|
||||
// Filter by metadata fields if specified
|
||||
if (filter.metadata) {
|
||||
let metadataMatch = true;
|
||||
for (const [key, value] of Object.entries(filter.metadata)) {
|
||||
if (metadata[key] !== value) {
|
||||
metadataMatch = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!metadataMatch)
|
||||
continue;
|
||||
}
|
||||
// If we got here, the noun matches all filters
|
||||
matchingIds.push(nounId);
|
||||
}
|
||||
// Calculate pagination
|
||||
const totalCount = matchingIds.length;
|
||||
const paginatedIds = matchingIds.slice(offset, offset + limit);
|
||||
const hasMore = offset + limit < totalCount;
|
||||
// Create cursor for next page if there are more results
|
||||
const nextCursor = hasMore ? `${offset + limit}` : undefined;
|
||||
// Fetch the actual nouns for the current page
|
||||
const items = [];
|
||||
for (const id of paginatedIds) {
|
||||
const noun = this.nouns.get(id);
|
||||
if (!noun)
|
||||
continue;
|
||||
// Create a deep copy to avoid reference issues
|
||||
const nounCopy = {
|
||||
id: noun.id,
|
||||
vector: [...noun.vector],
|
||||
connections: new Map(),
|
||||
level: noun.level || 0,
|
||||
metadata: noun.metadata
|
||||
};
|
||||
// Copy connections
|
||||
for (const [level, connections] of noun.connections.entries()) {
|
||||
nounCopy.connections.set(level, new Set(connections));
|
||||
}
|
||||
items.push(nounCopy);
|
||||
}
|
||||
return {
|
||||
items,
|
||||
totalCount,
|
||||
hasMore,
|
||||
nextCursor
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Get nouns with pagination - simplified interface for compatibility
|
||||
*/
|
||||
async getNounsWithPagination(options = {}) {
|
||||
// Convert to the getNouns format
|
||||
const result = await this.getNouns({
|
||||
pagination: {
|
||||
offset: options.cursor ? parseInt(options.cursor) : 0,
|
||||
limit: options.limit || 100
|
||||
},
|
||||
filter: options.filter
|
||||
});
|
||||
return {
|
||||
items: result.items,
|
||||
totalCount: result.totalCount || 0,
|
||||
hasMore: result.hasMore,
|
||||
nextCursor: result.nextCursor
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Get nouns by noun type
|
||||
* @param nounType The noun type to filter by
|
||||
* @returns Promise that resolves to an array of nouns of the specified noun type
|
||||
* @deprecated Use getNouns() with filter.nounType instead
|
||||
*/
|
||||
async getNounsByNounType_internal(nounType) {
|
||||
const result = await this.getNouns({
|
||||
filter: {
|
||||
nounType
|
||||
}
|
||||
});
|
||||
return result.items;
|
||||
}
|
||||
/**
|
||||
* Delete a noun from storage
|
||||
*/
|
||||
async deleteNoun_internal(id) {
|
||||
this.nouns.delete(id);
|
||||
}
|
||||
/**
|
||||
* Save a verb to storage
|
||||
*/
|
||||
async saveVerb_internal(verb) {
|
||||
// Create a deep copy to avoid reference issues
|
||||
const verbCopy = {
|
||||
id: verb.id,
|
||||
vector: [...verb.vector],
|
||||
connections: new Map()
|
||||
};
|
||||
// Copy connections
|
||||
for (const [level, connections] of verb.connections.entries()) {
|
||||
verbCopy.connections.set(level, new Set(connections));
|
||||
}
|
||||
// Save the verb directly in the verbs map
|
||||
this.verbs.set(verb.id, verbCopy);
|
||||
}
|
||||
/**
|
||||
* Get a verb from storage
|
||||
*/
|
||||
async getVerb_internal(id) {
|
||||
// Get the verb directly from the verbs map
|
||||
const verb = this.verbs.get(id);
|
||||
// If not found, return null
|
||||
if (!verb) {
|
||||
return null;
|
||||
}
|
||||
// Create default timestamp if not present
|
||||
const defaultTimestamp = {
|
||||
seconds: Math.floor(Date.now() / 1000),
|
||||
nanoseconds: (Date.now() % 1000) * 1000000
|
||||
};
|
||||
// Create default createdBy if not present
|
||||
const defaultCreatedBy = {
|
||||
augmentation: 'unknown',
|
||||
version: '1.0'
|
||||
};
|
||||
// Return a deep copy of the HNSWVerb
|
||||
const verbCopy = {
|
||||
id: verb.id,
|
||||
vector: [...verb.vector],
|
||||
connections: new Map()
|
||||
};
|
||||
// Copy connections
|
||||
for (const [level, connections] of verb.connections.entries()) {
|
||||
verbCopy.connections.set(level, new Set(connections));
|
||||
}
|
||||
return verbCopy;
|
||||
}
|
||||
/**
|
||||
* Get verbs with pagination and filtering
|
||||
* @param options Pagination and filtering options
|
||||
* @returns Promise that resolves to a paginated result of verbs
|
||||
*/
|
||||
async getVerbs(options = {}) {
|
||||
const pagination = options.pagination || {};
|
||||
const filter = options.filter || {};
|
||||
// Default values
|
||||
const offset = pagination.offset || 0;
|
||||
const limit = pagination.limit || 100;
|
||||
// Convert string types to arrays for consistent handling
|
||||
const verbTypes = filter.verbType
|
||||
? Array.isArray(filter.verbType) ? filter.verbType : [filter.verbType]
|
||||
: undefined;
|
||||
const sourceIds = filter.sourceId
|
||||
? Array.isArray(filter.sourceId) ? filter.sourceId : [filter.sourceId]
|
||||
: undefined;
|
||||
const targetIds = filter.targetId
|
||||
? Array.isArray(filter.targetId) ? filter.targetId : [filter.targetId]
|
||||
: undefined;
|
||||
const services = filter.service
|
||||
? Array.isArray(filter.service) ? filter.service : [filter.service]
|
||||
: undefined;
|
||||
// First, collect all verb IDs that match the filter criteria
|
||||
const matchingIds = [];
|
||||
// Iterate through all verbs to find matches
|
||||
for (const [verbId, hnswVerb] of this.verbs.entries()) {
|
||||
// Get the metadata for this verb to do filtering
|
||||
const metadata = this.verbMetadata.get(verbId);
|
||||
// Filter by verb type if specified
|
||||
if (verbTypes && metadata && !verbTypes.includes(metadata.type || metadata.verb || '')) {
|
||||
continue;
|
||||
}
|
||||
// Filter by source ID if specified
|
||||
if (sourceIds && metadata && !sourceIds.includes(metadata.sourceId || metadata.source || '')) {
|
||||
continue;
|
||||
}
|
||||
// Filter by target ID if specified
|
||||
if (targetIds && metadata && !targetIds.includes(metadata.targetId || metadata.target || '')) {
|
||||
continue;
|
||||
}
|
||||
// Filter by metadata fields if specified
|
||||
if (filter.metadata && metadata && metadata.data) {
|
||||
let metadataMatch = true;
|
||||
for (const [key, value] of Object.entries(filter.metadata)) {
|
||||
if (metadata.data[key] !== value) {
|
||||
metadataMatch = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!metadataMatch)
|
||||
continue;
|
||||
}
|
||||
// Filter by service if specified
|
||||
if (services && metadata && metadata.createdBy && metadata.createdBy.augmentation &&
|
||||
!services.includes(metadata.createdBy.augmentation)) {
|
||||
continue;
|
||||
}
|
||||
// If we got here, the verb matches all filters
|
||||
matchingIds.push(verbId);
|
||||
}
|
||||
// Calculate pagination
|
||||
const totalCount = matchingIds.length;
|
||||
const paginatedIds = matchingIds.slice(offset, offset + limit);
|
||||
const hasMore = offset + limit < totalCount;
|
||||
// Create cursor for next page if there are more results
|
||||
const nextCursor = hasMore ? `${offset + limit}` : undefined;
|
||||
// Fetch the actual verbs for the current page
|
||||
const items = [];
|
||||
for (const id of paginatedIds) {
|
||||
const hnswVerb = this.verbs.get(id);
|
||||
const metadata = this.verbMetadata.get(id);
|
||||
if (!hnswVerb)
|
||||
continue;
|
||||
if (!metadata) {
|
||||
console.warn(`Verb ${id} found but no metadata - creating minimal GraphVerb`);
|
||||
// Return minimal GraphVerb if metadata is missing
|
||||
items.push({
|
||||
id: hnswVerb.id,
|
||||
vector: hnswVerb.vector,
|
||||
sourceId: '',
|
||||
targetId: ''
|
||||
});
|
||||
continue;
|
||||
}
|
||||
// Create a complete GraphVerb by combining HNSWVerb with metadata
|
||||
const graphVerb = {
|
||||
id: hnswVerb.id,
|
||||
vector: [...hnswVerb.vector],
|
||||
sourceId: metadata.sourceId,
|
||||
targetId: metadata.targetId,
|
||||
source: metadata.source,
|
||||
target: metadata.target,
|
||||
verb: metadata.verb,
|
||||
type: metadata.type,
|
||||
weight: metadata.weight,
|
||||
createdAt: metadata.createdAt,
|
||||
updatedAt: metadata.updatedAt,
|
||||
createdBy: metadata.createdBy,
|
||||
data: metadata.data,
|
||||
metadata: metadata.data // Alias for backward compatibility
|
||||
};
|
||||
items.push(graphVerb);
|
||||
}
|
||||
return {
|
||||
items,
|
||||
totalCount,
|
||||
hasMore,
|
||||
nextCursor
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Get verbs by source
|
||||
* @deprecated Use getVerbs() with filter.sourceId instead
|
||||
*/
|
||||
async getVerbsBySource_internal(sourceId) {
|
||||
const result = await this.getVerbs({
|
||||
filter: {
|
||||
sourceId
|
||||
}
|
||||
});
|
||||
return result.items;
|
||||
}
|
||||
/**
|
||||
* Get verbs by target
|
||||
* @deprecated Use getVerbs() with filter.targetId instead
|
||||
*/
|
||||
async getVerbsByTarget_internal(targetId) {
|
||||
const result = await this.getVerbs({
|
||||
filter: {
|
||||
targetId
|
||||
}
|
||||
});
|
||||
return result.items;
|
||||
}
|
||||
/**
|
||||
* Get verbs by type
|
||||
* @deprecated Use getVerbs() with filter.verbType instead
|
||||
*/
|
||||
async getVerbsByType_internal(type) {
|
||||
const result = await this.getVerbs({
|
||||
filter: {
|
||||
verbType: type
|
||||
}
|
||||
});
|
||||
return result.items;
|
||||
}
|
||||
/**
|
||||
* Delete a verb from storage
|
||||
*/
|
||||
async deleteVerb_internal(id) {
|
||||
// Delete the verb directly from the verbs map
|
||||
this.verbs.delete(id);
|
||||
}
|
||||
/**
|
||||
* Save metadata to storage
|
||||
*/
|
||||
async saveMetadata(id, metadata) {
|
||||
this.metadata.set(id, JSON.parse(JSON.stringify(metadata)));
|
||||
}
|
||||
/**
|
||||
* Get metadata from storage
|
||||
*/
|
||||
async getMetadata(id) {
|
||||
const metadata = this.metadata.get(id);
|
||||
if (!metadata) {
|
||||
return null;
|
||||
}
|
||||
return JSON.parse(JSON.stringify(metadata));
|
||||
}
|
||||
/**
|
||||
* Get multiple metadata objects in batches (CRITICAL: Prevents socket exhaustion)
|
||||
* Memory storage implementation is simple since all data is already in memory
|
||||
*/
|
||||
async getMetadataBatch(ids) {
|
||||
const results = new Map();
|
||||
// Memory storage can handle all IDs at once since it's in-memory
|
||||
for (const id of ids) {
|
||||
const metadata = this.metadata.get(id);
|
||||
if (metadata) {
|
||||
// Deep clone to prevent mutation
|
||||
results.set(id, JSON.parse(JSON.stringify(metadata)));
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
/**
|
||||
* Save noun metadata to storage (internal implementation)
|
||||
*/
|
||||
async saveNounMetadata_internal(id, metadata) {
|
||||
this.nounMetadata.set(id, JSON.parse(JSON.stringify(metadata)));
|
||||
}
|
||||
/**
|
||||
* Get noun metadata from storage
|
||||
*/
|
||||
async getNounMetadata(id) {
|
||||
const metadata = this.nounMetadata.get(id);
|
||||
if (!metadata) {
|
||||
return null;
|
||||
}
|
||||
return JSON.parse(JSON.stringify(metadata));
|
||||
}
|
||||
/**
|
||||
* Save verb metadata to storage (internal implementation)
|
||||
*/
|
||||
async saveVerbMetadata_internal(id, metadata) {
|
||||
this.verbMetadata.set(id, JSON.parse(JSON.stringify(metadata)));
|
||||
}
|
||||
/**
|
||||
* Get verb metadata from storage
|
||||
*/
|
||||
async getVerbMetadata(id) {
|
||||
const metadata = this.verbMetadata.get(id);
|
||||
if (!metadata) {
|
||||
return null;
|
||||
}
|
||||
return JSON.parse(JSON.stringify(metadata));
|
||||
}
|
||||
/**
|
||||
* Clear all data from storage
|
||||
*/
|
||||
async clear() {
|
||||
this.nouns.clear();
|
||||
this.verbs.clear();
|
||||
this.metadata.clear();
|
||||
this.nounMetadata.clear();
|
||||
this.verbMetadata.clear();
|
||||
this.statistics = null;
|
||||
// Clear the statistics cache
|
||||
this.statisticsCache = null;
|
||||
this.statisticsModified = false;
|
||||
}
|
||||
/**
|
||||
* Get information about storage usage and capacity
|
||||
*/
|
||||
async getStorageStatus() {
|
||||
return {
|
||||
type: 'memory',
|
||||
used: 0, // In-memory storage doesn't have a meaningful size
|
||||
quota: null, // In-memory storage doesn't have a quota
|
||||
details: {
|
||||
nodeCount: this.nouns.size,
|
||||
edgeCount: this.verbs.size,
|
||||
metadataCount: this.metadata.size
|
||||
}
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Save statistics data to storage
|
||||
* @param statistics The statistics data to save
|
||||
*/
|
||||
async saveStatisticsData(statistics) {
|
||||
// For memory storage, we just need to store the statistics in memory
|
||||
// Create a deep copy to avoid reference issues
|
||||
this.statistics = {
|
||||
nounCount: { ...statistics.nounCount },
|
||||
verbCount: { ...statistics.verbCount },
|
||||
metadataCount: { ...statistics.metadataCount },
|
||||
hnswIndexSize: statistics.hnswIndexSize,
|
||||
lastUpdated: statistics.lastUpdated,
|
||||
// Include serviceActivity if present
|
||||
...(statistics.serviceActivity && {
|
||||
serviceActivity: Object.fromEntries(Object.entries(statistics.serviceActivity).map(([k, v]) => [k, { ...v }]))
|
||||
}),
|
||||
// Include services if present
|
||||
...(statistics.services && {
|
||||
services: statistics.services.map(s => ({ ...s }))
|
||||
}),
|
||||
// Include distributedConfig if present
|
||||
...(statistics.distributedConfig && {
|
||||
distributedConfig: JSON.parse(JSON.stringify(statistics.distributedConfig))
|
||||
})
|
||||
};
|
||||
// Since this is in-memory, there's no need for time-based partitioning
|
||||
// or legacy file handling
|
||||
}
|
||||
/**
|
||||
* Get statistics data from storage
|
||||
* @returns Promise that resolves to the statistics data or null if not found
|
||||
*/
|
||||
async getStatisticsData() {
|
||||
if (!this.statistics) {
|
||||
return null;
|
||||
}
|
||||
// Return a deep copy to avoid reference issues
|
||||
return {
|
||||
nounCount: { ...this.statistics.nounCount },
|
||||
verbCount: { ...this.statistics.verbCount },
|
||||
metadataCount: { ...this.statistics.metadataCount },
|
||||
hnswIndexSize: this.statistics.hnswIndexSize,
|
||||
lastUpdated: this.statistics.lastUpdated,
|
||||
// Include serviceActivity if present
|
||||
...(this.statistics.serviceActivity && {
|
||||
serviceActivity: Object.fromEntries(Object.entries(this.statistics.serviceActivity).map(([k, v]) => [k, { ...v }]))
|
||||
}),
|
||||
// Include services if present
|
||||
...(this.statistics.services && {
|
||||
services: this.statistics.services.map(s => ({ ...s }))
|
||||
}),
|
||||
// Include distributedConfig if present
|
||||
...(this.statistics.distributedConfig && {
|
||||
distributedConfig: JSON.parse(JSON.stringify(this.statistics.distributedConfig))
|
||||
})
|
||||
};
|
||||
// Since this is in-memory, there's no need for fallback mechanisms
|
||||
// to check multiple storage locations
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=memoryStorage.js.map
|
||||
File diff suppressed because one or more lines are too long
258
.recovery-workspace/dist-backup-20250910-141917/storage/adapters/opfsStorage.d.ts
vendored
Normal file
258
.recovery-workspace/dist-backup-20250910-141917/storage/adapters/opfsStorage.d.ts
vendored
Normal file
|
|
@ -0,0 +1,258 @@
|
|||
/**
|
||||
* OPFS (Origin Private File System) Storage Adapter
|
||||
* Provides persistent storage for the vector database using the Origin Private File System API
|
||||
*/
|
||||
import { GraphVerb, HNSWNoun, HNSWVerb, StatisticsData } from '../../coreTypes.js';
|
||||
import { BaseStorage } from '../baseStorage.js';
|
||||
import '../../types/fileSystemTypes.js';
|
||||
type HNSWNode = HNSWNoun;
|
||||
/**
|
||||
* Type alias for HNSWVerb to make the code more readable
|
||||
*/
|
||||
type Edge = HNSWVerb;
|
||||
type HNSWNoun_internal = HNSWNoun;
|
||||
/**
|
||||
* OPFS storage adapter for browser environments
|
||||
* Uses the Origin Private File System API to store data persistently
|
||||
*/
|
||||
export declare class OPFSStorage extends BaseStorage {
|
||||
private rootDir;
|
||||
private nounsDir;
|
||||
private verbsDir;
|
||||
private metadataDir;
|
||||
private nounMetadataDir;
|
||||
private verbMetadataDir;
|
||||
private indexDir;
|
||||
private isAvailable;
|
||||
private isPersistentRequested;
|
||||
private isPersistentGranted;
|
||||
private statistics;
|
||||
private activeLocks;
|
||||
private lockPrefix;
|
||||
constructor();
|
||||
/**
|
||||
* Initialize the storage adapter
|
||||
*/
|
||||
init(): Promise<void>;
|
||||
/**
|
||||
* Check if OPFS is available in the current environment
|
||||
*/
|
||||
isOPFSAvailable(): boolean;
|
||||
/**
|
||||
* Request persistent storage permission from the user
|
||||
* @returns Promise that resolves to true if permission was granted, false otherwise
|
||||
*/
|
||||
requestPersistentStorage(): Promise<boolean>;
|
||||
/**
|
||||
* Check if persistent storage is granted
|
||||
* @returns Promise that resolves to true if persistent storage is granted, false otherwise
|
||||
*/
|
||||
isPersistent(): Promise<boolean>;
|
||||
/**
|
||||
* Save a noun to storage
|
||||
*/
|
||||
protected saveNoun_internal(noun: HNSWNoun_internal): Promise<void>;
|
||||
/**
|
||||
* Get a noun from storage
|
||||
*/
|
||||
protected getNoun_internal(id: string): Promise<HNSWNoun_internal | null>;
|
||||
/**
|
||||
* Get nouns by noun type (internal implementation)
|
||||
* @param nounType The noun type to filter by
|
||||
* @returns Promise that resolves to an array of nouns of the specified noun type
|
||||
*/
|
||||
protected getNounsByNounType_internal(nounType: string): Promise<HNSWNoun[]>;
|
||||
/**
|
||||
* Get nodes by noun type
|
||||
* @param nounType The noun type to filter by
|
||||
* @returns Promise that resolves to an array of nodes of the specified noun type
|
||||
*/
|
||||
protected getNodesByNounType(nounType: string): Promise<HNSWNode[]>;
|
||||
/**
|
||||
* Delete a noun from storage (internal implementation)
|
||||
*/
|
||||
protected deleteNoun_internal(id: string): Promise<void>;
|
||||
/**
|
||||
* Delete a node from storage
|
||||
*/
|
||||
protected deleteNode(id: string): Promise<void>;
|
||||
/**
|
||||
* Save a verb to storage (internal implementation)
|
||||
*/
|
||||
protected saveVerb_internal(verb: HNSWVerb): Promise<void>;
|
||||
/**
|
||||
* Save an edge to storage
|
||||
*/
|
||||
protected saveEdge(edge: Edge): Promise<void>;
|
||||
/**
|
||||
* Get a verb from storage (internal implementation)
|
||||
*/
|
||||
protected getVerb_internal(id: string): Promise<HNSWVerb | null>;
|
||||
/**
|
||||
* Get an edge from storage
|
||||
*/
|
||||
protected getEdge(id: string): Promise<Edge | null>;
|
||||
/**
|
||||
* Get all edges from storage
|
||||
*/
|
||||
protected getAllEdges(): Promise<Edge[]>;
|
||||
/**
|
||||
* Get verbs by source (internal implementation)
|
||||
*/
|
||||
protected getVerbsBySource_internal(sourceId: string): Promise<GraphVerb[]>;
|
||||
/**
|
||||
* Get edges by source
|
||||
*/
|
||||
protected getEdgesBySource(sourceId: string): Promise<GraphVerb[]>;
|
||||
/**
|
||||
* Get verbs by target (internal implementation)
|
||||
*/
|
||||
protected getVerbsByTarget_internal(targetId: string): Promise<GraphVerb[]>;
|
||||
/**
|
||||
* Get edges by target
|
||||
*/
|
||||
protected getEdgesByTarget(targetId: string): Promise<GraphVerb[]>;
|
||||
/**
|
||||
* Get verbs by type (internal implementation)
|
||||
*/
|
||||
protected getVerbsByType_internal(type: string): Promise<GraphVerb[]>;
|
||||
/**
|
||||
* Get edges by type
|
||||
*/
|
||||
protected getEdgesByType(type: string): Promise<GraphVerb[]>;
|
||||
/**
|
||||
* Delete a verb from storage (internal implementation)
|
||||
*/
|
||||
protected deleteVerb_internal(id: string): Promise<void>;
|
||||
/**
|
||||
* Delete an edge from storage
|
||||
*/
|
||||
protected deleteEdge(id: string): Promise<void>;
|
||||
/**
|
||||
* Save metadata to storage
|
||||
*/
|
||||
saveMetadata(id: string, metadata: any): Promise<void>;
|
||||
/**
|
||||
* Get metadata from storage
|
||||
*/
|
||||
getMetadata(id: string): Promise<any | null>;
|
||||
/**
|
||||
* Get multiple metadata objects in batches (CRITICAL: Prevents socket exhaustion)
|
||||
* OPFS implementation uses controlled concurrency for file operations
|
||||
*/
|
||||
getMetadataBatch(ids: string[]): Promise<Map<string, any>>;
|
||||
/**
|
||||
* Save verb metadata to storage
|
||||
*/
|
||||
protected saveVerbMetadata_internal(id: string, metadata: any): Promise<void>;
|
||||
/**
|
||||
* Get verb metadata from storage
|
||||
*/
|
||||
getVerbMetadata(id: string): Promise<any | null>;
|
||||
/**
|
||||
* Save noun metadata to storage
|
||||
*/
|
||||
protected saveNounMetadata_internal(id: string, metadata: any): Promise<void>;
|
||||
/**
|
||||
* Get noun metadata from storage
|
||||
*/
|
||||
getNounMetadata(id: string): Promise<any | null>;
|
||||
/**
|
||||
* Clear all data from storage
|
||||
*/
|
||||
clear(): Promise<void>;
|
||||
/**
|
||||
* Get information about storage usage and capacity
|
||||
*/
|
||||
getStorageStatus(): Promise<{
|
||||
type: string;
|
||||
used: number;
|
||||
quota: number | null;
|
||||
details?: Record<string, any>;
|
||||
}>;
|
||||
/**
|
||||
* Get the statistics key for a specific date
|
||||
* @param date The date to get the key for
|
||||
* @returns The statistics key for the specified date
|
||||
*/
|
||||
private getStatisticsKeyForDate;
|
||||
/**
|
||||
* Get the current statistics key
|
||||
* @returns The current statistics key
|
||||
*/
|
||||
private getCurrentStatisticsKey;
|
||||
/**
|
||||
* Get the legacy statistics key (for backward compatibility)
|
||||
* @returns The legacy statistics key
|
||||
*/
|
||||
private getLegacyStatisticsKey;
|
||||
/**
|
||||
* Acquire a browser-based lock for coordinating operations across multiple tabs
|
||||
* @param lockKey The key to lock on
|
||||
* @param ttl Time to live for the lock in milliseconds (default: 30 seconds)
|
||||
* @returns Promise that resolves to true if lock was acquired, false otherwise
|
||||
*/
|
||||
private acquireLock;
|
||||
/**
|
||||
* Release a browser-based lock
|
||||
* @param lockKey The key to unlock
|
||||
* @param lockValue The value used when acquiring the lock (for verification)
|
||||
* @returns Promise that resolves when lock is released
|
||||
*/
|
||||
private releaseLock;
|
||||
/**
|
||||
* Clean up expired locks from localStorage
|
||||
*/
|
||||
private cleanupExpiredLocks;
|
||||
/**
|
||||
* Save statistics data to storage with browser-based locking
|
||||
* @param statistics The statistics data to save
|
||||
*/
|
||||
protected saveStatisticsData(statistics: StatisticsData): Promise<void>;
|
||||
/**
|
||||
* Get statistics data from storage
|
||||
* @returns Promise that resolves to the statistics data or null if not found
|
||||
*/
|
||||
protected getStatisticsData(): Promise<StatisticsData | null>;
|
||||
/**
|
||||
* Get nouns with pagination support
|
||||
* @param options Pagination and filter options
|
||||
* @returns Promise that resolves to a paginated result of nouns
|
||||
*/
|
||||
getNounsWithPagination(options?: {
|
||||
limit?: number;
|
||||
cursor?: string;
|
||||
filter?: {
|
||||
nounType?: string | string[];
|
||||
service?: string | string[];
|
||||
metadata?: Record<string, any>;
|
||||
};
|
||||
}): Promise<{
|
||||
items: HNSWNoun[];
|
||||
totalCount?: number;
|
||||
hasMore: boolean;
|
||||
nextCursor?: string;
|
||||
}>;
|
||||
/**
|
||||
* Get verbs with pagination support
|
||||
* @param options Pagination and filter options
|
||||
* @returns Promise that resolves to a paginated result of verbs
|
||||
*/
|
||||
getVerbsWithPagination(options?: {
|
||||
limit?: number;
|
||||
cursor?: string;
|
||||
filter?: {
|
||||
verbType?: string | string[];
|
||||
sourceId?: string | string[];
|
||||
targetId?: string | string[];
|
||||
service?: string | string[];
|
||||
metadata?: Record<string, any>;
|
||||
};
|
||||
}): Promise<{
|
||||
items: GraphVerb[];
|
||||
totalCount?: number;
|
||||
hasMore: boolean;
|
||||
nextCursor?: string;
|
||||
}>;
|
||||
}
|
||||
export {};
|
||||
File diff suppressed because it is too large
Load diff
File diff suppressed because one or more lines are too long
79
.recovery-workspace/dist-backup-20250910-141917/storage/adapters/optimizedS3Search.d.ts
vendored
Normal file
79
.recovery-workspace/dist-backup-20250910-141917/storage/adapters/optimizedS3Search.d.ts
vendored
Normal file
|
|
@ -0,0 +1,79 @@
|
|||
/**
|
||||
* Optimized S3 Search and Pagination
|
||||
* Provides efficient search and pagination capabilities for S3-compatible storage
|
||||
*/
|
||||
import { HNSWNoun, GraphVerb } from '../../coreTypes.js';
|
||||
/**
|
||||
* Pagination result interface
|
||||
*/
|
||||
export interface PaginationResult<T> {
|
||||
items: T[];
|
||||
totalCount?: number;
|
||||
hasMore: boolean;
|
||||
nextCursor?: string;
|
||||
}
|
||||
/**
|
||||
* Filter interface for nouns
|
||||
*/
|
||||
export interface NounFilter {
|
||||
nounType?: string | string[];
|
||||
service?: string | string[];
|
||||
metadata?: Record<string, any>;
|
||||
}
|
||||
/**
|
||||
* Filter interface for verbs
|
||||
*/
|
||||
export interface VerbFilter {
|
||||
verbType?: string | string[];
|
||||
sourceId?: string | string[];
|
||||
targetId?: string | string[];
|
||||
service?: string | string[];
|
||||
metadata?: Record<string, any>;
|
||||
}
|
||||
/**
|
||||
* Interface for storage operations needed by optimized search
|
||||
*/
|
||||
export interface StorageOperations {
|
||||
listObjectKeys(prefix: string, limit: number, cursor?: string): Promise<{
|
||||
keys: string[];
|
||||
hasMore: boolean;
|
||||
nextCursor?: string;
|
||||
}>;
|
||||
getObject<T>(key: string): Promise<T | null>;
|
||||
getMetadata(id: string, type: 'noun' | 'verb'): Promise<any | null>;
|
||||
}
|
||||
/**
|
||||
* Optimized search implementation for S3-compatible storage
|
||||
*/
|
||||
export declare class OptimizedS3Search {
|
||||
private storage;
|
||||
constructor(storage: StorageOperations);
|
||||
/**
|
||||
* Get nouns with optimized pagination and filtering
|
||||
*/
|
||||
getNounsWithPagination(options?: {
|
||||
limit?: number;
|
||||
cursor?: string;
|
||||
filter?: NounFilter;
|
||||
}): Promise<PaginationResult<HNSWNoun>>;
|
||||
/**
|
||||
* Get verbs with optimized pagination and filtering
|
||||
*/
|
||||
getVerbsWithPagination(options?: {
|
||||
limit?: number;
|
||||
cursor?: string;
|
||||
filter?: VerbFilter;
|
||||
}): Promise<PaginationResult<GraphVerb>>;
|
||||
/**
|
||||
* Check if a noun matches the filter criteria
|
||||
*/
|
||||
private matchesNounFilter;
|
||||
/**
|
||||
* Check if a verb matches the filter criteria
|
||||
*/
|
||||
private matchesVerbFilter;
|
||||
/**
|
||||
* Combine HNSWVerb data with metadata to create GraphVerb
|
||||
*/
|
||||
private combineVerbWithMetadata;
|
||||
}
|
||||
|
|
@ -0,0 +1,249 @@
|
|||
/**
|
||||
* Optimized S3 Search and Pagination
|
||||
* Provides efficient search and pagination capabilities for S3-compatible storage
|
||||
*/
|
||||
import { createModuleLogger } from '../../utils/logger.js';
|
||||
import { getDirectoryPath } from '../baseStorage.js';
|
||||
const logger = createModuleLogger('OptimizedS3Search');
|
||||
/**
|
||||
* Optimized search implementation for S3-compatible storage
|
||||
*/
|
||||
export class OptimizedS3Search {
|
||||
constructor(storage) {
|
||||
this.storage = storage;
|
||||
}
|
||||
/**
|
||||
* Get nouns with optimized pagination and filtering
|
||||
*/
|
||||
async getNounsWithPagination(options = {}) {
|
||||
const limit = options.limit || 100;
|
||||
const cursor = options.cursor;
|
||||
try {
|
||||
// List noun objects with pagination
|
||||
const listResult = await this.storage.listObjectKeys(`${getDirectoryPath('noun', 'vector')}/`, limit * 2, cursor);
|
||||
if (!listResult.keys.length) {
|
||||
return {
|
||||
items: [],
|
||||
hasMore: false
|
||||
};
|
||||
}
|
||||
// Load nouns in parallel batches
|
||||
const nouns = [];
|
||||
const batchSize = 10;
|
||||
for (let i = 0; i < listResult.keys.length && nouns.length < limit; i += batchSize) {
|
||||
const batch = listResult.keys.slice(i, i + batchSize);
|
||||
const batchPromises = batch.map(key => this.storage.getObject(key));
|
||||
const batchResults = await Promise.all(batchPromises);
|
||||
for (const noun of batchResults) {
|
||||
if (!noun)
|
||||
continue;
|
||||
// Apply filters
|
||||
if (options.filter && !(await this.matchesNounFilter(noun, options.filter))) {
|
||||
continue;
|
||||
}
|
||||
nouns.push(noun);
|
||||
if (nouns.length >= limit) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Determine if there are more items
|
||||
const hasMore = listResult.hasMore || nouns.length >= limit;
|
||||
// Set next cursor
|
||||
let nextCursor;
|
||||
if (hasMore && nouns.length > 0) {
|
||||
nextCursor = nouns[nouns.length - 1].id;
|
||||
}
|
||||
return {
|
||||
items: nouns.slice(0, limit),
|
||||
hasMore,
|
||||
nextCursor
|
||||
};
|
||||
}
|
||||
catch (error) {
|
||||
logger.error('Failed to get nouns with pagination:', error);
|
||||
return {
|
||||
items: [],
|
||||
hasMore: false
|
||||
};
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Get verbs with optimized pagination and filtering
|
||||
*/
|
||||
async getVerbsWithPagination(options = {}) {
|
||||
const limit = options.limit || 100;
|
||||
const cursor = options.cursor;
|
||||
try {
|
||||
// List verb objects with pagination
|
||||
const listResult = await this.storage.listObjectKeys(`${getDirectoryPath('verb', 'vector')}/`, limit * 2, cursor);
|
||||
if (!listResult.keys.length) {
|
||||
return {
|
||||
items: [],
|
||||
hasMore: false
|
||||
};
|
||||
}
|
||||
// Load verbs in parallel batches
|
||||
const verbs = [];
|
||||
const batchSize = 10;
|
||||
for (let i = 0; i < listResult.keys.length && verbs.length < limit; i += batchSize) {
|
||||
const batch = listResult.keys.slice(i, i + batchSize);
|
||||
// Load verbs and their metadata in parallel
|
||||
const batchPromises = batch.map(async (key) => {
|
||||
const verbData = await this.storage.getObject(key);
|
||||
if (!verbData)
|
||||
return null;
|
||||
// Get metadata
|
||||
const verbId = key.replace(`${getDirectoryPath('verb', 'vector')}/`, '').replace('.json', '');
|
||||
const metadata = await this.storage.getMetadata(verbId, 'verb');
|
||||
// Combine into GraphVerb
|
||||
return this.combineVerbWithMetadata(verbData, metadata);
|
||||
});
|
||||
const batchResults = await Promise.all(batchPromises);
|
||||
for (const verb of batchResults) {
|
||||
if (!verb)
|
||||
continue;
|
||||
// Apply filters
|
||||
if (options.filter && !this.matchesVerbFilter(verb, options.filter)) {
|
||||
continue;
|
||||
}
|
||||
verbs.push(verb);
|
||||
if (verbs.length >= limit) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Determine if there are more items
|
||||
const hasMore = listResult.hasMore || verbs.length >= limit;
|
||||
// Set next cursor
|
||||
let nextCursor;
|
||||
if (hasMore && verbs.length > 0) {
|
||||
nextCursor = verbs[verbs.length - 1].id;
|
||||
}
|
||||
return {
|
||||
items: verbs.slice(0, limit),
|
||||
hasMore,
|
||||
nextCursor
|
||||
};
|
||||
}
|
||||
catch (error) {
|
||||
logger.error('Failed to get verbs with pagination:', error);
|
||||
return {
|
||||
items: [],
|
||||
hasMore: false
|
||||
};
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Check if a noun matches the filter criteria
|
||||
*/
|
||||
async matchesNounFilter(noun, filter) {
|
||||
// Get metadata for filtering
|
||||
const metadata = await this.storage.getMetadata(noun.id, 'noun');
|
||||
// Filter by noun type
|
||||
if (filter.nounType) {
|
||||
const nounTypes = Array.isArray(filter.nounType) ? filter.nounType : [filter.nounType];
|
||||
const nounType = metadata?.type || metadata?.noun;
|
||||
if (!nounType || !nounTypes.includes(nounType)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
// Filter by service
|
||||
if (filter.service) {
|
||||
const services = Array.isArray(filter.service) ? filter.service : [filter.service];
|
||||
if (!metadata?.service || !services.includes(metadata.service)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
// Filter by metadata
|
||||
if (filter.metadata) {
|
||||
if (!metadata)
|
||||
return false;
|
||||
for (const [key, value] of Object.entries(filter.metadata)) {
|
||||
if (metadata[key] !== value) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
/**
|
||||
* Check if a verb matches the filter criteria
|
||||
*/
|
||||
matchesVerbFilter(verb, filter) {
|
||||
// Filter by verb type
|
||||
if (filter.verbType) {
|
||||
const verbTypes = Array.isArray(filter.verbType) ? filter.verbType : [filter.verbType];
|
||||
if (!verb.type || !verbTypes.includes(verb.type)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
// Filter by source ID
|
||||
if (filter.sourceId) {
|
||||
const sourceIds = Array.isArray(filter.sourceId) ? filter.sourceId : [filter.sourceId];
|
||||
if (!verb.sourceId || !sourceIds.includes(verb.sourceId)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
// Filter by target ID
|
||||
if (filter.targetId) {
|
||||
const targetIds = Array.isArray(filter.targetId) ? filter.targetId : [filter.targetId];
|
||||
if (!verb.targetId || !targetIds.includes(verb.targetId)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
// Filter by service
|
||||
if (filter.service) {
|
||||
const services = Array.isArray(filter.service) ? filter.service : [filter.service];
|
||||
if (!verb.metadata?.service || !services.includes(verb.metadata.service)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
// Filter by metadata
|
||||
if (filter.metadata) {
|
||||
if (!verb.metadata)
|
||||
return false;
|
||||
for (const [key, value] of Object.entries(filter.metadata)) {
|
||||
if (verb.metadata[key] !== value) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
/**
|
||||
* Combine HNSWVerb data with metadata to create GraphVerb
|
||||
*/
|
||||
combineVerbWithMetadata(verbData, metadata) {
|
||||
if (!verbData || !metadata)
|
||||
return null;
|
||||
// Create default timestamp if not present
|
||||
const defaultTimestamp = {
|
||||
seconds: Math.floor(Date.now() / 1000),
|
||||
nanoseconds: (Date.now() % 1000) * 1000000
|
||||
};
|
||||
// Create default createdBy if not present
|
||||
const defaultCreatedBy = {
|
||||
augmentation: 'unknown',
|
||||
version: '1.0'
|
||||
};
|
||||
return {
|
||||
id: verbData.id,
|
||||
vector: verbData.vector,
|
||||
sourceId: metadata.sourceId,
|
||||
targetId: metadata.targetId,
|
||||
source: metadata.source,
|
||||
target: metadata.target,
|
||||
verb: metadata.verb,
|
||||
type: metadata.type,
|
||||
weight: metadata.weight || 1.0,
|
||||
metadata: metadata.metadata || {},
|
||||
createdAt: metadata.createdAt || defaultTimestamp,
|
||||
updatedAt: metadata.updatedAt || defaultTimestamp,
|
||||
createdBy: metadata.createdBy || defaultCreatedBy,
|
||||
data: metadata.data,
|
||||
embedding: verbData.vector
|
||||
};
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=optimizedS3Search.js.map
|
||||
File diff suppressed because one or more lines are too long
498
.recovery-workspace/dist-backup-20250910-141917/storage/adapters/s3CompatibleStorage.d.ts
vendored
Normal file
498
.recovery-workspace/dist-backup-20250910-141917/storage/adapters/s3CompatibleStorage.d.ts
vendored
Normal file
|
|
@ -0,0 +1,498 @@
|
|||
/**
|
||||
* S3-Compatible Storage Adapter
|
||||
* Uses the AWS S3 client to interact with S3-compatible storage services
|
||||
* including Amazon S3, Cloudflare R2, and Google Cloud Storage
|
||||
*/
|
||||
import { GraphVerb, HNSWNoun, HNSWVerb, StatisticsData } from '../../coreTypes.js';
|
||||
import { BaseStorage } from '../baseStorage.js';
|
||||
import { OperationConfig } from '../../utils/operationUtils.js';
|
||||
type HNSWNode = HNSWNoun;
|
||||
type Edge = HNSWVerb;
|
||||
interface ChangeLogEntry {
|
||||
timestamp: number;
|
||||
operation: 'add' | 'update' | 'delete';
|
||||
entityType: 'noun' | 'verb' | 'metadata';
|
||||
entityId: string;
|
||||
data?: any;
|
||||
instanceId?: string;
|
||||
}
|
||||
export { S3CompatibleStorage as R2Storage };
|
||||
/**
|
||||
* S3-compatible storage adapter for server environments
|
||||
* Uses the AWS S3 client to interact with S3-compatible storage services
|
||||
* including Amazon S3, Cloudflare R2, and Google Cloud Storage
|
||||
*
|
||||
* To use this adapter with Amazon S3, you need to provide:
|
||||
* - region: AWS region (e.g., 'us-east-1')
|
||||
* - credentials: AWS credentials (accessKeyId and secretAccessKey)
|
||||
* - bucketName: S3 bucket name
|
||||
*
|
||||
* To use this adapter with Cloudflare R2, you need to provide:
|
||||
* - accountId: Cloudflare account ID
|
||||
* - accessKeyId: R2 access key ID
|
||||
* - secretAccessKey: R2 secret access key
|
||||
* - bucketName: R2 bucket name
|
||||
*
|
||||
* To use this adapter with Google Cloud Storage, you need to provide:
|
||||
* - region: GCS region (e.g., 'us-central1')
|
||||
* - credentials: GCS credentials (accessKeyId and secretAccessKey)
|
||||
* - endpoint: GCS endpoint (e.g., 'https://storage.googleapis.com')
|
||||
* - bucketName: GCS bucket name
|
||||
*/
|
||||
export declare class S3CompatibleStorage extends BaseStorage {
|
||||
private s3Client;
|
||||
private bucketName;
|
||||
private serviceType;
|
||||
private region;
|
||||
private endpoint?;
|
||||
private accountId?;
|
||||
private accessKeyId;
|
||||
private secretAccessKey;
|
||||
private sessionToken?;
|
||||
private nounPrefix;
|
||||
private verbPrefix;
|
||||
private metadataPrefix;
|
||||
private verbMetadataPrefix;
|
||||
private indexPrefix;
|
||||
private systemPrefix;
|
||||
private useDualWrite;
|
||||
protected statisticsCache: StatisticsData | null;
|
||||
private lockPrefix;
|
||||
private activeLocks;
|
||||
private changeLogPrefix;
|
||||
private pendingOperations;
|
||||
private maxConcurrentOperations;
|
||||
private baseBatchSize;
|
||||
private currentBatchSize;
|
||||
private lastMemoryCheck;
|
||||
private memoryCheckInterval;
|
||||
private consecutiveErrors;
|
||||
private lastErrorReset;
|
||||
private socketManager;
|
||||
private backpressure;
|
||||
private nounWriteBuffer;
|
||||
private verbWriteBuffer;
|
||||
private requestCoalescer;
|
||||
private highVolumeMode;
|
||||
private lastVolumeCheck;
|
||||
private volumeCheckInterval;
|
||||
private forceHighVolumeMode;
|
||||
private operationExecutors;
|
||||
private nounCacheManager;
|
||||
private verbCacheManager;
|
||||
private logger;
|
||||
/**
|
||||
* Initialize the storage adapter
|
||||
* @param options Configuration options for the S3-compatible storage
|
||||
*/
|
||||
constructor(options: {
|
||||
bucketName: string;
|
||||
region?: string;
|
||||
endpoint?: string;
|
||||
accountId?: string;
|
||||
accessKeyId: string;
|
||||
secretAccessKey: string;
|
||||
sessionToken?: string;
|
||||
serviceType?: string;
|
||||
operationConfig?: OperationConfig;
|
||||
cacheConfig?: {
|
||||
hotCacheMaxSize?: number;
|
||||
hotCacheEvictionThreshold?: number;
|
||||
warmCacheTTL?: number;
|
||||
};
|
||||
readOnly?: boolean;
|
||||
});
|
||||
/**
|
||||
* Initialize the storage adapter
|
||||
*/
|
||||
init(): Promise<void>;
|
||||
/**
|
||||
* Override base class method to detect S3-specific throttling errors
|
||||
*/
|
||||
protected isThrottlingError(error: any): boolean;
|
||||
/**
|
||||
* Override to add S3-specific logging
|
||||
*/
|
||||
handleThrottling(error: any, service?: string): Promise<void>;
|
||||
/**
|
||||
* Smart delay based on current throttling status
|
||||
*/
|
||||
private smartDelay;
|
||||
/**
|
||||
* Auto-cleanup legacy /index folder during initialization
|
||||
* This removes old index data that has been migrated to _system
|
||||
*/
|
||||
private cleanupLegacyIndexFolder;
|
||||
/**
|
||||
* Initialize write buffers for high-volume scenarios
|
||||
*/
|
||||
private initializeBuffers;
|
||||
/**
|
||||
* Initialize request coalescer
|
||||
*/
|
||||
private initializeCoalescer;
|
||||
/**
|
||||
* Check if we should enable high-volume mode
|
||||
*/
|
||||
private checkVolumeMode;
|
||||
/**
|
||||
* Bulk write nouns to S3
|
||||
*/
|
||||
private bulkWriteNouns;
|
||||
/**
|
||||
* Bulk write verbs to S3
|
||||
*/
|
||||
private bulkWriteVerbs;
|
||||
/**
|
||||
* Process coalesced batch of operations
|
||||
*/
|
||||
private processCoalescedBatch;
|
||||
/**
|
||||
* Process bulk deletes
|
||||
*/
|
||||
private processBulkDeletes;
|
||||
/**
|
||||
* Process bulk writes
|
||||
*/
|
||||
private processBulkWrites;
|
||||
/**
|
||||
* Process bulk reads
|
||||
*/
|
||||
private processBulkReads;
|
||||
/**
|
||||
* Dynamically adjust batch size based on memory pressure and error rates
|
||||
*/
|
||||
private adjustBatchSize;
|
||||
/**
|
||||
* Apply backpressure when system is under load
|
||||
*/
|
||||
private applyBackpressure;
|
||||
/**
|
||||
* Release backpressure after operation completes
|
||||
*/
|
||||
private releaseBackpressure;
|
||||
/**
|
||||
* Get current batch size for operations
|
||||
*/
|
||||
private getBatchSize;
|
||||
/**
|
||||
* Save a noun to storage (internal implementation)
|
||||
*/
|
||||
protected saveNoun_internal(noun: HNSWNoun): Promise<void>;
|
||||
/**
|
||||
* Save a node to storage
|
||||
*/
|
||||
protected saveNode(node: HNSWNode): Promise<void>;
|
||||
/**
|
||||
* Get a noun from storage (internal implementation)
|
||||
*/
|
||||
protected getNoun_internal(id: string): Promise<HNSWNoun | null>;
|
||||
/**
|
||||
* Get a node from storage
|
||||
*/
|
||||
protected getNode(id: string): Promise<HNSWNode | null>;
|
||||
private nodeCache;
|
||||
/**
|
||||
* Get all nodes from storage
|
||||
* @deprecated This method is deprecated and will be removed in a future version.
|
||||
* It can cause memory issues with large datasets. Use getNodesWithPagination() instead.
|
||||
*/
|
||||
protected getAllNodes(): Promise<HNSWNode[]>;
|
||||
/**
|
||||
* Get nodes with pagination
|
||||
* @param options Pagination options
|
||||
* @returns Promise that resolves to a paginated result of nodes
|
||||
*/
|
||||
protected getNodesWithPagination(options?: {
|
||||
limit?: number;
|
||||
cursor?: string;
|
||||
useCache?: boolean;
|
||||
}): Promise<{
|
||||
nodes: HNSWNode[];
|
||||
hasMore: boolean;
|
||||
nextCursor?: string;
|
||||
}>;
|
||||
/**
|
||||
* Get nouns by noun type (internal implementation)
|
||||
* @param nounType The noun type to filter by
|
||||
* @returns Promise that resolves to an array of nouns of the specified noun type
|
||||
*/
|
||||
protected getNounsByNounType_internal(nounType: string): Promise<HNSWNoun[]>;
|
||||
/**
|
||||
* Get nodes by noun type
|
||||
* @param nounType The noun type to filter by
|
||||
* @returns Promise that resolves to an array of nodes of the specified noun type
|
||||
*/
|
||||
protected getNodesByNounType(nounType: string): Promise<HNSWNode[]>;
|
||||
/**
|
||||
* Delete a noun from storage (internal implementation)
|
||||
*/
|
||||
protected deleteNoun_internal(id: string): Promise<void>;
|
||||
/**
|
||||
* Delete a node from storage
|
||||
*/
|
||||
protected deleteNode(id: string): Promise<void>;
|
||||
/**
|
||||
* Save a verb to storage (internal implementation)
|
||||
*/
|
||||
protected saveVerb_internal(verb: HNSWVerb): Promise<void>;
|
||||
/**
|
||||
* Save an edge to storage
|
||||
*/
|
||||
protected saveEdge(edge: Edge): Promise<void>;
|
||||
/**
|
||||
* Get a verb from storage (internal implementation)
|
||||
*/
|
||||
protected getVerb_internal(id: string): Promise<HNSWVerb | null>;
|
||||
/**
|
||||
* Get an edge from storage
|
||||
*/
|
||||
protected getEdge(id: string): Promise<Edge | null>;
|
||||
/**
|
||||
* Get all edges from storage
|
||||
* @deprecated This method is deprecated and will be removed in a future version.
|
||||
* It can cause memory issues with large datasets. Use getEdgesWithPagination() instead.
|
||||
*/
|
||||
protected getAllEdges(): Promise<Edge[]>;
|
||||
/**
|
||||
* Get edges with pagination
|
||||
* @param options Pagination options
|
||||
* @returns Promise that resolves to a paginated result of edges
|
||||
*/
|
||||
protected getEdgesWithPagination(options?: {
|
||||
limit?: number;
|
||||
cursor?: string;
|
||||
useCache?: boolean;
|
||||
filter?: {
|
||||
sourceId?: string;
|
||||
targetId?: string;
|
||||
type?: string;
|
||||
};
|
||||
}): Promise<{
|
||||
edges: Edge[];
|
||||
hasMore: boolean;
|
||||
nextCursor?: string;
|
||||
}>;
|
||||
/**
|
||||
* Filter an edge based on filter criteria
|
||||
* @param edge The edge to filter
|
||||
* @param filter The filter criteria
|
||||
* @returns True if the edge matches the filter, false otherwise
|
||||
*/
|
||||
private filterEdge;
|
||||
/**
|
||||
* Get verbs with pagination
|
||||
* @param options Pagination options
|
||||
* @returns Promise that resolves to a paginated result of verbs
|
||||
*/
|
||||
getVerbsWithPagination(options?: {
|
||||
limit?: number;
|
||||
cursor?: string;
|
||||
filter?: {
|
||||
verbType?: string | string[];
|
||||
sourceId?: string | string[];
|
||||
targetId?: string | string[];
|
||||
service?: string | string[];
|
||||
metadata?: Record<string, any>;
|
||||
};
|
||||
}): Promise<{
|
||||
items: GraphVerb[];
|
||||
totalCount?: number;
|
||||
hasMore: boolean;
|
||||
nextCursor?: string;
|
||||
}>;
|
||||
/**
|
||||
* Get verbs by source (internal implementation)
|
||||
*/
|
||||
protected getVerbsBySource_internal(sourceId: string): Promise<GraphVerb[]>;
|
||||
/**
|
||||
* Get verbs by target (internal implementation)
|
||||
*/
|
||||
protected getVerbsByTarget_internal(targetId: string): Promise<GraphVerb[]>;
|
||||
/**
|
||||
* Get verbs by type (internal implementation)
|
||||
*/
|
||||
protected getVerbsByType_internal(type: string): Promise<GraphVerb[]>;
|
||||
/**
|
||||
* Delete a verb from storage (internal implementation)
|
||||
*/
|
||||
protected deleteVerb_internal(id: string): Promise<void>;
|
||||
/**
|
||||
* Delete an edge from storage
|
||||
*/
|
||||
protected deleteEdge(id: string): Promise<void>;
|
||||
/**
|
||||
* Save metadata to storage
|
||||
*/
|
||||
saveMetadata(id: string, metadata: any): Promise<void>;
|
||||
/**
|
||||
* Save verb metadata to storage
|
||||
*/
|
||||
protected saveVerbMetadata_internal(id: string, metadata: any): Promise<void>;
|
||||
/**
|
||||
* Get verb metadata from storage
|
||||
*/
|
||||
getVerbMetadata(id: string): Promise<any | null>;
|
||||
/**
|
||||
* Save noun metadata to storage
|
||||
*/
|
||||
protected saveNounMetadata_internal(id: string, metadata: any): Promise<void>;
|
||||
/**
|
||||
* Get multiple metadata objects in batches (CRITICAL: Prevents socket exhaustion)
|
||||
* This is the solution to the metadata reading socket exhaustion during initialization
|
||||
*/
|
||||
getMetadataBatch(ids: string[]): Promise<Map<string, any>>;
|
||||
/**
|
||||
* Get multiple verb metadata objects in batches (prevents socket exhaustion)
|
||||
*/
|
||||
getVerbMetadataBatch(ids: string[]): Promise<Map<string, any>>;
|
||||
/**
|
||||
* Get noun metadata from storage
|
||||
*/
|
||||
getNounMetadata(id: string): Promise<any | null>;
|
||||
/**
|
||||
* Get metadata from storage
|
||||
*/
|
||||
getMetadata(id: string): Promise<any | null>;
|
||||
/**
|
||||
* Clear all data from storage
|
||||
*/
|
||||
clear(): Promise<void>;
|
||||
/**
|
||||
* Enhanced clear operation with safety mechanisms and performance optimizations
|
||||
* Provides progress tracking, backup options, and instance name confirmation
|
||||
*/
|
||||
clearEnhanced(options?: import('../enhancedClearOperations.js').ClearOptions): Promise<import('../enhancedClearOperations.js').ClearResult>;
|
||||
/**
|
||||
* Get information about storage usage and capacity
|
||||
* Optimized version that uses cached statistics instead of expensive full scans
|
||||
*/
|
||||
getStorageStatus(): Promise<{
|
||||
type: string;
|
||||
used: number;
|
||||
quota: number | null;
|
||||
details?: Record<string, any>;
|
||||
}>;
|
||||
protected statisticsBatchUpdateTimerId: NodeJS.Timeout | null;
|
||||
protected statisticsModified: boolean;
|
||||
protected lastStatisticsFlushTime: number;
|
||||
protected readonly MIN_FLUSH_INTERVAL_MS = 5000;
|
||||
protected readonly MAX_FLUSH_DELAY_MS = 30000;
|
||||
/**
|
||||
* Get the statistics key for a specific date
|
||||
* @param date The date to get the key for
|
||||
* @returns The statistics key for the specified date
|
||||
*/
|
||||
private getStatisticsKeyForDate;
|
||||
/**
|
||||
* Get the current statistics key
|
||||
* @returns The current statistics key
|
||||
*/
|
||||
private getCurrentStatisticsKey;
|
||||
/**
|
||||
* Get the legacy statistics key (DEPRECATED - /index folder is auto-cleaned)
|
||||
* @returns The legacy statistics key
|
||||
* @deprecated Legacy /index folder is automatically cleaned on initialization
|
||||
*/
|
||||
private getLegacyStatisticsKey;
|
||||
/**
|
||||
* Schedule a batch update of statistics
|
||||
*/
|
||||
protected scheduleBatchUpdate(): void;
|
||||
/**
|
||||
* Flush statistics to storage with distributed locking
|
||||
*/
|
||||
protected flushStatistics(): Promise<void>;
|
||||
/**
|
||||
* Merge statistics from storage with local statistics
|
||||
* @param storageStats Statistics from storage
|
||||
* @param localStats Local statistics to merge
|
||||
* @returns Merged statistics data
|
||||
*/
|
||||
private mergeStatistics;
|
||||
/**
|
||||
* Save statistics data to storage
|
||||
* @param statistics The statistics data to save
|
||||
*/
|
||||
protected saveStatisticsData(statistics: StatisticsData): Promise<void>;
|
||||
/**
|
||||
* Get statistics data from storage
|
||||
* @returns Promise that resolves to the statistics data or null if not found
|
||||
*/
|
||||
protected getStatisticsData(): Promise<StatisticsData | null>;
|
||||
/**
|
||||
* Check if we should try yesterday's statistics file
|
||||
* Only try within 2 hours of midnight to avoid unnecessary calls
|
||||
*/
|
||||
private shouldTryYesterday;
|
||||
/**
|
||||
* Get yesterday's date
|
||||
*/
|
||||
private getYesterday;
|
||||
/**
|
||||
* Try to get statistics from a specific key
|
||||
* @param key The key to try to get statistics from
|
||||
* @returns The statistics data or null if not found
|
||||
*/
|
||||
private tryGetStatisticsFromKey;
|
||||
/**
|
||||
* Append an entry to the change log for efficient synchronization
|
||||
* @param entry The change log entry to append
|
||||
*/
|
||||
private appendToChangeLog;
|
||||
/**
|
||||
* Get changes from the change log since a specific timestamp
|
||||
* @param sinceTimestamp Timestamp to get changes since
|
||||
* @param maxEntries Maximum number of entries to return (default: 1000)
|
||||
* @returns Array of change log entries
|
||||
*/
|
||||
getChangesSince(sinceTimestamp: number, maxEntries?: number): Promise<ChangeLogEntry[]>;
|
||||
/**
|
||||
* Clean up old change log entries to prevent unlimited growth
|
||||
* @param olderThanTimestamp Remove entries older than this timestamp
|
||||
*/
|
||||
cleanupOldChangeLogs(olderThanTimestamp: number): Promise<void>;
|
||||
/**
|
||||
* Sample-based storage estimation as fallback when statistics unavailable
|
||||
* Much faster than full scans - samples first 50 objects per prefix
|
||||
*/
|
||||
private getSampleBasedStorageEstimate;
|
||||
/**
|
||||
* Acquire a distributed lock for coordinating operations across multiple instances
|
||||
* @param lockKey The key to lock on
|
||||
* @param ttl Time to live for the lock in milliseconds (default: 30 seconds)
|
||||
* @returns Promise that resolves to true if lock was acquired, false otherwise
|
||||
*/
|
||||
private acquireLock;
|
||||
/**
|
||||
* Release a distributed lock
|
||||
* @param lockKey The key to unlock
|
||||
* @param lockValue The value used when acquiring the lock (for verification)
|
||||
* @returns Promise that resolves when lock is released
|
||||
*/
|
||||
private releaseLock;
|
||||
/**
|
||||
* Clean up expired locks to prevent lock leakage
|
||||
* This method should be called periodically
|
||||
*/
|
||||
private cleanupExpiredLocks;
|
||||
/**
|
||||
* Get nouns with pagination support
|
||||
* @param options Pagination options
|
||||
* @returns Promise that resolves to a paginated result of nouns
|
||||
*/
|
||||
getNounsWithPagination(options?: {
|
||||
limit?: number;
|
||||
cursor?: string;
|
||||
filter?: {
|
||||
nounType?: string | string[];
|
||||
service?: string | string[];
|
||||
metadata?: Record<string, any>;
|
||||
};
|
||||
}): Promise<{
|
||||
items: HNSWNoun[];
|
||||
totalCount?: number;
|
||||
hasMore: boolean;
|
||||
nextCursor?: string;
|
||||
}>;
|
||||
}
|
||||
File diff suppressed because it is too large
Load diff
File diff suppressed because one or more lines are too long
Loading…
Add table
Add a link
Reference in a new issue