David Snelling
f8c45f2d8d
Open source vector database with HNSW indexing, graph relationships, and metadata facets. Features CLI with professional augmentation registry integration for discovering extensions and capabilities.
493 lines
16 KiB
TypeScript
493 lines
16 KiB
TypeScript
/**
|
|
* S3-Compatible Storage Adapter
|
|
* Uses the AWS S3 client to interact with S3-compatible storage services
|
|
* including Amazon S3, Cloudflare R2, and Google Cloud Storage
|
|
*/
|
|
import { GraphVerb, HNSWNoun, HNSWVerb, StatisticsData } from '../../coreTypes.js';
|
|
import { BaseStorage } from '../baseStorage.js';
|
|
import { OperationConfig } from '../../utils/operationUtils.js';
|
|
type HNSWNode = HNSWNoun;
|
|
type Edge = HNSWVerb;
|
|
interface ChangeLogEntry {
|
|
timestamp: number;
|
|
operation: 'add' | 'update' | 'delete';
|
|
entityType: 'noun' | 'verb' | 'metadata';
|
|
entityId: string;
|
|
data?: any;
|
|
instanceId?: string;
|
|
}
|
|
export { S3CompatibleStorage as R2Storage };
|
|
/**
|
|
* S3-compatible storage adapter for server environments
|
|
* Uses the AWS S3 client to interact with S3-compatible storage services
|
|
* including Amazon S3, Cloudflare R2, and Google Cloud Storage
|
|
*
|
|
* To use this adapter with Amazon S3, you need to provide:
|
|
* - region: AWS region (e.g., 'us-east-1')
|
|
* - credentials: AWS credentials (accessKeyId and secretAccessKey)
|
|
* - bucketName: S3 bucket name
|
|
*
|
|
* To use this adapter with Cloudflare R2, you need to provide:
|
|
* - accountId: Cloudflare account ID
|
|
* - accessKeyId: R2 access key ID
|
|
* - secretAccessKey: R2 secret access key
|
|
* - bucketName: R2 bucket name
|
|
*
|
|
* To use this adapter with Google Cloud Storage, you need to provide:
|
|
* - region: GCS region (e.g., 'us-central1')
|
|
* - credentials: GCS credentials (accessKeyId and secretAccessKey)
|
|
* - endpoint: GCS endpoint (e.g., 'https://storage.googleapis.com')
|
|
* - bucketName: GCS bucket name
|
|
*/
|
|
export declare class S3CompatibleStorage extends BaseStorage {
|
|
private s3Client;
|
|
private bucketName;
|
|
private serviceType;
|
|
private region;
|
|
private endpoint?;
|
|
private accountId?;
|
|
private accessKeyId;
|
|
private secretAccessKey;
|
|
private sessionToken?;
|
|
private nounPrefix;
|
|
private verbPrefix;
|
|
private metadataPrefix;
|
|
private verbMetadataPrefix;
|
|
private indexPrefix;
|
|
private systemPrefix;
|
|
private useDualWrite;
|
|
protected statisticsCache: StatisticsData | null;
|
|
private lockPrefix;
|
|
private activeLocks;
|
|
private changeLogPrefix;
|
|
private pendingOperations;
|
|
private maxConcurrentOperations;
|
|
private baseBatchSize;
|
|
private currentBatchSize;
|
|
private lastMemoryCheck;
|
|
private memoryCheckInterval;
|
|
private consecutiveErrors;
|
|
private lastErrorReset;
|
|
private socketManager;
|
|
private backpressure;
|
|
private nounWriteBuffer;
|
|
private verbWriteBuffer;
|
|
private requestCoalescer;
|
|
private highVolumeMode;
|
|
private lastVolumeCheck;
|
|
private volumeCheckInterval;
|
|
private forceHighVolumeMode;
|
|
private operationExecutors;
|
|
private nounCacheManager;
|
|
private verbCacheManager;
|
|
private logger;
|
|
/**
|
|
* Initialize the storage adapter
|
|
* @param options Configuration options for the S3-compatible storage
|
|
*/
|
|
constructor(options: {
|
|
bucketName: string;
|
|
region?: string;
|
|
endpoint?: string;
|
|
accountId?: string;
|
|
accessKeyId: string;
|
|
secretAccessKey: string;
|
|
sessionToken?: string;
|
|
serviceType?: string;
|
|
operationConfig?: OperationConfig;
|
|
cacheConfig?: {
|
|
hotCacheMaxSize?: number;
|
|
hotCacheEvictionThreshold?: number;
|
|
warmCacheTTL?: number;
|
|
};
|
|
readOnly?: boolean;
|
|
});
|
|
/**
|
|
* Initialize the storage adapter
|
|
*/
|
|
init(): Promise<void>;
|
|
/**
|
|
* Override base class method to detect S3-specific throttling errors
|
|
*/
|
|
protected isThrottlingError(error: any): boolean;
|
|
/**
|
|
* Override to add S3-specific logging
|
|
*/
|
|
handleThrottling(error: any, service?: string): Promise<void>;
|
|
/**
|
|
* Smart delay based on current throttling status
|
|
*/
|
|
private smartDelay;
|
|
/**
|
|
* Auto-cleanup legacy /index folder during initialization
|
|
* This removes old index data that has been migrated to _system
|
|
*/
|
|
private cleanupLegacyIndexFolder;
|
|
/**
|
|
* Initialize write buffers for high-volume scenarios
|
|
*/
|
|
private initializeBuffers;
|
|
/**
|
|
* Initialize request coalescer
|
|
*/
|
|
private initializeCoalescer;
|
|
/**
|
|
* Check if we should enable high-volume mode
|
|
*/
|
|
private checkVolumeMode;
|
|
/**
|
|
* Bulk write nouns to S3
|
|
*/
|
|
private bulkWriteNouns;
|
|
/**
|
|
* Bulk write verbs to S3
|
|
*/
|
|
private bulkWriteVerbs;
|
|
/**
|
|
* Process coalesced batch of operations
|
|
*/
|
|
private processCoalescedBatch;
|
|
/**
|
|
* Process bulk deletes
|
|
*/
|
|
private processBulkDeletes;
|
|
/**
|
|
* Process bulk writes
|
|
*/
|
|
private processBulkWrites;
|
|
/**
|
|
* Process bulk reads
|
|
*/
|
|
private processBulkReads;
|
|
/**
|
|
* Dynamically adjust batch size based on memory pressure and error rates
|
|
*/
|
|
private adjustBatchSize;
|
|
/**
|
|
* Apply backpressure when system is under load
|
|
*/
|
|
private applyBackpressure;
|
|
/**
|
|
* Release backpressure after operation completes
|
|
*/
|
|
private releaseBackpressure;
|
|
/**
|
|
* Get current batch size for operations
|
|
*/
|
|
private getBatchSize;
|
|
/**
|
|
* Save a noun to storage (internal implementation)
|
|
*/
|
|
protected saveNoun_internal(noun: HNSWNoun): Promise<void>;
|
|
/**
|
|
* Save a node to storage
|
|
*/
|
|
protected saveNode(node: HNSWNode): Promise<void>;
|
|
/**
|
|
* Get a noun from storage (internal implementation)
|
|
*/
|
|
protected getNoun_internal(id: string): Promise<HNSWNoun | null>;
|
|
/**
|
|
* Get a node from storage
|
|
*/
|
|
protected getNode(id: string): Promise<HNSWNode | null>;
|
|
private nodeCache;
|
|
/**
|
|
* Get all nodes from storage
|
|
* @deprecated This method is deprecated and will be removed in a future version.
|
|
* It can cause memory issues with large datasets. Use getNodesWithPagination() instead.
|
|
*/
|
|
protected getAllNodes(): Promise<HNSWNode[]>;
|
|
/**
|
|
* Get nodes with pagination
|
|
* @param options Pagination options
|
|
* @returns Promise that resolves to a paginated result of nodes
|
|
*/
|
|
protected getNodesWithPagination(options?: {
|
|
limit?: number;
|
|
cursor?: string;
|
|
useCache?: boolean;
|
|
}): Promise<{
|
|
nodes: HNSWNode[];
|
|
hasMore: boolean;
|
|
nextCursor?: string;
|
|
}>;
|
|
/**
|
|
* Get nouns by noun type (internal implementation)
|
|
* @param nounType The noun type to filter by
|
|
* @returns Promise that resolves to an array of nouns of the specified noun type
|
|
*/
|
|
protected getNounsByNounType_internal(nounType: string): Promise<HNSWNoun[]>;
|
|
/**
|
|
* Get nodes by noun type
|
|
* @param nounType The noun type to filter by
|
|
* @returns Promise that resolves to an array of nodes of the specified noun type
|
|
*/
|
|
protected getNodesByNounType(nounType: string): Promise<HNSWNode[]>;
|
|
/**
|
|
* Delete a noun from storage (internal implementation)
|
|
*/
|
|
protected deleteNoun_internal(id: string): Promise<void>;
|
|
/**
|
|
* Delete a node from storage
|
|
*/
|
|
protected deleteNode(id: string): Promise<void>;
|
|
/**
|
|
* Save a verb to storage (internal implementation)
|
|
*/
|
|
protected saveVerb_internal(verb: HNSWVerb): Promise<void>;
|
|
/**
|
|
* Save an edge to storage
|
|
*/
|
|
protected saveEdge(edge: Edge): Promise<void>;
|
|
/**
|
|
* Get a verb from storage (internal implementation)
|
|
*/
|
|
protected getVerb_internal(id: string): Promise<HNSWVerb | null>;
|
|
/**
|
|
* Get an edge from storage
|
|
*/
|
|
protected getEdge(id: string): Promise<Edge | null>;
|
|
/**
|
|
* Get all edges from storage
|
|
* @deprecated This method is deprecated and will be removed in a future version.
|
|
* It can cause memory issues with large datasets. Use getEdgesWithPagination() instead.
|
|
*/
|
|
protected getAllEdges(): Promise<Edge[]>;
|
|
/**
|
|
* Get edges with pagination
|
|
* @param options Pagination options
|
|
* @returns Promise that resolves to a paginated result of edges
|
|
*/
|
|
protected getEdgesWithPagination(options?: {
|
|
limit?: number;
|
|
cursor?: string;
|
|
useCache?: boolean;
|
|
filter?: {
|
|
sourceId?: string;
|
|
targetId?: string;
|
|
type?: string;
|
|
};
|
|
}): Promise<{
|
|
edges: Edge[];
|
|
hasMore: boolean;
|
|
nextCursor?: string;
|
|
}>;
|
|
/**
|
|
* Filter an edge based on filter criteria
|
|
* @param edge The edge to filter
|
|
* @param filter The filter criteria
|
|
* @returns True if the edge matches the filter, false otherwise
|
|
*/
|
|
private filterEdge;
|
|
/**
|
|
* Get verbs with pagination
|
|
* @param options Pagination options
|
|
* @returns Promise that resolves to a paginated result of verbs
|
|
*/
|
|
getVerbsWithPagination(options?: {
|
|
limit?: number;
|
|
cursor?: string;
|
|
filter?: {
|
|
verbType?: string | string[];
|
|
sourceId?: string | string[];
|
|
targetId?: string | string[];
|
|
service?: string | string[];
|
|
metadata?: Record<string, any>;
|
|
};
|
|
}): Promise<{
|
|
items: GraphVerb[];
|
|
totalCount?: number;
|
|
hasMore: boolean;
|
|
nextCursor?: string;
|
|
}>;
|
|
/**
|
|
* Get verbs by source (internal implementation)
|
|
*/
|
|
protected getVerbsBySource_internal(sourceId: string): Promise<GraphVerb[]>;
|
|
/**
|
|
* Get verbs by target (internal implementation)
|
|
*/
|
|
protected getVerbsByTarget_internal(targetId: string): Promise<GraphVerb[]>;
|
|
/**
|
|
* Get verbs by type (internal implementation)
|
|
*/
|
|
protected getVerbsByType_internal(type: string): Promise<GraphVerb[]>;
|
|
/**
|
|
* Delete a verb from storage (internal implementation)
|
|
*/
|
|
protected deleteVerb_internal(id: string): Promise<void>;
|
|
/**
|
|
* Delete an edge from storage
|
|
*/
|
|
protected deleteEdge(id: string): Promise<void>;
|
|
/**
|
|
* Save metadata to storage
|
|
*/
|
|
saveMetadata(id: string, metadata: any): Promise<void>;
|
|
/**
|
|
* Save verb metadata to storage
|
|
*/
|
|
saveVerbMetadata(id: string, metadata: any): Promise<void>;
|
|
/**
|
|
* Get verb metadata from storage
|
|
*/
|
|
getVerbMetadata(id: string): Promise<any | null>;
|
|
/**
|
|
* Save noun metadata to storage
|
|
*/
|
|
saveNounMetadata(id: string, metadata: any): Promise<void>;
|
|
/**
|
|
* Get multiple metadata objects in batches (CRITICAL: Prevents socket exhaustion)
|
|
* This is the solution to the metadata reading socket exhaustion during initialization
|
|
*/
|
|
getMetadataBatch(ids: string[]): Promise<Map<string, any>>;
|
|
/**
|
|
* Get multiple verb metadata objects in batches (prevents socket exhaustion)
|
|
*/
|
|
getVerbMetadataBatch(ids: string[]): Promise<Map<string, any>>;
|
|
/**
|
|
* Get noun metadata from storage
|
|
*/
|
|
getNounMetadata(id: string): Promise<any | null>;
|
|
/**
|
|
* Get metadata from storage
|
|
*/
|
|
getMetadata(id: string): Promise<any | null>;
|
|
/**
|
|
* Clear all data from storage
|
|
*/
|
|
clear(): Promise<void>;
|
|
/**
|
|
* Get information about storage usage and capacity
|
|
* Optimized version that uses cached statistics instead of expensive full scans
|
|
*/
|
|
getStorageStatus(): Promise<{
|
|
type: string;
|
|
used: number;
|
|
quota: number | null;
|
|
details?: Record<string, any>;
|
|
}>;
|
|
protected statisticsBatchUpdateTimerId: NodeJS.Timeout | null;
|
|
protected statisticsModified: boolean;
|
|
protected lastStatisticsFlushTime: number;
|
|
protected readonly MIN_FLUSH_INTERVAL_MS = 5000;
|
|
protected readonly MAX_FLUSH_DELAY_MS = 30000;
|
|
/**
|
|
* Get the statistics key for a specific date
|
|
* @param date The date to get the key for
|
|
* @returns The statistics key for the specified date
|
|
*/
|
|
private getStatisticsKeyForDate;
|
|
/**
|
|
* Get the current statistics key
|
|
* @returns The current statistics key
|
|
*/
|
|
private getCurrentStatisticsKey;
|
|
/**
|
|
* Get the legacy statistics key (DEPRECATED - /index folder is auto-cleaned)
|
|
* @returns The legacy statistics key
|
|
* @deprecated Legacy /index folder is automatically cleaned on initialization
|
|
*/
|
|
private getLegacyStatisticsKey;
|
|
/**
|
|
* Schedule a batch update of statistics
|
|
*/
|
|
protected scheduleBatchUpdate(): void;
|
|
/**
|
|
* Flush statistics to storage with distributed locking
|
|
*/
|
|
protected flushStatistics(): Promise<void>;
|
|
/**
|
|
* Merge statistics from storage with local statistics
|
|
* @param storageStats Statistics from storage
|
|
* @param localStats Local statistics to merge
|
|
* @returns Merged statistics data
|
|
*/
|
|
private mergeStatistics;
|
|
/**
|
|
* Save statistics data to storage
|
|
* @param statistics The statistics data to save
|
|
*/
|
|
protected saveStatisticsData(statistics: StatisticsData): Promise<void>;
|
|
/**
|
|
* Get statistics data from storage
|
|
* @returns Promise that resolves to the statistics data or null if not found
|
|
*/
|
|
protected getStatisticsData(): Promise<StatisticsData | null>;
|
|
/**
|
|
* Check if we should try yesterday's statistics file
|
|
* Only try within 2 hours of midnight to avoid unnecessary calls
|
|
*/
|
|
private shouldTryYesterday;
|
|
/**
|
|
* Get yesterday's date
|
|
*/
|
|
private getYesterday;
|
|
/**
|
|
* Try to get statistics from a specific key
|
|
* @param key The key to try to get statistics from
|
|
* @returns The statistics data or null if not found
|
|
*/
|
|
private tryGetStatisticsFromKey;
|
|
/**
|
|
* Append an entry to the change log for efficient synchronization
|
|
* @param entry The change log entry to append
|
|
*/
|
|
private appendToChangeLog;
|
|
/**
|
|
* Get changes from the change log since a specific timestamp
|
|
* @param sinceTimestamp Timestamp to get changes since
|
|
* @param maxEntries Maximum number of entries to return (default: 1000)
|
|
* @returns Array of change log entries
|
|
*/
|
|
getChangesSince(sinceTimestamp: number, maxEntries?: number): Promise<ChangeLogEntry[]>;
|
|
/**
|
|
* Clean up old change log entries to prevent unlimited growth
|
|
* @param olderThanTimestamp Remove entries older than this timestamp
|
|
*/
|
|
cleanupOldChangeLogs(olderThanTimestamp: number): Promise<void>;
|
|
/**
|
|
* Sample-based storage estimation as fallback when statistics unavailable
|
|
* Much faster than full scans - samples first 50 objects per prefix
|
|
*/
|
|
private getSampleBasedStorageEstimate;
|
|
/**
|
|
* Acquire a distributed lock for coordinating operations across multiple instances
|
|
* @param lockKey The key to lock on
|
|
* @param ttl Time to live for the lock in milliseconds (default: 30 seconds)
|
|
* @returns Promise that resolves to true if lock was acquired, false otherwise
|
|
*/
|
|
private acquireLock;
|
|
/**
|
|
* Release a distributed lock
|
|
* @param lockKey The key to unlock
|
|
* @param lockValue The value used when acquiring the lock (for verification)
|
|
* @returns Promise that resolves when lock is released
|
|
*/
|
|
private releaseLock;
|
|
/**
|
|
* Clean up expired locks to prevent lock leakage
|
|
* This method should be called periodically
|
|
*/
|
|
private cleanupExpiredLocks;
|
|
/**
|
|
* Get nouns with pagination support
|
|
* @param options Pagination options
|
|
* @returns Promise that resolves to a paginated result of nouns
|
|
*/
|
|
getNounsWithPagination(options?: {
|
|
limit?: number;
|
|
cursor?: string;
|
|
filter?: {
|
|
nounType?: string | string[];
|
|
service?: string | string[];
|
|
metadata?: Record<string, any>;
|
|
};
|
|
}): Promise<{
|
|
items: HNSWNoun[];
|
|
totalCount?: number;
|
|
hasMore: boolean;
|
|
nextCursor?: string;
|
|
}>;
|
|
}
|