Initial commit: Brainy - Multi-Dimensional AI Database
Open source vector database with HNSW indexing, graph relationships, and metadata facets. Features CLI with professional augmentation registry integration for discovering extensions and capabilities.
This commit is contained in:
commit
f8c45f2d8d
448 changed files with 103294 additions and 0 deletions
214
dist/storage/adapters/baseStorageAdapter.d.ts
vendored
Normal file
214
dist/storage/adapters/baseStorageAdapter.d.ts
vendored
Normal file
|
|
@ -0,0 +1,214 @@
|
|||
/**
|
||||
* Base Storage Adapter
|
||||
* Provides common functionality for all storage adapters, including statistics tracking
|
||||
*/
|
||||
import { StatisticsData, StorageAdapter } from '../../coreTypes.js';
|
||||
/**
|
||||
* Base class for storage adapters that implements statistics tracking
|
||||
*/
|
||||
export declare abstract class BaseStorageAdapter implements StorageAdapter {
|
||||
abstract init(): Promise<void>;
|
||||
abstract saveNoun(noun: any): Promise<void>;
|
||||
abstract getNoun(id: string): Promise<any | null>;
|
||||
abstract getNounsByNounType(nounType: string): Promise<any[]>;
|
||||
abstract deleteNoun(id: string): Promise<void>;
|
||||
abstract saveVerb(verb: any): Promise<void>;
|
||||
abstract getVerb(id: string): Promise<any | null>;
|
||||
abstract getVerbsBySource(sourceId: string): Promise<any[]>;
|
||||
abstract getVerbsByTarget(targetId: string): Promise<any[]>;
|
||||
abstract getVerbsByType(type: string): Promise<any[]>;
|
||||
abstract deleteVerb(id: string): Promise<void>;
|
||||
abstract saveMetadata(id: string, metadata: any): Promise<void>;
|
||||
abstract getMetadata(id: string): Promise<any | null>;
|
||||
abstract saveVerbMetadata(id: string, metadata: any): Promise<void>;
|
||||
abstract getVerbMetadata(id: string): Promise<any | null>;
|
||||
abstract clear(): Promise<void>;
|
||||
abstract getStorageStatus(): Promise<{
|
||||
type: string;
|
||||
used: number;
|
||||
quota: number | null;
|
||||
details?: Record<string, any>;
|
||||
}>;
|
||||
/**
|
||||
* Get nouns with pagination and filtering
|
||||
* @param options Pagination and filtering options
|
||||
* @returns Promise that resolves to a paginated result of nouns
|
||||
*/
|
||||
abstract getNouns(options?: {
|
||||
pagination?: {
|
||||
offset?: number;
|
||||
limit?: number;
|
||||
cursor?: string;
|
||||
};
|
||||
filter?: {
|
||||
nounType?: string | string[];
|
||||
service?: string | string[];
|
||||
metadata?: Record<string, any>;
|
||||
};
|
||||
}): Promise<{
|
||||
items: any[];
|
||||
totalCount?: number;
|
||||
hasMore: boolean;
|
||||
nextCursor?: string;
|
||||
}>;
|
||||
/**
|
||||
* Get verbs with pagination and filtering
|
||||
* @param options Pagination and filtering options
|
||||
* @returns Promise that resolves to a paginated result of verbs
|
||||
*/
|
||||
abstract getVerbs(options?: {
|
||||
pagination?: {
|
||||
offset?: number;
|
||||
limit?: number;
|
||||
cursor?: string;
|
||||
};
|
||||
filter?: {
|
||||
verbType?: string | string[];
|
||||
sourceId?: string | string[];
|
||||
targetId?: string | string[];
|
||||
service?: string | string[];
|
||||
metadata?: Record<string, any>;
|
||||
};
|
||||
}): Promise<{
|
||||
items: any[];
|
||||
totalCount?: number;
|
||||
hasMore: boolean;
|
||||
nextCursor?: string;
|
||||
}>;
|
||||
protected statisticsCache: StatisticsData | null;
|
||||
protected statisticsBatchUpdateTimerId: NodeJS.Timeout | null;
|
||||
protected statisticsModified: boolean;
|
||||
protected lastStatisticsFlushTime: number;
|
||||
protected readonly MIN_FLUSH_INTERVAL_MS = 5000;
|
||||
protected readonly MAX_FLUSH_DELAY_MS = 30000;
|
||||
protected throttlingDetected: boolean;
|
||||
protected throttlingBackoffMs: number;
|
||||
protected maxBackoffMs: number;
|
||||
protected consecutiveThrottleEvents: number;
|
||||
protected lastThrottleTime: number;
|
||||
protected totalThrottleEvents: number;
|
||||
protected throttleEventsByHour: number[];
|
||||
protected throttleReasons: Record<string, number>;
|
||||
protected lastThrottleHourIndex: number;
|
||||
protected delayedOperations: number;
|
||||
protected retriedOperations: number;
|
||||
protected failedDueToThrottling: number;
|
||||
protected totalDelayMs: number;
|
||||
protected serviceThrottling: Map<string, {
|
||||
throttleCount: number;
|
||||
lastThrottle: number;
|
||||
status: 'normal' | 'throttled' | 'recovering';
|
||||
}>;
|
||||
protected abstract saveStatisticsData(statistics: StatisticsData): Promise<void>;
|
||||
protected abstract getStatisticsData(): Promise<StatisticsData | null>;
|
||||
/**
|
||||
* Save statistics data
|
||||
* @param statistics The statistics data to save
|
||||
*/
|
||||
saveStatistics(statistics: StatisticsData): Promise<void>;
|
||||
/**
|
||||
* Get statistics data
|
||||
* @returns Promise that resolves to the statistics data
|
||||
*/
|
||||
getStatistics(): Promise<StatisticsData | null>;
|
||||
/**
|
||||
* Schedule a batch update of statistics
|
||||
*/
|
||||
protected scheduleBatchUpdate(): void;
|
||||
/**
|
||||
* Flush statistics to storage
|
||||
*/
|
||||
protected flushStatistics(): Promise<void>;
|
||||
/**
|
||||
* Increment a statistic counter
|
||||
* @param type The type of statistic to increment ('noun', 'verb', 'metadata')
|
||||
* @param service The service that inserted the data
|
||||
* @param amount The amount to increment by (default: 1)
|
||||
*/
|
||||
incrementStatistic(type: 'noun' | 'verb' | 'metadata', service: string, amount?: number): Promise<void>;
|
||||
/**
|
||||
* Track service activity (first/last activity, operation counts)
|
||||
* @param service The service name
|
||||
* @param operation The operation type
|
||||
*/
|
||||
protected trackServiceActivity(service: string, operation: 'add' | 'update' | 'delete'): void;
|
||||
/**
|
||||
* Decrement a statistic counter
|
||||
* @param type The type of statistic to decrement ('noun', 'verb', 'metadata')
|
||||
* @param service The service that inserted the data
|
||||
* @param amount The amount to decrement by (default: 1)
|
||||
*/
|
||||
decrementStatistic(type: 'noun' | 'verb' | 'metadata', service: string, amount?: number): Promise<void>;
|
||||
/**
|
||||
* Update the HNSW index size statistic
|
||||
* @param size The new size of the HNSW index
|
||||
*/
|
||||
updateHnswIndexSize(size: number): Promise<void>;
|
||||
/**
|
||||
* Force an immediate flush of statistics to storage
|
||||
* This ensures that any pending statistics updates are written to persistent storage
|
||||
*/
|
||||
flushStatisticsToStorage(): Promise<void>;
|
||||
/**
|
||||
* Track field names from a JSON document
|
||||
* @param jsonDocument The JSON document to extract field names from
|
||||
* @param service The service that inserted the data
|
||||
*/
|
||||
trackFieldNames(jsonDocument: any, service: string): Promise<void>;
|
||||
/**
|
||||
* Get available field names by service
|
||||
* @returns Record of field names by service
|
||||
*/
|
||||
getAvailableFieldNames(): Promise<Record<string, string[]>>;
|
||||
/**
|
||||
* Get standard field mappings
|
||||
* @returns Record of standard field mappings
|
||||
*/
|
||||
getStandardFieldMappings(): Promise<Record<string, Record<string, string[]>>>;
|
||||
/**
|
||||
* Create default statistics data
|
||||
* @returns Default statistics data
|
||||
*/
|
||||
protected createDefaultStatistics(): StatisticsData;
|
||||
/**
|
||||
* Detect if an error is a throttling error
|
||||
* Override this method in specific adapters for custom detection
|
||||
*/
|
||||
protected isThrottlingError(error: any): boolean;
|
||||
/**
|
||||
* Track a throttling event
|
||||
* @param error The error that caused throttling
|
||||
* @param service Optional service that was throttled
|
||||
*/
|
||||
protected trackThrottlingEvent(error: any, service?: string): void;
|
||||
/**
|
||||
* Get the reason for throttling from an error
|
||||
*/
|
||||
protected getThrottleReason(error: any): string;
|
||||
/**
|
||||
* Clear throttling state after successful operations
|
||||
*/
|
||||
protected clearThrottlingState(): void;
|
||||
/**
|
||||
* Handle throttling by implementing exponential backoff
|
||||
* @param error The error that triggered throttling
|
||||
* @param service Optional service that was throttled
|
||||
*/
|
||||
handleThrottling(error: any, service?: string): Promise<void>;
|
||||
/**
|
||||
* Track a retried operation
|
||||
*/
|
||||
protected trackRetriedOperation(): void;
|
||||
/**
|
||||
* Track an operation that failed due to throttling
|
||||
*/
|
||||
protected trackFailedDueToThrottling(): void;
|
||||
/**
|
||||
* Get current throttling metrics
|
||||
*/
|
||||
protected getThrottlingMetrics(): StatisticsData['throttlingMetrics'];
|
||||
/**
|
||||
* Include throttling metrics in statistics
|
||||
*/
|
||||
getStatisticsWithThrottling(): Promise<StatisticsData | null>;
|
||||
}
|
||||
613
dist/storage/adapters/baseStorageAdapter.js
vendored
Normal file
613
dist/storage/adapters/baseStorageAdapter.js
vendored
Normal file
|
|
@ -0,0 +1,613 @@
|
|||
/**
|
||||
* Base Storage Adapter
|
||||
* Provides common functionality for all storage adapters, including statistics tracking
|
||||
*/
|
||||
import { extractFieldNamesFromJson, mapToStandardField } from '../../utils/fieldNameTracking.js';
|
||||
/**
|
||||
* Base class for storage adapters that implements statistics tracking
|
||||
*/
|
||||
export class BaseStorageAdapter {
|
||||
constructor() {
|
||||
// Statistics cache
|
||||
this.statisticsCache = null;
|
||||
// Batch update timer ID
|
||||
this.statisticsBatchUpdateTimerId = null;
|
||||
// Flag to indicate if statistics have been modified since last save
|
||||
this.statisticsModified = false;
|
||||
// Time of last statistics flush to storage
|
||||
this.lastStatisticsFlushTime = 0;
|
||||
// Minimum time between statistics flushes (5 seconds)
|
||||
this.MIN_FLUSH_INTERVAL_MS = 5000;
|
||||
// Maximum time to wait before flushing statistics (30 seconds)
|
||||
this.MAX_FLUSH_DELAY_MS = 30000;
|
||||
// Throttling tracking properties
|
||||
this.throttlingDetected = false;
|
||||
this.throttlingBackoffMs = 1000; // Start with 1 second
|
||||
this.maxBackoffMs = 30000; // Max 30 seconds
|
||||
this.consecutiveThrottleEvents = 0;
|
||||
this.lastThrottleTime = 0;
|
||||
this.totalThrottleEvents = 0;
|
||||
this.throttleEventsByHour = new Array(24).fill(0);
|
||||
this.throttleReasons = {};
|
||||
this.lastThrottleHourIndex = -1;
|
||||
// Operation impact tracking
|
||||
this.delayedOperations = 0;
|
||||
this.retriedOperations = 0;
|
||||
this.failedDueToThrottling = 0;
|
||||
this.totalDelayMs = 0;
|
||||
// Service-level throttling
|
||||
this.serviceThrottling = new Map();
|
||||
}
|
||||
/**
|
||||
* Save statistics data
|
||||
* @param statistics The statistics data to save
|
||||
*/
|
||||
async saveStatistics(statistics) {
|
||||
// Update the cache with a deep copy to avoid reference issues
|
||||
this.statisticsCache = {
|
||||
nounCount: { ...statistics.nounCount },
|
||||
verbCount: { ...statistics.verbCount },
|
||||
metadataCount: { ...statistics.metadataCount },
|
||||
hnswIndexSize: statistics.hnswIndexSize,
|
||||
lastUpdated: statistics.lastUpdated,
|
||||
// Include serviceActivity if present
|
||||
...(statistics.serviceActivity && {
|
||||
serviceActivity: Object.fromEntries(Object.entries(statistics.serviceActivity).map(([k, v]) => [k, { ...v }]))
|
||||
}),
|
||||
// Include services if present
|
||||
...(statistics.services && {
|
||||
services: statistics.services.map(s => ({ ...s }))
|
||||
})
|
||||
};
|
||||
// Schedule a batch update instead of saving immediately
|
||||
this.scheduleBatchUpdate();
|
||||
}
|
||||
/**
|
||||
* Get statistics data
|
||||
* @returns Promise that resolves to the statistics data
|
||||
*/
|
||||
async getStatistics() {
|
||||
// If we have cached statistics, return a deep copy
|
||||
if (this.statisticsCache) {
|
||||
return {
|
||||
nounCount: { ...this.statisticsCache.nounCount },
|
||||
verbCount: { ...this.statisticsCache.verbCount },
|
||||
metadataCount: { ...this.statisticsCache.metadataCount },
|
||||
hnswIndexSize: this.statisticsCache.hnswIndexSize,
|
||||
lastUpdated: this.statisticsCache.lastUpdated
|
||||
};
|
||||
}
|
||||
// Otherwise, get from storage
|
||||
const statistics = await this.getStatisticsData();
|
||||
// If we found statistics, update the cache
|
||||
if (statistics) {
|
||||
// Update the cache with a deep copy
|
||||
this.statisticsCache = {
|
||||
nounCount: { ...statistics.nounCount },
|
||||
verbCount: { ...statistics.verbCount },
|
||||
metadataCount: { ...statistics.metadataCount },
|
||||
hnswIndexSize: statistics.hnswIndexSize,
|
||||
lastUpdated: statistics.lastUpdated
|
||||
};
|
||||
}
|
||||
return statistics;
|
||||
}
|
||||
/**
|
||||
* Schedule a batch update of statistics
|
||||
*/
|
||||
scheduleBatchUpdate() {
|
||||
// Mark statistics as modified
|
||||
this.statisticsModified = true;
|
||||
// If a timer is already set, don't set another one
|
||||
if (this.statisticsBatchUpdateTimerId !== null) {
|
||||
return;
|
||||
}
|
||||
// Calculate time since last flush
|
||||
const now = Date.now();
|
||||
const timeSinceLastFlush = now - this.lastStatisticsFlushTime;
|
||||
// If we've recently flushed, wait longer before the next flush
|
||||
const delayMs = timeSinceLastFlush < this.MIN_FLUSH_INTERVAL_MS
|
||||
? this.MAX_FLUSH_DELAY_MS
|
||||
: this.MIN_FLUSH_INTERVAL_MS;
|
||||
// Schedule the batch update
|
||||
this.statisticsBatchUpdateTimerId = setTimeout(() => {
|
||||
this.flushStatistics();
|
||||
}, delayMs);
|
||||
}
|
||||
/**
|
||||
* Flush statistics to storage
|
||||
*/
|
||||
async flushStatistics() {
|
||||
// Clear the timer
|
||||
if (this.statisticsBatchUpdateTimerId !== null) {
|
||||
clearTimeout(this.statisticsBatchUpdateTimerId);
|
||||
this.statisticsBatchUpdateTimerId = null;
|
||||
}
|
||||
// If statistics haven't been modified, no need to flush
|
||||
if (!this.statisticsModified || !this.statisticsCache) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
// Save the statistics to storage
|
||||
await this.saveStatisticsData(this.statisticsCache);
|
||||
// Update the last flush time
|
||||
this.lastStatisticsFlushTime = Date.now();
|
||||
// Reset the modified flag
|
||||
this.statisticsModified = false;
|
||||
}
|
||||
catch (error) {
|
||||
console.error('Failed to flush statistics data:', error);
|
||||
// Mark as still modified so we'll try again later
|
||||
this.statisticsModified = true;
|
||||
// Don't throw the error to avoid disrupting the application
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Increment a statistic counter
|
||||
* @param type The type of statistic to increment ('noun', 'verb', 'metadata')
|
||||
* @param service The service that inserted the data
|
||||
* @param amount The amount to increment by (default: 1)
|
||||
*/
|
||||
async incrementStatistic(type, service, amount = 1) {
|
||||
// Get current statistics from cache or storage
|
||||
let statistics = this.statisticsCache;
|
||||
if (!statistics) {
|
||||
statistics = await this.getStatisticsData();
|
||||
if (!statistics) {
|
||||
statistics = this.createDefaultStatistics();
|
||||
}
|
||||
// Update the cache
|
||||
this.statisticsCache = {
|
||||
nounCount: { ...statistics.nounCount },
|
||||
verbCount: { ...statistics.verbCount },
|
||||
metadataCount: { ...statistics.metadataCount },
|
||||
hnswIndexSize: statistics.hnswIndexSize,
|
||||
lastUpdated: statistics.lastUpdated,
|
||||
// Include serviceActivity if present
|
||||
...(statistics.serviceActivity && {
|
||||
serviceActivity: Object.fromEntries(Object.entries(statistics.serviceActivity).map(([k, v]) => [k, { ...v }]))
|
||||
}),
|
||||
// Include services if present
|
||||
...(statistics.services && {
|
||||
services: statistics.services.map(s => ({ ...s }))
|
||||
})
|
||||
};
|
||||
}
|
||||
// Increment the appropriate counter
|
||||
const counterMap = {
|
||||
noun: this.statisticsCache.nounCount,
|
||||
verb: this.statisticsCache.verbCount,
|
||||
metadata: this.statisticsCache.metadataCount
|
||||
};
|
||||
const counter = counterMap[type];
|
||||
counter[service] = (counter[service] || 0) + amount;
|
||||
// Track service activity
|
||||
this.trackServiceActivity(service, 'add');
|
||||
// Update timestamp
|
||||
this.statisticsCache.lastUpdated = new Date().toISOString();
|
||||
// Schedule a batch update instead of saving immediately
|
||||
this.scheduleBatchUpdate();
|
||||
}
|
||||
/**
|
||||
* Track service activity (first/last activity, operation counts)
|
||||
* @param service The service name
|
||||
* @param operation The operation type
|
||||
*/
|
||||
trackServiceActivity(service, operation) {
|
||||
if (!this.statisticsCache) {
|
||||
return;
|
||||
}
|
||||
// Initialize serviceActivity if it doesn't exist
|
||||
if (!this.statisticsCache.serviceActivity) {
|
||||
this.statisticsCache.serviceActivity = {};
|
||||
}
|
||||
const now = new Date().toISOString();
|
||||
const activity = this.statisticsCache.serviceActivity[service];
|
||||
if (!activity) {
|
||||
// First activity for this service
|
||||
this.statisticsCache.serviceActivity[service] = {
|
||||
firstActivity: now,
|
||||
lastActivity: now,
|
||||
totalOperations: 1
|
||||
};
|
||||
}
|
||||
else {
|
||||
// Update existing activity
|
||||
activity.lastActivity = now;
|
||||
activity.totalOperations++;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Decrement a statistic counter
|
||||
* @param type The type of statistic to decrement ('noun', 'verb', 'metadata')
|
||||
* @param service The service that inserted the data
|
||||
* @param amount The amount to decrement by (default: 1)
|
||||
*/
|
||||
async decrementStatistic(type, service, amount = 1) {
|
||||
// Get current statistics from cache or storage
|
||||
let statistics = this.statisticsCache;
|
||||
if (!statistics) {
|
||||
statistics = await this.getStatisticsData();
|
||||
if (!statistics) {
|
||||
statistics = this.createDefaultStatistics();
|
||||
}
|
||||
// Update the cache
|
||||
this.statisticsCache = {
|
||||
nounCount: { ...statistics.nounCount },
|
||||
verbCount: { ...statistics.verbCount },
|
||||
metadataCount: { ...statistics.metadataCount },
|
||||
hnswIndexSize: statistics.hnswIndexSize,
|
||||
lastUpdated: statistics.lastUpdated,
|
||||
// Include serviceActivity if present
|
||||
...(statistics.serviceActivity && {
|
||||
serviceActivity: Object.fromEntries(Object.entries(statistics.serviceActivity).map(([k, v]) => [k, { ...v }]))
|
||||
}),
|
||||
// Include services if present
|
||||
...(statistics.services && {
|
||||
services: statistics.services.map(s => ({ ...s }))
|
||||
})
|
||||
};
|
||||
}
|
||||
// Decrement the appropriate counter
|
||||
const counterMap = {
|
||||
noun: this.statisticsCache.nounCount,
|
||||
verb: this.statisticsCache.verbCount,
|
||||
metadata: this.statisticsCache.metadataCount
|
||||
};
|
||||
const counter = counterMap[type];
|
||||
counter[service] = Math.max(0, (counter[service] || 0) - amount);
|
||||
// Track service activity
|
||||
this.trackServiceActivity(service, 'delete');
|
||||
// Update timestamp
|
||||
this.statisticsCache.lastUpdated = new Date().toISOString();
|
||||
// Schedule a batch update instead of saving immediately
|
||||
this.scheduleBatchUpdate();
|
||||
}
|
||||
/**
|
||||
* Update the HNSW index size statistic
|
||||
* @param size The new size of the HNSW index
|
||||
*/
|
||||
async updateHnswIndexSize(size) {
|
||||
// Get current statistics from cache or storage
|
||||
let statistics = this.statisticsCache;
|
||||
if (!statistics) {
|
||||
statistics = await this.getStatisticsData();
|
||||
if (!statistics) {
|
||||
statistics = this.createDefaultStatistics();
|
||||
}
|
||||
// Update the cache
|
||||
this.statisticsCache = {
|
||||
nounCount: { ...statistics.nounCount },
|
||||
verbCount: { ...statistics.verbCount },
|
||||
metadataCount: { ...statistics.metadataCount },
|
||||
hnswIndexSize: statistics.hnswIndexSize,
|
||||
lastUpdated: statistics.lastUpdated,
|
||||
// Include serviceActivity if present
|
||||
...(statistics.serviceActivity && {
|
||||
serviceActivity: Object.fromEntries(Object.entries(statistics.serviceActivity).map(([k, v]) => [k, { ...v }]))
|
||||
}),
|
||||
// Include services if present
|
||||
...(statistics.services && {
|
||||
services: statistics.services.map(s => ({ ...s }))
|
||||
})
|
||||
};
|
||||
}
|
||||
// Update HNSW index size
|
||||
this.statisticsCache.hnswIndexSize = size;
|
||||
// Update timestamp
|
||||
this.statisticsCache.lastUpdated = new Date().toISOString();
|
||||
// Schedule a batch update instead of saving immediately
|
||||
this.scheduleBatchUpdate();
|
||||
}
|
||||
/**
|
||||
* Force an immediate flush of statistics to storage
|
||||
* This ensures that any pending statistics updates are written to persistent storage
|
||||
*/
|
||||
async flushStatisticsToStorage() {
|
||||
// If there are no statistics in cache or they haven't been modified, nothing to flush
|
||||
if (!this.statisticsCache || !this.statisticsModified) {
|
||||
return;
|
||||
}
|
||||
// Call the protected flushStatistics method to immediately write to storage
|
||||
await this.flushStatistics();
|
||||
}
|
||||
/**
|
||||
* Track field names from a JSON document
|
||||
* @param jsonDocument The JSON document to extract field names from
|
||||
* @param service The service that inserted the data
|
||||
*/
|
||||
async trackFieldNames(jsonDocument, service) {
|
||||
// Skip if not a JSON object
|
||||
if (typeof jsonDocument !== 'object' || jsonDocument === null || Array.isArray(jsonDocument)) {
|
||||
return;
|
||||
}
|
||||
// Get current statistics from cache or storage
|
||||
let statistics = this.statisticsCache;
|
||||
if (!statistics) {
|
||||
statistics = await this.getStatisticsData();
|
||||
if (!statistics) {
|
||||
statistics = this.createDefaultStatistics();
|
||||
}
|
||||
// Update the cache
|
||||
this.statisticsCache = {
|
||||
...statistics,
|
||||
nounCount: { ...statistics.nounCount },
|
||||
verbCount: { ...statistics.verbCount },
|
||||
metadataCount: { ...statistics.metadataCount },
|
||||
fieldNames: { ...statistics.fieldNames },
|
||||
standardFieldMappings: { ...statistics.standardFieldMappings }
|
||||
};
|
||||
}
|
||||
// Ensure fieldNames exists
|
||||
if (!this.statisticsCache.fieldNames) {
|
||||
this.statisticsCache.fieldNames = {};
|
||||
}
|
||||
// Ensure standardFieldMappings exists
|
||||
if (!this.statisticsCache.standardFieldMappings) {
|
||||
this.statisticsCache.standardFieldMappings = {};
|
||||
}
|
||||
// Extract field names from the JSON document
|
||||
const fieldNames = extractFieldNamesFromJson(jsonDocument);
|
||||
// Initialize service entry if it doesn't exist
|
||||
if (!this.statisticsCache.fieldNames[service]) {
|
||||
this.statisticsCache.fieldNames[service] = [];
|
||||
}
|
||||
// Add new field names to the service's list
|
||||
for (const fieldName of fieldNames) {
|
||||
if (!this.statisticsCache.fieldNames[service].includes(fieldName)) {
|
||||
this.statisticsCache.fieldNames[service].push(fieldName);
|
||||
}
|
||||
// Map to standard field if possible
|
||||
const standardField = mapToStandardField(fieldName);
|
||||
if (standardField) {
|
||||
// Initialize standard field entry if it doesn't exist
|
||||
if (!this.statisticsCache.standardFieldMappings[standardField]) {
|
||||
this.statisticsCache.standardFieldMappings[standardField] = {};
|
||||
}
|
||||
// Initialize service entry if it doesn't exist
|
||||
if (!this.statisticsCache.standardFieldMappings[standardField][service]) {
|
||||
this.statisticsCache.standardFieldMappings[standardField][service] = [];
|
||||
}
|
||||
// Add field name to standard field mapping if not already there
|
||||
if (!this.statisticsCache.standardFieldMappings[standardField][service].includes(fieldName)) {
|
||||
this.statisticsCache.standardFieldMappings[standardField][service].push(fieldName);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Update timestamp
|
||||
this.statisticsCache.lastUpdated = new Date().toISOString();
|
||||
// Schedule a batch update
|
||||
this.statisticsModified = true;
|
||||
this.scheduleBatchUpdate();
|
||||
}
|
||||
/**
|
||||
* Get available field names by service
|
||||
* @returns Record of field names by service
|
||||
*/
|
||||
async getAvailableFieldNames() {
|
||||
// Get current statistics from cache or storage
|
||||
let statistics = this.statisticsCache;
|
||||
if (!statistics) {
|
||||
statistics = await this.getStatisticsData();
|
||||
if (!statistics) {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
// Return field names by service
|
||||
return statistics.fieldNames || {};
|
||||
}
|
||||
/**
|
||||
* Get standard field mappings
|
||||
* @returns Record of standard field mappings
|
||||
*/
|
||||
async getStandardFieldMappings() {
|
||||
// Get current statistics from cache or storage
|
||||
let statistics = this.statisticsCache;
|
||||
if (!statistics) {
|
||||
statistics = await this.getStatisticsData();
|
||||
if (!statistics) {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
// Return standard field mappings
|
||||
return statistics.standardFieldMappings || {};
|
||||
}
|
||||
/**
|
||||
* Create default statistics data
|
||||
* @returns Default statistics data
|
||||
*/
|
||||
createDefaultStatistics() {
|
||||
return {
|
||||
nounCount: {},
|
||||
verbCount: {},
|
||||
metadataCount: {},
|
||||
hnswIndexSize: 0,
|
||||
fieldNames: {},
|
||||
standardFieldMappings: {},
|
||||
lastUpdated: new Date().toISOString()
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Detect if an error is a throttling error
|
||||
* Override this method in specific adapters for custom detection
|
||||
*/
|
||||
isThrottlingError(error) {
|
||||
const statusCode = error.$metadata?.httpStatusCode || error.statusCode || error.code;
|
||||
const message = error.message?.toLowerCase() || '';
|
||||
return (statusCode === 429 || // Too Many Requests
|
||||
statusCode === 503 || // Service Unavailable / Slow Down
|
||||
statusCode === 'ECONNRESET' || // Connection reset
|
||||
statusCode === 'ETIMEDOUT' || // Timeout
|
||||
message.includes('throttl') ||
|
||||
message.includes('slow down') ||
|
||||
message.includes('rate limit') ||
|
||||
message.includes('too many requests') ||
|
||||
message.includes('quota exceeded'));
|
||||
}
|
||||
/**
|
||||
* Track a throttling event
|
||||
* @param error The error that caused throttling
|
||||
* @param service Optional service that was throttled
|
||||
*/
|
||||
trackThrottlingEvent(error, service) {
|
||||
this.throttlingDetected = true;
|
||||
this.consecutiveThrottleEvents++;
|
||||
this.lastThrottleTime = Date.now();
|
||||
this.totalThrottleEvents++;
|
||||
// Track by hour
|
||||
const hourIndex = new Date().getHours();
|
||||
if (hourIndex !== this.lastThrottleHourIndex) {
|
||||
// Reset hour tracking if we've moved to a new hour
|
||||
this.throttleEventsByHour = new Array(24).fill(0);
|
||||
this.lastThrottleHourIndex = hourIndex;
|
||||
}
|
||||
this.throttleEventsByHour[hourIndex]++;
|
||||
// Track throttle reason
|
||||
const reason = this.getThrottleReason(error);
|
||||
this.throttleReasons[reason] = (this.throttleReasons[reason] || 0) + 1;
|
||||
// Track service-level throttling
|
||||
if (service) {
|
||||
const serviceInfo = this.serviceThrottling.get(service) || {
|
||||
throttleCount: 0,
|
||||
lastThrottle: 0,
|
||||
status: 'normal'
|
||||
};
|
||||
serviceInfo.throttleCount++;
|
||||
serviceInfo.lastThrottle = Date.now();
|
||||
serviceInfo.status = 'throttled';
|
||||
this.serviceThrottling.set(service, serviceInfo);
|
||||
}
|
||||
// Exponential backoff
|
||||
this.throttlingBackoffMs = Math.min(this.throttlingBackoffMs * 2, this.maxBackoffMs);
|
||||
}
|
||||
/**
|
||||
* Get the reason for throttling from an error
|
||||
*/
|
||||
getThrottleReason(error) {
|
||||
const statusCode = error.$metadata?.httpStatusCode || error.statusCode || error.code;
|
||||
if (statusCode === 429)
|
||||
return '429_TooManyRequests';
|
||||
if (statusCode === 503)
|
||||
return '503_ServiceUnavailable';
|
||||
if (statusCode === 'ECONNRESET')
|
||||
return 'ConnectionReset';
|
||||
if (statusCode === 'ETIMEDOUT')
|
||||
return 'Timeout';
|
||||
const message = error.message?.toLowerCase() || '';
|
||||
if (message.includes('throttl'))
|
||||
return 'Throttled';
|
||||
if (message.includes('slow down'))
|
||||
return 'SlowDown';
|
||||
if (message.includes('rate limit'))
|
||||
return 'RateLimit';
|
||||
if (message.includes('quota exceeded'))
|
||||
return 'QuotaExceeded';
|
||||
return 'Unknown';
|
||||
}
|
||||
/**
|
||||
* Clear throttling state after successful operations
|
||||
*/
|
||||
clearThrottlingState() {
|
||||
if (this.consecutiveThrottleEvents > 0) {
|
||||
this.consecutiveThrottleEvents = 0;
|
||||
this.throttlingBackoffMs = 1000; // Reset to initial backoff
|
||||
if (this.throttlingDetected) {
|
||||
this.throttlingDetected = false;
|
||||
// Update service statuses
|
||||
for (const [service, info] of this.serviceThrottling) {
|
||||
if (info.status === 'throttled') {
|
||||
info.status = 'recovering';
|
||||
}
|
||||
else if (info.status === 'recovering') {
|
||||
const timeSinceThrottle = Date.now() - info.lastThrottle;
|
||||
if (timeSinceThrottle > 60000) { // 1 minute recovery period
|
||||
info.status = 'normal';
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Handle throttling by implementing exponential backoff
|
||||
* @param error The error that triggered throttling
|
||||
* @param service Optional service that was throttled
|
||||
*/
|
||||
async handleThrottling(error, service) {
|
||||
if (this.isThrottlingError(error)) {
|
||||
this.trackThrottlingEvent(error, service);
|
||||
// Add delay for retry
|
||||
const delayMs = this.throttlingBackoffMs;
|
||||
this.totalDelayMs += delayMs;
|
||||
this.delayedOperations++;
|
||||
await new Promise(resolve => setTimeout(resolve, delayMs));
|
||||
}
|
||||
else {
|
||||
// Clear throttling state on non-throttling errors
|
||||
this.clearThrottlingState();
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Track a retried operation
|
||||
*/
|
||||
trackRetriedOperation() {
|
||||
this.retriedOperations++;
|
||||
}
|
||||
/**
|
||||
* Track an operation that failed due to throttling
|
||||
*/
|
||||
trackFailedDueToThrottling() {
|
||||
this.failedDueToThrottling++;
|
||||
}
|
||||
/**
|
||||
* Get current throttling metrics
|
||||
*/
|
||||
getThrottlingMetrics() {
|
||||
const averageDelayMs = this.delayedOperations > 0
|
||||
? this.totalDelayMs / this.delayedOperations
|
||||
: 0;
|
||||
// Convert service throttling map to record
|
||||
const serviceThrottlingRecord = {};
|
||||
for (const [service, info] of this.serviceThrottling) {
|
||||
serviceThrottlingRecord[service] = {
|
||||
throttleCount: info.throttleCount,
|
||||
lastThrottle: new Date(info.lastThrottle).toISOString(),
|
||||
status: info.status
|
||||
};
|
||||
}
|
||||
return {
|
||||
storage: {
|
||||
currentlyThrottled: this.throttlingDetected,
|
||||
lastThrottleTime: this.lastThrottleTime > 0
|
||||
? new Date(this.lastThrottleTime).toISOString()
|
||||
: undefined,
|
||||
consecutiveThrottleEvents: this.consecutiveThrottleEvents,
|
||||
currentBackoffMs: this.throttlingBackoffMs,
|
||||
totalThrottleEvents: this.totalThrottleEvents,
|
||||
throttleEventsByHour: [...this.throttleEventsByHour],
|
||||
throttleReasons: { ...this.throttleReasons }
|
||||
},
|
||||
operationImpact: {
|
||||
delayedOperations: this.delayedOperations,
|
||||
retriedOperations: this.retriedOperations,
|
||||
failedDueToThrottling: this.failedDueToThrottling,
|
||||
averageDelayMs,
|
||||
totalDelayMs: this.totalDelayMs
|
||||
},
|
||||
serviceThrottling: Object.keys(serviceThrottlingRecord).length > 0
|
||||
? serviceThrottlingRecord
|
||||
: undefined
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Include throttling metrics in statistics
|
||||
*/
|
||||
async getStatisticsWithThrottling() {
|
||||
const stats = await this.getStatistics();
|
||||
if (stats) {
|
||||
stats.throttlingMetrics = this.getThrottlingMetrics();
|
||||
}
|
||||
return stats;
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=baseStorageAdapter.js.map
|
||||
1
dist/storage/adapters/baseStorageAdapter.js.map
vendored
Normal file
1
dist/storage/adapters/baseStorageAdapter.js.map
vendored
Normal file
File diff suppressed because one or more lines are too long
71
dist/storage/adapters/batchS3Operations.d.ts
vendored
Normal file
71
dist/storage/adapters/batchS3Operations.d.ts
vendored
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
/**
|
||||
* Enhanced Batch S3 Operations for High-Performance Vector Retrieval
|
||||
* Implements optimized batch operations to reduce S3 API calls and latency
|
||||
*/
|
||||
import { HNSWNoun } from '../../coreTypes.js';
|
||||
type S3Client = any;
|
||||
export interface BatchRetrievalOptions {
|
||||
maxConcurrency?: number;
|
||||
prefetchSize?: number;
|
||||
useS3Select?: boolean;
|
||||
compressionEnabled?: boolean;
|
||||
}
|
||||
export interface BatchResult<T> {
|
||||
items: Map<string, T>;
|
||||
errors: Map<string, Error>;
|
||||
statistics: {
|
||||
totalRequested: number;
|
||||
totalRetrieved: number;
|
||||
totalErrors: number;
|
||||
duration: number;
|
||||
apiCalls: number;
|
||||
};
|
||||
}
|
||||
/**
|
||||
* High-performance batch operations for S3-compatible storage
|
||||
* Optimizes retrieval patterns for HNSW search operations
|
||||
*/
|
||||
export declare class BatchS3Operations {
|
||||
private s3Client;
|
||||
private bucketName;
|
||||
private options;
|
||||
constructor(s3Client: S3Client, bucketName: string, options?: BatchRetrievalOptions);
|
||||
/**
|
||||
* Batch retrieve HNSW nodes with intelligent prefetching
|
||||
*/
|
||||
batchGetNodes(nodeIds: string[], prefix?: string): Promise<BatchResult<HNSWNoun>>;
|
||||
/**
|
||||
* Parallel GetObject operations for small batches
|
||||
*/
|
||||
private parallelGetObjects;
|
||||
/**
|
||||
* Chunked parallel retrieval with intelligent batching
|
||||
*/
|
||||
private chunkedParallelGet;
|
||||
/**
|
||||
* List-based batch retrieval for large datasets
|
||||
* Uses S3 ListObjects to reduce API calls
|
||||
*/
|
||||
private listBasedBatchGet;
|
||||
/**
|
||||
* Intelligent prefetch based on HNSW graph connectivity
|
||||
*/
|
||||
prefetchConnectedNodes(currentNodeIds: string[], connectionMap: Map<string, Set<string>>, prefix?: string): Promise<BatchResult<HNSWNoun>>;
|
||||
/**
|
||||
* S3 Select-based retrieval for filtered queries
|
||||
*/
|
||||
selectiveRetrieve(prefix: string, filter: {
|
||||
vectorDimension?: number;
|
||||
metadataKey?: string;
|
||||
metadataValue?: any;
|
||||
}): Promise<BatchResult<HNSWNoun>>;
|
||||
/**
|
||||
* Parse stored object from JSON string
|
||||
*/
|
||||
private parseStoredObject;
|
||||
/**
|
||||
* Utility function to chunk arrays
|
||||
*/
|
||||
private chunkArray;
|
||||
}
|
||||
export {};
|
||||
287
dist/storage/adapters/batchS3Operations.js
vendored
Normal file
287
dist/storage/adapters/batchS3Operations.js
vendored
Normal file
|
|
@ -0,0 +1,287 @@
|
|||
/**
|
||||
* Enhanced Batch S3 Operations for High-Performance Vector Retrieval
|
||||
* Implements optimized batch operations to reduce S3 API calls and latency
|
||||
*/
|
||||
/**
|
||||
* High-performance batch operations for S3-compatible storage
|
||||
* Optimizes retrieval patterns for HNSW search operations
|
||||
*/
|
||||
export class BatchS3Operations {
|
||||
constructor(s3Client, bucketName, options = {}) {
|
||||
this.s3Client = s3Client;
|
||||
this.bucketName = bucketName;
|
||||
this.options = {
|
||||
maxConcurrency: 50, // AWS S3 rate limit friendly
|
||||
prefetchSize: 100,
|
||||
useS3Select: false,
|
||||
compressionEnabled: false,
|
||||
...options
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Batch retrieve HNSW nodes with intelligent prefetching
|
||||
*/
|
||||
async batchGetNodes(nodeIds, prefix = 'nodes/') {
|
||||
const startTime = Date.now();
|
||||
const result = {
|
||||
items: new Map(),
|
||||
errors: new Map(),
|
||||
statistics: {
|
||||
totalRequested: nodeIds.length,
|
||||
totalRetrieved: 0,
|
||||
totalErrors: 0,
|
||||
duration: 0,
|
||||
apiCalls: 0
|
||||
}
|
||||
};
|
||||
if (nodeIds.length === 0) {
|
||||
result.statistics.duration = Date.now() - startTime;
|
||||
return result;
|
||||
}
|
||||
// Use different strategies based on request size
|
||||
if (nodeIds.length <= 10) {
|
||||
// Small batch - use parallel GetObject
|
||||
await this.parallelGetObjects(nodeIds, prefix, result);
|
||||
}
|
||||
else if (nodeIds.length <= 1000) {
|
||||
// Medium batch - use chunked parallel with prefetching
|
||||
await this.chunkedParallelGet(nodeIds, prefix, result);
|
||||
}
|
||||
else {
|
||||
// Large batch - use S3 list-based approach with filtering
|
||||
await this.listBasedBatchGet(nodeIds, prefix, result);
|
||||
}
|
||||
result.statistics.duration = Date.now() - startTime;
|
||||
return result;
|
||||
}
|
||||
/**
|
||||
* Parallel GetObject operations for small batches
|
||||
*/
|
||||
async parallelGetObjects(ids, prefix, result) {
|
||||
const { GetObjectCommand } = await import('@aws-sdk/client-s3');
|
||||
const semaphore = new Semaphore(this.options.maxConcurrency);
|
||||
const promises = ids.map(async (id) => {
|
||||
await semaphore.acquire();
|
||||
try {
|
||||
result.statistics.apiCalls++;
|
||||
const response = await this.s3Client.send(new GetObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: `${prefix}${id}.json`
|
||||
}));
|
||||
if (response.Body) {
|
||||
const content = await response.Body.transformToString();
|
||||
const item = this.parseStoredObject(content);
|
||||
if (item) {
|
||||
result.items.set(id, item);
|
||||
result.statistics.totalRetrieved++;
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
result.errors.set(id, error);
|
||||
result.statistics.totalErrors++;
|
||||
}
|
||||
finally {
|
||||
semaphore.release();
|
||||
}
|
||||
});
|
||||
await Promise.all(promises);
|
||||
}
|
||||
/**
|
||||
* Chunked parallel retrieval with intelligent batching
|
||||
*/
|
||||
async chunkedParallelGet(ids, prefix, result) {
|
||||
const chunkSize = Math.min(50, Math.ceil(ids.length / 10));
|
||||
const chunks = this.chunkArray(ids, chunkSize);
|
||||
// Process chunks with controlled concurrency
|
||||
const semaphore = new Semaphore(Math.min(5, chunks.length));
|
||||
const chunkPromises = chunks.map(async (chunk) => {
|
||||
await semaphore.acquire();
|
||||
try {
|
||||
await this.parallelGetObjects(chunk, prefix, result);
|
||||
}
|
||||
finally {
|
||||
semaphore.release();
|
||||
}
|
||||
});
|
||||
await Promise.all(chunkPromises);
|
||||
}
|
||||
/**
|
||||
* List-based batch retrieval for large datasets
|
||||
* Uses S3 ListObjects to reduce API calls
|
||||
*/
|
||||
async listBasedBatchGet(ids, prefix, result) {
|
||||
const { ListObjectsV2Command, GetObjectCommand } = await import('@aws-sdk/client-s3');
|
||||
// Create a set for O(1) lookup
|
||||
const idSet = new Set(ids);
|
||||
// List objects with the prefix
|
||||
let continuationToken;
|
||||
const maxKeys = 1000;
|
||||
do {
|
||||
result.statistics.apiCalls++;
|
||||
const listResponse = await this.s3Client.send(new ListObjectsV2Command({
|
||||
Bucket: this.bucketName,
|
||||
Prefix: prefix,
|
||||
MaxKeys: maxKeys,
|
||||
ContinuationToken: continuationToken
|
||||
}));
|
||||
if (listResponse.Contents) {
|
||||
// Filter objects that match our requested IDs
|
||||
const matchingObjects = listResponse.Contents.filter((obj) => {
|
||||
if (!obj.Key)
|
||||
return false;
|
||||
const id = obj.Key.replace(prefix, '').replace('.json', '');
|
||||
return idSet.has(id);
|
||||
});
|
||||
// Batch retrieve matching objects
|
||||
const semaphore = new Semaphore(this.options.maxConcurrency);
|
||||
const retrievalPromises = matchingObjects.map(async (obj) => {
|
||||
if (!obj.Key)
|
||||
return;
|
||||
await semaphore.acquire();
|
||||
try {
|
||||
result.statistics.apiCalls++;
|
||||
const response = await this.s3Client.send(new GetObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: obj.Key
|
||||
}));
|
||||
if (response.Body) {
|
||||
const content = await response.Body.transformToString();
|
||||
const item = this.parseStoredObject(content);
|
||||
if (item) {
|
||||
const id = obj.Key.replace(prefix, '').replace('.json', '');
|
||||
result.items.set(id, item);
|
||||
result.statistics.totalRetrieved++;
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
const id = obj.Key.replace(prefix, '').replace('.json', '');
|
||||
result.errors.set(id, error);
|
||||
result.statistics.totalErrors++;
|
||||
}
|
||||
finally {
|
||||
semaphore.release();
|
||||
}
|
||||
});
|
||||
await Promise.all(retrievalPromises);
|
||||
}
|
||||
continuationToken = listResponse.NextContinuationToken;
|
||||
} while (continuationToken && result.items.size < ids.length);
|
||||
}
|
||||
/**
|
||||
* Intelligent prefetch based on HNSW graph connectivity
|
||||
*/
|
||||
async prefetchConnectedNodes(currentNodeIds, connectionMap, prefix = 'nodes/') {
|
||||
// Analyze connection patterns to predict next nodes
|
||||
const predictedNodes = new Set();
|
||||
for (const nodeId of currentNodeIds) {
|
||||
const connections = connectionMap.get(nodeId);
|
||||
if (connections) {
|
||||
// Add immediate neighbors
|
||||
connections.forEach(connId => predictedNodes.add(connId));
|
||||
// Add second-degree neighbors (limited)
|
||||
let count = 0;
|
||||
for (const connId of connections) {
|
||||
if (count >= 5)
|
||||
break; // Limit prefetch scope
|
||||
const secondDegree = connectionMap.get(connId);
|
||||
if (secondDegree) {
|
||||
secondDegree.forEach(id => {
|
||||
if (count < 20) {
|
||||
predictedNodes.add(id);
|
||||
count++;
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Remove nodes we already have
|
||||
const nodesToPrefetch = Array.from(predictedNodes).filter(id => !currentNodeIds.includes(id));
|
||||
return this.batchGetNodes(nodesToPrefetch.slice(0, this.options.prefetchSize), prefix);
|
||||
}
|
||||
/**
|
||||
* S3 Select-based retrieval for filtered queries
|
||||
*/
|
||||
async selectiveRetrieve(prefix, filter) {
|
||||
// This would use S3 Select to filter objects server-side
|
||||
// Reducing data transfer for large-scale operations
|
||||
const startTime = Date.now();
|
||||
const result = {
|
||||
items: new Map(),
|
||||
errors: new Map(),
|
||||
statistics: {
|
||||
totalRequested: 0,
|
||||
totalRetrieved: 0,
|
||||
totalErrors: 0,
|
||||
duration: 0,
|
||||
apiCalls: 0
|
||||
}
|
||||
};
|
||||
// S3 Select implementation would go here
|
||||
// For now, fall back to list-based approach
|
||||
console.warn('S3 Select not implemented, falling back to list-based retrieval');
|
||||
result.statistics.duration = Date.now() - startTime;
|
||||
return result;
|
||||
}
|
||||
/**
|
||||
* Parse stored object from JSON string
|
||||
*/
|
||||
parseStoredObject(content) {
|
||||
try {
|
||||
const parsed = JSON.parse(content);
|
||||
// Reconstruct HNSW node structure
|
||||
if (parsed.connections && typeof parsed.connections === 'object') {
|
||||
const connections = new Map();
|
||||
for (const [level, nodeIds] of Object.entries(parsed.connections)) {
|
||||
connections.set(Number(level), new Set(nodeIds));
|
||||
}
|
||||
parsed.connections = connections;
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
catch (error) {
|
||||
console.error('Failed to parse stored object:', error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Utility function to chunk arrays
|
||||
*/
|
||||
chunkArray(array, chunkSize) {
|
||||
const chunks = [];
|
||||
for (let i = 0; i < array.length; i += chunkSize) {
|
||||
chunks.push(array.slice(i, i + chunkSize));
|
||||
}
|
||||
return chunks;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Simple semaphore implementation for concurrency control
|
||||
*/
|
||||
class Semaphore {
|
||||
constructor(permits) {
|
||||
this.waiting = [];
|
||||
this.permits = permits;
|
||||
}
|
||||
async acquire() {
|
||||
if (this.permits > 0) {
|
||||
this.permits--;
|
||||
return Promise.resolve();
|
||||
}
|
||||
return new Promise((resolve) => {
|
||||
this.waiting.push(resolve);
|
||||
});
|
||||
}
|
||||
release() {
|
||||
if (this.waiting.length > 0) {
|
||||
const resolve = this.waiting.shift();
|
||||
resolve();
|
||||
}
|
||||
else {
|
||||
this.permits++;
|
||||
}
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=batchS3Operations.js.map
|
||||
1
dist/storage/adapters/batchS3Operations.js.map
vendored
Normal file
1
dist/storage/adapters/batchS3Operations.js.map
vendored
Normal file
File diff suppressed because one or more lines are too long
226
dist/storage/adapters/fileSystemStorage.d.ts
vendored
Normal file
226
dist/storage/adapters/fileSystemStorage.d.ts
vendored
Normal file
|
|
@ -0,0 +1,226 @@
|
|||
/**
|
||||
* File System Storage Adapter
|
||||
* File system storage adapter for Node.js environments
|
||||
*/
|
||||
import { GraphVerb, HNSWNoun, HNSWVerb, StatisticsData } from '../../coreTypes.js';
|
||||
import { BaseStorage } from '../baseStorage.js';
|
||||
type HNSWNode = HNSWNoun;
|
||||
type Edge = HNSWVerb;
|
||||
/**
|
||||
* File system storage adapter for Node.js environments
|
||||
* Uses the file system to store data in the specified directory structure
|
||||
*/
|
||||
export declare class FileSystemStorage extends BaseStorage {
|
||||
private rootDir;
|
||||
private nounsDir;
|
||||
private verbsDir;
|
||||
private metadataDir;
|
||||
private nounMetadataDir;
|
||||
private verbMetadataDir;
|
||||
private indexDir;
|
||||
private systemDir;
|
||||
private lockDir;
|
||||
private useDualWrite;
|
||||
private activeLocks;
|
||||
/**
|
||||
* Initialize the storage adapter
|
||||
* @param rootDirectory The root directory for storage
|
||||
*/
|
||||
constructor(rootDirectory: string);
|
||||
/**
|
||||
* Initialize the storage adapter
|
||||
*/
|
||||
init(): Promise<void>;
|
||||
/**
|
||||
* Check if a directory exists
|
||||
*/
|
||||
private directoryExists;
|
||||
/**
|
||||
* Ensure a directory exists, creating it if necessary
|
||||
*/
|
||||
private ensureDirectoryExists;
|
||||
/**
|
||||
* Save a node to storage
|
||||
*/
|
||||
protected saveNode(node: HNSWNode): Promise<void>;
|
||||
/**
|
||||
* Get a node from storage
|
||||
*/
|
||||
protected getNode(id: string): Promise<HNSWNode | null>;
|
||||
/**
|
||||
* Get all nodes from storage
|
||||
*/
|
||||
protected getAllNodes(): Promise<HNSWNode[]>;
|
||||
/**
|
||||
* Get nodes by noun type
|
||||
* @param nounType The noun type to filter by
|
||||
* @returns Promise that resolves to an array of nodes of the specified noun type
|
||||
*/
|
||||
protected getNodesByNounType(nounType: string): Promise<HNSWNode[]>;
|
||||
/**
|
||||
* Delete a node from storage
|
||||
*/
|
||||
protected deleteNode(id: string): Promise<void>;
|
||||
/**
|
||||
* Save an edge to storage
|
||||
*/
|
||||
protected saveEdge(edge: Edge): Promise<void>;
|
||||
/**
|
||||
* Get an edge from storage
|
||||
*/
|
||||
protected getEdge(id: string): Promise<Edge | null>;
|
||||
/**
|
||||
* Get all edges from storage
|
||||
*/
|
||||
protected getAllEdges(): Promise<Edge[]>;
|
||||
/**
|
||||
* Get edges by source
|
||||
*/
|
||||
protected getEdgesBySource(sourceId: string): Promise<Edge[]>;
|
||||
/**
|
||||
* Get edges by target
|
||||
*/
|
||||
protected getEdgesByTarget(targetId: string): Promise<Edge[]>;
|
||||
/**
|
||||
* Get edges by type
|
||||
*/
|
||||
protected getEdgesByType(type: string): Promise<Edge[]>;
|
||||
/**
|
||||
* Delete an edge from storage
|
||||
*/
|
||||
protected deleteEdge(id: string): Promise<void>;
|
||||
/**
|
||||
* Save metadata to storage
|
||||
*/
|
||||
saveMetadata(id: string, metadata: any): Promise<void>;
|
||||
/**
|
||||
* Get metadata from storage
|
||||
*/
|
||||
getMetadata(id: string): Promise<any | null>;
|
||||
/**
|
||||
* Get multiple metadata objects in batches (CRITICAL: Prevents socket exhaustion)
|
||||
* FileSystem implementation uses controlled concurrency to prevent too many file reads
|
||||
*/
|
||||
getMetadataBatch(ids: string[]): Promise<Map<string, any>>;
|
||||
/**
|
||||
* Save noun metadata to storage
|
||||
*/
|
||||
saveNounMetadata(id: string, metadata: any): Promise<void>;
|
||||
/**
|
||||
* Get noun metadata from storage
|
||||
*/
|
||||
getNounMetadata(id: string): Promise<any | null>;
|
||||
/**
|
||||
* Save verb metadata to storage
|
||||
*/
|
||||
saveVerbMetadata(id: string, metadata: any): Promise<void>;
|
||||
/**
|
||||
* Get verb metadata from storage
|
||||
*/
|
||||
getVerbMetadata(id: string): Promise<any | null>;
|
||||
/**
|
||||
* Get nouns with pagination support
|
||||
* @param options Pagination options
|
||||
*/
|
||||
getNounsWithPagination(options?: {
|
||||
limit?: number;
|
||||
cursor?: string;
|
||||
filter?: any;
|
||||
}): Promise<{
|
||||
items: HNSWNoun[];
|
||||
totalCount: number;
|
||||
hasMore: boolean;
|
||||
nextCursor?: string;
|
||||
}>;
|
||||
/**
|
||||
* Clear all data from storage
|
||||
*/
|
||||
clear(): Promise<void>;
|
||||
/**
|
||||
* Get information about storage usage and capacity
|
||||
*/
|
||||
getStorageStatus(): Promise<{
|
||||
type: string;
|
||||
used: number;
|
||||
quota: number | null;
|
||||
details?: Record<string, any>;
|
||||
}>;
|
||||
/**
|
||||
* Implementation of abstract methods from BaseStorage
|
||||
*/
|
||||
/**
|
||||
* Save a noun to storage
|
||||
*/
|
||||
protected saveNoun_internal(noun: HNSWNoun): Promise<void>;
|
||||
/**
|
||||
* Get a noun from storage
|
||||
*/
|
||||
protected getNoun_internal(id: string): Promise<HNSWNoun | null>;
|
||||
/**
|
||||
* Get nouns by noun type
|
||||
*/
|
||||
protected getNounsByNounType_internal(nounType: string): Promise<HNSWNoun[]>;
|
||||
/**
|
||||
* Delete a noun from storage
|
||||
*/
|
||||
protected deleteNoun_internal(id: string): Promise<void>;
|
||||
/**
|
||||
* Save a verb to storage
|
||||
*/
|
||||
protected saveVerb_internal(verb: HNSWVerb): Promise<void>;
|
||||
/**
|
||||
* Get a verb from storage
|
||||
*/
|
||||
protected getVerb_internal(id: string): Promise<HNSWVerb | null>;
|
||||
/**
|
||||
* Get verbs by source
|
||||
*/
|
||||
protected getVerbsBySource_internal(sourceId: string): Promise<GraphVerb[]>;
|
||||
/**
|
||||
* Get verbs by target
|
||||
*/
|
||||
protected getVerbsByTarget_internal(targetId: string): Promise<GraphVerb[]>;
|
||||
/**
|
||||
* Get verbs by type
|
||||
*/
|
||||
protected getVerbsByType_internal(type: string): Promise<GraphVerb[]>;
|
||||
/**
|
||||
* Delete a verb from storage
|
||||
*/
|
||||
protected deleteVerb_internal(id: string): Promise<void>;
|
||||
/**
|
||||
* Acquire a file-based lock for coordinating operations across multiple processes
|
||||
* @param lockKey The key to lock on
|
||||
* @param ttl Time to live for the lock in milliseconds (default: 30 seconds)
|
||||
* @returns Promise that resolves to true if lock was acquired, false otherwise
|
||||
*/
|
||||
private acquireLock;
|
||||
/**
|
||||
* Release a file-based lock
|
||||
* @param lockKey The key to unlock
|
||||
* @param lockValue The value used when acquiring the lock (for verification)
|
||||
* @returns Promise that resolves when lock is released
|
||||
*/
|
||||
private releaseLock;
|
||||
/**
|
||||
* Clean up expired lock files
|
||||
*/
|
||||
private cleanupExpiredLocks;
|
||||
/**
|
||||
* Save statistics data to storage with file-based locking
|
||||
*/
|
||||
protected saveStatisticsData(statistics: StatisticsData): Promise<void>;
|
||||
/**
|
||||
* Get statistics data from storage
|
||||
*/
|
||||
protected getStatisticsData(): Promise<StatisticsData | null>;
|
||||
/**
|
||||
* Save statistics with backward compatibility (dual write)
|
||||
*/
|
||||
private saveStatisticsWithBackwardCompat;
|
||||
/**
|
||||
* Get statistics with backward compatibility (dual read)
|
||||
*/
|
||||
private getStatisticsWithBackwardCompat;
|
||||
}
|
||||
export {};
|
||||
1016
dist/storage/adapters/fileSystemStorage.js
vendored
Normal file
1016
dist/storage/adapters/fileSystemStorage.js
vendored
Normal file
File diff suppressed because it is too large
Load diff
1
dist/storage/adapters/fileSystemStorage.js.map
vendored
Normal file
1
dist/storage/adapters/fileSystemStorage.js.map
vendored
Normal file
File diff suppressed because one or more lines are too long
172
dist/storage/adapters/memoryStorage.d.ts
vendored
Normal file
172
dist/storage/adapters/memoryStorage.d.ts
vendored
Normal file
|
|
@ -0,0 +1,172 @@
|
|||
/**
|
||||
* Memory Storage Adapter
|
||||
* In-memory storage adapter for environments where persistent storage is not available or needed
|
||||
*/
|
||||
import { GraphVerb, HNSWNoun, HNSWVerb, StatisticsData } from '../../coreTypes.js';
|
||||
import { BaseStorage } from '../baseStorage.js';
|
||||
import { PaginatedResult } from '../../types/paginationTypes.js';
|
||||
/**
|
||||
* In-memory storage adapter
|
||||
* Uses Maps to store data in memory
|
||||
*/
|
||||
export declare class MemoryStorage extends BaseStorage {
|
||||
private nouns;
|
||||
private verbs;
|
||||
private metadata;
|
||||
private nounMetadata;
|
||||
private verbMetadata;
|
||||
private statistics;
|
||||
constructor();
|
||||
/**
|
||||
* Initialize the storage adapter
|
||||
* Nothing to initialize for in-memory storage
|
||||
*/
|
||||
init(): Promise<void>;
|
||||
/**
|
||||
* Save a noun to storage
|
||||
*/
|
||||
protected saveNoun_internal(noun: HNSWNoun): Promise<void>;
|
||||
/**
|
||||
* Get a noun from storage
|
||||
*/
|
||||
protected getNoun_internal(id: string): Promise<HNSWNoun | null>;
|
||||
/**
|
||||
* Get nouns with pagination and filtering
|
||||
* @param options Pagination and filtering options
|
||||
* @returns Promise that resolves to a paginated result of nouns
|
||||
*/
|
||||
getNouns(options?: {
|
||||
pagination?: {
|
||||
offset?: number;
|
||||
limit?: number;
|
||||
cursor?: string;
|
||||
};
|
||||
filter?: {
|
||||
nounType?: string | string[];
|
||||
service?: string | string[];
|
||||
metadata?: Record<string, any>;
|
||||
};
|
||||
}): Promise<PaginatedResult<HNSWNoun>>;
|
||||
/**
|
||||
* Get nouns with pagination - simplified interface for compatibility
|
||||
*/
|
||||
getNounsWithPagination(options?: {
|
||||
limit?: number;
|
||||
cursor?: string;
|
||||
filter?: any;
|
||||
}): Promise<{
|
||||
items: HNSWNoun[];
|
||||
totalCount: number;
|
||||
hasMore: boolean;
|
||||
nextCursor?: string;
|
||||
}>;
|
||||
/**
|
||||
* Get nouns by noun type
|
||||
* @param nounType The noun type to filter by
|
||||
* @returns Promise that resolves to an array of nouns of the specified noun type
|
||||
* @deprecated Use getNouns() with filter.nounType instead
|
||||
*/
|
||||
protected getNounsByNounType_internal(nounType: string): Promise<HNSWNoun[]>;
|
||||
/**
|
||||
* Delete a noun from storage
|
||||
*/
|
||||
protected deleteNoun_internal(id: string): Promise<void>;
|
||||
/**
|
||||
* Save a verb to storage
|
||||
*/
|
||||
protected saveVerb_internal(verb: HNSWVerb): Promise<void>;
|
||||
/**
|
||||
* Get a verb from storage
|
||||
*/
|
||||
protected getVerb_internal(id: string): Promise<HNSWVerb | null>;
|
||||
/**
|
||||
* Get verbs with pagination and filtering
|
||||
* @param options Pagination and filtering options
|
||||
* @returns Promise that resolves to a paginated result of verbs
|
||||
*/
|
||||
getVerbs(options?: {
|
||||
pagination?: {
|
||||
offset?: number;
|
||||
limit?: number;
|
||||
cursor?: string;
|
||||
};
|
||||
filter?: {
|
||||
verbType?: string | string[];
|
||||
sourceId?: string | string[];
|
||||
targetId?: string | string[];
|
||||
service?: string | string[];
|
||||
metadata?: Record<string, any>;
|
||||
};
|
||||
}): Promise<PaginatedResult<GraphVerb>>;
|
||||
/**
|
||||
* Get verbs by source
|
||||
* @deprecated Use getVerbs() with filter.sourceId instead
|
||||
*/
|
||||
protected getVerbsBySource_internal(sourceId: string): Promise<GraphVerb[]>;
|
||||
/**
|
||||
* Get verbs by target
|
||||
* @deprecated Use getVerbs() with filter.targetId instead
|
||||
*/
|
||||
protected getVerbsByTarget_internal(targetId: string): Promise<GraphVerb[]>;
|
||||
/**
|
||||
* Get verbs by type
|
||||
* @deprecated Use getVerbs() with filter.verbType instead
|
||||
*/
|
||||
protected getVerbsByType_internal(type: string): Promise<GraphVerb[]>;
|
||||
/**
|
||||
* Delete a verb from storage
|
||||
*/
|
||||
protected deleteVerb_internal(id: string): Promise<void>;
|
||||
/**
|
||||
* Save metadata to storage
|
||||
*/
|
||||
saveMetadata(id: string, metadata: any): Promise<void>;
|
||||
/**
|
||||
* Get metadata from storage
|
||||
*/
|
||||
getMetadata(id: string): Promise<any | null>;
|
||||
/**
|
||||
* Get multiple metadata objects in batches (CRITICAL: Prevents socket exhaustion)
|
||||
* Memory storage implementation is simple since all data is already in memory
|
||||
*/
|
||||
getMetadataBatch(ids: string[]): Promise<Map<string, any>>;
|
||||
/**
|
||||
* Save noun metadata to storage
|
||||
*/
|
||||
saveNounMetadata(id: string, metadata: any): Promise<void>;
|
||||
/**
|
||||
* Get noun metadata from storage
|
||||
*/
|
||||
getNounMetadata(id: string): Promise<any | null>;
|
||||
/**
|
||||
* Save verb metadata to storage
|
||||
*/
|
||||
saveVerbMetadata(id: string, metadata: any): Promise<void>;
|
||||
/**
|
||||
* Get verb metadata from storage
|
||||
*/
|
||||
getVerbMetadata(id: string): Promise<any | null>;
|
||||
/**
|
||||
* Clear all data from storage
|
||||
*/
|
||||
clear(): Promise<void>;
|
||||
/**
|
||||
* Get information about storage usage and capacity
|
||||
*/
|
||||
getStorageStatus(): Promise<{
|
||||
type: string;
|
||||
used: number;
|
||||
quota: number | null;
|
||||
details?: Record<string, any>;
|
||||
}>;
|
||||
/**
|
||||
* Save statistics data to storage
|
||||
* @param statistics The statistics data to save
|
||||
*/
|
||||
protected saveStatisticsData(statistics: StatisticsData): Promise<void>;
|
||||
/**
|
||||
* Get statistics data from storage
|
||||
* @returns Promise that resolves to the statistics data or null if not found
|
||||
*/
|
||||
protected getStatisticsData(): Promise<StatisticsData | null>;
|
||||
}
|
||||
548
dist/storage/adapters/memoryStorage.js
vendored
Normal file
548
dist/storage/adapters/memoryStorage.js
vendored
Normal file
|
|
@ -0,0 +1,548 @@
|
|||
/**
|
||||
* Memory Storage Adapter
|
||||
* In-memory storage adapter for environments where persistent storage is not available or needed
|
||||
*/
|
||||
import { BaseStorage } from '../baseStorage.js';
|
||||
// No type aliases needed - using the original types directly
|
||||
/**
|
||||
* In-memory storage adapter
|
||||
* Uses Maps to store data in memory
|
||||
*/
|
||||
export class MemoryStorage extends BaseStorage {
|
||||
constructor() {
|
||||
super();
|
||||
// Single map of noun ID to noun
|
||||
this.nouns = new Map();
|
||||
this.verbs = new Map();
|
||||
this.metadata = new Map();
|
||||
this.nounMetadata = new Map();
|
||||
this.verbMetadata = new Map();
|
||||
this.statistics = null;
|
||||
}
|
||||
/**
|
||||
* Initialize the storage adapter
|
||||
* Nothing to initialize for in-memory storage
|
||||
*/
|
||||
async init() {
|
||||
this.isInitialized = true;
|
||||
}
|
||||
/**
|
||||
* Save a noun to storage
|
||||
*/
|
||||
async saveNoun_internal(noun) {
|
||||
// Create a deep copy to avoid reference issues
|
||||
const nounCopy = {
|
||||
id: noun.id,
|
||||
vector: [...noun.vector],
|
||||
connections: new Map(),
|
||||
level: noun.level || 0
|
||||
};
|
||||
// Copy connections
|
||||
for (const [level, connections] of noun.connections.entries()) {
|
||||
nounCopy.connections.set(level, new Set(connections));
|
||||
}
|
||||
// Save the noun directly in the nouns map
|
||||
this.nouns.set(noun.id, nounCopy);
|
||||
}
|
||||
/**
|
||||
* Get a noun from storage
|
||||
*/
|
||||
async getNoun_internal(id) {
|
||||
// Get the noun directly from the nouns map
|
||||
const noun = this.nouns.get(id);
|
||||
// If not found, return null
|
||||
if (!noun) {
|
||||
return null;
|
||||
}
|
||||
// Return a deep copy to avoid reference issues
|
||||
const nounCopy = {
|
||||
id: noun.id,
|
||||
vector: [...noun.vector],
|
||||
connections: new Map(),
|
||||
level: noun.level || 0
|
||||
};
|
||||
// Copy connections
|
||||
for (const [level, connections] of noun.connections.entries()) {
|
||||
nounCopy.connections.set(level, new Set(connections));
|
||||
}
|
||||
return nounCopy;
|
||||
}
|
||||
/**
|
||||
* Get nouns with pagination and filtering
|
||||
* @param options Pagination and filtering options
|
||||
* @returns Promise that resolves to a paginated result of nouns
|
||||
*/
|
||||
async getNouns(options = {}) {
|
||||
const pagination = options.pagination || {};
|
||||
const filter = options.filter || {};
|
||||
// Default values
|
||||
const offset = pagination.offset || 0;
|
||||
const limit = pagination.limit || 100;
|
||||
// Convert string types to arrays for consistent handling
|
||||
const nounTypes = filter.nounType
|
||||
? Array.isArray(filter.nounType) ? filter.nounType : [filter.nounType]
|
||||
: undefined;
|
||||
const services = filter.service
|
||||
? Array.isArray(filter.service) ? filter.service : [filter.service]
|
||||
: undefined;
|
||||
// First, collect all noun IDs that match the filter criteria
|
||||
const matchingIds = [];
|
||||
// Iterate through all nouns to find matches
|
||||
for (const [nounId, noun] of this.nouns.entries()) {
|
||||
// Get the metadata to check filters
|
||||
const metadata = await this.getMetadata(nounId);
|
||||
if (!metadata)
|
||||
continue;
|
||||
// Filter by noun type if specified
|
||||
if (nounTypes && !nounTypes.includes(metadata.noun)) {
|
||||
continue;
|
||||
}
|
||||
// Filter by service if specified
|
||||
if (services && metadata.service && !services.includes(metadata.service)) {
|
||||
continue;
|
||||
}
|
||||
// Filter by metadata fields if specified
|
||||
if (filter.metadata) {
|
||||
let metadataMatch = true;
|
||||
for (const [key, value] of Object.entries(filter.metadata)) {
|
||||
if (metadata[key] !== value) {
|
||||
metadataMatch = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!metadataMatch)
|
||||
continue;
|
||||
}
|
||||
// If we got here, the noun matches all filters
|
||||
matchingIds.push(nounId);
|
||||
}
|
||||
// Calculate pagination
|
||||
const totalCount = matchingIds.length;
|
||||
const paginatedIds = matchingIds.slice(offset, offset + limit);
|
||||
const hasMore = offset + limit < totalCount;
|
||||
// Create cursor for next page if there are more results
|
||||
const nextCursor = hasMore ? `${offset + limit}` : undefined;
|
||||
// Fetch the actual nouns for the current page
|
||||
const items = [];
|
||||
for (const id of paginatedIds) {
|
||||
const noun = this.nouns.get(id);
|
||||
if (!noun)
|
||||
continue;
|
||||
// Create a deep copy to avoid reference issues
|
||||
const nounCopy = {
|
||||
id: noun.id,
|
||||
vector: [...noun.vector],
|
||||
connections: new Map(),
|
||||
level: noun.level || 0
|
||||
};
|
||||
// Copy connections
|
||||
for (const [level, connections] of noun.connections.entries()) {
|
||||
nounCopy.connections.set(level, new Set(connections));
|
||||
}
|
||||
items.push(nounCopy);
|
||||
}
|
||||
return {
|
||||
items,
|
||||
totalCount,
|
||||
hasMore,
|
||||
nextCursor
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Get nouns with pagination - simplified interface for compatibility
|
||||
*/
|
||||
async getNounsWithPagination(options = {}) {
|
||||
// Convert to the getNouns format
|
||||
const result = await this.getNouns({
|
||||
pagination: {
|
||||
offset: options.cursor ? parseInt(options.cursor) : 0,
|
||||
limit: options.limit || 100
|
||||
},
|
||||
filter: options.filter
|
||||
});
|
||||
return {
|
||||
items: result.items,
|
||||
totalCount: result.totalCount || 0,
|
||||
hasMore: result.hasMore,
|
||||
nextCursor: result.nextCursor
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Get nouns by noun type
|
||||
* @param nounType The noun type to filter by
|
||||
* @returns Promise that resolves to an array of nouns of the specified noun type
|
||||
* @deprecated Use getNouns() with filter.nounType instead
|
||||
*/
|
||||
async getNounsByNounType_internal(nounType) {
|
||||
const result = await this.getNouns({
|
||||
filter: {
|
||||
nounType
|
||||
}
|
||||
});
|
||||
return result.items;
|
||||
}
|
||||
/**
|
||||
* Delete a noun from storage
|
||||
*/
|
||||
async deleteNoun_internal(id) {
|
||||
this.nouns.delete(id);
|
||||
}
|
||||
/**
|
||||
* Save a verb to storage
|
||||
*/
|
||||
async saveVerb_internal(verb) {
|
||||
// Create a deep copy to avoid reference issues
|
||||
const verbCopy = {
|
||||
id: verb.id,
|
||||
vector: [...verb.vector],
|
||||
connections: new Map()
|
||||
};
|
||||
// Copy connections
|
||||
for (const [level, connections] of verb.connections.entries()) {
|
||||
verbCopy.connections.set(level, new Set(connections));
|
||||
}
|
||||
// Save the verb directly in the verbs map
|
||||
this.verbs.set(verb.id, verbCopy);
|
||||
}
|
||||
/**
|
||||
* Get a verb from storage
|
||||
*/
|
||||
async getVerb_internal(id) {
|
||||
// Get the verb directly from the verbs map
|
||||
const verb = this.verbs.get(id);
|
||||
// If not found, return null
|
||||
if (!verb) {
|
||||
return null;
|
||||
}
|
||||
// Create default timestamp if not present
|
||||
const defaultTimestamp = {
|
||||
seconds: Math.floor(Date.now() / 1000),
|
||||
nanoseconds: (Date.now() % 1000) * 1000000
|
||||
};
|
||||
// Create default createdBy if not present
|
||||
const defaultCreatedBy = {
|
||||
augmentation: 'unknown',
|
||||
version: '1.0'
|
||||
};
|
||||
// Return a deep copy of the HNSWVerb
|
||||
const verbCopy = {
|
||||
id: verb.id,
|
||||
vector: [...verb.vector],
|
||||
connections: new Map()
|
||||
};
|
||||
// Copy connections
|
||||
for (const [level, connections] of verb.connections.entries()) {
|
||||
verbCopy.connections.set(level, new Set(connections));
|
||||
}
|
||||
return verbCopy;
|
||||
}
|
||||
/**
|
||||
* Get verbs with pagination and filtering
|
||||
* @param options Pagination and filtering options
|
||||
* @returns Promise that resolves to a paginated result of verbs
|
||||
*/
|
||||
async getVerbs(options = {}) {
|
||||
const pagination = options.pagination || {};
|
||||
const filter = options.filter || {};
|
||||
// Default values
|
||||
const offset = pagination.offset || 0;
|
||||
const limit = pagination.limit || 100;
|
||||
// Convert string types to arrays for consistent handling
|
||||
const verbTypes = filter.verbType
|
||||
? Array.isArray(filter.verbType) ? filter.verbType : [filter.verbType]
|
||||
: undefined;
|
||||
const sourceIds = filter.sourceId
|
||||
? Array.isArray(filter.sourceId) ? filter.sourceId : [filter.sourceId]
|
||||
: undefined;
|
||||
const targetIds = filter.targetId
|
||||
? Array.isArray(filter.targetId) ? filter.targetId : [filter.targetId]
|
||||
: undefined;
|
||||
const services = filter.service
|
||||
? Array.isArray(filter.service) ? filter.service : [filter.service]
|
||||
: undefined;
|
||||
// First, collect all verb IDs that match the filter criteria
|
||||
const matchingIds = [];
|
||||
// Iterate through all verbs to find matches
|
||||
for (const [verbId, hnswVerb] of this.verbs.entries()) {
|
||||
// Get the metadata for this verb to do filtering
|
||||
const metadata = this.verbMetadata.get(verbId);
|
||||
// Filter by verb type if specified
|
||||
if (verbTypes && metadata && !verbTypes.includes(metadata.type || metadata.verb || '')) {
|
||||
continue;
|
||||
}
|
||||
// Filter by source ID if specified
|
||||
if (sourceIds && metadata && !sourceIds.includes(metadata.sourceId || metadata.source || '')) {
|
||||
continue;
|
||||
}
|
||||
// Filter by target ID if specified
|
||||
if (targetIds && metadata && !targetIds.includes(metadata.targetId || metadata.target || '')) {
|
||||
continue;
|
||||
}
|
||||
// Filter by metadata fields if specified
|
||||
if (filter.metadata && metadata && metadata.data) {
|
||||
let metadataMatch = true;
|
||||
for (const [key, value] of Object.entries(filter.metadata)) {
|
||||
if (metadata.data[key] !== value) {
|
||||
metadataMatch = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!metadataMatch)
|
||||
continue;
|
||||
}
|
||||
// Filter by service if specified
|
||||
if (services && metadata && metadata.createdBy && metadata.createdBy.augmentation &&
|
||||
!services.includes(metadata.createdBy.augmentation)) {
|
||||
continue;
|
||||
}
|
||||
// If we got here, the verb matches all filters
|
||||
matchingIds.push(verbId);
|
||||
}
|
||||
// Calculate pagination
|
||||
const totalCount = matchingIds.length;
|
||||
const paginatedIds = matchingIds.slice(offset, offset + limit);
|
||||
const hasMore = offset + limit < totalCount;
|
||||
// Create cursor for next page if there are more results
|
||||
const nextCursor = hasMore ? `${offset + limit}` : undefined;
|
||||
// Fetch the actual verbs for the current page
|
||||
const items = [];
|
||||
for (const id of paginatedIds) {
|
||||
const hnswVerb = this.verbs.get(id);
|
||||
const metadata = this.verbMetadata.get(id);
|
||||
if (!hnswVerb)
|
||||
continue;
|
||||
if (!metadata) {
|
||||
console.warn(`Verb ${id} found but no metadata - creating minimal GraphVerb`);
|
||||
// Return minimal GraphVerb if metadata is missing
|
||||
items.push({
|
||||
id: hnswVerb.id,
|
||||
vector: hnswVerb.vector,
|
||||
sourceId: '',
|
||||
targetId: ''
|
||||
});
|
||||
continue;
|
||||
}
|
||||
// Create a complete GraphVerb by combining HNSWVerb with metadata
|
||||
const graphVerb = {
|
||||
id: hnswVerb.id,
|
||||
vector: [...hnswVerb.vector],
|
||||
sourceId: metadata.sourceId,
|
||||
targetId: metadata.targetId,
|
||||
source: metadata.source,
|
||||
target: metadata.target,
|
||||
verb: metadata.verb,
|
||||
type: metadata.type,
|
||||
weight: metadata.weight,
|
||||
createdAt: metadata.createdAt,
|
||||
updatedAt: metadata.updatedAt,
|
||||
createdBy: metadata.createdBy,
|
||||
data: metadata.data,
|
||||
metadata: metadata.data // Alias for backward compatibility
|
||||
};
|
||||
items.push(graphVerb);
|
||||
}
|
||||
return {
|
||||
items,
|
||||
totalCount,
|
||||
hasMore,
|
||||
nextCursor
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Get verbs by source
|
||||
* @deprecated Use getVerbs() with filter.sourceId instead
|
||||
*/
|
||||
async getVerbsBySource_internal(sourceId) {
|
||||
const result = await this.getVerbs({
|
||||
filter: {
|
||||
sourceId
|
||||
}
|
||||
});
|
||||
return result.items;
|
||||
}
|
||||
/**
|
||||
* Get verbs by target
|
||||
* @deprecated Use getVerbs() with filter.targetId instead
|
||||
*/
|
||||
async getVerbsByTarget_internal(targetId) {
|
||||
const result = await this.getVerbs({
|
||||
filter: {
|
||||
targetId
|
||||
}
|
||||
});
|
||||
return result.items;
|
||||
}
|
||||
/**
|
||||
* Get verbs by type
|
||||
* @deprecated Use getVerbs() with filter.verbType instead
|
||||
*/
|
||||
async getVerbsByType_internal(type) {
|
||||
const result = await this.getVerbs({
|
||||
filter: {
|
||||
verbType: type
|
||||
}
|
||||
});
|
||||
return result.items;
|
||||
}
|
||||
/**
|
||||
* Delete a verb from storage
|
||||
*/
|
||||
async deleteVerb_internal(id) {
|
||||
// Delete the verb directly from the verbs map
|
||||
this.verbs.delete(id);
|
||||
}
|
||||
/**
|
||||
* Save metadata to storage
|
||||
*/
|
||||
async saveMetadata(id, metadata) {
|
||||
this.metadata.set(id, JSON.parse(JSON.stringify(metadata)));
|
||||
}
|
||||
/**
|
||||
* Get metadata from storage
|
||||
*/
|
||||
async getMetadata(id) {
|
||||
const metadata = this.metadata.get(id);
|
||||
if (!metadata) {
|
||||
return null;
|
||||
}
|
||||
return JSON.parse(JSON.stringify(metadata));
|
||||
}
|
||||
/**
|
||||
* Get multiple metadata objects in batches (CRITICAL: Prevents socket exhaustion)
|
||||
* Memory storage implementation is simple since all data is already in memory
|
||||
*/
|
||||
async getMetadataBatch(ids) {
|
||||
const results = new Map();
|
||||
// Memory storage can handle all IDs at once since it's in-memory
|
||||
for (const id of ids) {
|
||||
const metadata = this.metadata.get(id);
|
||||
if (metadata) {
|
||||
// Deep clone to prevent mutation
|
||||
results.set(id, JSON.parse(JSON.stringify(metadata)));
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
/**
|
||||
* Save noun metadata to storage
|
||||
*/
|
||||
async saveNounMetadata(id, metadata) {
|
||||
this.nounMetadata.set(id, JSON.parse(JSON.stringify(metadata)));
|
||||
}
|
||||
/**
|
||||
* Get noun metadata from storage
|
||||
*/
|
||||
async getNounMetadata(id) {
|
||||
const metadata = this.nounMetadata.get(id);
|
||||
if (!metadata) {
|
||||
return null;
|
||||
}
|
||||
return JSON.parse(JSON.stringify(metadata));
|
||||
}
|
||||
/**
|
||||
* Save verb metadata to storage
|
||||
*/
|
||||
async saveVerbMetadata(id, metadata) {
|
||||
this.verbMetadata.set(id, JSON.parse(JSON.stringify(metadata)));
|
||||
}
|
||||
/**
|
||||
* Get verb metadata from storage
|
||||
*/
|
||||
async getVerbMetadata(id) {
|
||||
const metadata = this.verbMetadata.get(id);
|
||||
if (!metadata) {
|
||||
return null;
|
||||
}
|
||||
return JSON.parse(JSON.stringify(metadata));
|
||||
}
|
||||
/**
|
||||
* Clear all data from storage
|
||||
*/
|
||||
async clear() {
|
||||
this.nouns.clear();
|
||||
this.verbs.clear();
|
||||
this.metadata.clear();
|
||||
this.nounMetadata.clear();
|
||||
this.verbMetadata.clear();
|
||||
this.statistics = null;
|
||||
// Clear the statistics cache
|
||||
this.statisticsCache = null;
|
||||
this.statisticsModified = false;
|
||||
}
|
||||
/**
|
||||
* Get information about storage usage and capacity
|
||||
*/
|
||||
async getStorageStatus() {
|
||||
return {
|
||||
type: 'memory',
|
||||
used: 0, // In-memory storage doesn't have a meaningful size
|
||||
quota: null, // In-memory storage doesn't have a quota
|
||||
details: {
|
||||
nodeCount: this.nouns.size,
|
||||
edgeCount: this.verbs.size,
|
||||
metadataCount: this.metadata.size
|
||||
}
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Save statistics data to storage
|
||||
* @param statistics The statistics data to save
|
||||
*/
|
||||
async saveStatisticsData(statistics) {
|
||||
// For memory storage, we just need to store the statistics in memory
|
||||
// Create a deep copy to avoid reference issues
|
||||
this.statistics = {
|
||||
nounCount: { ...statistics.nounCount },
|
||||
verbCount: { ...statistics.verbCount },
|
||||
metadataCount: { ...statistics.metadataCount },
|
||||
hnswIndexSize: statistics.hnswIndexSize,
|
||||
lastUpdated: statistics.lastUpdated,
|
||||
// Include serviceActivity if present
|
||||
...(statistics.serviceActivity && {
|
||||
serviceActivity: Object.fromEntries(Object.entries(statistics.serviceActivity).map(([k, v]) => [k, { ...v }]))
|
||||
}),
|
||||
// Include services if present
|
||||
...(statistics.services && {
|
||||
services: statistics.services.map(s => ({ ...s }))
|
||||
}),
|
||||
// Include distributedConfig if present
|
||||
...(statistics.distributedConfig && {
|
||||
distributedConfig: JSON.parse(JSON.stringify(statistics.distributedConfig))
|
||||
})
|
||||
};
|
||||
// Since this is in-memory, there's no need for time-based partitioning
|
||||
// or legacy file handling
|
||||
}
|
||||
/**
|
||||
* Get statistics data from storage
|
||||
* @returns Promise that resolves to the statistics data or null if not found
|
||||
*/
|
||||
async getStatisticsData() {
|
||||
if (!this.statistics) {
|
||||
return null;
|
||||
}
|
||||
// Return a deep copy to avoid reference issues
|
||||
return {
|
||||
nounCount: { ...this.statistics.nounCount },
|
||||
verbCount: { ...this.statistics.verbCount },
|
||||
metadataCount: { ...this.statistics.metadataCount },
|
||||
hnswIndexSize: this.statistics.hnswIndexSize,
|
||||
lastUpdated: this.statistics.lastUpdated,
|
||||
// Include serviceActivity if present
|
||||
...(this.statistics.serviceActivity && {
|
||||
serviceActivity: Object.fromEntries(Object.entries(this.statistics.serviceActivity).map(([k, v]) => [k, { ...v }]))
|
||||
}),
|
||||
// Include services if present
|
||||
...(this.statistics.services && {
|
||||
services: this.statistics.services.map(s => ({ ...s }))
|
||||
}),
|
||||
// Include distributedConfig if present
|
||||
...(this.statistics.distributedConfig && {
|
||||
distributedConfig: JSON.parse(JSON.stringify(this.statistics.distributedConfig))
|
||||
})
|
||||
};
|
||||
// Since this is in-memory, there's no need for fallback mechanisms
|
||||
// to check multiple storage locations
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=memoryStorage.js.map
|
||||
1
dist/storage/adapters/memoryStorage.js.map
vendored
Normal file
1
dist/storage/adapters/memoryStorage.js.map
vendored
Normal file
File diff suppressed because one or more lines are too long
258
dist/storage/adapters/opfsStorage.d.ts
vendored
Normal file
258
dist/storage/adapters/opfsStorage.d.ts
vendored
Normal file
|
|
@ -0,0 +1,258 @@
|
|||
/**
|
||||
* OPFS (Origin Private File System) Storage Adapter
|
||||
* Provides persistent storage for the vector database using the Origin Private File System API
|
||||
*/
|
||||
import { GraphVerb, HNSWNoun, HNSWVerb, StatisticsData } from '../../coreTypes.js';
|
||||
import { BaseStorage } from '../baseStorage.js';
|
||||
import '../../types/fileSystemTypes.js';
|
||||
type HNSWNode = HNSWNoun;
|
||||
/**
|
||||
* Type alias for HNSWVerb to make the code more readable
|
||||
*/
|
||||
type Edge = HNSWVerb;
|
||||
type HNSWNoun_internal = HNSWNoun;
|
||||
/**
|
||||
* OPFS storage adapter for browser environments
|
||||
* Uses the Origin Private File System API to store data persistently
|
||||
*/
|
||||
export declare class OPFSStorage extends BaseStorage {
|
||||
private rootDir;
|
||||
private nounsDir;
|
||||
private verbsDir;
|
||||
private metadataDir;
|
||||
private nounMetadataDir;
|
||||
private verbMetadataDir;
|
||||
private indexDir;
|
||||
private isAvailable;
|
||||
private isPersistentRequested;
|
||||
private isPersistentGranted;
|
||||
private statistics;
|
||||
private activeLocks;
|
||||
private lockPrefix;
|
||||
constructor();
|
||||
/**
|
||||
* Initialize the storage adapter
|
||||
*/
|
||||
init(): Promise<void>;
|
||||
/**
|
||||
* Check if OPFS is available in the current environment
|
||||
*/
|
||||
isOPFSAvailable(): boolean;
|
||||
/**
|
||||
* Request persistent storage permission from the user
|
||||
* @returns Promise that resolves to true if permission was granted, false otherwise
|
||||
*/
|
||||
requestPersistentStorage(): Promise<boolean>;
|
||||
/**
|
||||
* Check if persistent storage is granted
|
||||
* @returns Promise that resolves to true if persistent storage is granted, false otherwise
|
||||
*/
|
||||
isPersistent(): Promise<boolean>;
|
||||
/**
|
||||
* Save a noun to storage
|
||||
*/
|
||||
protected saveNoun_internal(noun: HNSWNoun_internal): Promise<void>;
|
||||
/**
|
||||
* Get a noun from storage
|
||||
*/
|
||||
protected getNoun_internal(id: string): Promise<HNSWNoun_internal | null>;
|
||||
/**
|
||||
* Get nouns by noun type (internal implementation)
|
||||
* @param nounType The noun type to filter by
|
||||
* @returns Promise that resolves to an array of nouns of the specified noun type
|
||||
*/
|
||||
protected getNounsByNounType_internal(nounType: string): Promise<HNSWNoun[]>;
|
||||
/**
|
||||
* Get nodes by noun type
|
||||
* @param nounType The noun type to filter by
|
||||
* @returns Promise that resolves to an array of nodes of the specified noun type
|
||||
*/
|
||||
protected getNodesByNounType(nounType: string): Promise<HNSWNode[]>;
|
||||
/**
|
||||
* Delete a noun from storage (internal implementation)
|
||||
*/
|
||||
protected deleteNoun_internal(id: string): Promise<void>;
|
||||
/**
|
||||
* Delete a node from storage
|
||||
*/
|
||||
protected deleteNode(id: string): Promise<void>;
|
||||
/**
|
||||
* Save a verb to storage (internal implementation)
|
||||
*/
|
||||
protected saveVerb_internal(verb: HNSWVerb): Promise<void>;
|
||||
/**
|
||||
* Save an edge to storage
|
||||
*/
|
||||
protected saveEdge(edge: Edge): Promise<void>;
|
||||
/**
|
||||
* Get a verb from storage (internal implementation)
|
||||
*/
|
||||
protected getVerb_internal(id: string): Promise<HNSWVerb | null>;
|
||||
/**
|
||||
* Get an edge from storage
|
||||
*/
|
||||
protected getEdge(id: string): Promise<Edge | null>;
|
||||
/**
|
||||
* Get all edges from storage
|
||||
*/
|
||||
protected getAllEdges(): Promise<Edge[]>;
|
||||
/**
|
||||
* Get verbs by source (internal implementation)
|
||||
*/
|
||||
protected getVerbsBySource_internal(sourceId: string): Promise<GraphVerb[]>;
|
||||
/**
|
||||
* Get edges by source
|
||||
*/
|
||||
protected getEdgesBySource(sourceId: string): Promise<GraphVerb[]>;
|
||||
/**
|
||||
* Get verbs by target (internal implementation)
|
||||
*/
|
||||
protected getVerbsByTarget_internal(targetId: string): Promise<GraphVerb[]>;
|
||||
/**
|
||||
* Get edges by target
|
||||
*/
|
||||
protected getEdgesByTarget(targetId: string): Promise<GraphVerb[]>;
|
||||
/**
|
||||
* Get verbs by type (internal implementation)
|
||||
*/
|
||||
protected getVerbsByType_internal(type: string): Promise<GraphVerb[]>;
|
||||
/**
|
||||
* Get edges by type
|
||||
*/
|
||||
protected getEdgesByType(type: string): Promise<GraphVerb[]>;
|
||||
/**
|
||||
* Delete a verb from storage (internal implementation)
|
||||
*/
|
||||
protected deleteVerb_internal(id: string): Promise<void>;
|
||||
/**
|
||||
* Delete an edge from storage
|
||||
*/
|
||||
protected deleteEdge(id: string): Promise<void>;
|
||||
/**
|
||||
* Save metadata to storage
|
||||
*/
|
||||
saveMetadata(id: string, metadata: any): Promise<void>;
|
||||
/**
|
||||
* Get metadata from storage
|
||||
*/
|
||||
getMetadata(id: string): Promise<any | null>;
|
||||
/**
|
||||
* Get multiple metadata objects in batches (CRITICAL: Prevents socket exhaustion)
|
||||
* OPFS implementation uses controlled concurrency for file operations
|
||||
*/
|
||||
getMetadataBatch(ids: string[]): Promise<Map<string, any>>;
|
||||
/**
|
||||
* Save verb metadata to storage
|
||||
*/
|
||||
saveVerbMetadata(id: string, metadata: any): Promise<void>;
|
||||
/**
|
||||
* Get verb metadata from storage
|
||||
*/
|
||||
getVerbMetadata(id: string): Promise<any | null>;
|
||||
/**
|
||||
* Save noun metadata to storage
|
||||
*/
|
||||
saveNounMetadata(id: string, metadata: any): Promise<void>;
|
||||
/**
|
||||
* Get noun metadata from storage
|
||||
*/
|
||||
getNounMetadata(id: string): Promise<any | null>;
|
||||
/**
|
||||
* Clear all data from storage
|
||||
*/
|
||||
clear(): Promise<void>;
|
||||
/**
|
||||
* Get information about storage usage and capacity
|
||||
*/
|
||||
getStorageStatus(): Promise<{
|
||||
type: string;
|
||||
used: number;
|
||||
quota: number | null;
|
||||
details?: Record<string, any>;
|
||||
}>;
|
||||
/**
|
||||
* Get the statistics key for a specific date
|
||||
* @param date The date to get the key for
|
||||
* @returns The statistics key for the specified date
|
||||
*/
|
||||
private getStatisticsKeyForDate;
|
||||
/**
|
||||
* Get the current statistics key
|
||||
* @returns The current statistics key
|
||||
*/
|
||||
private getCurrentStatisticsKey;
|
||||
/**
|
||||
* Get the legacy statistics key (for backward compatibility)
|
||||
* @returns The legacy statistics key
|
||||
*/
|
||||
private getLegacyStatisticsKey;
|
||||
/**
|
||||
* Acquire a browser-based lock for coordinating operations across multiple tabs
|
||||
* @param lockKey The key to lock on
|
||||
* @param ttl Time to live for the lock in milliseconds (default: 30 seconds)
|
||||
* @returns Promise that resolves to true if lock was acquired, false otherwise
|
||||
*/
|
||||
private acquireLock;
|
||||
/**
|
||||
* Release a browser-based lock
|
||||
* @param lockKey The key to unlock
|
||||
* @param lockValue The value used when acquiring the lock (for verification)
|
||||
* @returns Promise that resolves when lock is released
|
||||
*/
|
||||
private releaseLock;
|
||||
/**
|
||||
* Clean up expired locks from localStorage
|
||||
*/
|
||||
private cleanupExpiredLocks;
|
||||
/**
|
||||
* Save statistics data to storage with browser-based locking
|
||||
* @param statistics The statistics data to save
|
||||
*/
|
||||
protected saveStatisticsData(statistics: StatisticsData): Promise<void>;
|
||||
/**
|
||||
* Get statistics data from storage
|
||||
* @returns Promise that resolves to the statistics data or null if not found
|
||||
*/
|
||||
protected getStatisticsData(): Promise<StatisticsData | null>;
|
||||
/**
|
||||
* Get nouns with pagination support
|
||||
* @param options Pagination and filter options
|
||||
* @returns Promise that resolves to a paginated result of nouns
|
||||
*/
|
||||
getNounsWithPagination(options?: {
|
||||
limit?: number;
|
||||
cursor?: string;
|
||||
filter?: {
|
||||
nounType?: string | string[];
|
||||
service?: string | string[];
|
||||
metadata?: Record<string, any>;
|
||||
};
|
||||
}): Promise<{
|
||||
items: HNSWNoun[];
|
||||
totalCount?: number;
|
||||
hasMore: boolean;
|
||||
nextCursor?: string;
|
||||
}>;
|
||||
/**
|
||||
* Get verbs with pagination support
|
||||
* @param options Pagination and filter options
|
||||
* @returns Promise that resolves to a paginated result of verbs
|
||||
*/
|
||||
getVerbsWithPagination(options?: {
|
||||
limit?: number;
|
||||
cursor?: string;
|
||||
filter?: {
|
||||
verbType?: string | string[];
|
||||
sourceId?: string | string[];
|
||||
targetId?: string | string[];
|
||||
service?: string | string[];
|
||||
metadata?: Record<string, any>;
|
||||
};
|
||||
}): Promise<{
|
||||
items: GraphVerb[];
|
||||
totalCount?: number;
|
||||
hasMore: boolean;
|
||||
nextCursor?: string;
|
||||
}>;
|
||||
}
|
||||
export {};
|
||||
1307
dist/storage/adapters/opfsStorage.js
vendored
Normal file
1307
dist/storage/adapters/opfsStorage.js
vendored
Normal file
File diff suppressed because it is too large
Load diff
1
dist/storage/adapters/opfsStorage.js.map
vendored
Normal file
1
dist/storage/adapters/opfsStorage.js.map
vendored
Normal file
File diff suppressed because one or more lines are too long
79
dist/storage/adapters/optimizedS3Search.d.ts
vendored
Normal file
79
dist/storage/adapters/optimizedS3Search.d.ts
vendored
Normal file
|
|
@ -0,0 +1,79 @@
|
|||
/**
|
||||
* Optimized S3 Search and Pagination
|
||||
* Provides efficient search and pagination capabilities for S3-compatible storage
|
||||
*/
|
||||
import { HNSWNoun, GraphVerb } from '../../coreTypes.js';
|
||||
/**
|
||||
* Pagination result interface
|
||||
*/
|
||||
export interface PaginationResult<T> {
|
||||
items: T[];
|
||||
totalCount?: number;
|
||||
hasMore: boolean;
|
||||
nextCursor?: string;
|
||||
}
|
||||
/**
|
||||
* Filter interface for nouns
|
||||
*/
|
||||
export interface NounFilter {
|
||||
nounType?: string | string[];
|
||||
service?: string | string[];
|
||||
metadata?: Record<string, any>;
|
||||
}
|
||||
/**
|
||||
* Filter interface for verbs
|
||||
*/
|
||||
export interface VerbFilter {
|
||||
verbType?: string | string[];
|
||||
sourceId?: string | string[];
|
||||
targetId?: string | string[];
|
||||
service?: string | string[];
|
||||
metadata?: Record<string, any>;
|
||||
}
|
||||
/**
|
||||
* Interface for storage operations needed by optimized search
|
||||
*/
|
||||
export interface StorageOperations {
|
||||
listObjectKeys(prefix: string, limit: number, cursor?: string): Promise<{
|
||||
keys: string[];
|
||||
hasMore: boolean;
|
||||
nextCursor?: string;
|
||||
}>;
|
||||
getObject<T>(key: string): Promise<T | null>;
|
||||
getMetadata(id: string, type: 'noun' | 'verb'): Promise<any | null>;
|
||||
}
|
||||
/**
|
||||
* Optimized search implementation for S3-compatible storage
|
||||
*/
|
||||
export declare class OptimizedS3Search {
|
||||
private storage;
|
||||
constructor(storage: StorageOperations);
|
||||
/**
|
||||
* Get nouns with optimized pagination and filtering
|
||||
*/
|
||||
getNounsWithPagination(options?: {
|
||||
limit?: number;
|
||||
cursor?: string;
|
||||
filter?: NounFilter;
|
||||
}): Promise<PaginationResult<HNSWNoun>>;
|
||||
/**
|
||||
* Get verbs with optimized pagination and filtering
|
||||
*/
|
||||
getVerbsWithPagination(options?: {
|
||||
limit?: number;
|
||||
cursor?: string;
|
||||
filter?: VerbFilter;
|
||||
}): Promise<PaginationResult<GraphVerb>>;
|
||||
/**
|
||||
* Check if a noun matches the filter criteria
|
||||
*/
|
||||
private matchesNounFilter;
|
||||
/**
|
||||
* Check if a verb matches the filter criteria
|
||||
*/
|
||||
private matchesVerbFilter;
|
||||
/**
|
||||
* Combine HNSWVerb data with metadata to create GraphVerb
|
||||
*/
|
||||
private combineVerbWithMetadata;
|
||||
}
|
||||
249
dist/storage/adapters/optimizedS3Search.js
vendored
Normal file
249
dist/storage/adapters/optimizedS3Search.js
vendored
Normal file
|
|
@ -0,0 +1,249 @@
|
|||
/**
|
||||
* Optimized S3 Search and Pagination
|
||||
* Provides efficient search and pagination capabilities for S3-compatible storage
|
||||
*/
|
||||
import { createModuleLogger } from '../../utils/logger.js';
|
||||
import { getDirectoryPath } from '../baseStorage.js';
|
||||
const logger = createModuleLogger('OptimizedS3Search');
|
||||
/**
|
||||
* Optimized search implementation for S3-compatible storage
|
||||
*/
|
||||
export class OptimizedS3Search {
|
||||
constructor(storage) {
|
||||
this.storage = storage;
|
||||
}
|
||||
/**
|
||||
* Get nouns with optimized pagination and filtering
|
||||
*/
|
||||
async getNounsWithPagination(options = {}) {
|
||||
const limit = options.limit || 100;
|
||||
const cursor = options.cursor;
|
||||
try {
|
||||
// List noun objects with pagination
|
||||
const listResult = await this.storage.listObjectKeys(`${getDirectoryPath('noun', 'vector')}/`, limit * 2, cursor);
|
||||
if (!listResult.keys.length) {
|
||||
return {
|
||||
items: [],
|
||||
hasMore: false
|
||||
};
|
||||
}
|
||||
// Load nouns in parallel batches
|
||||
const nouns = [];
|
||||
const batchSize = 10;
|
||||
for (let i = 0; i < listResult.keys.length && nouns.length < limit; i += batchSize) {
|
||||
const batch = listResult.keys.slice(i, i + batchSize);
|
||||
const batchPromises = batch.map(key => this.storage.getObject(key));
|
||||
const batchResults = await Promise.all(batchPromises);
|
||||
for (const noun of batchResults) {
|
||||
if (!noun)
|
||||
continue;
|
||||
// Apply filters
|
||||
if (options.filter && !(await this.matchesNounFilter(noun, options.filter))) {
|
||||
continue;
|
||||
}
|
||||
nouns.push(noun);
|
||||
if (nouns.length >= limit) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Determine if there are more items
|
||||
const hasMore = listResult.hasMore || nouns.length >= limit;
|
||||
// Set next cursor
|
||||
let nextCursor;
|
||||
if (hasMore && nouns.length > 0) {
|
||||
nextCursor = nouns[nouns.length - 1].id;
|
||||
}
|
||||
return {
|
||||
items: nouns.slice(0, limit),
|
||||
hasMore,
|
||||
nextCursor
|
||||
};
|
||||
}
|
||||
catch (error) {
|
||||
logger.error('Failed to get nouns with pagination:', error);
|
||||
return {
|
||||
items: [],
|
||||
hasMore: false
|
||||
};
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Get verbs with optimized pagination and filtering
|
||||
*/
|
||||
async getVerbsWithPagination(options = {}) {
|
||||
const limit = options.limit || 100;
|
||||
const cursor = options.cursor;
|
||||
try {
|
||||
// List verb objects with pagination
|
||||
const listResult = await this.storage.listObjectKeys(`${getDirectoryPath('verb', 'vector')}/`, limit * 2, cursor);
|
||||
if (!listResult.keys.length) {
|
||||
return {
|
||||
items: [],
|
||||
hasMore: false
|
||||
};
|
||||
}
|
||||
// Load verbs in parallel batches
|
||||
const verbs = [];
|
||||
const batchSize = 10;
|
||||
for (let i = 0; i < listResult.keys.length && verbs.length < limit; i += batchSize) {
|
||||
const batch = listResult.keys.slice(i, i + batchSize);
|
||||
// Load verbs and their metadata in parallel
|
||||
const batchPromises = batch.map(async (key) => {
|
||||
const verbData = await this.storage.getObject(key);
|
||||
if (!verbData)
|
||||
return null;
|
||||
// Get metadata
|
||||
const verbId = key.replace(`${getDirectoryPath('verb', 'vector')}/`, '').replace('.json', '');
|
||||
const metadata = await this.storage.getMetadata(verbId, 'verb');
|
||||
// Combine into GraphVerb
|
||||
return this.combineVerbWithMetadata(verbData, metadata);
|
||||
});
|
||||
const batchResults = await Promise.all(batchPromises);
|
||||
for (const verb of batchResults) {
|
||||
if (!verb)
|
||||
continue;
|
||||
// Apply filters
|
||||
if (options.filter && !this.matchesVerbFilter(verb, options.filter)) {
|
||||
continue;
|
||||
}
|
||||
verbs.push(verb);
|
||||
if (verbs.length >= limit) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Determine if there are more items
|
||||
const hasMore = listResult.hasMore || verbs.length >= limit;
|
||||
// Set next cursor
|
||||
let nextCursor;
|
||||
if (hasMore && verbs.length > 0) {
|
||||
nextCursor = verbs[verbs.length - 1].id;
|
||||
}
|
||||
return {
|
||||
items: verbs.slice(0, limit),
|
||||
hasMore,
|
||||
nextCursor
|
||||
};
|
||||
}
|
||||
catch (error) {
|
||||
logger.error('Failed to get verbs with pagination:', error);
|
||||
return {
|
||||
items: [],
|
||||
hasMore: false
|
||||
};
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Check if a noun matches the filter criteria
|
||||
*/
|
||||
async matchesNounFilter(noun, filter) {
|
||||
// Get metadata for filtering
|
||||
const metadata = await this.storage.getMetadata(noun.id, 'noun');
|
||||
// Filter by noun type
|
||||
if (filter.nounType) {
|
||||
const nounTypes = Array.isArray(filter.nounType) ? filter.nounType : [filter.nounType];
|
||||
const nounType = metadata?.type || metadata?.noun;
|
||||
if (!nounType || !nounTypes.includes(nounType)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
// Filter by service
|
||||
if (filter.service) {
|
||||
const services = Array.isArray(filter.service) ? filter.service : [filter.service];
|
||||
if (!metadata?.service || !services.includes(metadata.service)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
// Filter by metadata
|
||||
if (filter.metadata) {
|
||||
if (!metadata)
|
||||
return false;
|
||||
for (const [key, value] of Object.entries(filter.metadata)) {
|
||||
if (metadata[key] !== value) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
/**
|
||||
* Check if a verb matches the filter criteria
|
||||
*/
|
||||
matchesVerbFilter(verb, filter) {
|
||||
// Filter by verb type
|
||||
if (filter.verbType) {
|
||||
const verbTypes = Array.isArray(filter.verbType) ? filter.verbType : [filter.verbType];
|
||||
if (!verb.type || !verbTypes.includes(verb.type)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
// Filter by source ID
|
||||
if (filter.sourceId) {
|
||||
const sourceIds = Array.isArray(filter.sourceId) ? filter.sourceId : [filter.sourceId];
|
||||
if (!verb.sourceId || !sourceIds.includes(verb.sourceId)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
// Filter by target ID
|
||||
if (filter.targetId) {
|
||||
const targetIds = Array.isArray(filter.targetId) ? filter.targetId : [filter.targetId];
|
||||
if (!verb.targetId || !targetIds.includes(verb.targetId)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
// Filter by service
|
||||
if (filter.service) {
|
||||
const services = Array.isArray(filter.service) ? filter.service : [filter.service];
|
||||
if (!verb.metadata?.service || !services.includes(verb.metadata.service)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
// Filter by metadata
|
||||
if (filter.metadata) {
|
||||
if (!verb.metadata)
|
||||
return false;
|
||||
for (const [key, value] of Object.entries(filter.metadata)) {
|
||||
if (verb.metadata[key] !== value) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
/**
|
||||
* Combine HNSWVerb data with metadata to create GraphVerb
|
||||
*/
|
||||
combineVerbWithMetadata(verbData, metadata) {
|
||||
if (!verbData || !metadata)
|
||||
return null;
|
||||
// Create default timestamp if not present
|
||||
const defaultTimestamp = {
|
||||
seconds: Math.floor(Date.now() / 1000),
|
||||
nanoseconds: (Date.now() % 1000) * 1000000
|
||||
};
|
||||
// Create default createdBy if not present
|
||||
const defaultCreatedBy = {
|
||||
augmentation: 'unknown',
|
||||
version: '1.0'
|
||||
};
|
||||
return {
|
||||
id: verbData.id,
|
||||
vector: verbData.vector,
|
||||
sourceId: metadata.sourceId,
|
||||
targetId: metadata.targetId,
|
||||
source: metadata.source,
|
||||
target: metadata.target,
|
||||
verb: metadata.verb,
|
||||
type: metadata.type,
|
||||
weight: metadata.weight || 1.0,
|
||||
metadata: metadata.metadata || {},
|
||||
createdAt: metadata.createdAt || defaultTimestamp,
|
||||
updatedAt: metadata.updatedAt || defaultTimestamp,
|
||||
createdBy: metadata.createdBy || defaultCreatedBy,
|
||||
data: metadata.data,
|
||||
embedding: verbData.vector
|
||||
};
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=optimizedS3Search.js.map
|
||||
1
dist/storage/adapters/optimizedS3Search.js.map
vendored
Normal file
1
dist/storage/adapters/optimizedS3Search.js.map
vendored
Normal file
File diff suppressed because one or more lines are too long
493
dist/storage/adapters/s3CompatibleStorage.d.ts
vendored
Normal file
493
dist/storage/adapters/s3CompatibleStorage.d.ts
vendored
Normal file
|
|
@ -0,0 +1,493 @@
|
|||
/**
|
||||
* S3-Compatible Storage Adapter
|
||||
* Uses the AWS S3 client to interact with S3-compatible storage services
|
||||
* including Amazon S3, Cloudflare R2, and Google Cloud Storage
|
||||
*/
|
||||
import { GraphVerb, HNSWNoun, HNSWVerb, StatisticsData } from '../../coreTypes.js';
|
||||
import { BaseStorage } from '../baseStorage.js';
|
||||
import { OperationConfig } from '../../utils/operationUtils.js';
|
||||
type HNSWNode = HNSWNoun;
|
||||
type Edge = HNSWVerb;
|
||||
interface ChangeLogEntry {
|
||||
timestamp: number;
|
||||
operation: 'add' | 'update' | 'delete';
|
||||
entityType: 'noun' | 'verb' | 'metadata';
|
||||
entityId: string;
|
||||
data?: any;
|
||||
instanceId?: string;
|
||||
}
|
||||
export { S3CompatibleStorage as R2Storage };
|
||||
/**
|
||||
* S3-compatible storage adapter for server environments
|
||||
* Uses the AWS S3 client to interact with S3-compatible storage services
|
||||
* including Amazon S3, Cloudflare R2, and Google Cloud Storage
|
||||
*
|
||||
* To use this adapter with Amazon S3, you need to provide:
|
||||
* - region: AWS region (e.g., 'us-east-1')
|
||||
* - credentials: AWS credentials (accessKeyId and secretAccessKey)
|
||||
* - bucketName: S3 bucket name
|
||||
*
|
||||
* To use this adapter with Cloudflare R2, you need to provide:
|
||||
* - accountId: Cloudflare account ID
|
||||
* - accessKeyId: R2 access key ID
|
||||
* - secretAccessKey: R2 secret access key
|
||||
* - bucketName: R2 bucket name
|
||||
*
|
||||
* To use this adapter with Google Cloud Storage, you need to provide:
|
||||
* - region: GCS region (e.g., 'us-central1')
|
||||
* - credentials: GCS credentials (accessKeyId and secretAccessKey)
|
||||
* - endpoint: GCS endpoint (e.g., 'https://storage.googleapis.com')
|
||||
* - bucketName: GCS bucket name
|
||||
*/
|
||||
export declare class S3CompatibleStorage extends BaseStorage {
|
||||
private s3Client;
|
||||
private bucketName;
|
||||
private serviceType;
|
||||
private region;
|
||||
private endpoint?;
|
||||
private accountId?;
|
||||
private accessKeyId;
|
||||
private secretAccessKey;
|
||||
private sessionToken?;
|
||||
private nounPrefix;
|
||||
private verbPrefix;
|
||||
private metadataPrefix;
|
||||
private verbMetadataPrefix;
|
||||
private indexPrefix;
|
||||
private systemPrefix;
|
||||
private useDualWrite;
|
||||
protected statisticsCache: StatisticsData | null;
|
||||
private lockPrefix;
|
||||
private activeLocks;
|
||||
private changeLogPrefix;
|
||||
private pendingOperations;
|
||||
private maxConcurrentOperations;
|
||||
private baseBatchSize;
|
||||
private currentBatchSize;
|
||||
private lastMemoryCheck;
|
||||
private memoryCheckInterval;
|
||||
private consecutiveErrors;
|
||||
private lastErrorReset;
|
||||
private socketManager;
|
||||
private backpressure;
|
||||
private nounWriteBuffer;
|
||||
private verbWriteBuffer;
|
||||
private requestCoalescer;
|
||||
private highVolumeMode;
|
||||
private lastVolumeCheck;
|
||||
private volumeCheckInterval;
|
||||
private forceHighVolumeMode;
|
||||
private operationExecutors;
|
||||
private nounCacheManager;
|
||||
private verbCacheManager;
|
||||
private logger;
|
||||
/**
|
||||
* Initialize the storage adapter
|
||||
* @param options Configuration options for the S3-compatible storage
|
||||
*/
|
||||
constructor(options: {
|
||||
bucketName: string;
|
||||
region?: string;
|
||||
endpoint?: string;
|
||||
accountId?: string;
|
||||
accessKeyId: string;
|
||||
secretAccessKey: string;
|
||||
sessionToken?: string;
|
||||
serviceType?: string;
|
||||
operationConfig?: OperationConfig;
|
||||
cacheConfig?: {
|
||||
hotCacheMaxSize?: number;
|
||||
hotCacheEvictionThreshold?: number;
|
||||
warmCacheTTL?: number;
|
||||
};
|
||||
readOnly?: boolean;
|
||||
});
|
||||
/**
|
||||
* Initialize the storage adapter
|
||||
*/
|
||||
init(): Promise<void>;
|
||||
/**
|
||||
* Override base class method to detect S3-specific throttling errors
|
||||
*/
|
||||
protected isThrottlingError(error: any): boolean;
|
||||
/**
|
||||
* Override to add S3-specific logging
|
||||
*/
|
||||
handleThrottling(error: any, service?: string): Promise<void>;
|
||||
/**
|
||||
* Smart delay based on current throttling status
|
||||
*/
|
||||
private smartDelay;
|
||||
/**
|
||||
* Auto-cleanup legacy /index folder during initialization
|
||||
* This removes old index data that has been migrated to _system
|
||||
*/
|
||||
private cleanupLegacyIndexFolder;
|
||||
/**
|
||||
* Initialize write buffers for high-volume scenarios
|
||||
*/
|
||||
private initializeBuffers;
|
||||
/**
|
||||
* Initialize request coalescer
|
||||
*/
|
||||
private initializeCoalescer;
|
||||
/**
|
||||
* Check if we should enable high-volume mode
|
||||
*/
|
||||
private checkVolumeMode;
|
||||
/**
|
||||
* Bulk write nouns to S3
|
||||
*/
|
||||
private bulkWriteNouns;
|
||||
/**
|
||||
* Bulk write verbs to S3
|
||||
*/
|
||||
private bulkWriteVerbs;
|
||||
/**
|
||||
* Process coalesced batch of operations
|
||||
*/
|
||||
private processCoalescedBatch;
|
||||
/**
|
||||
* Process bulk deletes
|
||||
*/
|
||||
private processBulkDeletes;
|
||||
/**
|
||||
* Process bulk writes
|
||||
*/
|
||||
private processBulkWrites;
|
||||
/**
|
||||
* Process bulk reads
|
||||
*/
|
||||
private processBulkReads;
|
||||
/**
|
||||
* Dynamically adjust batch size based on memory pressure and error rates
|
||||
*/
|
||||
private adjustBatchSize;
|
||||
/**
|
||||
* Apply backpressure when system is under load
|
||||
*/
|
||||
private applyBackpressure;
|
||||
/**
|
||||
* Release backpressure after operation completes
|
||||
*/
|
||||
private releaseBackpressure;
|
||||
/**
|
||||
* Get current batch size for operations
|
||||
*/
|
||||
private getBatchSize;
|
||||
/**
|
||||
* Save a noun to storage (internal implementation)
|
||||
*/
|
||||
protected saveNoun_internal(noun: HNSWNoun): Promise<void>;
|
||||
/**
|
||||
* Save a node to storage
|
||||
*/
|
||||
protected saveNode(node: HNSWNode): Promise<void>;
|
||||
/**
|
||||
* Get a noun from storage (internal implementation)
|
||||
*/
|
||||
protected getNoun_internal(id: string): Promise<HNSWNoun | null>;
|
||||
/**
|
||||
* Get a node from storage
|
||||
*/
|
||||
protected getNode(id: string): Promise<HNSWNode | null>;
|
||||
private nodeCache;
|
||||
/**
|
||||
* Get all nodes from storage
|
||||
* @deprecated This method is deprecated and will be removed in a future version.
|
||||
* It can cause memory issues with large datasets. Use getNodesWithPagination() instead.
|
||||
*/
|
||||
protected getAllNodes(): Promise<HNSWNode[]>;
|
||||
/**
|
||||
* Get nodes with pagination
|
||||
* @param options Pagination options
|
||||
* @returns Promise that resolves to a paginated result of nodes
|
||||
*/
|
||||
protected getNodesWithPagination(options?: {
|
||||
limit?: number;
|
||||
cursor?: string;
|
||||
useCache?: boolean;
|
||||
}): Promise<{
|
||||
nodes: HNSWNode[];
|
||||
hasMore: boolean;
|
||||
nextCursor?: string;
|
||||
}>;
|
||||
/**
|
||||
* Get nouns by noun type (internal implementation)
|
||||
* @param nounType The noun type to filter by
|
||||
* @returns Promise that resolves to an array of nouns of the specified noun type
|
||||
*/
|
||||
protected getNounsByNounType_internal(nounType: string): Promise<HNSWNoun[]>;
|
||||
/**
|
||||
* Get nodes by noun type
|
||||
* @param nounType The noun type to filter by
|
||||
* @returns Promise that resolves to an array of nodes of the specified noun type
|
||||
*/
|
||||
protected getNodesByNounType(nounType: string): Promise<HNSWNode[]>;
|
||||
/**
|
||||
* Delete a noun from storage (internal implementation)
|
||||
*/
|
||||
protected deleteNoun_internal(id: string): Promise<void>;
|
||||
/**
|
||||
* Delete a node from storage
|
||||
*/
|
||||
protected deleteNode(id: string): Promise<void>;
|
||||
/**
|
||||
* Save a verb to storage (internal implementation)
|
||||
*/
|
||||
protected saveVerb_internal(verb: HNSWVerb): Promise<void>;
|
||||
/**
|
||||
* Save an edge to storage
|
||||
*/
|
||||
protected saveEdge(edge: Edge): Promise<void>;
|
||||
/**
|
||||
* Get a verb from storage (internal implementation)
|
||||
*/
|
||||
protected getVerb_internal(id: string): Promise<HNSWVerb | null>;
|
||||
/**
|
||||
* Get an edge from storage
|
||||
*/
|
||||
protected getEdge(id: string): Promise<Edge | null>;
|
||||
/**
|
||||
* Get all edges from storage
|
||||
* @deprecated This method is deprecated and will be removed in a future version.
|
||||
* It can cause memory issues with large datasets. Use getEdgesWithPagination() instead.
|
||||
*/
|
||||
protected getAllEdges(): Promise<Edge[]>;
|
||||
/**
|
||||
* Get edges with pagination
|
||||
* @param options Pagination options
|
||||
* @returns Promise that resolves to a paginated result of edges
|
||||
*/
|
||||
protected getEdgesWithPagination(options?: {
|
||||
limit?: number;
|
||||
cursor?: string;
|
||||
useCache?: boolean;
|
||||
filter?: {
|
||||
sourceId?: string;
|
||||
targetId?: string;
|
||||
type?: string;
|
||||
};
|
||||
}): Promise<{
|
||||
edges: Edge[];
|
||||
hasMore: boolean;
|
||||
nextCursor?: string;
|
||||
}>;
|
||||
/**
|
||||
* Filter an edge based on filter criteria
|
||||
* @param edge The edge to filter
|
||||
* @param filter The filter criteria
|
||||
* @returns True if the edge matches the filter, false otherwise
|
||||
*/
|
||||
private filterEdge;
|
||||
/**
|
||||
* Get verbs with pagination
|
||||
* @param options Pagination options
|
||||
* @returns Promise that resolves to a paginated result of verbs
|
||||
*/
|
||||
getVerbsWithPagination(options?: {
|
||||
limit?: number;
|
||||
cursor?: string;
|
||||
filter?: {
|
||||
verbType?: string | string[];
|
||||
sourceId?: string | string[];
|
||||
targetId?: string | string[];
|
||||
service?: string | string[];
|
||||
metadata?: Record<string, any>;
|
||||
};
|
||||
}): Promise<{
|
||||
items: GraphVerb[];
|
||||
totalCount?: number;
|
||||
hasMore: boolean;
|
||||
nextCursor?: string;
|
||||
}>;
|
||||
/**
|
||||
* Get verbs by source (internal implementation)
|
||||
*/
|
||||
protected getVerbsBySource_internal(sourceId: string): Promise<GraphVerb[]>;
|
||||
/**
|
||||
* Get verbs by target (internal implementation)
|
||||
*/
|
||||
protected getVerbsByTarget_internal(targetId: string): Promise<GraphVerb[]>;
|
||||
/**
|
||||
* Get verbs by type (internal implementation)
|
||||
*/
|
||||
protected getVerbsByType_internal(type: string): Promise<GraphVerb[]>;
|
||||
/**
|
||||
* Delete a verb from storage (internal implementation)
|
||||
*/
|
||||
protected deleteVerb_internal(id: string): Promise<void>;
|
||||
/**
|
||||
* Delete an edge from storage
|
||||
*/
|
||||
protected deleteEdge(id: string): Promise<void>;
|
||||
/**
|
||||
* Save metadata to storage
|
||||
*/
|
||||
saveMetadata(id: string, metadata: any): Promise<void>;
|
||||
/**
|
||||
* Save verb metadata to storage
|
||||
*/
|
||||
saveVerbMetadata(id: string, metadata: any): Promise<void>;
|
||||
/**
|
||||
* Get verb metadata from storage
|
||||
*/
|
||||
getVerbMetadata(id: string): Promise<any | null>;
|
||||
/**
|
||||
* Save noun metadata to storage
|
||||
*/
|
||||
saveNounMetadata(id: string, metadata: any): Promise<void>;
|
||||
/**
|
||||
* Get multiple metadata objects in batches (CRITICAL: Prevents socket exhaustion)
|
||||
* This is the solution to the metadata reading socket exhaustion during initialization
|
||||
*/
|
||||
getMetadataBatch(ids: string[]): Promise<Map<string, any>>;
|
||||
/**
|
||||
* Get multiple verb metadata objects in batches (prevents socket exhaustion)
|
||||
*/
|
||||
getVerbMetadataBatch(ids: string[]): Promise<Map<string, any>>;
|
||||
/**
|
||||
* Get noun metadata from storage
|
||||
*/
|
||||
getNounMetadata(id: string): Promise<any | null>;
|
||||
/**
|
||||
* Get metadata from storage
|
||||
*/
|
||||
getMetadata(id: string): Promise<any | null>;
|
||||
/**
|
||||
* Clear all data from storage
|
||||
*/
|
||||
clear(): Promise<void>;
|
||||
/**
|
||||
* Get information about storage usage and capacity
|
||||
* Optimized version that uses cached statistics instead of expensive full scans
|
||||
*/
|
||||
getStorageStatus(): Promise<{
|
||||
type: string;
|
||||
used: number;
|
||||
quota: number | null;
|
||||
details?: Record<string, any>;
|
||||
}>;
|
||||
protected statisticsBatchUpdateTimerId: NodeJS.Timeout | null;
|
||||
protected statisticsModified: boolean;
|
||||
protected lastStatisticsFlushTime: number;
|
||||
protected readonly MIN_FLUSH_INTERVAL_MS = 5000;
|
||||
protected readonly MAX_FLUSH_DELAY_MS = 30000;
|
||||
/**
|
||||
* Get the statistics key for a specific date
|
||||
* @param date The date to get the key for
|
||||
* @returns The statistics key for the specified date
|
||||
*/
|
||||
private getStatisticsKeyForDate;
|
||||
/**
|
||||
* Get the current statistics key
|
||||
* @returns The current statistics key
|
||||
*/
|
||||
private getCurrentStatisticsKey;
|
||||
/**
|
||||
* Get the legacy statistics key (DEPRECATED - /index folder is auto-cleaned)
|
||||
* @returns The legacy statistics key
|
||||
* @deprecated Legacy /index folder is automatically cleaned on initialization
|
||||
*/
|
||||
private getLegacyStatisticsKey;
|
||||
/**
|
||||
* Schedule a batch update of statistics
|
||||
*/
|
||||
protected scheduleBatchUpdate(): void;
|
||||
/**
|
||||
* Flush statistics to storage with distributed locking
|
||||
*/
|
||||
protected flushStatistics(): Promise<void>;
|
||||
/**
|
||||
* Merge statistics from storage with local statistics
|
||||
* @param storageStats Statistics from storage
|
||||
* @param localStats Local statistics to merge
|
||||
* @returns Merged statistics data
|
||||
*/
|
||||
private mergeStatistics;
|
||||
/**
|
||||
* Save statistics data to storage
|
||||
* @param statistics The statistics data to save
|
||||
*/
|
||||
protected saveStatisticsData(statistics: StatisticsData): Promise<void>;
|
||||
/**
|
||||
* Get statistics data from storage
|
||||
* @returns Promise that resolves to the statistics data or null if not found
|
||||
*/
|
||||
protected getStatisticsData(): Promise<StatisticsData | null>;
|
||||
/**
|
||||
* Check if we should try yesterday's statistics file
|
||||
* Only try within 2 hours of midnight to avoid unnecessary calls
|
||||
*/
|
||||
private shouldTryYesterday;
|
||||
/**
|
||||
* Get yesterday's date
|
||||
*/
|
||||
private getYesterday;
|
||||
/**
|
||||
* Try to get statistics from a specific key
|
||||
* @param key The key to try to get statistics from
|
||||
* @returns The statistics data or null if not found
|
||||
*/
|
||||
private tryGetStatisticsFromKey;
|
||||
/**
|
||||
* Append an entry to the change log for efficient synchronization
|
||||
* @param entry The change log entry to append
|
||||
*/
|
||||
private appendToChangeLog;
|
||||
/**
|
||||
* Get changes from the change log since a specific timestamp
|
||||
* @param sinceTimestamp Timestamp to get changes since
|
||||
* @param maxEntries Maximum number of entries to return (default: 1000)
|
||||
* @returns Array of change log entries
|
||||
*/
|
||||
getChangesSince(sinceTimestamp: number, maxEntries?: number): Promise<ChangeLogEntry[]>;
|
||||
/**
|
||||
* Clean up old change log entries to prevent unlimited growth
|
||||
* @param olderThanTimestamp Remove entries older than this timestamp
|
||||
*/
|
||||
cleanupOldChangeLogs(olderThanTimestamp: number): Promise<void>;
|
||||
/**
|
||||
* Sample-based storage estimation as fallback when statistics unavailable
|
||||
* Much faster than full scans - samples first 50 objects per prefix
|
||||
*/
|
||||
private getSampleBasedStorageEstimate;
|
||||
/**
|
||||
* Acquire a distributed lock for coordinating operations across multiple instances
|
||||
* @param lockKey The key to lock on
|
||||
* @param ttl Time to live for the lock in milliseconds (default: 30 seconds)
|
||||
* @returns Promise that resolves to true if lock was acquired, false otherwise
|
||||
*/
|
||||
private acquireLock;
|
||||
/**
|
||||
* Release a distributed lock
|
||||
* @param lockKey The key to unlock
|
||||
* @param lockValue The value used when acquiring the lock (for verification)
|
||||
* @returns Promise that resolves when lock is released
|
||||
*/
|
||||
private releaseLock;
|
||||
/**
|
||||
* Clean up expired locks to prevent lock leakage
|
||||
* This method should be called periodically
|
||||
*/
|
||||
private cleanupExpiredLocks;
|
||||
/**
|
||||
* Get nouns with pagination support
|
||||
* @param options Pagination options
|
||||
* @returns Promise that resolves to a paginated result of nouns
|
||||
*/
|
||||
getNounsWithPagination(options?: {
|
||||
limit?: number;
|
||||
cursor?: string;
|
||||
filter?: {
|
||||
nounType?: string | string[];
|
||||
service?: string | string[];
|
||||
metadata?: Record<string, any>;
|
||||
};
|
||||
}): Promise<{
|
||||
items: HNSWNoun[];
|
||||
totalCount?: number;
|
||||
hasMore: boolean;
|
||||
nextCursor?: string;
|
||||
}>;
|
||||
}
|
||||
2648
dist/storage/adapters/s3CompatibleStorage.js
vendored
Normal file
2648
dist/storage/adapters/s3CompatibleStorage.js
vendored
Normal file
File diff suppressed because it is too large
Load diff
1
dist/storage/adapters/s3CompatibleStorage.js.map
vendored
Normal file
1
dist/storage/adapters/s3CompatibleStorage.js.map
vendored
Normal file
File diff suppressed because one or more lines are too long
Loading…
Add table
Add a link
Reference in a new issue