diff --git a/src/brainyData.ts b/src/brainyData.ts index e2bff7b0..53c9510c 100644 --- a/src/brainyData.ts +++ b/src/brainyData.ts @@ -34,6 +34,41 @@ export interface BrainyDataConfig { */ storageAdapter?: StorageAdapter + /** + * Storage configuration options + * These will be passed to createStorage if storageAdapter is not provided + */ + storage?: { + requestPersistentStorage?: boolean; + r2Storage?: { + bucketName?: string; + accountId?: string; + accessKeyId?: string; + secretAccessKey?: string; + }; + s3Storage?: { + bucketName?: string; + accessKeyId?: string; + secretAccessKey?: string; + region?: string; + }; + gcsStorage?: { + bucketName?: string; + accessKeyId?: string; + secretAccessKey?: string; + endpoint?: string; + }; + customS3Storage?: { + bucketName?: string; + accessKeyId?: string; + secretAccessKey?: string; + endpoint?: string; + region?: string; + }; + forceFileSystemStorage?: boolean; + forceMemoryStorage?: boolean; + } + /** * Embedding function to convert data to vectors */ @@ -42,8 +77,15 @@ export interface BrainyDataConfig { /** * Request persistent storage when running in a browser * This will prompt the user for permission to use persistent storage + * @deprecated Use storage.requestPersistentStorage instead */ requestPersistentStorage?: boolean + + /** + * Set the database to read-only mode + * When true, all write operations will throw an error + */ + readOnly?: boolean } export class BrainyData { @@ -52,6 +94,8 @@ export class BrainyData { private isInitialized = false private embeddingFunction: EmbeddingFunction private requestPersistentStorage: boolean + private readOnly: boolean + private storageConfig: BrainyDataConfig['storage'] = {} /** * Create a new vector database @@ -69,8 +113,27 @@ export class BrainyData { // Set embedding function if provided, otherwise use default this.embeddingFunction = config.embeddingFunction || defaultEmbeddingFunction - // Set persistent storage request flag - this.requestPersistentStorage = config.requestPersistentStorage || false + // Set persistent storage request flag (support both new and deprecated options) + this.requestPersistentStorage = + (config.storage?.requestPersistentStorage !== undefined) + ? config.storage.requestPersistentStorage + : (config.requestPersistentStorage || false) + + // Set read-only flag + this.readOnly = config.readOnly || false + + // Store storage configuration for later use in init() + this.storageConfig = config.storage || {} + } + + /** + * Check if the database is in read-only mode and throw an error if it is + * @throws Error if the database is in read-only mode + */ + private checkReadOnly(): void { + if (this.readOnly) { + throw new Error('Cannot perform write operation: database is in read-only mode') + } } /** @@ -85,9 +148,13 @@ export class BrainyData { try { // Initialize storage if not provided in constructor if (!this.storage) { - this.storage = await createStorage({ + // Combine storage config with requestPersistentStorage for backward compatibility + const storageOptions = { + ...this.storageConfig, requestPersistentStorage: this.requestPersistentStorage - }) + }; + + this.storage = await createStorage(storageOptions); } // Initialize storage @@ -130,6 +197,9 @@ export class BrainyData { ): Promise { await this.ensureInitialized() + // Check if database is in read-only mode + this.checkReadOnly() + try { let vector: Vector @@ -200,6 +270,9 @@ export class BrainyData { ): Promise { await this.ensureInitialized() + // Check if database is in read-only mode + this.checkReadOnly() + const ids: string[] = [] try { @@ -395,6 +468,9 @@ export class BrainyData { public async delete(id: string): Promise { await this.ensureInitialized() + // Check if database is in read-only mode + this.checkReadOnly() + try { // Remove from index const removed = this.index.removeItem(id) @@ -425,6 +501,9 @@ export class BrainyData { public async updateMetadata(id: string, metadata: T): Promise { await this.ensureInitialized() + // Check if database is in read-only mode + this.checkReadOnly() + try { // Check if a vector exists const node = this.index.getNodes().get(id) @@ -457,6 +536,9 @@ export class BrainyData { ): Promise { await this.ensureInitialized() + // Check if database is in read-only mode + this.checkReadOnly() + try { // Check if source and target nodes exist const sourceNode = this.index.getNodes().get(sourceId) @@ -591,6 +673,9 @@ export class BrainyData { public async deleteEdge(id: string): Promise { await this.ensureInitialized() + // Check if database is in read-only mode + this.checkReadOnly() + try { // Remove from index const removed = this.index.removeItem(id) @@ -614,6 +699,9 @@ export class BrainyData { public async clear(): Promise { await this.ensureInitialized() + // Check if database is in read-only mode + this.checkReadOnly() + try { // Clear index this.index.clear() @@ -633,6 +721,22 @@ export class BrainyData { return this.index.size() } + /** + * Check if the database is in read-only mode + * @returns True if the database is in read-only mode, false otherwise + */ + public isReadOnly(): boolean { + return this.readOnly + } + + /** + * Set the database to read-only mode + * @param readOnly True to set the database to read-only mode, false to allow writes + */ + public setReadOnly(readOnly: boolean): void { + this.readOnly = readOnly + } + /** * Embed text or data into a vector using the same embedding function used by this instance * This allows clients to use the same TensorFlow Universal Sentence Encoder throughout their application diff --git a/src/storage/opfsStorage.ts b/src/storage/opfsStorage.ts index 87d7c92c..8a1ce68d 100644 --- a/src/storage/opfsStorage.ts +++ b/src/storage/opfsStorage.ts @@ -1442,12 +1442,164 @@ export class MemoryStorage implements StorageAdapter { * @returns Promise that resolves to a StorageAdapter instance */ export async function createStorage(options: { - requestPersistentStorage?: boolean + requestPersistentStorage?: boolean; + r2Storage?: { + bucketName?: string; + accountId?: string; + accessKeyId?: string; + secretAccessKey?: string; + }; + s3Storage?: { + bucketName?: string; + accessKeyId?: string; + secretAccessKey?: string; + region?: string; + }; + gcsStorage?: { + bucketName?: string; + accessKeyId?: string; + secretAccessKey?: string; + endpoint?: string; + }; + customS3Storage?: { + bucketName?: string; + accessKeyId?: string; + secretAccessKey?: string; + endpoint?: string; + region?: string; + }; + forceFileSystemStorage?: boolean; + forceMemoryStorage?: boolean; } = {}): Promise { - // Check if we're in a Node.js environment - const isNode = typeof process !== 'undefined' && - process.versions != null && - process.versions.node != null + // Check for environment variables for cloud storage (only in Node.js environment) + const isNode = typeof process !== 'undefined' && + process.versions != null && + process.versions.node != null; + + // Default empty values + const defaultEnvStorage = { + bucketName: undefined, + accountId: undefined, + accessKeyId: undefined, + secretAccessKey: undefined, + region: undefined, + endpoint: undefined + }; + + // Only try to access process.env in Node.js environment + const envR2Storage = isNode ? { + bucketName: process.env.R2_BUCKET_NAME, + accountId: process.env.R2_ACCOUNT_ID, + accessKeyId: process.env.R2_ACCESS_KEY_ID, + secretAccessKey: process.env.R2_SECRET_ACCESS_KEY + } : defaultEnvStorage; + + const envS3Storage = isNode ? { + bucketName: process.env.S3_BUCKET_NAME, + accessKeyId: process.env.S3_ACCESS_KEY_ID || process.env.AWS_ACCESS_KEY_ID, + secretAccessKey: process.env.S3_SECRET_ACCESS_KEY || process.env.AWS_SECRET_ACCESS_KEY, + region: process.env.S3_REGION || process.env.AWS_REGION + } : defaultEnvStorage; + + const envGCSStorage = isNode ? { + bucketName: process.env.GCS_BUCKET_NAME, + accessKeyId: process.env.GCS_ACCESS_KEY_ID, + secretAccessKey: process.env.GCS_SECRET_ACCESS_KEY, + endpoint: process.env.GCS_ENDPOINT + } : defaultEnvStorage; + + // Merge environment variables with provided options + const mergedOptions = { + ...options, + r2Storage: options.r2Storage ? { + ...envR2Storage, + ...options.r2Storage + } : ( + // Only use environment variables if they have the required fields + envR2Storage.bucketName && envR2Storage.accessKeyId && envR2Storage.secretAccessKey ? + envR2Storage : undefined + ), + s3Storage: options.s3Storage ? { + ...envS3Storage, + ...options.s3Storage + } : ( + // Only use environment variables if they have the required fields + envS3Storage.bucketName && envS3Storage.accessKeyId && envS3Storage.secretAccessKey ? + envS3Storage : undefined + ), + gcsStorage: options.gcsStorage ? { + ...envGCSStorage, + ...options.gcsStorage + } : ( + // Only use environment variables if they have the required fields + envGCSStorage.bucketName && envGCSStorage.accessKeyId && envGCSStorage.secretAccessKey ? + envGCSStorage : undefined + ) + }; + + // If any S3-compatible storage options are provided, use S3CompatibleStorage + if (mergedOptions.r2Storage || mergedOptions.s3Storage || mergedOptions.gcsStorage || mergedOptions.customS3Storage) { + try { + const s3Module = await import('./s3CompatibleStorage.js') + + if (mergedOptions.r2Storage && + mergedOptions.r2Storage.bucketName && + mergedOptions.r2Storage.accountId && + mergedOptions.r2Storage.accessKeyId && + mergedOptions.r2Storage.secretAccessKey) { + return new s3Module.R2Storage({ + bucketName: mergedOptions.r2Storage.bucketName, + accountId: mergedOptions.r2Storage.accountId, + accessKeyId: mergedOptions.r2Storage.accessKeyId, + secretAccessKey: mergedOptions.r2Storage.secretAccessKey + }) + } else if (mergedOptions.s3Storage && + mergedOptions.s3Storage.bucketName && + mergedOptions.s3Storage.accessKeyId && + mergedOptions.s3Storage.secretAccessKey) { + return new s3Module.S3CompatibleStorage({ + bucketName: mergedOptions.s3Storage.bucketName, + accessKeyId: mergedOptions.s3Storage.accessKeyId, + secretAccessKey: mergedOptions.s3Storage.secretAccessKey, + region: mergedOptions.s3Storage.region, + serviceType: 's3' + }) + } else if (mergedOptions.gcsStorage && + mergedOptions.gcsStorage.bucketName && + mergedOptions.gcsStorage.accessKeyId && + mergedOptions.gcsStorage.secretAccessKey) { + return new s3Module.S3CompatibleStorage({ + bucketName: mergedOptions.gcsStorage.bucketName, + accessKeyId: mergedOptions.gcsStorage.accessKeyId, + secretAccessKey: mergedOptions.gcsStorage.secretAccessKey, + endpoint: mergedOptions.gcsStorage.endpoint, + serviceType: 'gcs' + }) + } else if (mergedOptions.customS3Storage && + mergedOptions.customS3Storage.bucketName && + mergedOptions.customS3Storage.accessKeyId && + mergedOptions.customS3Storage.secretAccessKey) { + return new s3Module.S3CompatibleStorage({ + bucketName: mergedOptions.customS3Storage.bucketName, + accessKeyId: mergedOptions.customS3Storage.accessKeyId, + secretAccessKey: mergedOptions.customS3Storage.secretAccessKey, + endpoint: mergedOptions.customS3Storage.endpoint, + region: mergedOptions.customS3Storage.region, + serviceType: 'custom' + }) + } + } catch (error) { + console.warn('Failed to load S3CompatibleStorage, falling back to default storage:', error) + // Continue to default storage selection + } + } + + // If force memory storage is specified, use MemoryStorage + if (options.forceMemoryStorage) { + return new MemoryStorage() + } + + // Reuse the isNode variable from above if (isNode) { // In Node.js, use FileSystemStorage first, then fall back to memory @@ -1459,31 +1611,33 @@ export async function createStorage(options: { return new MemoryStorage() } } else { - // In browser, try OPFS first - const opfsStorage = new OPFSStorage() + // In browser, try OPFS first (unless force FileSystem is specified) + if (!options.forceFileSystemStorage) { + const opfsStorage = new OPFSStorage() - if (opfsStorage.isOPFSAvailable()) { - // Request persistent storage if specified - if (options.requestPersistentStorage) { - const isPersistentGranted = await opfsStorage.requestPersistentStorage() - if (isPersistentGranted) { - console.log('Persistent storage permission granted') - } else { - console.warn('Persistent storage permission denied') + if (opfsStorage.isOPFSAvailable()) { + // Request persistent storage if specified + if (options.requestPersistentStorage) { + const isPersistentGranted = await opfsStorage.requestPersistentStorage() + if (isPersistentGranted) { + console.log('Persistent storage permission granted') + } else { + console.warn('Persistent storage permission denied') + } } + return opfsStorage } - return opfsStorage - } else { - // OPFS is not available, try to use FileSystem API if available - try { - // Try to load FileSystemStorage for browser environments - // Note: This will likely fail as FileSystemStorage is designed for Node.js - const fileSystemModule = await import('./fileSystemStorage.js') - return new fileSystemModule.FileSystemStorage() - } catch (error) { - console.warn('FileSystem storage is not available, falling back to in-memory storage') - return new MemoryStorage() - } + } + + // OPFS is not available or force FileSystem is specified, try to use FileSystem API if available + try { + // Try to load FileSystemStorage for browser environments + // Note: This will likely fail as FileSystemStorage is designed for Node.js + const fileSystemModule = await import('./fileSystemStorage.js') + return new fileSystemModule.FileSystemStorage() + } catch (error) { + console.warn('FileSystem storage is not available, falling back to in-memory storage') + return new MemoryStorage() } } } diff --git a/src/storage/s3CompatibleStorage.ts b/src/storage/s3CompatibleStorage.ts new file mode 100644 index 00000000..200ea61f --- /dev/null +++ b/src/storage/s3CompatibleStorage.ts @@ -0,0 +1,1037 @@ +import { Edge, HNSWNode, StorageAdapter } from '../coreTypes.js' + +// Constants for S3 bucket prefixes +const NODES_PREFIX = 'nodes/' +const EDGES_PREFIX = 'edges/' +const METADATA_PREFIX = 'metadata/' + +// Constants for noun type prefixes +const PERSON_PREFIX = 'nodes/person/' +const PLACE_PREFIX = 'place/' +const THING_PREFIX = 'thing/' +const EVENT_PREFIX = 'event/' +const CONCEPT_PREFIX = 'concept/' +const CONTENT_PREFIX = 'content/' +const DEFAULT_PREFIX = 'default/' // For nodes without a noun type + +/** + * S3-compatible storage adapter for server environments + * Uses the AWS S3 client to interact with S3-compatible storage services + * including Amazon S3, Cloudflare R2, and Google Cloud Storage + * + * To use this adapter with Cloudflare R2, you need to provide: + * - bucketName: Your bucket name + * - accountId: Your Cloudflare account ID + * - accessKeyId: R2 access key ID + * - secretAccessKey: R2 secret access key + * - serviceType: 'r2' + * + * To use this adapter with Amazon S3, you need to provide: + * - bucketName: Your S3 bucket name + * - accessKeyId: AWS access key ID + * - secretAccessKey: AWS secret access key + * - region: AWS region (e.g., 'us-east-1') + * - serviceType: 's3' + * + * To use this adapter with Google Cloud Storage, you need to provide: + * - bucketName: Your GCS bucket name + * - accessKeyId: HMAC access key + * - secretAccessKey: HMAC secret + * - endpoint: GCS endpoint (e.g., 'https://storage.googleapis.com') + * - serviceType: 'gcs' + * + * For other S3-compatible services, provide: + * - bucketName: Your bucket name + * - accessKeyId: Access key ID + * - secretAccessKey: Secret access key + * - endpoint: Service endpoint URL + * - region: Region (if required) + * - serviceType: 'custom' + */ +export class S3CompatibleStorage implements StorageAdapter { + private bucketName: string + private accessKeyId: string + private secretAccessKey: string + private endpoint?: string + private region?: string + private accountId?: string + private serviceType: 'r2' | 's3' | 'gcs' | 'custom' + private s3Client: any // Will be initialized in init() + private isInitialized = false + + constructor(options: { + bucketName: string + accessKeyId: string + secretAccessKey: string + serviceType: 'r2' | 's3' | 'gcs' | 'custom' + accountId?: string + region?: string + endpoint?: string + }) { + this.bucketName = options.bucketName + this.accessKeyId = options.accessKeyId + this.secretAccessKey = options.secretAccessKey + this.serviceType = options.serviceType + this.accountId = options.accountId + this.region = options.region + this.endpoint = options.endpoint + } + + /** + * Initialize the storage adapter + */ + public async init(): Promise { + if (this.isInitialized) { + return + } + + try { + // Dynamically import the AWS SDK to avoid bundling it in browser environments + try { + const { S3Client } = await import('@aws-sdk/client-s3') + + // Configure the S3 client based on the service type + const clientConfig: any = { + credentials: { + accessKeyId: this.accessKeyId, + secretAccessKey: this.secretAccessKey + } + } + + switch (this.serviceType) { + case 'r2': + if (!this.accountId) { + throw new Error('accountId is required for Cloudflare R2') + } + clientConfig.region = 'auto' // R2 uses 'auto' as the region + clientConfig.endpoint = `https://${this.accountId}.r2.cloudflarestorage.com` + break + + case 's3': + if (!this.region) { + throw new Error('region is required for Amazon S3') + } + clientConfig.region = this.region + // No endpoint needed for standard S3 + break + + case 'gcs': + if (!this.endpoint) { + // Default GCS endpoint if not provided + this.endpoint = 'https://storage.googleapis.com' + } + clientConfig.endpoint = this.endpoint + clientConfig.region = this.region || 'auto' + break + + case 'custom': + if (!this.endpoint) { + throw new Error('endpoint is required for custom S3-compatible services') + } + clientConfig.endpoint = this.endpoint + if (this.region) { + clientConfig.region = this.region + } + break + + default: + throw new Error(`Unsupported service type: ${this.serviceType}`) + } + + // Initialize the S3 client with the configured options + this.s3Client = new S3Client(clientConfig) + + this.isInitialized = true + } catch (importError) { + throw new Error(`Failed to import AWS SDK: ${importError}. Make sure @aws-sdk/client-s3 is installed.`) + } + } catch (error) { + console.error(`Failed to initialize ${this.serviceType} storage:`, error) + throw new Error(`Failed to initialize ${this.serviceType} storage: ${error}`) + } + } + + /** + * Save a node to storage + */ + public async saveNode(node: HNSWNode): Promise { + await this.ensureInitialized() + + try { + // Convert connections Map to a serializable format + const serializableNode = { + ...node, + connections: this.mapToObject(node.connections, (set) => Array.from(set)) + } + + // Get the appropriate prefix based on the node's metadata + const nodePrefix = await this.getNodePrefix(node.id) + + // Import the PutObjectCommand only when needed + const { PutObjectCommand } = await import('@aws-sdk/client-s3') + + // Save the node to S3-compatible storage + await this.s3Client.send( + new PutObjectCommand({ + Bucket: this.bucketName, + Key: `${nodePrefix}${node.id}.json`, + Body: JSON.stringify(serializableNode, null, 2), + ContentType: 'application/json' + }) + ) + } catch (error) { + console.error(`Failed to save node ${node.id}:`, error) + throw new Error(`Failed to save node ${node.id}: ${error}`) + } + } + + /** + * Get a node from storage + */ + public async getNode(id: string): Promise { + await this.ensureInitialized() + + try { + // Get the appropriate prefix based on the node's metadata + const nodePrefix = await this.getNodePrefix(id) + + // Import the GetObjectCommand only when needed + const { GetObjectCommand } = await import('@aws-sdk/client-s3') + + try { + // Try to get the node from S3-compatible storage + const response = await this.s3Client.send( + new GetObjectCommand({ + Bucket: this.bucketName, + Key: `${nodePrefix}${id}.json` + }) + ) + + // Convert the response body to a string + const bodyContents = await response.Body.transformToString() + const parsedNode = JSON.parse(bodyContents) + + // Convert serialized connections back to Map> + const connections = new Map>() + for (const [level, nodeIds] of Object.entries(parsedNode.connections)) { + connections.set(Number(level), new Set(nodeIds as string[])) + } + + return { + id: parsedNode.id, + vector: parsedNode.vector, + connections + } + } catch (error) { + // If the node is not found in the expected prefix, try other prefixes + if (nodePrefix !== DEFAULT_PREFIX) { + // Try the default prefix + try { + const response = await this.s3Client.send( + new GetObjectCommand({ + Bucket: this.bucketName, + Key: `${NODES_PREFIX}${DEFAULT_PREFIX}${id}.json` + }) + ) + + const bodyContents = await response.Body.transformToString() + const parsedNode = JSON.parse(bodyContents) + + // Convert serialized connections back to Map> + const connections = new Map>() + for (const [level, nodeIds] of Object.entries(parsedNode.connections)) { + connections.set(Number(level), new Set(nodeIds as string[])) + } + + return { + id: parsedNode.id, + vector: parsedNode.vector, + connections + } + } catch { + // If not found in default prefix, try all other prefixes + const prefixes = [ + PERSON_PREFIX, + PLACE_PREFIX, + THING_PREFIX, + EVENT_PREFIX, + CONCEPT_PREFIX, + CONTENT_PREFIX + ] + + for (const prefix of prefixes) { + if (prefix === nodePrefix) continue // Skip the already checked prefix + + try { + const response = await this.s3Client.send( + new GetObjectCommand({ + Bucket: this.bucketName, + Key: `${NODES_PREFIX}${prefix}${id}.json` + }) + ) + + const bodyContents = await response.Body.transformToString() + const parsedNode = JSON.parse(bodyContents) + + // Convert serialized connections back to Map> + const connections = new Map>() + for (const [level, nodeIds] of Object.entries(parsedNode.connections)) { + connections.set(Number(level), new Set(nodeIds as string[])) + } + + return { + id: parsedNode.id, + vector: parsedNode.vector, + connections + } + } catch { + // Continue to the next prefix + } + } + } + } + + return null // Node not found in any prefix + } + } catch (error) { + console.error(`Failed to get node ${id}:`, error) + return null + } + } + + /** + * Get nodes by noun type + * @param nounType The noun type to filter by + * @returns Promise that resolves to an array of nodes of the specified noun type + */ + public async getNodesByNounType(nounType: string): Promise { + await this.ensureInitialized() + + try { + // Determine the prefix based on the noun type + let prefix: string + switch (nounType) { + case 'person': + prefix = PERSON_PREFIX + break + case 'place': + prefix = PLACE_PREFIX + break + case 'thing': + prefix = THING_PREFIX + break + case 'event': + prefix = EVENT_PREFIX + break + case 'concept': + prefix = CONCEPT_PREFIX + break + case 'content': + prefix = CONTENT_PREFIX + break + default: + prefix = DEFAULT_PREFIX + } + + // Import the ListObjectsV2Command and GetObjectCommand only when needed + const { ListObjectsV2Command, GetObjectCommand } = await import('@aws-sdk/client-s3') + + // List all objects with the specified prefix + const listResponse = await this.s3Client.send( + new ListObjectsV2Command({ + Bucket: this.bucketName, + Prefix: `${NODES_PREFIX}${prefix}` + }) + ) + + const nodes: HNSWNode[] = [] + + // If there are no objects, return an empty array + if (!listResponse.Contents || listResponse.Contents.length === 0) { + return nodes + } + + // Get each node + const nodePromises = listResponse.Contents.map(async (object: { Key: string }) => { + try { + const response = await this.s3Client.send( + new GetObjectCommand({ + Bucket: this.bucketName, + Key: object.Key + }) + ) + + const bodyContents = await response.Body.transformToString() + const parsedNode = JSON.parse(bodyContents) + + // Convert serialized connections back to Map> + const connections = new Map>() + for (const [level, nodeIds] of Object.entries(parsedNode.connections)) { + connections.set(Number(level), new Set(nodeIds as string[])) + } + + return { + id: parsedNode.id, + vector: parsedNode.vector, + connections + } + } catch (error) { + console.error(`Failed to get node from ${object.Key}:`, error) + return null + } + }) + + const nodeResults = await Promise.all(nodePromises) + return nodeResults.filter((node): node is HNSWNode => node !== null) + } catch (error) { + console.error(`Failed to get nodes for noun type ${nounType}:`, error) + throw new Error(`Failed to get nodes for noun type ${nounType}: ${error}`) + } + } + + /** + * Get all nodes from storage + */ + public async getAllNodes(): Promise { + await this.ensureInitialized() + + try { + // Get all noun types + const nounTypes = [ + 'person', + 'place', + 'thing', + 'event', + 'concept', + 'content', + 'default' + ] + + // Run searches in parallel for all noun types + const nodePromises = nounTypes.map(nounType => this.getNodesByNounType(nounType)) + const nodeArrays = await Promise.all(nodePromises) + + // Combine all results + const allNodes: HNSWNode[] = [] + for (const nodes of nodeArrays) { + allNodes.push(...nodes) + } + + return allNodes + } catch (error) { + console.error('Failed to get all nodes:', error) + throw new Error(`Failed to get all nodes: ${error}`) + } + } + + /** + * Delete a node from storage + */ + public async deleteNode(id: string): Promise { + await this.ensureInitialized() + + try { + // Get the appropriate prefix based on the node's metadata + const nodePrefix = await this.getNodePrefix(id) + + // Import the DeleteObjectCommand only when needed + const { DeleteObjectCommand, GetObjectCommand } = await import('@aws-sdk/client-s3') + + try { + // Check if the node exists before deleting + await this.s3Client.send( + new GetObjectCommand({ + Bucket: this.bucketName, + Key: `${nodePrefix}${id}.json` + }) + ) + + // Delete the node + await this.s3Client.send( + new DeleteObjectCommand({ + Bucket: this.bucketName, + Key: `${nodePrefix}${id}.json` + }) + ) + return // Node found and deleted + } catch { + // If the node is not found in the expected prefix, try other prefixes + if (nodePrefix !== DEFAULT_PREFIX) { + try { + // Try the default prefix + await this.s3Client.send( + new GetObjectCommand({ + Bucket: this.bucketName, + Key: `${NODES_PREFIX}${DEFAULT_PREFIX}${id}.json` + }) + ) + + // Delete the node + await this.s3Client.send( + new DeleteObjectCommand({ + Bucket: this.bucketName, + Key: `${NODES_PREFIX}${DEFAULT_PREFIX}${id}.json` + }) + ) + return // Node found and deleted + } catch { + // If not found in default prefix, try all other prefixes + const prefixes = [ + PERSON_PREFIX, + PLACE_PREFIX, + THING_PREFIX, + EVENT_PREFIX, + CONCEPT_PREFIX, + CONTENT_PREFIX + ] + + for (const prefix of prefixes) { + if (prefix === nodePrefix) continue // Skip the already checked prefix + + try { + await this.s3Client.send( + new GetObjectCommand({ + Bucket: this.bucketName, + Key: `${NODES_PREFIX}${prefix}${id}.json` + }) + ) + + // Delete the node + await this.s3Client.send( + new DeleteObjectCommand({ + Bucket: this.bucketName, + Key: `${NODES_PREFIX}${prefix}${id}.json` + }) + ) + return // Node found and deleted + } catch { + // Continue to the next prefix + } + } + } + } + + return // Node not found in any prefix, nothing to delete + } + } catch (error) { + console.error(`Failed to delete node ${id}:`, error) + throw new Error(`Failed to delete node ${id}: ${error}`) + } + } + + /** + * Save an edge to storage + */ + public async saveEdge(edge: Edge): Promise { + await this.ensureInitialized() + + try { + // Convert connections Map to a serializable format + const serializableEdge = { + ...edge, + connections: this.mapToObject(edge.connections, (set) => Array.from(set)) + } + + // Import the PutObjectCommand only when needed + const { PutObjectCommand } = await import('@aws-sdk/client-s3') + + // Save the edge to S3-compatible storage + await this.s3Client.send( + new PutObjectCommand({ + Bucket: this.bucketName, + Key: `${EDGES_PREFIX}${edge.id}.json`, + Body: JSON.stringify(serializableEdge, null, 2), + ContentType: 'application/json' + }) + ) + } catch (error) { + console.error(`Failed to save edge ${edge.id}:`, error) + throw new Error(`Failed to save edge ${edge.id}: ${error}`) + } + } + + /** + * Get an edge from storage + */ + public async getEdge(id: string): Promise { + await this.ensureInitialized() + + try { + // Import the GetObjectCommand only when needed + const { GetObjectCommand } = await import('@aws-sdk/client-s3') + + try { + // Try to get the edge from S3-compatible storage + const response = await this.s3Client.send( + new GetObjectCommand({ + Bucket: this.bucketName, + Key: `${EDGES_PREFIX}${id}.json` + }) + ) + + // Convert the response body to a string + const bodyContents = await response.Body.transformToString() + const parsedEdge = JSON.parse(bodyContents) + + // Convert serialized connections back to Map> + const connections = new Map>() + for (const [level, nodeIds] of Object.entries(parsedEdge.connections)) { + connections.set(Number(level), new Set(nodeIds as string[])) + } + + return { + id: parsedEdge.id, + vector: parsedEdge.vector, + connections, + sourceId: parsedEdge.sourceId, + targetId: parsedEdge.targetId, + type: parsedEdge.type, + weight: parsedEdge.weight, + metadata: parsedEdge.metadata + } + } catch { + return null // Edge not found + } + } catch (error) { + console.error(`Failed to get edge ${id}:`, error) + return null + } + } + + /** + * Get all edges from storage + */ + public async getAllEdges(): Promise { + await this.ensureInitialized() + + try { + // Import the ListObjectsV2Command and GetObjectCommand only when needed + const { ListObjectsV2Command, GetObjectCommand } = await import('@aws-sdk/client-s3') + + // List all objects with the edges prefix + const listResponse = await this.s3Client.send( + new ListObjectsV2Command({ + Bucket: this.bucketName, + Prefix: EDGES_PREFIX + }) + ) + + const edges: Edge[] = [] + + // If there are no objects, return an empty array + if (!listResponse.Contents || listResponse.Contents.length === 0) { + return edges + } + + // Get each edge + const edgePromises = listResponse.Contents.map(async (object: { Key: string }) => { + try { + const response = await this.s3Client.send( + new GetObjectCommand({ + Bucket: this.bucketName, + Key: object.Key + }) + ) + + const bodyContents = await response.Body.transformToString() + const parsedEdge = JSON.parse(bodyContents) + + // Convert serialized connections back to Map> + const connections = new Map>() + for (const [level, nodeIds] of Object.entries(parsedEdge.connections)) { + connections.set(Number(level), new Set(nodeIds as string[])) + } + + return { + id: parsedEdge.id, + vector: parsedEdge.vector, + connections, + sourceId: parsedEdge.sourceId, + targetId: parsedEdge.targetId, + type: parsedEdge.type, + weight: parsedEdge.weight, + metadata: parsedEdge.metadata + } + } catch (error) { + console.error(`Failed to get edge from ${object.Key}:`, error) + return null + } + }) + + const edgeResults = await Promise.all(edgePromises) + return edgeResults.filter((edge): edge is Edge => edge !== null) + } catch (error) { + console.error('Failed to get all edges:', error) + throw new Error(`Failed to get all edges: ${error}`) + } + } + + /** + * Get edges by source node ID + */ + public async getEdgesBySource(sourceId: string): Promise { + await this.ensureInitialized() + + try { + const allEdges = await this.getAllEdges() + return allEdges.filter(edge => edge.sourceId === sourceId) + } catch (error) { + console.error(`Failed to get edges by source ${sourceId}:`, error) + throw new Error(`Failed to get edges by source ${sourceId}: ${error}`) + } + } + + /** + * Get edges by target node ID + */ + public async getEdgesByTarget(targetId: string): Promise { + await this.ensureInitialized() + + try { + const allEdges = await this.getAllEdges() + return allEdges.filter(edge => edge.targetId === targetId) + } catch (error) { + console.error(`Failed to get edges by target ${targetId}:`, error) + throw new Error(`Failed to get edges by target ${targetId}: ${error}`) + } + } + + /** + * Get edges by type + */ + public async getEdgesByType(type: string): Promise { + await this.ensureInitialized() + + try { + const allEdges = await this.getAllEdges() + return allEdges.filter(edge => edge.type === type) + } catch (error) { + console.error(`Failed to get edges by type ${type}:`, error) + throw new Error(`Failed to get edges by type ${type}: ${error}`) + } + } + + /** + * Delete an edge from storage + */ + public async deleteEdge(id: string): Promise { + await this.ensureInitialized() + + try { + // Import the DeleteObjectCommand and GetObjectCommand only when needed + const { DeleteObjectCommand, GetObjectCommand } = await import('@aws-sdk/client-s3') + + try { + // Check if the edge exists before deleting + await this.s3Client.send( + new GetObjectCommand({ + Bucket: this.bucketName, + Key: `${EDGES_PREFIX}${id}.json` + }) + ) + + // Delete the edge + await this.s3Client.send( + new DeleteObjectCommand({ + Bucket: this.bucketName, + Key: `${EDGES_PREFIX}${id}.json` + }) + ) + } catch { + // Edge not found, nothing to delete + return + } + } catch (error) { + console.error(`Failed to delete edge ${id}:`, error) + throw new Error(`Failed to delete edge ${id}: ${error}`) + } + } + + /** + * Save metadata to storage + */ + public async saveMetadata(id: string, metadata: any): Promise { + await this.ensureInitialized() + + try { + // Import the PutObjectCommand only when needed + const { PutObjectCommand } = await import('@aws-sdk/client-s3') + + // Save the metadata to S3-compatible storage + await this.s3Client.send( + new PutObjectCommand({ + Bucket: this.bucketName, + Key: `${METADATA_PREFIX}${id}.json`, + Body: JSON.stringify(metadata, null, 2), + ContentType: 'application/json' + }) + ) + } catch (error) { + console.error(`Failed to save metadata for ${id}:`, error) + throw new Error(`Failed to save metadata for ${id}: ${error}`) + } + } + + /** + * Get metadata from storage + */ + public async getMetadata(id: string): Promise { + await this.ensureInitialized() + + try { + // Import the GetObjectCommand only when needed + const { GetObjectCommand } = await import('@aws-sdk/client-s3') + + try { + // Try to get the metadata from S3-compatible storage + const response = await this.s3Client.send( + new GetObjectCommand({ + Bucket: this.bucketName, + Key: `${METADATA_PREFIX}${id}.json` + }) + ) + + // Convert the response body to a string + const bodyContents = await response.Body.transformToString() + return JSON.parse(bodyContents) + } catch { + return null // Metadata not found + } + } catch (error) { + console.error(`Failed to get metadata for ${id}:`, error) + return null + } + } + + /** + * Clear all data from storage + */ + public async clear(): Promise { + await this.ensureInitialized() + + try { + // Import the ListObjectsV2Command and DeleteObjectCommand only when needed + const { ListObjectsV2Command, DeleteObjectCommand } = await import('@aws-sdk/client-s3') + + // List all objects in the bucket + const listResponse = await this.s3Client.send( + new ListObjectsV2Command({ + Bucket: this.bucketName + }) + ) + + // If there are no objects, return + if (!listResponse.Contents || listResponse.Contents.length === 0) { + return + } + + // Delete each object + const deletePromises = listResponse.Contents.map(async (object: { Key: string }) => { + try { + await this.s3Client.send( + new DeleteObjectCommand({ + Bucket: this.bucketName, + Key: object.Key + }) + ) + } catch (error) { + console.error(`Failed to delete object ${object.Key}:`, error) + } + }) + + await Promise.all(deletePromises) + } catch (error) { + console.error('Failed to clear storage:', error) + throw new Error(`Failed to clear storage: ${error}`) + } + } + + /** + * Get information about storage usage and capacity + */ + public async getStorageStatus(): Promise<{ + type: string; + used: number; + quota: number | null; + details?: Record; + }> { + await this.ensureInitialized() + + try { + // Import the ListObjectsV2Command only when needed + const { ListObjectsV2Command } = await import('@aws-sdk/client-s3') + + // List all objects in the bucket to calculate total size + const listResponse = await this.s3Client.send( + new ListObjectsV2Command({ + Bucket: this.bucketName + }) + ) + + let totalSize = 0 + let nodeCount = 0 + let edgeCount = 0 + let metadataCount = 0 + + // Calculate total size and counts + if (listResponse.Contents) { + for (const object of listResponse.Contents) { + totalSize += object.Size || 0 + + const key = object.Key || '' + if (key.startsWith(NODES_PREFIX)) { + nodeCount++ + } else if (key.startsWith(EDGES_PREFIX)) { + edgeCount++ + } else if (key.startsWith(METADATA_PREFIX)) { + metadataCount++ + } + } + } + + // Count nodes by noun type + const nounTypeCounts: Record = { + person: 0, + place: 0, + thing: 0, + event: 0, + concept: 0, + content: 0, + default: 0 + } + + // List objects for each noun type prefix + const nounTypes = [ + { type: 'person', prefix: PERSON_PREFIX }, + { type: 'place', prefix: PLACE_PREFIX }, + { type: 'thing', prefix: THING_PREFIX }, + { type: 'event', prefix: EVENT_PREFIX }, + { type: 'concept', prefix: CONCEPT_PREFIX }, + { type: 'content', prefix: CONTENT_PREFIX }, + { type: 'default', prefix: DEFAULT_PREFIX } + ] + + for (const { type, prefix } of nounTypes) { + const listResponse = await this.s3Client.send( + new ListObjectsV2Command({ + Bucket: this.bucketName, + Prefix: `${NODES_PREFIX}${prefix}` + }) + ) + + nounTypeCounts[type] = listResponse.Contents?.length || 0 + } + + return { + type: this.serviceType, + used: totalSize, + quota: null, // S3-compatible services typically don't provide quota information through the API + details: { + nodeCount, + edgeCount, + metadataCount, + nounTypes: { + person: { count: nounTypeCounts.person }, + place: { count: nounTypeCounts.place }, + thing: { count: nounTypeCounts.thing }, + event: { count: nounTypeCounts.event }, + concept: { count: nounTypeCounts.concept }, + content: { count: nounTypeCounts.content }, + default: { count: nounTypeCounts.default } + } + } + } + } catch (error) { + console.error('Failed to get storage status:', error) + return { + type: this.serviceType, + used: 0, + quota: null, + details: { error: String(error) } + } + } + } + + /** + * Ensure the storage adapter is initialized + */ + private async ensureInitialized(): Promise { + if (!this.isInitialized) { + await this.init() + } + } + + /** + * Get the appropriate prefix for a node based on its metadata + */ + private async getNodePrefix(id: string): Promise { + try { + // Try to get the metadata for the node + const metadata = await this.getMetadata(id) + + // If metadata exists and has a noun field, use the corresponding prefix + if (metadata && metadata.noun) { + switch (metadata.noun) { + case 'person': + return PERSON_PREFIX + case 'place': + return PLACE_PREFIX + case 'thing': + return THING_PREFIX + case 'event': + return EVENT_PREFIX + case 'concept': + return CONCEPT_PREFIX + case 'content': + return CONTENT_PREFIX + default: + return DEFAULT_PREFIX + } + } + + // If no metadata or no noun field, use the default prefix + return DEFAULT_PREFIX + } catch (error) { + // If there's an error getting the metadata, use the default prefix + return DEFAULT_PREFIX + } + } + + /** + * Convert a Map to a plain object for serialization + */ + private mapToObject( + map: Map, + valueTransformer: (value: V) => any = (v) => v + ): Record { + const obj: Record = {} + for (const [key, value] of map.entries()) { + obj[key.toString()] = valueTransformer(value) + } + return obj + } +} + +/** + * Cloudflare R2 storage adapter for server environments + * This class is maintained for backward compatibility + * It is recommended to use S3CompatibleStorage directly for new code + * + * @deprecated Use S3CompatibleStorage with serviceType: 'r2' instead + */ +export class R2Storage extends S3CompatibleStorage { + constructor(options: { + bucketName: string + accountId: string + accessKeyId: string + secretAccessKey: string + }) { + super({ + bucketName: options.bucketName, + accessKeyId: options.accessKeyId, + secretAccessKey: options.secretAccessKey, + accountId: options.accountId, + serviceType: 'r2' + }); + } +}