diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts new file mode 100644 index 00000000..2ee1fccb --- /dev/null +++ b/src/storage/adapters/fileSystemStorage.ts @@ -0,0 +1,555 @@ +/** + * File System Storage Adapter + * File system storage adapter for Node.js environments + */ + +import { GraphVerb, HNSWNoun } from '../../coreTypes.js' +import { BaseStorage, NOUNS_DIR, VERBS_DIR, METADATA_DIR, INDEX_DIR } from '../baseStorage.js' + +// Type aliases for better readability +type HNSWNode = HNSWNoun +type Edge = GraphVerb + +// Node.js modules - dynamically imported to avoid issues in browser environments +let fs: any +let path: any + +// Try to load Node.js modules +try { + // Using dynamic imports to avoid issues in browser environments + const fsPromise = import('fs') + const pathPromise = import('path') + + Promise.all([fsPromise, pathPromise]).then(([fsModule, pathModule]) => { + fs = fsModule + path = pathModule.default + }).catch(error => { + console.error('Failed to load Node.js modules:', error) + }) +} catch (error) { + console.error( + 'FileSystemStorage: Failed to load Node.js modules. This adapter is not supported in this environment.', + error + ) +} + +/** + * File system storage adapter for Node.js environments + * Uses the file system to store data in the specified directory structure + */ +export class FileSystemStorage extends BaseStorage { + private rootDir: string + private nounsDir: string + private verbsDir: string + private metadataDir: string + private indexDir: string + + /** + * Initialize the storage adapter + * @param rootDirectory The root directory for storage + */ + constructor(rootDirectory: string) { + super() + this.rootDir = rootDirectory + this.nounsDir = path.join(this.rootDir, NOUNS_DIR) + this.verbsDir = path.join(this.rootDir, VERBS_DIR) + this.metadataDir = path.join(this.rootDir, METADATA_DIR) + this.indexDir = path.join(this.rootDir, INDEX_DIR) + } + + /** + * Initialize the storage adapter + */ + public async init(): Promise { + if (this.isInitialized) { + return + } + + // Check if Node.js modules are available + if (!fs || !path) { + throw new Error( + 'FileSystemStorage requires a Node.js environment, but `fs` and `path` modules could not be loaded.' + ) + } + + try { + // Create the root directory if it doesn't exist + await this.ensureDirectoryExists(this.rootDir) + + // Create the nouns directory if it doesn't exist + await this.ensureDirectoryExists(this.nounsDir) + + // Create the verbs directory if it doesn't exist + await this.ensureDirectoryExists(this.verbsDir) + + // Create the metadata directory if it doesn't exist + await this.ensureDirectoryExists(this.metadataDir) + + // Create the index directory if it doesn't exist + await this.ensureDirectoryExists(this.indexDir) + + this.isInitialized = true + } catch (error) { + console.error('Error initializing FileSystemStorage:', error) + throw error + } + } + + /** + * Ensure a directory exists, creating it if necessary + */ + private async ensureDirectoryExists(dirPath: string): Promise { + try { + await fs.promises.mkdir(dirPath, { recursive: true }) + } catch (error: any) { + // Ignore EEXIST error, which means the directory already exists + if (error.code !== 'EEXIST') { + throw error + } + } + } + + /** + * Save a node to storage + */ + protected async saveNode(node: HNSWNode): Promise { + await this.ensureInitialized() + + // Convert connections Map to a serializable format + const serializableNode = { + ...node, + connections: this.mapToObject(node.connections, (set) => + Array.from(set as Set) + ) + } + + const filePath = path.join(this.nounsDir, `${node.id}.json`) + await fs.promises.writeFile(filePath, JSON.stringify(serializableNode, null, 2)) + } + + /** + * Get a node from storage + */ + protected async getNode(id: string): Promise { + await this.ensureInitialized() + + const filePath = path.join(this.nounsDir, `${id}.json`) + try { + const data = await fs.promises.readFile(filePath, 'utf-8') + const parsedNode = JSON.parse(data) + + // Convert serialized connections back to Map> + const connections = new Map>() + for (const [level, nodeIds] of Object.entries(parsedNode.connections)) { + connections.set(Number(level), new Set(nodeIds as string[])) + } + + return { + id: parsedNode.id, + vector: parsedNode.vector, + connections + } + } catch (error: any) { + if (error.code !== 'ENOENT') { + console.error(`Error reading node ${id}:`, error) + } + return null + } + } + + /** + * Get all nodes from storage + */ + protected async getAllNodes(): Promise { + await this.ensureInitialized() + + const allNodes: HNSWNode[] = [] + try { + const files = await fs.promises.readdir(this.nounsDir) + for (const file of files) { + if (file.endsWith('.json')) { + const filePath = path.join(this.nounsDir, file) + const data = await fs.promises.readFile(filePath, 'utf-8') + const parsedNode = JSON.parse(data) + + // Convert serialized connections back to Map> + const connections = new Map>() + for (const [level, nodeIds] of Object.entries(parsedNode.connections)) { + connections.set(Number(level), new Set(nodeIds as string[])) + } + + allNodes.push({ + id: parsedNode.id, + vector: parsedNode.vector, + connections + }) + } + } + } catch (error: any) { + if (error.code !== 'ENOENT') { + console.error(`Error reading directory ${this.nounsDir}:`, error) + } + } + return allNodes + } + + /** + * Get nodes by noun type + * @param nounType The noun type to filter by + * @returns Promise that resolves to an array of nodes of the specified noun type + */ + protected async getNodesByNounType(nounType: string): Promise { + await this.ensureInitialized() + + const nouns: HNSWNode[] = [] + try { + const files = await fs.promises.readdir(this.nounsDir) + for (const file of files) { + if (file.endsWith('.json')) { + const filePath = path.join(this.nounsDir, file) + const data = await fs.promises.readFile(filePath, 'utf-8') + const parsedNode = JSON.parse(data) + + // Filter by noun type using metadata + const nodeId = parsedNode.id + const metadata = await this.getMetadata(nodeId) + if (metadata && metadata.noun === nounType) { + // Convert serialized connections back to Map> + const connections = new Map>() + for (const [level, nodeIds] of Object.entries(parsedNode.connections)) { + connections.set(Number(level), new Set(nodeIds as string[])) + } + + nouns.push({ + id: parsedNode.id, + vector: parsedNode.vector, + connections + }) + } + } + } + } catch (error: any) { + if (error.code !== 'ENOENT') { + console.error(`Error reading directory ${this.nounsDir}:`, error) + } + } + + return nouns + } + + /** + * Delete a node from storage + */ + protected async deleteNode(id: string): Promise { + await this.ensureInitialized() + + const filePath = path.join(this.nounsDir, `${id}.json`) + try { + await fs.promises.unlink(filePath) + } catch (error: any) { + if (error.code !== 'ENOENT') { + console.error(`Error deleting node file ${filePath}:`, error) + throw error + } + } + } + + /** + * Save an edge to storage + */ + protected async saveEdge(edge: Edge): Promise { + await this.ensureInitialized() + + // Convert connections Map to a serializable format + const serializableEdge = { + ...edge, + connections: this.mapToObject(edge.connections, (set) => + Array.from(set as Set) + ) + } + + const filePath = path.join(this.verbsDir, `${edge.id}.json`) + await fs.promises.writeFile(filePath, JSON.stringify(serializableEdge, null, 2)) + } + + /** + * Get an edge from storage + */ + protected async getEdge(id: string): Promise { + await this.ensureInitialized() + + const filePath = path.join(this.verbsDir, `${id}.json`) + try { + const data = await fs.promises.readFile(filePath, 'utf-8') + const parsedEdge = JSON.parse(data) + + // Convert serialized connections back to Map> + const connections = new Map>() + for (const [level, nodeIds] of Object.entries(parsedEdge.connections)) { + connections.set(Number(level), new Set(nodeIds as string[])) + } + + return { + id: parsedEdge.id, + vector: parsedEdge.vector, + connections, + sourceId: parsedEdge.sourceId, + targetId: parsedEdge.targetId, + type: parsedEdge.type, + weight: parsedEdge.weight, + metadata: parsedEdge.metadata + } + } catch (error: any) { + if (error.code !== 'ENOENT') { + console.error(`Error reading edge ${id}:`, error) + } + return null + } + } + + /** + * Get all edges from storage + */ + protected async getAllEdges(): Promise { + await this.ensureInitialized() + + const allEdges: Edge[] = [] + try { + const files = await fs.promises.readdir(this.verbsDir) + for (const file of files) { + if (file.endsWith('.json')) { + const filePath = path.join(this.verbsDir, file) + const data = await fs.promises.readFile(filePath, 'utf-8') + const parsedEdge = JSON.parse(data) + + // Convert serialized connections back to Map> + const connections = new Map>() + for (const [level, nodeIds] of Object.entries(parsedEdge.connections)) { + connections.set(Number(level), new Set(nodeIds as string[])) + } + + allEdges.push({ + id: parsedEdge.id, + vector: parsedEdge.vector, + connections, + sourceId: parsedEdge.sourceId, + targetId: parsedEdge.targetId, + type: parsedEdge.type, + weight: parsedEdge.weight, + metadata: parsedEdge.metadata + }) + } + } + } catch (error: any) { + if (error.code !== 'ENOENT') { + console.error(`Error reading directory ${this.verbsDir}:`, error) + } + } + return allEdges + } + + /** + * Get edges by source + */ + protected async getEdgesBySource(sourceId: string): Promise { + const edges = await this.getAllEdges() + return edges.filter((edge) => edge.sourceId === sourceId) + } + + /** + * Get edges by target + */ + protected async getEdgesByTarget(targetId: string): Promise { + const edges = await this.getAllEdges() + return edges.filter((edge) => edge.targetId === targetId) + } + + /** + * Get edges by type + */ + protected async getEdgesByType(type: string): Promise { + const edges = await this.getAllEdges() + return edges.filter((edge) => edge.type === type) + } + + /** + * Delete an edge from storage + */ + protected async deleteEdge(id: string): Promise { + await this.ensureInitialized() + + const filePath = path.join(this.verbsDir, `${id}.json`) + try { + await fs.promises.unlink(filePath) + } catch (error: any) { + if (error.code !== 'ENOENT') { + console.error(`Error deleting edge file ${filePath}:`, error) + throw error + } + } + } + + /** + * Save metadata to storage + */ + public async saveMetadata(id: string, metadata: any): Promise { + await this.ensureInitialized() + + const filePath = path.join(this.metadataDir, `${id}.json`) + await fs.promises.writeFile(filePath, JSON.stringify(metadata, null, 2)) + } + + /** + * Get metadata from storage + */ + public async getMetadata(id: string): Promise { + await this.ensureInitialized() + + const filePath = path.join(this.metadataDir, `${id}.json`) + try { + const data = await fs.promises.readFile(filePath, 'utf-8') + return JSON.parse(data) + } catch (error: any) { + if (error.code !== 'ENOENT') { + console.error(`Error reading metadata ${id}:`, error) + } + return null + } + } + + /** + * Clear all data from storage + */ + public async clear(): Promise { + await this.ensureInitialized() + + // Helper function to remove all files in a directory + const removeDirectoryContents = async (dirPath: string): Promise => { + try { + const files = await fs.promises.readdir(dirPath) + for (const file of files) { + const filePath = path.join(dirPath, file) + const stats = await fs.promises.stat(filePath) + if (stats.isDirectory()) { + await removeDirectoryContents(filePath) + await fs.promises.rmdir(filePath) + } else { + await fs.promises.unlink(filePath) + } + } + } catch (error: any) { + if (error.code !== 'ENOENT') { + console.error(`Error removing directory contents ${dirPath}:`, error) + throw error + } + } + } + + // Remove all files in the nouns directory + await removeDirectoryContents(this.nounsDir) + + // Remove all files in the verbs directory + await removeDirectoryContents(this.verbsDir) + + // Remove all files in the metadata directory + await removeDirectoryContents(this.metadataDir) + + // Remove all files in the index directory + await removeDirectoryContents(this.indexDir) + } + + /** + * Get information about storage usage and capacity + */ + public async getStorageStatus(): Promise<{ + type: string + used: number + quota: number | null + details?: Record + }> { + await this.ensureInitialized() + + try { + // Calculate the total size of all files in the storage directories + let totalSize = 0 + + // Helper function to calculate directory size + const calculateSize = async (dirPath: string): Promise => { + let size = 0 + try { + const files = await fs.promises.readdir(dirPath) + for (const file of files) { + const filePath = path.join(dirPath, file) + const stats = await fs.promises.stat(filePath) + if (stats.isDirectory()) { + size += await calculateSize(filePath) + } else { + size += stats.size + } + } + } catch (error: any) { + if (error.code !== 'ENOENT') { + console.error(`Error calculating size for directory ${dirPath}:`, error) + } + } + return size + } + + // Calculate size for each directory + const nounsDirSize = await calculateSize(this.nounsDir) + const verbsDirSize = await calculateSize(this.verbsDir) + const metadataDirSize = await calculateSize(this.metadataDir) + const indexDirSize = await calculateSize(this.indexDir) + + totalSize = nounsDirSize + verbsDirSize + metadataDirSize + indexDirSize + + // Count files in each directory + const nounsCount = (await fs.promises.readdir(this.nounsDir)).filter((file: string) => file.endsWith('.json')).length + const verbsCount = (await fs.promises.readdir(this.verbsDir)).filter((file: string) => file.endsWith('.json')).length + const metadataCount = (await fs.promises.readdir(this.metadataDir)).filter((file: string) => file.endsWith('.json')).length + + // Count nouns by type using metadata + const nounTypeCounts: Record = {} + const metadataFiles = await fs.promises.readdir(this.metadataDir) + for (const file of metadataFiles) { + if (file.endsWith('.json')) { + try { + const filePath = path.join(this.metadataDir, file) + const data = await fs.promises.readFile(filePath, 'utf-8') + const metadata = JSON.parse(data) + if (metadata.noun) { + nounTypeCounts[metadata.noun] = (nounTypeCounts[metadata.noun] || 0) + 1 + } + } catch (error) { + console.error(`Error reading metadata file ${file}:`, error) + } + } + } + + return { + type: 'filesystem', + used: totalSize, + quota: null, // File system doesn't provide quota information + details: { + rootDirectory: this.rootDir, + nounsCount, + verbsCount, + metadataCount, + nounsDirSize, + verbsDirSize, + metadataDirSize, + indexDirSize, + nounTypes: nounTypeCounts + } + } + } catch (error) { + console.error('Failed to get storage status:', error) + return { + type: 'filesystem', + used: 0, + quota: null, + details: { error: String(error) } + } + } + } +} \ No newline at end of file diff --git a/src/storage/adapters/memoryStorage.ts b/src/storage/adapters/memoryStorage.ts new file mode 100644 index 00000000..9ba12c55 --- /dev/null +++ b/src/storage/adapters/memoryStorage.ts @@ -0,0 +1,324 @@ +/** + * Memory Storage Adapter + * In-memory storage adapter for environments where persistent storage is not available or needed + */ + +import { GraphVerb, HNSWNoun } from '../../coreTypes.js' +import { BaseStorage } from '../baseStorage.js' + +/** + * Type alias for HNSWNoun to make the code more readable + */ +type HNSWNode = HNSWNoun + +/** + * Type alias for GraphVerb to make the code more readable + */ +type Edge = GraphVerb + +/** + * In-memory storage adapter + * Uses Maps to store data in memory + */ +export class MemoryStorage extends BaseStorage { + // Single map of noun ID to noun + private nouns: Map = new Map() + private verbs: Map = new Map() + private metadata: Map = new Map() + + constructor() { + super() + } + + /** + * Initialize the storage adapter + * Nothing to initialize for in-memory storage + */ + public async init(): Promise { + this.isInitialized = true + } + + /** + * Save a node to storage + */ + protected async saveNode(node: HNSWNode): Promise { + // Create a deep copy to avoid reference issues + const nodeCopy: HNSWNode = { + id: node.id, + vector: [...node.vector], + connections: new Map() + } + + // Copy connections + for (const [level, connections] of node.connections.entries()) { + nodeCopy.connections.set(level, new Set(connections)) + } + + // Save the node directly in the nouns map + this.nouns.set(node.id, nodeCopy) + } + + /** + * Get a node from storage + */ + protected async getNode(id: string): Promise { + // Get the node directly from the nouns map + const node = this.nouns.get(id) + + // If not found, return null + if (!node) { + return null + } + + // Return a deep copy to avoid reference issues + const nodeCopy: HNSWNode = { + id: node.id, + vector: [...node.vector], + connections: new Map() + } + + // Copy connections + for (const [level, connections] of node.connections.entries()) { + nodeCopy.connections.set(level, new Set(connections)) + } + + return nodeCopy + } + + /** + * Get all nodes from storage + */ + protected async getAllNodes(): Promise { + const allNodes: HNSWNode[] = [] + + // Iterate through all nodes in the nouns map + for (const [nodeId, node] of this.nouns.entries()) { + // Return a deep copy to avoid reference issues + const nodeCopy: HNSWNode = { + id: node.id, + vector: [...node.vector], + connections: new Map() + } + + // Copy connections + for (const [level, connections] of node.connections.entries()) { + nodeCopy.connections.set(level, new Set(connections)) + } + + allNodes.push(nodeCopy) + } + + return allNodes + } + + /** + * Get nodes by noun type + * @param nounType The noun type to filter by + * @returns Promise that resolves to an array of nodes of the specified noun type + */ + protected async getNodesByNounType(nounType: string): Promise { + const nodes: HNSWNode[] = [] + + // Iterate through all nodes and filter by noun type using metadata + for (const [nodeId, node] of this.nouns.entries()) { + // Get the metadata to check the noun type + const metadata = await this.getMetadata(nodeId) + + // Include the node if its noun type matches the requested type + if (metadata && metadata.noun === nounType) { + // Return a deep copy to avoid reference issues + const nodeCopy: HNSWNode = { + id: node.id, + vector: [...node.vector], + connections: new Map() + } + + // Copy connections + for (const [level, connections] of node.connections.entries()) { + nodeCopy.connections.set(level, new Set(connections)) + } + + nodes.push(nodeCopy) + } + } + + return nodes + } + + /** + * Delete a node from storage + */ + protected async deleteNode(id: string): Promise { + // Delete the node directly from the nouns map + this.nouns.delete(id) + } + + /** + * Save an edge to storage + */ + protected async saveEdge(edge: Edge): Promise { + // Create a deep copy to avoid reference issues + const edgeCopy: Edge = { + id: edge.id, + vector: [...edge.vector], + connections: new Map(), + sourceId: edge.sourceId, + targetId: edge.targetId, + type: edge.type, + weight: edge.weight, + metadata: edge.metadata + } + + // Copy connections + for (const [level, connections] of edge.connections.entries()) { + edgeCopy.connections.set(level, new Set(connections)) + } + + // Save the edge directly in the verbs map + this.verbs.set(edge.id, edgeCopy) + } + + /** + * Get an edge from storage + */ + protected async getEdge(id: string): Promise { + // Get the edge directly from the verbs map + const edge = this.verbs.get(id) + + // If not found, return null + if (!edge) { + return null + } + + // Return a deep copy to avoid reference issues + const edgeCopy: Edge = { + id: edge.id, + vector: [...edge.vector], + connections: new Map(), + sourceId: edge.sourceId, + targetId: edge.targetId, + type: edge.type, + weight: edge.weight, + metadata: edge.metadata + } + + // Copy connections + for (const [level, connections] of edge.connections.entries()) { + edgeCopy.connections.set(level, new Set(connections)) + } + + return edgeCopy + } + + /** + * Get all edges from storage + */ + protected async getAllEdges(): Promise { + const allEdges: Edge[] = [] + + // Iterate through all edges in the verbs map + for (const [edgeId, edge] of this.verbs.entries()) { + // Return a deep copy to avoid reference issues + const edgeCopy: Edge = { + id: edge.id, + vector: [...edge.vector], + connections: new Map(), + sourceId: edge.sourceId, + targetId: edge.targetId, + type: edge.type, + weight: edge.weight, + metadata: edge.metadata + } + + // Copy connections + for (const [level, connections] of edge.connections.entries()) { + edgeCopy.connections.set(level, new Set(connections)) + } + + allEdges.push(edgeCopy) + } + + return allEdges + } + + /** + * Get edges by source + */ + protected async getEdgesBySource(sourceId: string): Promise { + const edges = await this.getAllEdges() + return edges.filter((edge) => edge.sourceId === sourceId) + } + + /** + * Get edges by target + */ + protected async getEdgesByTarget(targetId: string): Promise { + const edges = await this.getAllEdges() + return edges.filter((edge) => edge.targetId === targetId) + } + + /** + * Get edges by type + */ + protected async getEdgesByType(type: string): Promise { + const edges = await this.getAllEdges() + return edges.filter((edge) => edge.type === type) + } + + /** + * Delete an edge from storage + */ + protected async deleteEdge(id: string): Promise { + // Delete the edge directly from the verbs map + this.verbs.delete(id) + } + + /** + * Save metadata to storage + */ + public async saveMetadata(id: string, metadata: any): Promise { + this.metadata.set(id, JSON.parse(JSON.stringify(metadata))) + } + + /** + * Get metadata from storage + */ + public async getMetadata(id: string): Promise { + const metadata = this.metadata.get(id) + if (!metadata) { + return null + } + + return JSON.parse(JSON.stringify(metadata)) + } + + /** + * Clear all data from storage + */ + public async clear(): Promise { + this.nouns.clear() + this.verbs.clear() + this.metadata.clear() + } + + /** + * Get information about storage usage and capacity + */ + public async getStorageStatus(): Promise<{ + type: string + used: number + quota: number | null + details?: Record + }> { + return { + type: 'memory', + used: 0, // In-memory storage doesn't have a meaningful size + quota: null, // In-memory storage doesn't have a quota + details: { + nodeCount: this.nouns.size, + edgeCount: this.verbs.size, + metadataCount: this.metadata.size + } + } + } +} \ No newline at end of file diff --git a/src/storage/adapters/opfsStorage.ts b/src/storage/adapters/opfsStorage.ts new file mode 100644 index 00000000..d9037576 --- /dev/null +++ b/src/storage/adapters/opfsStorage.ts @@ -0,0 +1,691 @@ +/** + * OPFS (Origin Private File System) Storage Adapter + * Provides persistent storage for the vector database using the Origin Private File System API + */ + +import {GraphVerb, HNSWNoun} from '../../coreTypes.js' +import {BaseStorage, NOUNS_DIR, VERBS_DIR, METADATA_DIR, INDEX_DIR} from '../baseStorage.js' +import '../../types/fileSystemTypes.js' + +/** + * Helper function to safely get a file from a FileSystemHandle + * This is needed because TypeScript doesn't recognize that a FileSystemHandle + * can be a FileSystemFileHandle which has the getFile method + */ +async function safeGetFile(handle: FileSystemHandle): Promise { + // Type cast to any to avoid TypeScript error + return (handle as any).getFile() +} + +// Type aliases for better readability +type HNSWNode = HNSWNoun +type Edge = GraphVerb + +// Root directory name for OPFS storage +const ROOT_DIR = 'opfs-vector-db' + +/** + * OPFS storage adapter for browser environments + * Uses the Origin Private File System API to store data persistently + */ +export class OPFSStorage extends BaseStorage { + private rootDir: FileSystemDirectoryHandle | null = null + private nounsDir: FileSystemDirectoryHandle | null = null + private verbsDir: FileSystemDirectoryHandle | null = null + private metadataDir: FileSystemDirectoryHandle | null = null + private indexDir: FileSystemDirectoryHandle | null = null + private isAvailable = false + private isPersistentRequested = false + private isPersistentGranted = false + + constructor() { + super() + // Check if OPFS is available + this.isAvailable = + typeof navigator !== 'undefined' && + 'storage' in navigator && + 'getDirectory' in navigator.storage + } + + /** + * Initialize the storage adapter + */ + public async init(): Promise { + if (this.isInitialized) { + return + } + + if (!this.isAvailable) { + throw new Error( + 'Origin Private File System is not available in this environment' + ) + } + + try { + // Get the root directory + const root = await navigator.storage.getDirectory() + + // Create or get our app's root directory + this.rootDir = await root.getDirectoryHandle(ROOT_DIR, {create: true}) + + // Create or get nouns directory + this.nounsDir = await this.rootDir.getDirectoryHandle(NOUNS_DIR, { + create: true + }) + + // Create or get verbs directory + this.verbsDir = await this.rootDir.getDirectoryHandle(VERBS_DIR, { + create: true + }) + + // Create or get metadata directory + this.metadataDir = await this.rootDir.getDirectoryHandle(METADATA_DIR, { + create: true + }) + + // Create or get index directory + this.indexDir = await this.rootDir.getDirectoryHandle(INDEX_DIR, { + create: true + }) + + this.isInitialized = true + } catch (error) { + console.error('Failed to initialize OPFS storage:', error) + throw new Error(`Failed to initialize OPFS storage: ${error}`) + } + } + + /** + * Check if OPFS is available in the current environment + */ + public isOPFSAvailable(): boolean { + return this.isAvailable + } + + /** + * Request persistent storage permission from the user + * @returns Promise that resolves to true if permission was granted, false otherwise + */ + public async requestPersistentStorage(): Promise { + if (!this.isAvailable) { + console.warn('Cannot request persistent storage: OPFS is not available') + return false + } + + try { + // Check if persistence is already granted + this.isPersistentGranted = await navigator.storage.persisted() + + if (!this.isPersistentGranted) { + // Request permission for persistent storage + this.isPersistentGranted = await navigator.storage.persist() + } + + this.isPersistentRequested = true + return this.isPersistentGranted + } catch (error) { + console.warn('Failed to request persistent storage:', error) + return false + } + } + + /** + * Check if persistent storage is granted + * @returns Promise that resolves to true if persistent storage is granted, false otherwise + */ + public async isPersistent(): Promise { + if (!this.isAvailable) { + return false + } + + try { + this.isPersistentGranted = await navigator.storage.persisted() + return this.isPersistentGranted + } catch (error) { + console.warn('Failed to check persistent storage status:', error) + return false + } + } + + /** + * Save a node to storage + */ + protected async saveNode(node: HNSWNode): Promise { + await this.ensureInitialized() + + try { + // Convert connections Map to a serializable format + const serializableNode = { + ...node, + connections: this.mapToObject(node.connections, (set) => + Array.from(set as Set) + ) + } + + // Create or get the file for this noun + const fileHandle = await this.nounsDir!.getFileHandle(node.id, { + create: true + }) + + // Write the noun data to the file + const writable = await fileHandle.createWritable() + await writable.write(JSON.stringify(serializableNode)) + await writable.close() + } catch (error) { + console.error(`Failed to save node ${node.id}:`, error) + throw new Error(`Failed to save node ${node.id}: ${error}`) + } + } + + /** + * Get a node from storage + */ + protected async getNode(id: string): Promise { + await this.ensureInitialized() + + try { + // Get the file handle for this node + const fileHandle = await this.nounsDir!.getFileHandle(id) + + // Read the node data from the file + const file = await fileHandle.getFile() + const text = await file.text() + const data = JSON.parse(text) + + // Convert serialized connections back to Map> + const connections = new Map>() + for (const [level, nodeIds] of Object.entries(data.connections)) { + connections.set(Number(level), new Set(nodeIds as string[])) + } + + return { + id: data.id, + vector: data.vector, + connections + } + } catch (error) { + // Node not found or other error + return null + } + } + + /** + * Get all nodes from storage + */ + protected async getAllNodes(): Promise { + await this.ensureInitialized() + + const allNodes: HNSWNode[] = [] + try { + // Iterate through all files in the nouns directory + for await (const [name, handle] of this.nounsDir!.entries()) { + if (handle.kind === 'file') { + try { + // Read the node data from the file + const file = await safeGetFile(handle) + const text = await file.text() + const data = JSON.parse(text) + + // Convert serialized connections back to Map> + const connections = new Map>() + for (const [level, nodeIds] of Object.entries(data.connections)) { + connections.set(Number(level), new Set(nodeIds as string[])) + } + + allNodes.push({ + id: data.id, + vector: data.vector, + connections + }) + } catch (error) { + console.error(`Error reading node file ${name}:`, error) + } + } + } + } catch (error) { + console.error('Error reading nouns directory:', error) + } + + return allNodes + } + + /** + * Get nodes by noun type + * @param nounType The noun type to filter by + * @returns Promise that resolves to an array of nodes of the specified noun type + */ + protected async getNodesByNounType(nounType: string): Promise { + await this.ensureInitialized() + + const nodes: HNSWNode[] = [] + + try { + // Iterate through all files in the nouns directory + for await (const [name, handle] of this.nounsDir!.entries()) { + if (handle.kind === 'file') { + try { + // Read the node data from the file + const file = await safeGetFile(handle) + const text = await file.text() + const data = JSON.parse(text) + + // Get the metadata to check the noun type + const metadata = await this.getMetadata(data.id) + + // Include the node if its noun type matches the requested type + if (metadata && metadata.noun === nounType) { + // Convert serialized connections back to Map> + const connections = new Map>() + for (const [level, nodeIds] of Object.entries(data.connections)) { + connections.set(Number(level), new Set(nodeIds as string[])) + } + + nodes.push({ + id: data.id, + vector: data.vector, + connections + }) + } + } catch (error) { + console.error(`Error reading node file ${name}:`, error) + } + } + } + } catch (error) { + console.error('Error reading nouns directory:', error) + } + + return nodes + } + + /** + * Delete a node from storage + */ + protected async deleteNode(id: string): Promise { + await this.ensureInitialized() + + try { + await this.nounsDir!.removeEntry(id) + } catch (error: any) { + // Ignore NotFoundError, which means the file doesn't exist + if (error.name !== 'NotFoundError') { + console.error(`Error deleting node ${id}:`, error) + throw error + } + } + } + + /** + * Save an edge to storage + */ + protected async saveEdge(edge: Edge): Promise { + await this.ensureInitialized() + + try { + // Convert connections Map to a serializable format + const serializableEdge = { + ...edge, + connections: this.mapToObject(edge.connections, (set) => + Array.from(set as Set) + ) + } + + // Create or get the file for this verb + const fileHandle = await this.verbsDir!.getFileHandle(edge.id, { + create: true + }) + + // Write the verb data to the file + const writable = await fileHandle.createWritable() + await writable.write(JSON.stringify(serializableEdge)) + await writable.close() + } catch (error) { + console.error(`Failed to save edge ${edge.id}:`, error) + throw new Error(`Failed to save edge ${edge.id}: ${error}`) + } + } + + /** + * Get an edge from storage + */ + protected async getEdge(id: string): Promise { + await this.ensureInitialized() + + try { + // Get the file handle for this edge + const fileHandle = await this.verbsDir!.getFileHandle(id) + + // Read the edge data from the file + const file = await fileHandle.getFile() + const text = await file.text() + const data = JSON.parse(text) + + // Convert serialized connections back to Map> + const connections = new Map>() + for (const [level, nodeIds] of Object.entries(data.connections)) { + connections.set(Number(level), new Set(nodeIds as string[])) + } + + return { + id: data.id, + vector: data.vector, + connections, + sourceId: data.sourceId, + targetId: data.targetId, + type: data.type, + weight: data.weight, + metadata: data.metadata + } + } catch (error) { + // Edge not found or other error + return null + } + } + + /** + * Get all edges from storage + */ + protected async getAllEdges(): Promise { + await this.ensureInitialized() + + const allEdges: Edge[] = [] + try { + // Iterate through all files in the verbs directory + for await (const [name, handle] of this.verbsDir!.entries()) { + if (handle.kind === 'file') { + try { + // Read the edge data from the file + const file = await safeGetFile(handle) + const text = await file.text() + const data = JSON.parse(text) + + // Convert serialized connections back to Map> + const connections = new Map>() + for (const [level, nodeIds] of Object.entries(data.connections)) { + connections.set(Number(level), new Set(nodeIds as string[])) + } + + allEdges.push({ + id: data.id, + vector: data.vector, + connections, + sourceId: data.sourceId, + targetId: data.targetId, + type: data.type, + weight: data.weight, + metadata: data.metadata + }) + } catch (error) { + console.error(`Error reading edge file ${name}:`, error) + } + } + } + } catch (error) { + console.error('Error reading verbs directory:', error) + } + + return allEdges + } + + /** + * Get edges by source + */ + protected async getEdgesBySource(sourceId: string): Promise { + const edges = await this.getAllEdges() + return edges.filter((edge) => edge.sourceId === sourceId) + } + + /** + * Get edges by target + */ + protected async getEdgesByTarget(targetId: string): Promise { + const edges = await this.getAllEdges() + return edges.filter((edge) => edge.targetId === targetId) + } + + /** + * Get edges by type + */ + protected async getEdgesByType(type: string): Promise { + const edges = await this.getAllEdges() + return edges.filter((edge) => edge.type === type) + } + + /** + * Delete an edge from storage + */ + protected async deleteEdge(id: string): Promise { + await this.ensureInitialized() + + try { + await this.verbsDir!.removeEntry(id) + } catch (error: any) { + // Ignore NotFoundError, which means the file doesn't exist + if (error.name !== 'NotFoundError') { + console.error(`Error deleting edge ${id}:`, error) + throw error + } + } + } + + /** + * Save metadata to storage + */ + public async saveMetadata(id: string, metadata: any): Promise { + await this.ensureInitialized() + + try { + // Create or get the file for this metadata + const fileHandle = await this.metadataDir!.getFileHandle(id, { + create: true + }) + + // Write the metadata to the file + const writable = await fileHandle.createWritable() + await writable.write(JSON.stringify(metadata)) + await writable.close() + } catch (error) { + console.error(`Failed to save metadata ${id}:`, error) + throw new Error(`Failed to save metadata ${id}: ${error}`) + } + } + + /** + * Get metadata from storage + */ + public async getMetadata(id: string): Promise { + await this.ensureInitialized() + + try { + // Get the file handle for this metadata + const fileHandle = await this.metadataDir!.getFileHandle(id) + + // Read the metadata from the file + const file = await fileHandle.getFile() + const text = await file.text() + return JSON.parse(text) + } catch (error) { + // Metadata not found or other error + return null + } + } + + /** + * Clear all data from storage + */ + public async clear(): Promise { + await this.ensureInitialized() + + // Helper function to remove all files in a directory + const removeDirectoryContents = async ( + dirHandle: FileSystemDirectoryHandle + ): Promise => { + try { + for await (const [name, handle] of dirHandle.entries()) { + await dirHandle.removeEntry(name) + } + } catch (error) { + console.error(`Error removing directory contents:`, error) + throw error + } + } + + try { + // Remove all files in the nouns directory + await removeDirectoryContents(this.nounsDir!) + + // Remove all files in the verbs directory + await removeDirectoryContents(this.verbsDir!) + + // Remove all files in the metadata directory + await removeDirectoryContents(this.metadataDir!) + + // Remove all files in the index directory + await removeDirectoryContents(this.indexDir!) + } catch (error) { + console.error('Error clearing storage:', error) + throw error + } + } + + /** + * Get information about storage usage and capacity + */ + public async getStorageStatus(): Promise<{ + type: string + used: number + quota: number | null + details?: Record + }> { + await this.ensureInitialized() + + try { + // Calculate the total size of all files in the storage directories + let totalSize = 0 + + // Helper function to calculate directory size + const calculateDirSize = async ( + dirHandle: FileSystemDirectoryHandle + ): Promise => { + let size = 0 + try { + for await (const [name, handle] of dirHandle.entries()) { + if (handle.kind === 'file') { + const file = await (handle as FileSystemFileHandle).getFile() + size += file.size + } else if (handle.kind === 'directory') { + size += await calculateDirSize( + handle as FileSystemDirectoryHandle + ) + } + } + } catch (error) { + console.warn(`Error calculating size for directory:`, error) + } + return size + } + + // Helper function to count files in a directory + const countFilesInDirectory = async ( + dirHandle: FileSystemDirectoryHandle + ): Promise => { + let count = 0 + try { + for await (const [name, handle] of dirHandle.entries()) { + if (handle.kind === 'file') { + count++ + } + } + } catch (error) { + console.warn(`Error counting files in directory:`, error) + } + return count + } + + // Calculate size for each directory + if (this.nounsDir) { + totalSize += await calculateDirSize(this.nounsDir) + } + if (this.verbsDir) { + totalSize += await calculateDirSize(this.verbsDir) + } + if (this.metadataDir) { + totalSize += await calculateDirSize(this.metadataDir) + } + if (this.indexDir) { + totalSize += await calculateDirSize(this.indexDir) + } + + // Get storage quota information using the Storage API + let quota = null + let details: Record = { + isPersistent: await this.isPersistent(), + nounTypes: {} + } + + try { + if (navigator.storage && navigator.storage.estimate) { + const estimate = await navigator.storage.estimate() + quota = estimate.quota || null + details = { + ...details, + usage: estimate.usage, + quota: estimate.quota, + freePercentage: estimate.quota + ? ((estimate.quota - (estimate.usage || 0)) / estimate.quota) * + 100 + : null + } + } + } catch (error) { + console.warn('Unable to get storage estimate:', error) + } + + // Count files in each directory + if (this.nounsDir) { + details.nounsCount = await countFilesInDirectory(this.nounsDir) + } + if (this.verbsDir) { + details.verbsCount = await countFilesInDirectory(this.verbsDir) + } + if (this.metadataDir) { + details.metadataCount = await countFilesInDirectory(this.metadataDir) + } + + // Count nouns by type using metadata + const nounTypeCounts: Record = {} + if (this.metadataDir) { + for await (const [name, handle] of this.metadataDir.entries()) { + if (handle.kind === 'file') { + try { + const file = await safeGetFile(handle) + const text = await file.text() + const metadata = JSON.parse(text) + if (metadata.noun) { + nounTypeCounts[metadata.noun] = (nounTypeCounts[metadata.noun] || 0) + 1 + } + } catch (error) { + console.error(`Error reading metadata file ${name}:`, error) + } + } + } + } + details.nounTypes = nounTypeCounts + + return { + type: 'opfs', + used: totalSize, + quota, + details + } + } catch (error) { + console.error('Failed to get storage status:', error) + return { + type: 'opfs', + used: 0, + quota: null, + details: {error: String(error)} + } + } + } +} \ No newline at end of file diff --git a/src/storage/adapters/s3CompatibleStorage.ts b/src/storage/adapters/s3CompatibleStorage.ts new file mode 100644 index 00000000..1d16b22f --- /dev/null +++ b/src/storage/adapters/s3CompatibleStorage.ts @@ -0,0 +1,776 @@ +/** + * S3-Compatible Storage Adapter + * Uses the AWS S3 client to interact with S3-compatible storage services + * including Amazon S3, Cloudflare R2, and Google Cloud Storage + */ + +import { GraphVerb, HNSWNoun } from '../../coreTypes.js' +import { BaseStorage, NOUNS_DIR, VERBS_DIR, METADATA_DIR, INDEX_DIR } from '../baseStorage.js' + +// Type aliases for better readability +type HNSWNode = HNSWNoun +type Edge = GraphVerb + +// Export R2Storage as an alias for S3CompatibleStorage +export { S3CompatibleStorage as R2Storage } + +// S3 client and command types - dynamically imported to avoid issues in browser environments +type S3Client = any +type S3Command = any + +/** + * S3-compatible storage adapter for server environments + * Uses the AWS S3 client to interact with S3-compatible storage services + * including Amazon S3, Cloudflare R2, and Google Cloud Storage + * + * To use this adapter with Amazon S3, you need to provide: + * - region: AWS region (e.g., 'us-east-1') + * - credentials: AWS credentials (accessKeyId and secretAccessKey) + * - bucketName: S3 bucket name + * + * To use this adapter with Cloudflare R2, you need to provide: + * - accountId: Cloudflare account ID + * - accessKeyId: R2 access key ID + * - secretAccessKey: R2 secret access key + * - bucketName: R2 bucket name + * + * To use this adapter with Google Cloud Storage, you need to provide: + * - region: GCS region (e.g., 'us-central1') + * - credentials: GCS credentials (accessKeyId and secretAccessKey) + * - endpoint: GCS endpoint (e.g., 'https://storage.googleapis.com') + * - bucketName: GCS bucket name + */ +export class S3CompatibleStorage extends BaseStorage { + private s3Client: S3Client | null = null + private bucketName: string + private serviceType: string + private region: string + private endpoint?: string + private accountId?: string + private accessKeyId: string + private secretAccessKey: string + private sessionToken?: string + + // Prefixes for different types of data + private nounPrefix: string + private verbPrefix: string + private metadataPrefix: string + private indexPrefix: string + + /** + * Initialize the storage adapter + * @param options Configuration options for the S3-compatible storage + */ + constructor(options: { + bucketName: string + region?: string + endpoint?: string + accountId?: string + accessKeyId: string + secretAccessKey: string + sessionToken?: string + serviceType?: string + }) { + super() + this.bucketName = options.bucketName + this.region = options.region || 'auto' + this.endpoint = options.endpoint + this.accountId = options.accountId + this.accessKeyId = options.accessKeyId + this.secretAccessKey = options.secretAccessKey + this.sessionToken = options.sessionToken + this.serviceType = options.serviceType || 's3' + + // Set up prefixes for different types of data + this.nounPrefix = `${NOUNS_DIR}/` + this.verbPrefix = `${VERBS_DIR}/` + this.metadataPrefix = `${METADATA_DIR}/` + this.indexPrefix = `${INDEX_DIR}/` + } + + /** + * Initialize the storage adapter + */ + public async init(): Promise { + if (this.isInitialized) { + return + } + + try { + // Import AWS SDK modules only when needed + const { S3Client } = await import('@aws-sdk/client-s3') + + // Configure the S3 client based on the service type + const clientConfig: any = { + region: this.region, + credentials: { + accessKeyId: this.accessKeyId, + secretAccessKey: this.secretAccessKey + } + } + + // Add session token if provided + if (this.sessionToken) { + clientConfig.credentials.sessionToken = this.sessionToken + } + + // Add endpoint if provided (for R2, GCS, etc.) + if (this.endpoint) { + clientConfig.endpoint = this.endpoint + } + + // Special configuration for Cloudflare R2 + if (this.serviceType === 'r2' && this.accountId) { + clientConfig.endpoint = `https://${this.accountId}.r2.cloudflarestorage.com` + } + + // Create the S3 client + this.s3Client = new S3Client(clientConfig) + + // Ensure the bucket exists and is accessible + const { HeadBucketCommand } = await import('@aws-sdk/client-s3') + await this.s3Client.send( + new HeadBucketCommand({ + Bucket: this.bucketName + }) + ) + + this.isInitialized = true + } catch (error) { + console.error(`Failed to initialize ${this.serviceType} storage:`, error) + throw new Error( + `Failed to initialize ${this.serviceType} storage: ${error}` + ) + } + } + + /** + * Save a node to storage + */ + protected async saveNode(node: HNSWNode): Promise { + await this.ensureInitialized() + + try { + // Convert connections Map to a serializable format + const serializableNode = { + ...node, + connections: this.mapToObject(node.connections, (set) => + Array.from(set as Set) + ) + } + + // Import the PutObjectCommand only when needed + const { PutObjectCommand } = await import('@aws-sdk/client-s3') + + // Save the node to S3-compatible storage + await this.s3Client!.send( + new PutObjectCommand({ + Bucket: this.bucketName, + Key: `${this.nounPrefix}${node.id}.json`, + Body: JSON.stringify(serializableNode, null, 2), + ContentType: 'application/json' + }) + ) + } catch (error) { + console.error(`Failed to save node ${node.id}:`, error) + throw new Error(`Failed to save node ${node.id}: ${error}`) + } + } + + /** + * Get a node from storage + */ + protected async getNode(id: string): Promise { + await this.ensureInitialized() + + try { + // Import the GetObjectCommand only when needed + const { GetObjectCommand } = await import('@aws-sdk/client-s3') + + // Try to get the node from the nouns directory + const response = await this.s3Client!.send( + new GetObjectCommand({ + Bucket: this.bucketName, + Key: `${this.nounPrefix}${id}.json` + }) + ) + + // Convert the response body to a string + const bodyContents = await response.Body.transformToString() + const parsedNode = JSON.parse(bodyContents) + + // Convert serialized connections back to Map> + const connections = new Map>() + for (const [level, nodeIds] of Object.entries(parsedNode.connections)) { + connections.set(Number(level), new Set(nodeIds as string[])) + } + + return { + id: parsedNode.id, + vector: parsedNode.vector, + connections + } + } catch (error) { + // Node not found or other error + return null + } + } + + /** + * Get all nodes from storage + */ + protected async getAllNodes(): Promise { + await this.ensureInitialized() + + try { + // Import the ListObjectsV2Command and GetObjectCommand only when needed + const { ListObjectsV2Command, GetObjectCommand } = await import( + '@aws-sdk/client-s3' + ) + + // List all objects in the nouns directory + const listResponse = await this.s3Client!.send( + new ListObjectsV2Command({ + Bucket: this.bucketName, + Prefix: this.nounPrefix + }) + ) + + const nodes: HNSWNode[] = [] + + // If there are no objects, return an empty array + if (!listResponse.Contents || listResponse.Contents.length === 0) { + return nodes + } + + // Get each node + const nodePromises = listResponse.Contents.map( + async (object: { Key: string }) => { + try { + // Extract node ID from the key (remove prefix and .json extension) + const nodeId = object.Key.replace(this.nounPrefix, '').replace('.json', '') + + // Get the node data + const response = await this.s3Client!.send( + new GetObjectCommand({ + Bucket: this.bucketName, + Key: object.Key + }) + ) + + // Convert the response body to a string + const bodyContents = await response.Body.transformToString() + const parsedNode = JSON.parse(bodyContents) + + // Convert serialized connections back to Map> + const connections = new Map>() + for (const [level, nodeIds] of Object.entries(parsedNode.connections)) { + connections.set(Number(level), new Set(nodeIds as string[])) + } + + return { + id: parsedNode.id, + vector: parsedNode.vector, + connections + } + } catch (error) { + console.error(`Error getting node from ${object.Key}:`, error) + return null + } + } + ) + + // Wait for all promises to resolve and filter out nulls + const resolvedNodes = await Promise.all(nodePromises) + return resolvedNodes.filter((node): node is HNSWNode => node !== null) + } catch (error) { + console.error('Failed to get all nodes:', error) + return [] + } + } + + /** + * Get nodes by noun type + * @param nounType The noun type to filter by + * @returns Promise that resolves to an array of nodes of the specified noun type + */ + protected async getNodesByNounType(nounType: string): Promise { + await this.ensureInitialized() + + try { + // Get all nodes + const allNodes = await this.getAllNodes() + + // Filter nodes by noun type using metadata + const filteredNodes: HNSWNode[] = [] + for (const node of allNodes) { + const metadata = await this.getMetadata(node.id) + if (metadata && metadata.noun === nounType) { + filteredNodes.push(node) + } + } + + return filteredNodes + } catch (error) { + console.error(`Failed to get nodes by noun type ${nounType}:`, error) + return [] + } + } + + /** + * Delete a node from storage + */ + protected async deleteNode(id: string): Promise { + await this.ensureInitialized() + + try { + // Import the DeleteObjectCommand only when needed + const { DeleteObjectCommand } = await import('@aws-sdk/client-s3') + + // Delete the node from S3-compatible storage + await this.s3Client!.send( + new DeleteObjectCommand({ + Bucket: this.bucketName, + Key: `${this.nounPrefix}${id}.json` + }) + ) + } catch (error) { + console.error(`Failed to delete node ${id}:`, error) + throw new Error(`Failed to delete node ${id}: ${error}`) + } + } + + /** + * Save an edge to storage + */ + protected async saveEdge(edge: Edge): Promise { + await this.ensureInitialized() + + try { + // Convert connections Map to a serializable format + const serializableEdge = { + ...edge, + connections: this.mapToObject(edge.connections, (set) => + Array.from(set as Set) + ) + } + + // Import the PutObjectCommand only when needed + const { PutObjectCommand } = await import('@aws-sdk/client-s3') + + // Save the edge to S3-compatible storage + await this.s3Client!.send( + new PutObjectCommand({ + Bucket: this.bucketName, + Key: `${this.verbPrefix}${edge.id}.json`, + Body: JSON.stringify(serializableEdge, null, 2), + ContentType: 'application/json' + }) + ) + } catch (error) { + console.error(`Failed to save edge ${edge.id}:`, error) + throw new Error(`Failed to save edge ${edge.id}: ${error}`) + } + } + + /** + * Get an edge from storage + */ + protected async getEdge(id: string): Promise { + await this.ensureInitialized() + + try { + // Import the GetObjectCommand only when needed + const { GetObjectCommand } = await import('@aws-sdk/client-s3') + + // Try to get the edge from the verbs directory + const response = await this.s3Client!.send( + new GetObjectCommand({ + Bucket: this.bucketName, + Key: `${this.verbPrefix}${id}.json` + }) + ) + + // Convert the response body to a string + const bodyContents = await response.Body.transformToString() + const parsedEdge = JSON.parse(bodyContents) + + // Convert serialized connections back to Map> + const connections = new Map>() + for (const [level, nodeIds] of Object.entries(parsedEdge.connections)) { + connections.set(Number(level), new Set(nodeIds as string[])) + } + + return { + id: parsedEdge.id, + vector: parsedEdge.vector, + connections, + sourceId: parsedEdge.sourceId, + targetId: parsedEdge.targetId, + type: parsedEdge.type, + weight: parsedEdge.weight, + metadata: parsedEdge.metadata + } + } catch (error) { + // Edge not found or other error + return null + } + } + + /** + * Get all edges from storage + */ + protected async getAllEdges(): Promise { + await this.ensureInitialized() + + try { + // Import the ListObjectsV2Command and GetObjectCommand only when needed + const { ListObjectsV2Command, GetObjectCommand } = await import( + '@aws-sdk/client-s3' + ) + + // List all objects in the verbs directory + const listResponse = await this.s3Client!.send( + new ListObjectsV2Command({ + Bucket: this.bucketName, + Prefix: this.verbPrefix + }) + ) + + const edges: Edge[] = [] + + // If there are no objects, return an empty array + if (!listResponse.Contents || listResponse.Contents.length === 0) { + return edges + } + + // Get each edge + const edgePromises = listResponse.Contents.map( + async (object: { Key: string }) => { + try { + // Extract edge ID from the key (remove prefix and .json extension) + const edgeId = object.Key.replace(this.verbPrefix, '').replace('.json', '') + + // Get the edge data + const response = await this.s3Client!.send( + new GetObjectCommand({ + Bucket: this.bucketName, + Key: object.Key + }) + ) + + // Convert the response body to a string + const bodyContents = await response.Body.transformToString() + const parsedEdge = JSON.parse(bodyContents) + + // Convert serialized connections back to Map> + const connections = new Map>() + for (const [level, nodeIds] of Object.entries(parsedEdge.connections)) { + connections.set(Number(level), new Set(nodeIds as string[])) + } + + return { + id: parsedEdge.id, + vector: parsedEdge.vector, + connections, + sourceId: parsedEdge.sourceId, + targetId: parsedEdge.targetId, + type: parsedEdge.type, + weight: parsedEdge.weight, + metadata: parsedEdge.metadata + } + } catch (error) { + console.error(`Error getting edge from ${object.Key}:`, error) + return null + } + } + ) + + // Wait for all promises to resolve and filter out nulls + const resolvedEdges = await Promise.all(edgePromises) + return resolvedEdges.filter((edge): edge is Edge => edge !== null) + } catch (error) { + console.error('Failed to get all edges:', error) + return [] + } + } + + /** + * Get edges by source + */ + protected async getEdgesBySource(sourceId: string): Promise { + const edges = await this.getAllEdges() + return edges.filter((edge) => edge.sourceId === sourceId) + } + + /** + * Get edges by target + */ + protected async getEdgesByTarget(targetId: string): Promise { + const edges = await this.getAllEdges() + return edges.filter((edge) => edge.targetId === targetId) + } + + /** + * Get edges by type + */ + protected async getEdgesByType(type: string): Promise { + const edges = await this.getAllEdges() + return edges.filter((edge) => edge.type === type) + } + + /** + * Delete an edge from storage + */ + protected async deleteEdge(id: string): Promise { + await this.ensureInitialized() + + try { + // Import the DeleteObjectCommand only when needed + const { DeleteObjectCommand } = await import('@aws-sdk/client-s3') + + // Delete the edge from S3-compatible storage + await this.s3Client!.send( + new DeleteObjectCommand({ + Bucket: this.bucketName, + Key: `${this.verbPrefix}${id}.json` + }) + ) + } catch (error) { + console.error(`Failed to delete edge ${id}:`, error) + throw new Error(`Failed to delete edge ${id}: ${error}`) + } + } + + /** + * Save metadata to storage + */ + public async saveMetadata(id: string, metadata: any): Promise { + await this.ensureInitialized() + + try { + // Import the PutObjectCommand only when needed + const { PutObjectCommand } = await import('@aws-sdk/client-s3') + + // Save the metadata to S3-compatible storage + await this.s3Client!.send( + new PutObjectCommand({ + Bucket: this.bucketName, + Key: `${this.metadataPrefix}${id}.json`, + Body: JSON.stringify(metadata, null, 2), + ContentType: 'application/json' + }) + ) + } catch (error) { + console.error(`Failed to save metadata ${id}:`, error) + throw new Error(`Failed to save metadata ${id}: ${error}`) + } + } + + /** + * Get metadata from storage + */ + public async getMetadata(id: string): Promise { + await this.ensureInitialized() + + try { + // Import the GetObjectCommand only when needed + const { GetObjectCommand } = await import('@aws-sdk/client-s3') + + // Try to get the metadata from the metadata directory + const response = await this.s3Client!.send( + new GetObjectCommand({ + Bucket: this.bucketName, + Key: `${this.metadataPrefix}${id}.json` + }) + ) + + // Convert the response body to a string + const bodyContents = await response.Body.transformToString() + return JSON.parse(bodyContents) + } catch (error) { + // Metadata not found or other error + return null + } + } + + /** + * Clear all data from storage + */ + public async clear(): Promise { + await this.ensureInitialized() + + try { + // Import the ListObjectsV2Command and DeleteObjectCommand only when needed + const { ListObjectsV2Command, DeleteObjectCommand } = await import( + '@aws-sdk/client-s3' + ) + + // Helper function to delete all objects with a given prefix + const deleteObjectsWithPrefix = async (prefix: string): Promise => { + // List all objects with the given prefix + const listResponse = await this.s3Client!.send( + new ListObjectsV2Command({ + Bucket: this.bucketName, + Prefix: prefix + }) + ) + + // If there are no objects, return + if (!listResponse.Contents || listResponse.Contents.length === 0) { + return + } + + // Delete each object + for (const object of listResponse.Contents) { + await this.s3Client!.send( + new DeleteObjectCommand({ + Bucket: this.bucketName, + Key: object.Key + }) + ) + } + } + + // Delete all objects in the nouns directory + await deleteObjectsWithPrefix(this.nounPrefix) + + // Delete all objects in the verbs directory + await deleteObjectsWithPrefix(this.verbPrefix) + + // Delete all objects in the metadata directory + await deleteObjectsWithPrefix(this.metadataPrefix) + + // Delete all objects in the index directory + await deleteObjectsWithPrefix(this.indexPrefix) + } catch (error) { + console.error('Failed to clear storage:', error) + throw new Error(`Failed to clear storage: ${error}`) + } + } + + /** + * Get information about storage usage and capacity + */ + public async getStorageStatus(): Promise<{ + type: string + used: number + quota: number | null + details?: Record + }> { + await this.ensureInitialized() + + try { + // Import the ListObjectsV2Command only when needed + const { ListObjectsV2Command } = await import('@aws-sdk/client-s3') + + // Calculate the total size of all objects in the storage + let totalSize = 0 + let nodeCount = 0 + let edgeCount = 0 + let metadataCount = 0 + + // Helper function to calculate size and count for a given prefix + const calculateSizeAndCount = async ( + prefix: string + ): Promise<{ size: number; count: number }> => { + let size = 0 + let count = 0 + + // List all objects with the given prefix + const listResponse = await this.s3Client!.send( + new ListObjectsV2Command({ + Bucket: this.bucketName, + Prefix: prefix + }) + ) + + // If there are no objects, return + if (!listResponse.Contents || listResponse.Contents.length === 0) { + return { size, count } + } + + // Calculate size and count + for (const object of listResponse.Contents) { + size += object.Size || 0 + count++ + } + + return { size, count } + } + + // Calculate size and count for each directory + const nounsResult = await calculateSizeAndCount(this.nounPrefix) + const verbsResult = await calculateSizeAndCount(this.verbPrefix) + const metadataResult = await calculateSizeAndCount(this.metadataPrefix) + const indexResult = await calculateSizeAndCount(this.indexPrefix) + + totalSize = nounsResult.size + verbsResult.size + metadataResult.size + indexResult.size + nodeCount = nounsResult.count + edgeCount = verbsResult.count + metadataCount = metadataResult.count + + // Count nouns by type using metadata + const nounTypeCounts: Record = {} + + // List all objects in the metadata directory + const metadataListResponse = await this.s3Client!.send( + new ListObjectsV2Command({ + Bucket: this.bucketName, + Prefix: this.metadataPrefix + }) + ) + + if (metadataListResponse.Contents) { + // Import the GetObjectCommand only when needed + const { GetObjectCommand } = await import('@aws-sdk/client-s3') + + for (const object of metadataListResponse.Contents) { + try { + // Get the metadata + const response = await this.s3Client!.send( + new GetObjectCommand({ + Bucket: this.bucketName, + Key: object.Key + }) + ) + + // Convert the response body to a string + const bodyContents = await response.Body.transformToString() + const metadata = JSON.parse(bodyContents) + + // Count by noun type + if (metadata.noun) { + nounTypeCounts[metadata.noun] = (nounTypeCounts[metadata.noun] || 0) + 1 + } + } catch (error) { + console.error(`Error getting metadata from ${object.Key}:`, error) + } + } + } + + return { + type: this.serviceType, + used: totalSize, + quota: null, // S3-compatible services typically don't provide quota information through the API + details: { + bucketName: this.bucketName, + region: this.region, + endpoint: this.endpoint, + nodeCount, + edgeCount, + metadataCount, + nounTypes: nounTypeCounts + } + } + } catch (error) { + console.error('Failed to get storage status:', error) + return { + type: this.serviceType, + used: 0, + quota: null, + details: { error: String(error) } + } + } + } +} \ No newline at end of file diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts new file mode 100644 index 00000000..bc957161 --- /dev/null +++ b/src/storage/baseStorage.ts @@ -0,0 +1,248 @@ +/** + * Base Storage Adapter + * Provides common functionality for all storage adapters + */ + +import { GraphVerb, HNSWNoun, StorageAdapter } from '../coreTypes.js' + +// Common directory/prefix names +export const NOUNS_DIR = 'nouns' +export const VERBS_DIR = 'verbs' +export const METADATA_DIR = 'metadata' +export const INDEX_DIR = 'index' + +/** + * Base storage adapter that implements common functionality + * This is an abstract class that should be extended by specific storage adapters + */ +export abstract class BaseStorage implements StorageAdapter { + protected isInitialized = false + + /** + * Initialize the storage adapter + * This method should be implemented by each specific adapter + */ + public abstract init(): Promise + + /** + * Ensure the storage adapter is initialized + */ + protected async ensureInitialized(): Promise { + if (!this.isInitialized) { + await this.init() + } + } + + /** + * Save a noun to storage + */ + public async saveNoun(noun: HNSWNoun): Promise { + await this.ensureInitialized() + return this.saveNode(noun) + } + + /** + * Get a noun from storage + */ + public async getNoun(id: string): Promise { + await this.ensureInitialized() + return this.getNode(id) + } + + /** + * Get all nouns from storage + */ + public async getAllNouns(): Promise { + await this.ensureInitialized() + return this.getAllNodes() + } + + /** + * Get nouns by noun type + * @param nounType The noun type to filter by + * @returns Promise that resolves to an array of nouns of the specified noun type + */ + public async getNounsByNounType(nounType: string): Promise { + await this.ensureInitialized() + return this.getNodesByNounType(nounType) + } + + /** + * Delete a noun from storage + */ + public async deleteNoun(id: string): Promise { + await this.ensureInitialized() + return this.deleteNode(id) + } + + /** + * Save a verb to storage + */ + public async saveVerb(verb: GraphVerb): Promise { + await this.ensureInitialized() + return this.saveEdge(verb) + } + + /** + * Get a verb from storage + */ + public async getVerb(id: string): Promise { + await this.ensureInitialized() + return this.getEdge(id) + } + + /** + * Get all verbs from storage + */ + public async getAllVerbs(): Promise { + await this.ensureInitialized() + return this.getAllEdges() + } + + /** + * Get verbs by source + */ + public async getVerbsBySource(sourceId: string): Promise { + await this.ensureInitialized() + return this.getEdgesBySource(sourceId) + } + + /** + * Get verbs by target + */ + public async getVerbsByTarget(targetId: string): Promise { + await this.ensureInitialized() + return this.getEdgesByTarget(targetId) + } + + /** + * Get verbs by type + */ + public async getVerbsByType(type: string): Promise { + await this.ensureInitialized() + return this.getEdgesByType(type) + } + + /** + * Delete a verb from storage + */ + public async deleteVerb(id: string): Promise { + await this.ensureInitialized() + return this.deleteEdge(id) + } + + /** + * Clear all data from storage + * This method should be implemented by each specific adapter + */ + public abstract clear(): Promise + + /** + * Get information about storage usage and capacity + * This method should be implemented by each specific adapter + */ + public abstract getStorageStatus(): Promise<{ + type: string + used: number + quota: number | null + details?: Record + }> + + /** + * Save metadata to storage + * This method should be implemented by each specific adapter + */ + public abstract saveMetadata(id: string, metadata: any): Promise + + /** + * Get metadata from storage + * This method should be implemented by each specific adapter + */ + public abstract getMetadata(id: string): Promise + + /** + * Save a node to storage + * This method should be implemented by each specific adapter + */ + protected abstract saveNode(node: HNSWNoun): Promise + + /** + * Get a node from storage + * This method should be implemented by each specific adapter + */ + protected abstract getNode(id: string): Promise + + /** + * Get all nodes from storage + * This method should be implemented by each specific adapter + */ + protected abstract getAllNodes(): Promise + + /** + * Get nodes by noun type + * This method should be implemented by each specific adapter + */ + protected abstract getNodesByNounType(nounType: string): Promise + + /** + * Delete a node from storage + * This method should be implemented by each specific adapter + */ + protected abstract deleteNode(id: string): Promise + + /** + * Save an edge to storage + * This method should be implemented by each specific adapter + */ + protected abstract saveEdge(edge: GraphVerb): Promise + + /** + * Get an edge from storage + * This method should be implemented by each specific adapter + */ + protected abstract getEdge(id: string): Promise + + /** + * Get all edges from storage + * This method should be implemented by each specific adapter + */ + protected abstract getAllEdges(): Promise + + /** + * Get edges by source + * This method should be implemented by each specific adapter + */ + protected abstract getEdgesBySource(sourceId: string): Promise + + /** + * Get edges by target + * This method should be implemented by each specific adapter + */ + protected abstract getEdgesByTarget(targetId: string): Promise + + /** + * Get edges by type + * This method should be implemented by each specific adapter + */ + protected abstract getEdgesByType(type: string): Promise + + /** + * Delete an edge from storage + * This method should be implemented by each specific adapter + */ + protected abstract deleteEdge(id: string): Promise + + /** + * Helper method to convert a Map to a plain object for serialization + */ + protected mapToObject( + map: Map, + valueTransformer: (value: V) => any = (v) => v + ): Record { + const obj: Record = {} + for (const [key, value] of map.entries()) { + obj[key.toString()] = valueTransformer(value) + } + return obj + } +} \ No newline at end of file diff --git a/src/storage/fileSystemStorage.ts b/src/storage/fileSystemStorage.ts index 1935638d..07ea78ce 100644 --- a/src/storage/fileSystemStorage.ts +++ b/src/storage/fileSystemStorage.ts @@ -49,14 +49,7 @@ const NOUNS_DIR = 'nouns' const VERBS_DIR = 'verbs' const METADATA_DIR = 'metadata' -// Constants for noun type directories -const PERSON_DIR = 'person' -const PLACE_DIR = 'place' -const THING_DIR = 'thing' -const EVENT_DIR = 'event' -const CONCEPT_DIR = 'concept' -const CONTENT_DIR = 'content' -const DEFAULT_DIR = 'default' // For nodes without a noun type +// All nouns now use the same directory - no separate directories per noun type /** * File system storage adapter for Node.js environments @@ -66,13 +59,6 @@ export class FileSystemStorage implements StorageAdapter { private nounsDir: string private verbsDir: string private metadataDir: string - private personDir: string - private placeDir: string - private thingDir: string - private eventDir: string - private conceptDir: string - private contentDir: string - private defaultDir: string private isInitialized = false constructor(rootDirectory?: string) { @@ -81,13 +67,6 @@ export class FileSystemStorage implements StorageAdapter { this.nounsDir = '' this.verbsDir = '' this.metadataDir = '' - this.personDir = '' - this.placeDir = '' - this.thingDir = '' - this.eventDir = '' - this.conceptDir = '' - this.contentDir = '' - this.defaultDir = '' } /** @@ -123,27 +102,11 @@ export class FileSystemStorage implements StorageAdapter { this.verbsDir = path.join(this.rootDir, VERBS_DIR) this.metadataDir = path.join(this.rootDir, METADATA_DIR) - // Set up noun type directory paths - this.personDir = path.join(this.nounsDir, PERSON_DIR) - this.placeDir = path.join(this.nounsDir, PLACE_DIR) - this.thingDir = path.join(this.nounsDir, THING_DIR) - this.eventDir = path.join(this.nounsDir, EVENT_DIR) - this.conceptDir = path.join(this.nounsDir, CONCEPT_DIR) - this.contentDir = path.join(this.nounsDir, CONTENT_DIR) - this.defaultDir = path.join(this.nounsDir, DEFAULT_DIR) - // Create directories if they don't exist await this.ensureDirectoryExists(this.rootDir) await this.ensureDirectoryExists(this.nounsDir) await this.ensureDirectoryExists(this.verbsDir) await this.ensureDirectoryExists(this.metadataDir) - await this.ensureDirectoryExists(this.personDir) - await this.ensureDirectoryExists(this.placeDir) - await this.ensureDirectoryExists(this.thingDir) - await this.ensureDirectoryExists(this.eventDir) - await this.ensureDirectoryExists(this.conceptDir) - await this.ensureDirectoryExists(this.contentDir) - await this.ensureDirectoryExists(this.defaultDir) this.isInitialized = true } catch (error: any) { @@ -164,32 +127,8 @@ export class FileSystemStorage implements StorageAdapter { } private getNounPath(id: string, nounType?: string): string { - let typeDir = this.defaultDir - if (nounType) { - switch (nounType.toLowerCase()) { - case 'person': - typeDir = this.personDir - break - case 'place': - typeDir = this.placeDir - break - case 'thing': - typeDir = this.thingDir - break - case 'event': - typeDir = this.eventDir - break - case 'concept': - typeDir = this.conceptDir - break - case 'content': - typeDir = this.contentDir - break - default: - typeDir = this.defaultDir - } - } - return path.join(typeDir, `${id}.json`) + // All nouns now use the same directory regardless of type + return path.join(this.nounsDir, `${id}.json`) } public async saveNoun( @@ -204,27 +143,16 @@ export class FileSystemStorage implements StorageAdapter { public async getNoun(id: string): Promise { if (!this.isInitialized) await this.init() - const nounDirs = [ - this.personDir, - this.placeDir, - this.thingDir, - this.eventDir, - this.conceptDir, - this.contentDir, - this.defaultDir - ] - for (const dir of nounDirs) { - const filePath = path.join(dir, `${id}.json`) - try { - const data = await fs.promises.readFile(filePath, 'utf-8') - return JSON.parse(data) - } catch (error: any) { - if (error.code !== 'ENOENT') { - console.error(`Error reading noun ${id}:`, error) - } + const filePath = path.join(this.nounsDir, `${id}.json`) + try { + const data = await fs.promises.readFile(filePath, 'utf-8') + return JSON.parse(data) + } catch (error: any) { + if (error.code !== 'ENOENT') { + console.error(`Error reading noun ${id}:`, error) } + return null } - return null } public async deleteNoun(id: string): Promise { @@ -247,30 +175,19 @@ export class FileSystemStorage implements StorageAdapter { public async getAllNouns(): Promise { if (!this.isInitialized) await this.init() const allNouns: HNSWNoun[] = [] - const nounDirs = [ - this.personDir, - this.placeDir, - this.thingDir, - this.eventDir, - this.conceptDir, - this.contentDir, - this.defaultDir - ] - for (const dir of nounDirs) { - try { - const files = await fs.promises.readdir(dir) - for (const file of files) { - if (file.endsWith('.json')) { - const filePath = path.join(dir, file) - const data = await fs.promises.readFile(filePath, 'utf-8') - allNouns.push(JSON.parse(data)) - } - } - } catch (error: any) { - if (error.code !== 'ENOENT') { - console.error(`Error reading directory ${dir}:`, error) + try { + const files = await fs.promises.readdir(this.nounsDir) + for (const file of files) { + if (file.endsWith('.json')) { + const filePath = path.join(this.nounsDir, file) + const data = await fs.promises.readFile(filePath, 'utf-8') + allNouns.push(JSON.parse(data)) } } + } catch (error: any) { + if (error.code !== 'ENOENT') { + console.error(`Error reading directory ${this.nounsDir}:`, error) + } } return allNouns } @@ -283,43 +200,25 @@ export class FileSystemStorage implements StorageAdapter { public async getNounsByNounType(nounType: string): Promise { if (!this.isInitialized) await this.init() - let typeDir: string - switch (nounType.toLowerCase()) { - case 'person': - typeDir = this.personDir - break - case 'place': - typeDir = this.placeDir - break - case 'thing': - typeDir = this.thingDir - break - case 'event': - typeDir = this.eventDir - break - case 'concept': - typeDir = this.conceptDir - break - case 'content': - typeDir = this.contentDir - break - default: - typeDir = this.defaultDir - } - const nouns: HNSWNoun[] = [] try { - const files = await fs.promises.readdir(typeDir) + const files = await fs.promises.readdir(this.nounsDir) for (const file of files) { if (file.endsWith('.json')) { - const filePath = path.join(typeDir, file) + const filePath = path.join(this.nounsDir, file) const data = await fs.promises.readFile(filePath, 'utf-8') - nouns.push(JSON.parse(data)) + const noun = JSON.parse(data) + + // Filter by noun type using metadata + const metadata = await this.getMetadata(noun.id) + if (metadata && metadata.noun === nounType) { + nouns.push(noun) + } } } } catch (error: any) { if (error.code !== 'ENOENT') { - console.error(`Error reading directory ${typeDir}:`, error) + console.error(`Error reading directory ${this.nounsDir}:`, error) } } diff --git a/src/storage/opfsStorage.ts b/src/storage/opfsStorage.ts index 4f43d6cb..137a9a12 100644 --- a/src/storage/opfsStorage.ts +++ b/src/storage/opfsStorage.ts @@ -3,13 +3,13 @@ * Provides persistent storage for the vector database using the Origin Private File System API */ -import { GraphVerb, HNSWNoun, StorageAdapter } from '../coreTypes.js' +import {GraphVerb, HNSWNoun, StorageAdapter} from '../coreTypes.js' import '../types/fileSystemTypes.js' // Make sure TypeScript recognizes the FileSystemDirectoryHandle interface extension declare global { - interface FileSystemDirectoryHandle { - entries(): AsyncIterableIterator<[string, FileSystemHandle]> - } + interface FileSystemDirectoryHandle { + entries(): AsyncIterableIterator<[string, FileSystemHandle]> + } } // Type aliases for compatibility @@ -24,261 +24,167 @@ const VERBS_DIR = 'verbs' const METADATA_DIR = 'metadata' const DB_INFO_FILE = 'db-info.json' -// Constants for noun type directories -const PERSON_DIR = 'person' -const PLACE_DIR = 'place' -const THING_DIR = 'thing' -const EVENT_DIR = 'event' -const CONCEPT_DIR = 'concept' -const CONTENT_DIR = 'content' -const DEFAULT_DIR = 'default' // For nodes without a noun type +// All nouns now use the same directory - no separate directories per noun type export class OPFSStorage implements StorageAdapter { - private rootDir: FileSystemDirectoryHandle | null = null - private nounsDir: FileSystemDirectoryHandle | null = null - private verbsDir: FileSystemDirectoryHandle | null = null - private metadataDir: FileSystemDirectoryHandle | null = null - private personDir: FileSystemDirectoryHandle | null = null - private placeDir: FileSystemDirectoryHandle | null = null - private thingDir: FileSystemDirectoryHandle | null = null - private eventDir: FileSystemDirectoryHandle | null = null - private conceptDir: FileSystemDirectoryHandle | null = null - private contentDir: FileSystemDirectoryHandle | null = null - private defaultDir: FileSystemDirectoryHandle | null = null - private isInitialized = false - private isAvailable = false - private isPersistentRequested = false - private isPersistentGranted = false + private rootDir: FileSystemDirectoryHandle | null = null + private nounsDir: FileSystemDirectoryHandle | null = null + private verbsDir: FileSystemDirectoryHandle | null = null + private metadataDir: FileSystemDirectoryHandle | null = null + private isInitialized = false + private isAvailable = false + private isPersistentRequested = false + private isPersistentGranted = false - constructor() { - // Check if OPFS is available - this.isAvailable = - typeof navigator !== 'undefined' && - 'storage' in navigator && - 'getDirectory' in navigator.storage - } - - /** - * Initialize the storage adapter - */ - public async init(): Promise { - if (this.isInitialized) { - return + constructor() { + // Check if OPFS is available + this.isAvailable = + typeof navigator !== 'undefined' && + 'storage' in navigator && + 'getDirectory' in navigator.storage } - if (!this.isAvailable) { - throw new Error( - 'Origin Private File System is not available in this environment' - ) - } - - try { - // Get the root directory - const root = await navigator.storage.getDirectory() - - // Create or get our app's root directory - this.rootDir = await root.getDirectoryHandle(ROOT_DIR, { create: true }) - - // Create or get nouns directory - this.nounsDir = await this.rootDir.getDirectoryHandle(NOUNS_DIR, { - create: true - }) - - // Create or get verbs directory - this.verbsDir = await this.rootDir.getDirectoryHandle(VERBS_DIR, { - create: true - }) - - // Create or get metadata directory - this.metadataDir = await this.rootDir.getDirectoryHandle(METADATA_DIR, { - create: true - }) - - // Create or get noun type directories - this.personDir = await this.nounsDir.getDirectoryHandle(PERSON_DIR, { - create: true - }) - this.placeDir = await this.nounsDir.getDirectoryHandle(PLACE_DIR, { - create: true - }) - this.thingDir = await this.nounsDir.getDirectoryHandle(THING_DIR, { - create: true - }) - this.eventDir = await this.nounsDir.getDirectoryHandle(EVENT_DIR, { - create: true - }) - this.conceptDir = await this.nounsDir.getDirectoryHandle(CONCEPT_DIR, { - create: true - }) - this.contentDir = await this.nounsDir.getDirectoryHandle(CONTENT_DIR, { - create: true - }) - this.defaultDir = await this.nounsDir.getDirectoryHandle(DEFAULT_DIR, { - create: true - }) - - this.isInitialized = true - } catch (error) { - console.error('Failed to initialize OPFS storage:', error) - throw new Error(`Failed to initialize OPFS storage: ${error}`) - } - } - - /** - * Check if OPFS is available in the current environment - */ - public isOPFSAvailable(): boolean { - return this.isAvailable - } - - /** - * Request persistent storage permission from the user - * @returns Promise that resolves to true if permission was granted, false otherwise - */ - public async requestPersistentStorage(): Promise { - if (!this.isAvailable) { - console.warn('Cannot request persistent storage: OPFS is not available') - return false - } - - try { - // Check if persistence is already granted - this.isPersistentGranted = await navigator.storage.persisted() - - if (!this.isPersistentGranted) { - // Request permission for persistent storage - this.isPersistentGranted = await navigator.storage.persist() - } - - this.isPersistentRequested = true - return this.isPersistentGranted - } catch (error) { - console.warn('Failed to request persistent storage:', error) - return false - } - } - - /** - * Check if persistent storage is granted - * @returns Promise that resolves to true if persistent storage is granted, false otherwise - */ - public async isPersistent(): Promise { - if (!this.isAvailable) { - return false - } - - try { - this.isPersistentGranted = await navigator.storage.persisted() - return this.isPersistentGranted - } catch (error) { - console.warn('Failed to check persistent storage status:', error) - return false - } - } - - /** - * Save a noun to storage - */ - public async saveNoun(noun: HNSWNoun): Promise { - await this.ensureInitialized() - - try { - // Convert connections Map to a serializable format - const serializableNoun = { - ...noun, - connections: this.mapToObject(noun.connections, (set) => - Array.from(set as Set) - ) - } - - // Get the appropriate directory based on the noun's metadata - const nounDir = await this.getNodeDirectory(noun.id) - - // Create or get the file for this noun - const fileHandle = await nounDir.getFileHandle(noun.id, { - create: true - }) - - // Write the noun data to the file - const writable = await fileHandle.createWritable() - await writable.write(JSON.stringify(serializableNoun)) - await writable.close() - } catch (error) { - console.error(`Failed to save noun ${noun.id}:`, error) - throw new Error(`Failed to save noun ${noun.id}: ${error}`) - } - } - - /** - * Get a noun from storage - */ - public async getNoun(id: string): Promise { - await this.ensureInitialized() - - try { - // Get the appropriate directory based on the node's metadata - const nodeDir = await this.getNodeDirectory(id) - - try { - // Get the file handle for this node - const fileHandle = await nodeDir.getFileHandle(id) - - // Read the node data from the file - const file = await fileHandle.getFile() - const text = await file.text() - const data = JSON.parse(text) - - // Convert serialized connections back to Map> - const connections = new Map>() - for (const [level, nodeIds] of Object.entries(data.connections)) { - connections.set(Number(level), new Set(nodeIds as string[])) + /** + * Initialize the storage adapter + */ + public async init(): Promise { + if (this.isInitialized) { + return } - return { - id: data.id, - vector: data.vector, - connections + if (!this.isAvailable) { + throw new Error( + 'Origin Private File System is not available in this environment' + ) } - } catch (dirError) { - // If the file doesn't exist in the expected directory, try the default directory - if (nodeDir !== this.defaultDir) { - try { - // Get the file handle from the default directory - const fileHandle = await this.defaultDir!.getFileHandle(id) - // Read the node data from the file - const file = await fileHandle.getFile() - const text = await file.text() - const data = JSON.parse(text) + try { + // Get the root directory + const root = await navigator.storage.getDirectory() - // Convert serialized connections back to Map> - const connections = new Map>() - for (const [level, nodeIds] of Object.entries(data.connections)) { - connections.set(Number(level), new Set(nodeIds as string[])) + // Create or get our app's root directory + this.rootDir = await root.getDirectoryHandle(ROOT_DIR, {create: true}) + + // Create or get nouns directory + this.nounsDir = await this.rootDir.getDirectoryHandle(NOUNS_DIR, { + create: true + }) + + // Create or get verbs directory + this.verbsDir = await this.rootDir.getDirectoryHandle(VERBS_DIR, { + create: true + }) + + // Create or get metadata directory + this.metadataDir = await this.rootDir.getDirectoryHandle(METADATA_DIR, { + create: true + }) + + this.isInitialized = true + } catch (error) { + console.error('Failed to initialize OPFS storage:', error) + throw new Error(`Failed to initialize OPFS storage: ${error}`) + } + } + + /** + * Check if OPFS is available in the current environment + */ + public isOPFSAvailable(): boolean { + return this.isAvailable + } + + /** + * Request persistent storage permission from the user + * @returns Promise that resolves to true if permission was granted, false otherwise + */ + public async requestPersistentStorage(): Promise { + if (!this.isAvailable) { + console.warn('Cannot request persistent storage: OPFS is not available') + return false + } + + try { + // Check if persistence is already granted + this.isPersistentGranted = await navigator.storage.persisted() + + if (!this.isPersistentGranted) { + // Request permission for persistent storage + this.isPersistentGranted = await navigator.storage.persist() } - return { - id: data.id, - vector: data.vector, - connections + this.isPersistentRequested = true + return this.isPersistentGranted + } catch (error) { + console.warn('Failed to request persistent storage:', error) + return false + } + } + + /** + * Check if persistent storage is granted + * @returns Promise that resolves to true if persistent storage is granted, false otherwise + */ + public async isPersistent(): Promise { + if (!this.isAvailable) { + return false + } + + try { + this.isPersistentGranted = await navigator.storage.persisted() + return this.isPersistentGranted + } catch (error) { + console.warn('Failed to check persistent storage status:', error) + return false + } + } + + /** + * Save a noun to storage + */ + public async saveNoun(noun: HNSWNoun): Promise { + await this.ensureInitialized() + + try { + // Convert connections Map to a serializable format + const serializableNoun = { + ...noun, + connections: this.mapToObject(noun.connections, (set) => + Array.from(set as Set) + ) } - } catch (defaultDirError) { - // If not found in default directory either, try all noun type directories - const directories = [ - this.personDir!, - this.placeDir!, - this.thingDir!, - this.eventDir!, - this.conceptDir!, - this.contentDir! - ] - for (const dir of directories) { - if (dir === nodeDir) continue // Skip the already checked directory + // Get the appropriate directory based on the noun's metadata + const nounDir = await this.getNodeDirectory(noun.id) - try { - // Get the file handle from this directory - const fileHandle = await dir.getFileHandle(id) + // Create or get the file for this noun + const fileHandle = await nounDir.getFileHandle(noun.id, { + create: true + }) + + // Write the noun data to the file + const writable = await fileHandle.createWritable() + await writable.write(JSON.stringify(serializableNoun)) + await writable.close() + } catch (error) { + console.error(`Failed to save noun ${noun.id}:`, error) + throw new Error(`Failed to save noun ${noun.id}: ${error}`) + } + } + + /** + * Get a noun from storage + */ + public async getNoun(id: string): Promise { + await this.ensureInitialized() + + try { + // Get the nouns directory - all nouns are now in the same directory + const nounsDir = this.nounsDir! + + try { + // Get the file handle for this node + const fileHandle = await nounsDir.getFileHandle(id) // Read the node data from the file const file = await fileHandle.getFile() @@ -287,83 +193,226 @@ export class OPFSStorage implements StorageAdapter { // Convert serialized connections back to Map> const connections = new Map>() - for (const [level, nodeIds] of Object.entries( - data.connections - )) { - connections.set(Number(level), new Set(nodeIds as string[])) + for (const [level, nodeIds] of Object.entries(data.connections)) { + connections.set(Number(level), new Set(nodeIds as string[])) } return { - id: data.id, - vector: data.vector, - connections + id: data.id, + vector: data.vector, + connections } - } catch { - // Continue to the next directory - } + } catch (error) { + // File doesn't exist + return null + } + } catch (error) { + console.error(`Failed to get node ${id}:`, error) + throw new Error(`Failed to get node ${id}: ${error}`) + } + } + + /** + * Get nouns by noun type + * @param nounType The noun type to filter by + * @returns Promise that resolves to an array of nouns of the specified noun type + */ + public async getNounsByNounType(nounType: string): Promise { + await this.ensureInitialized() + + try { + const nouns: HNSWNoun[] = [] + + // Use the consolidated nouns directory + const dir = this.nounsDir! + + try { + // Get all entries (filename and handle pairs) in this directory + const entries = dir.entries() + + // Iterate through all entries and get the corresponding nodes + for await (const [name, handle] of entries) { + try { + // The handle is already a FileSystemHandle, but we need to ensure it's a file + if (handle.kind !== 'file') continue + + // Cast to FileSystemFileHandle + const fileHandle = handle as FileSystemFileHandle + + // Read the node data from the file + const file = await fileHandle.getFile() + const text = await file.text() + const data = JSON.parse(text) + + // Get the metadata to check the noun type + const metadata = await this.getMetadata(data.id) + + // Skip if metadata doesn't exist or noun type doesn't match + if (!metadata || metadata.noun !== nounType) { + continue + } + + // Convert serialized connections back to Map> + const connections = new Map>() + for (const [level, nodeIds] of Object.entries(data.connections)) { + connections.set(Number(level), new Set(nodeIds as string[])) + } + + nouns.push({ + id: data.id, + vector: data.vector, + connections + }) + } catch (nodeError) { + console.warn( + `Failed to read node ${name} from directory:`, + nodeError + ) + // Continue to the next node + } + } + } catch (dirError) { + console.warn( + `Failed to read directory for noun type ${nounType}:`, + dirError + ) } - return null // File doesn't exist in any directory - } + return nouns + } catch (error) { + console.error(`Failed to get nouns for noun type ${nounType}:`, error) + throw new Error(`Failed to get nouns for noun type ${nounType}: ${error}`) } - return null // File doesn't exist - } - } catch (error) { - console.error(`Failed to get node ${id}:`, error) - throw new Error(`Failed to get node ${id}: ${error}`) } - } - /** - * Get nouns by noun type - * @param nounType The noun type to filter by - * @returns Promise that resolves to an array of nouns of the specified noun type - */ - public async getNounsByNounType(nounType: string): Promise { - await this.ensureInitialized() + /** + * Get all nouns from storage + */ + public async getAllNouns(): Promise { + await this.ensureInitialized() - try { - const nouns: HNSWNoun[] = [] + try { + const nouns: HNSWNoun[] = [] - // Determine the directory based on the noun type - let dir: FileSystemDirectoryHandle - switch (nounType) { - case 'person': - dir = this.personDir! - break - case 'place': - dir = this.placeDir! - break - case 'thing': - dir = this.thingDir! - break - case 'event': - dir = this.eventDir! - break - case 'concept': - dir = this.conceptDir! - break - case 'content': - dir = this.contentDir! - break - default: - dir = this.defaultDir! - } + // Use the consolidated nouns directory + const dir = this.nounsDir! - try { - // Get all entries (filename and handle pairs) in this directory - const entries = dir.entries() + try { + // Get all entries (filename and handle pairs) in this directory + const entries = dir.entries() - // Iterate through all entries and get the corresponding nodes - for await (const [name, handle] of entries) { - try { - // The handle is already a FileSystemHandle, but we need to ensure it's a file - if (handle.kind !== 'file') continue + // Iterate through all entries and get the corresponding nodes + for await (const [name, handle] of entries) { + try { + // The handle is already a FileSystemHandle, but we need to ensure it's a file + if (handle.kind !== 'file') continue - // Cast to FileSystemFileHandle - const fileHandle = handle as FileSystemFileHandle + // Cast to FileSystemFileHandle + const fileHandle = handle as FileSystemFileHandle - // Read the node data from the file + // Read the node data from the file + const file = await fileHandle.getFile() + const text = await file.text() + const data = JSON.parse(text) + + // Convert serialized connections back to Map> + const connections = new Map>() + for (const [level, nodeIds] of Object.entries(data.connections)) { + connections.set(Number(level), new Set(nodeIds as string[])) + } + + nouns.push({ + id: data.id, + vector: data.vector, + connections + }) + } catch (nodeError) { + console.warn( + `Failed to read node ${name} from directory:`, + nodeError + ) + // Continue to the next node + } + } + } catch (dirError) { + console.warn( + `Failed to read nouns directory:`, + dirError + ) + } + + return nouns + } catch (error) { + console.error('Failed to get all nouns:', error) + throw new Error(`Failed to get all nouns: ${error}`) + } + } + + /** + * Delete a noun from storage + */ + public async deleteNoun(id: string): Promise { + await this.ensureInitialized() + + try { + // Get the nouns directory - all nouns are now in the same directory + const nounsDir = this.nounsDir! + + try { + // Try to delete the noun from the nouns directory + await nounsDir.removeEntry(id) + return // Noun deleted successfully + } catch (error) { + // If the file doesn't exist, that's fine + return + } + } catch (error) { + console.error(`Failed to delete noun ${id}:`, error) + throw new Error(`Failed to delete noun ${id}: ${error}`) + } + } + + /** + * Save a verb to storage + */ + public async saveVerb(verb: GraphVerb): Promise { + await this.ensureInitialized() + + try { + // Convert connections Map to a serializable format + const serializableVerb = { + ...verb, + connections: this.mapToObject(verb.connections, (set) => + Array.from(set as Set) + ) + } + + // Create or get the file for this verb + const fileHandle = await this.verbsDir!.getFileHandle(verb.id, { + create: true + }) + + // Write the verb data to the file + const writable = await fileHandle.createWritable() + await writable.write(JSON.stringify(serializableVerb)) + await writable.close() + } catch (error) { + console.error(`Failed to save verb ${verb.id}:`, error) + throw new Error(`Failed to save verb ${verb.id}: ${error}`) + } + } + + /** + * Get a verb from storage + */ + public async getVerb(id: string): Promise { + await this.ensureInitialized() + + try { + // Get the file handle for this verb + const fileHandle = await this.verbsDir!.getFileHandle(id) + + // Read the verb data from the file const file = await fileHandle.getFile() const text = await file.text() const data = JSON.parse(text) @@ -371,1257 +420,867 @@ export class OPFSStorage implements StorageAdapter { // Convert serialized connections back to Map> const connections = new Map>() for (const [level, nodeIds] of Object.entries(data.connections)) { - connections.set(Number(level), new Set(nodeIds as string[])) + connections.set(Number(level), new Set(nodeIds as string[])) } - nouns.push({ - id: data.id, - vector: data.vector, - connections + return { + id: data.id, + vector: data.vector, + connections, + sourceId: data.sourceId, + targetId: data.targetId, + type: data.type, + weight: data.weight, + metadata: data.metadata + } + } catch (error) { + // If the file doesn't exist, return null + if ((error as any).name === 'NotFoundError') { + return null + } + + console.error(`Failed to get verb ${id}:`, error) + throw new Error(`Failed to get verb ${id}: ${error}`) + } + } + + /** + * Get all edges from storage + */ + public async getAllEdges(): Promise { + await this.ensureInitialized() + + try { + const edges: Edge[] = [] + + // Get all entries (filename and handle pairs) in the verbs directory + const entries = this.verbsDir!.entries() + + // Iterate through all entries and get the corresponding edges + for await (const [name, handle] of entries) { + // Skip if not a file + if (handle.kind !== 'file') continue + + const edge = await this.getVerb(name) + if (edge) { + edges.push(edge) + } + } + + return edges + } catch (error) { + console.error('Failed to get all edges:', error) + throw new Error(`Failed to get all edges: ${error}`) + } + } + + /** + * Get all verbs from storage (alias for getAllEdges) + */ + public async getAllVerbs(): Promise { + return this.getAllEdges() + } + + /** + * Delete an edge from storage + */ + public async deleteEdge(id: string): Promise { + await this.ensureInitialized() + + try { + await this.verbsDir!.removeEntry(id) + } catch (error) { + // Ignore if the file doesn't exist + if ((error as any).name !== 'NotFoundError') { + console.error(`Failed to delete edge ${id}:`, error) + throw new Error(`Failed to delete edge ${id}: ${error}`) + } + } + } + + /** + * Delete a verb from storage (alias for deleteEdge) + */ + public async deleteVerb(id: string): Promise { + return this.deleteEdge(id) + } + + /** + * Get edges by source node ID + */ + public async getEdgesBySource(sourceId: string): Promise { + await this.ensureInitialized() + + try { + const allEdges = await this.getAllEdges() + return allEdges.filter((edge) => edge.sourceId === sourceId) + } catch (error) { + console.error(`Failed to get edges by source ${sourceId}:`, error) + throw new Error(`Failed to get edges by source ${sourceId}: ${error}`) + } + } + + /** + * Get verbs by source node ID (alias for getEdgesBySource) + */ + public async getVerbsBySource(sourceId: string): Promise { + return this.getEdgesBySource(sourceId) + } + + /** + * Get edges by target node ID + */ + public async getEdgesByTarget(targetId: string): Promise { + await this.ensureInitialized() + + try { + const allEdges = await this.getAllEdges() + return allEdges.filter((edge) => edge.targetId === targetId) + } catch (error) { + console.error(`Failed to get edges by target ${targetId}:`, error) + throw new Error(`Failed to get edges by target ${targetId}: ${error}`) + } + } + + /** + * Get verbs by target node ID (alias for getEdgesByTarget) + */ + public async getVerbsByTarget(targetId: string): Promise { + return this.getEdgesByTarget(targetId) + } + + /** + * Get edges by type + */ + public async getEdgesByType(type: string): Promise { + await this.ensureInitialized() + + try { + const allEdges = await this.getAllEdges() + return allEdges.filter((edge) => edge.type === type) + } catch (error) { + console.error(`Failed to get edges by type ${type}:`, error) + throw new Error(`Failed to get edges by type ${type}: ${error}`) + } + } + + /** + * Get verbs by type (alias for getEdgesByType) + */ + public async getVerbsByType(type: string): Promise { + return this.getEdgesByType(type) + } + + /** + * Save metadata for a node + */ + public async saveMetadata(id: string, metadata: any): Promise { + await this.ensureInitialized() + + try { + // Create or get the file for this metadata + const fileHandle = await this.metadataDir!.getFileHandle(id, { + create: true }) - } catch (nodeError) { - console.warn( - `Failed to read node ${name} from directory:`, - nodeError - ) - // Continue to the next node - } - } - } catch (dirError) { - console.warn( - `Failed to read directory for noun type ${nounType}:`, - dirError - ) - } - return nouns - } catch (error) { - console.error(`Failed to get nouns for noun type ${nounType}:`, error) - throw new Error(`Failed to get nouns for noun type ${nounType}: ${error}`) - } - } - - /** - * Get all nouns from storage - */ - public async getAllNouns(): Promise { - await this.ensureInitialized() - - try { - // Get all noun types - const nounTypes = [ - 'person', - 'place', - 'thing', - 'event', - 'concept', - 'content', - 'default' - ] - - // Run searches in parallel for all noun types - const nounPromises = nounTypes.map((nounType) => - this.getNounsByNounType(nounType) - ) - const nounArrays = await Promise.all(nounPromises) - - // Combine all results - const allNouns: HNSWNoun[] = [] - for (const nouns of nounArrays) { - allNouns.push(...nouns) - } - - return allNouns - } catch (error) { - console.error('Failed to get all nouns:', error) - throw new Error(`Failed to get all nouns: ${error}`) - } - } - - /** - * Delete a noun from storage - */ - public async deleteNoun(id: string): Promise { - await this.ensureInitialized() - - try { - // Get the appropriate directory based on the noun's metadata - const nounDir = await this.getNodeDirectory(id) - - try { - // Try to delete the noun from the appropriate directory - await nounDir.removeEntry(id) - return // Noun deleted successfully - } catch (dirError) { - // If the file doesn't exist in the expected directory, try the default directory - if (nounDir !== this.defaultDir) { - try { - await this.defaultDir!.removeEntry(id) - return // Noun deleted successfully - } catch (defaultDirError) { - // If not found in default directory either, try all noun type directories - const directories = [ - this.personDir!, - this.placeDir!, - this.thingDir!, - this.eventDir!, - this.conceptDir!, - this.contentDir! - ] - - for (const dir of directories) { - if (dir === nounDir) continue // Skip the already checked directory - - try { - await dir.removeEntry(id) - return // Node deleted successfully - } catch { - // Continue to the next directory - } - } - - // If we get here, the node wasn't found in any directory - return - } - } - // If the file doesn't exist, that's fine - return - } - } catch (error) { - console.error(`Failed to delete noun ${id}:`, error) - throw new Error(`Failed to delete noun ${id}: ${error}`) - } - } - - /** - * Save a verb to storage - */ - public async saveVerb(verb: GraphVerb): Promise { - await this.ensureInitialized() - - try { - // Convert connections Map to a serializable format - const serializableVerb = { - ...verb, - connections: this.mapToObject(verb.connections, (set) => - Array.from(set as Set) - ) - } - - // Create or get the file for this verb - const fileHandle = await this.verbsDir!.getFileHandle(verb.id, { - create: true - }) - - // Write the verb data to the file - const writable = await fileHandle.createWritable() - await writable.write(JSON.stringify(serializableVerb)) - await writable.close() - } catch (error) { - console.error(`Failed to save verb ${verb.id}:`, error) - throw new Error(`Failed to save verb ${verb.id}: ${error}`) - } - } - - /** - * Get a verb from storage - */ - public async getVerb(id: string): Promise { - await this.ensureInitialized() - - try { - // Get the file handle for this verb - const fileHandle = await this.verbsDir!.getFileHandle(id) - - // Read the verb data from the file - const file = await fileHandle.getFile() - const text = await file.text() - const data = JSON.parse(text) - - // Convert serialized connections back to Map> - const connections = new Map>() - for (const [level, nodeIds] of Object.entries(data.connections)) { - connections.set(Number(level), new Set(nodeIds as string[])) - } - - return { - id: data.id, - vector: data.vector, - connections, - sourceId: data.sourceId, - targetId: data.targetId, - type: data.type, - weight: data.weight, - metadata: data.metadata - } - } catch (error) { - // If the file doesn't exist, return null - if ((error as any).name === 'NotFoundError') { - return null - } - - console.error(`Failed to get verb ${id}:`, error) - throw new Error(`Failed to get verb ${id}: ${error}`) - } - } - - /** - * Get all edges from storage - */ - public async getAllEdges(): Promise { - await this.ensureInitialized() - - try { - const edges: Edge[] = [] - - // Get all entries (filename and handle pairs) in the verbs directory - const entries = this.verbsDir!.entries() - - // Iterate through all entries and get the corresponding edges - for await (const [name, handle] of entries) { - // Skip if not a file - if (handle.kind !== 'file') continue - - const edge = await this.getVerb(name) - if (edge) { - edges.push(edge) - } - } - - return edges - } catch (error) { - console.error('Failed to get all edges:', error) - throw new Error(`Failed to get all edges: ${error}`) - } - } - - /** - * Get all verbs from storage (alias for getAllEdges) - */ - public async getAllVerbs(): Promise { - return this.getAllEdges() - } - - /** - * Delete an edge from storage - */ - public async deleteEdge(id: string): Promise { - await this.ensureInitialized() - - try { - await this.verbsDir!.removeEntry(id) - } catch (error) { - // Ignore if the file doesn't exist - if ((error as any).name !== 'NotFoundError') { - console.error(`Failed to delete edge ${id}:`, error) - throw new Error(`Failed to delete edge ${id}: ${error}`) - } - } - } - - /** - * Delete a verb from storage (alias for deleteEdge) - */ - public async deleteVerb(id: string): Promise { - return this.deleteEdge(id) - } - - /** - * Get edges by source node ID - */ - public async getEdgesBySource(sourceId: string): Promise { - await this.ensureInitialized() - - try { - const allEdges = await this.getAllEdges() - return allEdges.filter((edge) => edge.sourceId === sourceId) - } catch (error) { - console.error(`Failed to get edges by source ${sourceId}:`, error) - throw new Error(`Failed to get edges by source ${sourceId}: ${error}`) - } - } - - /** - * Get verbs by source node ID (alias for getEdgesBySource) - */ - public async getVerbsBySource(sourceId: string): Promise { - return this.getEdgesBySource(sourceId) - } - - /** - * Get edges by target node ID - */ - public async getEdgesByTarget(targetId: string): Promise { - await this.ensureInitialized() - - try { - const allEdges = await this.getAllEdges() - return allEdges.filter((edge) => edge.targetId === targetId) - } catch (error) { - console.error(`Failed to get edges by target ${targetId}:`, error) - throw new Error(`Failed to get edges by target ${targetId}: ${error}`) - } - } - - /** - * Get verbs by target node ID (alias for getEdgesByTarget) - */ - public async getVerbsByTarget(targetId: string): Promise { - return this.getEdgesByTarget(targetId) - } - - /** - * Get edges by type - */ - public async getEdgesByType(type: string): Promise { - await this.ensureInitialized() - - try { - const allEdges = await this.getAllEdges() - return allEdges.filter((edge) => edge.type === type) - } catch (error) { - console.error(`Failed to get edges by type ${type}:`, error) - throw new Error(`Failed to get edges by type ${type}: ${error}`) - } - } - - /** - * Get verbs by type (alias for getEdgesByType) - */ - public async getVerbsByType(type: string): Promise { - return this.getEdgesByType(type) - } - - /** - * Save metadata for a node - */ - public async saveMetadata(id: string, metadata: any): Promise { - await this.ensureInitialized() - - try { - // Create or get the file for this metadata - const fileHandle = await this.metadataDir!.getFileHandle(id, { - create: true - }) - - // Write the metadata to the file - const writable = await fileHandle.createWritable() - await writable.write(JSON.stringify(metadata)) - await writable.close() - } catch (error) { - console.error(`Failed to save metadata for ${id}:`, error) - throw new Error(`Failed to save metadata for ${id}: ${error}`) - } - } - - /** - * Get metadata for a node - */ - public async getMetadata(id: string): Promise { - await this.ensureInitialized() - - try { - // Get the file handle for this metadata - const fileHandle = await this.metadataDir!.getFileHandle(id) - - // Read the metadata from the file - const file = await fileHandle.getFile() - const text = await file.text() - return JSON.parse(text) - } catch (error) { - // If the file doesn't exist, return null - if ((error as any).name === 'NotFoundError') { - return null - } - - console.error(`Failed to get metadata for ${id}:`, error) - throw new Error(`Failed to get metadata for ${id}: ${error}`) - } - } - - /** - * Clear all data from storage - */ - public async clear(): Promise { - await this.ensureInitialized() - - try { - // Delete and recreate the nouns directory - await this.rootDir!.removeEntry(NOUNS_DIR, { recursive: true }) - this.nounsDir = await this.rootDir!.getDirectoryHandle(NOUNS_DIR, { - create: true - }) - - // Create noun type directories - this.personDir = await this.nounsDir.getDirectoryHandle(PERSON_DIR, { - create: true - }) - this.placeDir = await this.nounsDir.getDirectoryHandle(PLACE_DIR, { - create: true - }) - this.thingDir = await this.nounsDir.getDirectoryHandle(THING_DIR, { - create: true - }) - this.eventDir = await this.nounsDir.getDirectoryHandle(EVENT_DIR, { - create: true - }) - this.conceptDir = await this.nounsDir.getDirectoryHandle(CONCEPT_DIR, { - create: true - }) - this.contentDir = await this.nounsDir.getDirectoryHandle(CONTENT_DIR, { - create: true - }) - this.defaultDir = await this.nounsDir.getDirectoryHandle(DEFAULT_DIR, { - create: true - }) - - // Delete and recreate the verbs directory - await this.rootDir!.removeEntry(VERBS_DIR, { recursive: true }) - this.verbsDir = await this.rootDir!.getDirectoryHandle(VERBS_DIR, { - create: true - }) - - // Delete and recreate the metadata directory - await this.rootDir!.removeEntry(METADATA_DIR, { recursive: true }) - this.metadataDir = await this.rootDir!.getDirectoryHandle(METADATA_DIR, { - create: true - }) - } catch (error) { - console.error('Failed to clear storage:', error) - throw new Error(`Failed to clear storage: ${error}`) - } - } - - /** - * Ensure the storage adapter is initialized - */ - private async ensureInitialized(): Promise { - if (!this.isInitialized) { - await this.init() - } - } - - /** - * Convert a Map to a plain object for serialization - */ - private mapToObject( - map: Map, - valueTransformer: (value: V) => any = (v) => v - ): Record { - const obj: Record = {} - for (const [key, value] of map.entries()) { - obj[key.toString()] = valueTransformer(value) - } - return obj - } - - /** - * Get the appropriate directory for a node based on its metadata - */ - private async getNodeDirectory( - id: string - ): Promise { - try { - // Try to get the metadata for the node - const metadata = await this.getMetadata(id) - - // If metadata exists and has a noun field, use the corresponding directory - if (metadata && metadata.noun) { - switch (metadata.noun) { - case 'person': - return this.personDir! - case 'place': - return this.placeDir! - case 'thing': - return this.thingDir! - case 'event': - return this.eventDir! - case 'concept': - return this.conceptDir! - case 'content': - return this.contentDir! - default: - return this.defaultDir! - } - } - - // If no metadata or no noun field, use the default directory - return this.defaultDir! - } catch (error) { - // If there's an error getting the metadata, use the default directory - return this.defaultDir! - } - } - - /** - * Get information about storage usage and capacity - */ - public async getStorageStatus(): Promise<{ - type: string - used: number - quota: number | null - details?: Record - }> { - await this.ensureInitialized() - - try { - // Calculate the total size of all files in the storage directories - let totalSize = 0 - - // Helper function to calculate directory size - const calculateDirSize = async ( - dirHandle: FileSystemDirectoryHandle - ): Promise => { - let size = 0 - try { - for await (const [name, handle] of dirHandle.entries()) { - if (handle.kind === 'file') { - const file = await (handle as FileSystemFileHandle).getFile() - size += file.size - } else if (handle.kind === 'directory') { - size += await calculateDirSize( - handle as FileSystemDirectoryHandle - ) - } - } + // Write the metadata to the file + const writable = await fileHandle.createWritable() + await writable.write(JSON.stringify(metadata)) + await writable.close() } catch (error) { - console.warn(`Error calculating size for directory:`, error) + console.error(`Failed to save metadata for ${id}:`, error) + throw new Error(`Failed to save metadata for ${id}: ${error}`) + } + } + + /** + * Get metadata for a node + */ + public async getMetadata(id: string): Promise { + await this.ensureInitialized() + + try { + // Get the file handle for this metadata + const fileHandle = await this.metadataDir!.getFileHandle(id) + + // Read the metadata from the file + const file = await fileHandle.getFile() + const text = await file.text() + return JSON.parse(text) + } catch (error) { + // If the file doesn't exist, return null + if ((error as any).name === 'NotFoundError') { + return null + } + + console.error(`Failed to get metadata for ${id}:`, error) + throw new Error(`Failed to get metadata for ${id}: ${error}`) + } + } + + /** + * Clear all data from storage + */ + public async clear(): Promise { + await this.ensureInitialized() + + try { + // Delete and recreate the nouns directory + await this.rootDir!.removeEntry(NOUNS_DIR, {recursive: true}) + this.nounsDir = await this.rootDir!.getDirectoryHandle(NOUNS_DIR, { + create: true + }) + + // Delete and recreate the verbs directory + await this.rootDir!.removeEntry(VERBS_DIR, {recursive: true}) + this.verbsDir = await this.rootDir!.getDirectoryHandle(VERBS_DIR, { + create: true + }) + + // Delete and recreate the metadata directory + await this.rootDir!.removeEntry(METADATA_DIR, {recursive: true}) + this.metadataDir = await this.rootDir!.getDirectoryHandle(METADATA_DIR, { + create: true + }) + } catch (error) { + console.error('Failed to clear storage:', error) + throw new Error(`Failed to clear storage: ${error}`) + } + } + + /** + * Ensure the storage adapter is initialized + */ + private async ensureInitialized(): Promise { + if (!this.isInitialized) { + await this.init() + } + } + + /** + * Convert a Map to a plain object for serialization + */ + private mapToObject( + map: Map, + valueTransformer: (value: V) => any = (v) => v + ): Record { + const obj: Record = {} + for (const [key, value] of map.entries()) { + obj[key.toString()] = valueTransformer(value) + } + return obj + } + + /** + * Get the directory for a node - now all nouns use the same directory + */ + private async getNodeDirectory( + id: string + ): Promise { + // All nouns now use the same directory regardless of type + return this.nounsDir! + } + + /** + * Get information about storage usage and capacity + */ + public async getStorageStatus(): Promise<{ + type: string + used: number + quota: number | null + details?: Record + }> { + await this.ensureInitialized() + + try { + // Calculate the total size of all files in the storage directories + let totalSize = 0 + + // Helper function to calculate directory size + const calculateDirSize = async ( + dirHandle: FileSystemDirectoryHandle + ): Promise => { + let size = 0 + try { + for await (const [name, handle] of dirHandle.entries()) { + if (handle.kind === 'file') { + const file = await (handle as FileSystemFileHandle).getFile() + size += file.size + } else if (handle.kind === 'directory') { + size += await calculateDirSize( + handle as FileSystemDirectoryHandle + ) + } + } + } catch (error) { + console.warn(`Error calculating size for directory:`, error) + } + return size + } + + // Helper function to count files in a directory + const countFilesInDirectory = async ( + dirHandle: FileSystemDirectoryHandle + ): Promise => { + let count = 0 + try { + for await (const [name, handle] of dirHandle.entries()) { + if (handle.kind === 'file') { + count++ + } + } + } catch (error) { + console.warn(`Error counting files in directory:`, error) + } + return count + } + + // Calculate size for each directory + if (this.nounsDir) { + totalSize += await calculateDirSize(this.nounsDir) + } + if (this.verbsDir) { + totalSize += await calculateDirSize(this.verbsDir) + } + if (this.metadataDir) { + totalSize += await calculateDirSize(this.metadataDir) + } + + // Get storage quota information using the Storage API + let quota = null + let details: Record = { + isPersistent: await this.isPersistent(), + nounTypes: { + // Count nouns by type using metadata + // This will be populated later if needed + } + } + + try { + if (navigator.storage && navigator.storage.estimate) { + const estimate = await navigator.storage.estimate() + quota = estimate.quota || null + details = { + ...details, + usage: estimate.usage, + quota: estimate.quota, + freePercentage: estimate.quota + ? ((estimate.quota - (estimate.usage || 0)) / estimate.quota) * + 100 + : null + } + } + } catch (error) { + console.warn('Unable to get storage estimate:', error) + } + + return { + type: 'opfs', + used: totalSize, + quota, + details + } + } catch (error) { + console.error('Failed to get storage status:', error) + return { + type: 'opfs', + used: 0, + quota: null, + details: {error: String(error)} + } } - return size - } - - // Helper function to count files in a directory - const countFilesInDirectory = async ( - dirHandle: FileSystemDirectoryHandle - ): Promise => { - let count = 0 - try { - for await (const [name, handle] of dirHandle.entries()) { - if (handle.kind === 'file') { - count++ - } - } - } catch (error) { - console.warn(`Error counting files in directory:`, error) - } - return count - } - - // Calculate size for each directory - if (this.nounsDir) { - totalSize += await calculateDirSize(this.nounsDir) - } - if (this.verbsDir) { - totalSize += await calculateDirSize(this.verbsDir) - } - if (this.metadataDir) { - totalSize += await calculateDirSize(this.metadataDir) - } - - // Calculate sizes of noun type directories - const personDirSize = this.personDir - ? await calculateDirSize(this.personDir) - : 0 - const placeDirSize = this.placeDir - ? await calculateDirSize(this.placeDir) - : 0 - const thingDirSize = this.thingDir - ? await calculateDirSize(this.thingDir) - : 0 - const eventDirSize = this.eventDir - ? await calculateDirSize(this.eventDir) - : 0 - const conceptDirSize = this.conceptDir - ? await calculateDirSize(this.conceptDir) - : 0 - const contentDirSize = this.contentDir - ? await calculateDirSize(this.contentDir) - : 0 - const defaultDirSize = this.defaultDir - ? await calculateDirSize(this.defaultDir) - : 0 - - // Get storage quota information using the Storage API - let quota = null - let details: Record = { - isPersistent: await this.isPersistent(), - nounTypes: { - person: { - size: personDirSize, - count: this.personDir - ? await countFilesInDirectory(this.personDir) - : 0 - }, - place: { - size: placeDirSize, - count: this.placeDir - ? await countFilesInDirectory(this.placeDir) - : 0 - }, - thing: { - size: thingDirSize, - count: this.thingDir - ? await countFilesInDirectory(this.thingDir) - : 0 - }, - event: { - size: eventDirSize, - count: this.eventDir - ? await countFilesInDirectory(this.eventDir) - : 0 - }, - concept: { - size: conceptDirSize, - count: this.conceptDir - ? await countFilesInDirectory(this.conceptDir) - : 0 - }, - content: { - size: contentDirSize, - count: this.contentDir - ? await countFilesInDirectory(this.contentDir) - : 0 - }, - default: { - size: defaultDirSize, - count: this.defaultDir - ? await countFilesInDirectory(this.defaultDir) - : 0 - } - } - } - - try { - if (navigator.storage && navigator.storage.estimate) { - const estimate = await navigator.storage.estimate() - quota = estimate.quota || null - details = { - ...details, - usage: estimate.usage, - quota: estimate.quota, - freePercentage: estimate.quota - ? ((estimate.quota - (estimate.usage || 0)) / estimate.quota) * - 100 - : null - } - } - } catch (error) { - console.warn('Unable to get storage estimate:', error) - } - - return { - type: 'opfs', - used: totalSize, - quota, - details - } - } catch (error) { - console.error('Failed to get storage status:', error) - return { - type: 'opfs', - used: 0, - quota: null, - details: { error: String(error) } - } } - } } /** * In-memory storage adapter for environments where OPFS is not available */ export class MemoryStorage implements StorageAdapter { - // Map of noun type to Map of noun ID to noun - private nouns: Map> = new Map() - private verbs: Map = new Map() - private metadata: Map = new Map() + // Single map of noun ID to noun + private nouns: Map = new Map() + private verbs: Map = new Map() + private metadata: Map = new Map() - // Initialize maps for each noun type - constructor() { - this.nouns.set(PERSON_DIR, new Map()) - this.nouns.set(PLACE_DIR, new Map()) - this.nouns.set(THING_DIR, new Map()) - this.nouns.set(EVENT_DIR, new Map()) - this.nouns.set(CONCEPT_DIR, new Map()) - this.nouns.set(CONTENT_DIR, new Map()) - this.nouns.set(DEFAULT_DIR, new Map()) - } + constructor() { + // No need to initialize separate maps for each noun type + } - // Alias methods to match StorageAdapter interface - public async saveNoun(noun: HNSWNoun): Promise { - return this.saveNode(noun) - } + // Alias methods to match StorageAdapter interface + public async saveNoun(noun: HNSWNoun): Promise { + return this.saveNode(noun) + } - public async getNoun(id: string): Promise { - return this.getNode(id) - } + public async getNoun(id: string): Promise { + return this.getNode(id) + } - public async getAllNouns(): Promise { - return this.getAllNodes() - } + public async getAllNouns(): Promise { + return this.getAllNodes() + } - public async getNounsByNounType(nounType: string): Promise { - return this.getNodesByNounType(nounType) - } + public async getNounsByNounType(nounType: string): Promise { + return this.getNodesByNounType(nounType) + } - public async deleteNoun(id: string): Promise { - return this.deleteNode(id) - } + public async deleteNoun(id: string): Promise { + return this.deleteNode(id) + } - public async saveVerb(verb: GraphVerb): Promise { - return this.saveEdge(verb) - } + public async saveVerb(verb: GraphVerb): Promise { + return this.saveEdge(verb) + } - public async getVerb(id: string): Promise { - return this.getEdge(id) - } + public async getVerb(id: string): Promise { + return this.getEdge(id) + } - public async getAllVerbs(): Promise { - return this.getAllEdges() - } + public async getAllVerbs(): Promise { + return this.getAllEdges() + } - public async getVerbsBySource(sourceId: string): Promise { - return this.getEdgesBySource(sourceId) - } + public async getVerbsBySource(sourceId: string): Promise { + return this.getEdgesBySource(sourceId) + } - public async getVerbsByTarget(targetId: string): Promise { - return this.getEdgesByTarget(targetId) - } + public async getVerbsByTarget(targetId: string): Promise { + return this.getEdgesByTarget(targetId) + } - public async getVerbsByType(type: string): Promise { - return this.getEdgesByType(type) - } + public async getVerbsByType(type: string): Promise { + return this.getEdgesByType(type) + } - public async deleteVerb(id: string): Promise { - return this.deleteEdge(id) - } + public async deleteVerb(id: string): Promise { + return this.deleteEdge(id) + } - public async init(): Promise { - // Nothing to initialize for in-memory storage - } + public async init(): Promise { + // Nothing to initialize for in-memory storage + } - /** - * Get the appropriate node type for a node based on its metadata - */ - private async getNodeType(id: string): Promise { - try { - // Try to get the metadata for the node - const metadata = await this.getMetadata(id) + /** + * Get the noun type for a node from its metadata + */ + private async getNounType(id: string): Promise { + try { + // Try to get the metadata for the node + const metadata = await this.getMetadata(id) - // If metadata exists and has a noun field, use the corresponding type - if (metadata && metadata.noun) { - switch (metadata.noun) { - case 'person': - return PERSON_DIR - case 'place': - return PLACE_DIR - case 'thing': - return THING_DIR - case 'event': - return EVENT_DIR - case 'concept': - return CONCEPT_DIR - case 'content': - return CONTENT_DIR - default: - return DEFAULT_DIR + // If metadata exists and has a noun field, return it + if (metadata && metadata.noun) { + return metadata.noun + } + + // If no metadata or no noun field, return null + return null + } catch (error) { + // If there's an error getting the metadata, return null + return null } - } - - // If no metadata or no noun field, use the default type - return DEFAULT_DIR - } catch (error) { - // If there's an error getting the metadata, use the default type - return DEFAULT_DIR - } - } - - public async saveNode(node: HNSWNode): Promise { - // Create a deep copy to avoid reference issues - const nodeCopy: HNSWNode = { - id: node.id, - vector: [...node.vector], - connections: new Map() } - // Copy connections - for (const [level, connections] of node.connections.entries()) { - nodeCopy.connections.set(level, new Set(connections)) + public async saveNode(node: HNSWNode): Promise { + // Create a deep copy to avoid reference issues + const nodeCopy: HNSWNode = { + id: node.id, + vector: [...node.vector], + connections: new Map() + } + + // Copy connections + for (const [level, connections] of node.connections.entries()) { + nodeCopy.connections.set(level, new Set(connections)) + } + + // Save the node directly in the nouns map + this.nouns.set(node.id, nodeCopy) } - // Get the appropriate node type based on the node's metadata - const nodeType = await this.getNodeType(node.id) + public async getNode(id: string): Promise { + // Get the node directly from the nouns map + const node = this.nouns.get(id) - // Get the map for this node type - const nodeMap = this.nouns.get(nodeType)! - - // Save the node in the appropriate map - nodeMap.set(node.id, nodeCopy) - } - - public async getNode(id: string): Promise { - // Get the appropriate node type based on the node's metadata - const nodeType = await this.getNodeType(id) - - // Get the map for this node type - const nodeMap = this.nouns.get(nodeType)! - - // Try to get the node from the appropriate map - let node = nodeMap.get(id) - - // If not found in the expected map, try other maps - if (!node) { - // If the node type is not the default type, try the default map - if (nodeType !== DEFAULT_DIR) { - const defaultMap = this.nouns.get(DEFAULT_DIR)! - node = defaultMap.get(id) - - // If still not found, try all other maps + // If not found, return null if (!node) { - const nodeTypes = [ - PERSON_DIR, - PLACE_DIR, - THING_DIR, - EVENT_DIR, - CONCEPT_DIR, - CONTENT_DIR - ] - for (const type of nodeTypes) { - if (type === nodeType) continue // Skip the already checked map - - const typeMap = this.nouns.get(type)! - node = typeMap.get(id) - if (node) break // Found the node, exit the loop - } - } - } - - // If still not found, return null - if (!node) { - return null - } - } - - // Return a deep copy to avoid reference issues - const nodeCopy: HNSWNode = { - id: node.id, - vector: [...node.vector], - connections: new Map() - } - - // Copy connections - for (const [level, connections] of node.connections.entries()) { - nodeCopy.connections.set(level, new Set(connections)) - } - - return nodeCopy - } - - /** - * Get nodes by noun type - * @param nounType The noun type to filter by - * @returns Promise that resolves to an array of nodes of the specified noun type - */ - public async getNodesByNounType(nounType: string): Promise { - const nodes: HNSWNode[] = [] - - // Get the map for this noun type - let typeMap: Map - switch (nounType) { - case 'person': - typeMap = this.nouns.get(PERSON_DIR)! - break - case 'place': - typeMap = this.nouns.get(PLACE_DIR)! - break - case 'thing': - typeMap = this.nouns.get(THING_DIR)! - break - case 'event': - typeMap = this.nouns.get(EVENT_DIR)! - break - case 'concept': - typeMap = this.nouns.get(CONCEPT_DIR)! - break - case 'content': - typeMap = this.nouns.get(CONTENT_DIR)! - break - default: - typeMap = this.nouns.get(DEFAULT_DIR)! - } - - // Get all nodes from this map - for (const nodeId of typeMap.keys()) { - const node = await this.getNode(nodeId) - if (node) { - nodes.push(node) - } - } - - return nodes - } - - public async getAllNodes(): Promise { - // Get all noun types - const nounTypes = [ - 'person', - 'place', - 'thing', - 'event', - 'concept', - 'content', - 'default' - ] - - // Run searches in parallel for all noun types - const nodePromises = nounTypes.map((nounType) => - this.getNodesByNounType(nounType) - ) - const nodeArrays = await Promise.all(nodePromises) - - // Combine all results - const allNodes: HNSWNode[] = [] - for (const nodes of nodeArrays) { - allNodes.push(...nodes) - } - - return allNodes - } - - public async deleteNode(id: string): Promise { - // Get the appropriate node type based on the node's metadata - const nodeType = await this.getNodeType(id) - - // Get the map for this node type - const nodeMap = this.nouns.get(nodeType)! - - // Try to delete the node from the appropriate map - const deleted = nodeMap.delete(id) - - // If not found in the expected map, try other maps - if (!deleted) { - // If the node type is not the default type, try the default map - if (nodeType !== DEFAULT_DIR) { - const defaultMap = this.nouns.get(DEFAULT_DIR)! - const deletedFromDefault = defaultMap.delete(id) - - // If still not found, try all other maps - if (!deletedFromDefault) { - const nodeTypes = [ - PERSON_DIR, - PLACE_DIR, - THING_DIR, - EVENT_DIR, - CONCEPT_DIR, - CONTENT_DIR - ] - for (const type of nodeTypes) { - if (type === nodeType) continue // Skip the already checked map - - const typeMap = this.nouns.get(type)! - const deletedFromType = typeMap.delete(id) - if (deletedFromType) break // Node deleted, exit the loop - } - } - } - } - } - - public async saveMetadata(id: string, metadata: any): Promise { - this.metadata.set(id, JSON.parse(JSON.stringify(metadata))) - } - - public async getMetadata(id: string): Promise { - const metadata = this.metadata.get(id) - if (!metadata) { - return null - } - - return JSON.parse(JSON.stringify(metadata)) - } - - public async saveEdge(edge: Edge): Promise { - // Create a deep copy to avoid reference issues - const edgeCopy: Edge = { - id: edge.id, - vector: [...edge.vector], - connections: new Map(), - sourceId: edge.sourceId, - targetId: edge.targetId, - type: edge.type, - weight: edge.weight, - metadata: edge.metadata - ? JSON.parse(JSON.stringify(edge.metadata)) - : undefined - } - - // Copy connections - for (const [level, connections] of edge.connections.entries()) { - edgeCopy.connections.set(level, new Set(connections)) - } - - this.verbs.set(edge.id, edgeCopy) - } - - public async getEdge(id: string): Promise { - const edge = this.verbs.get(id) - if (!edge) { - return null - } - - // Return a deep copy to avoid reference issues - const edgeCopy: Edge = { - id: edge.id, - vector: [...edge.vector], - connections: new Map(), - sourceId: edge.sourceId, - targetId: edge.targetId, - type: edge.type, - weight: edge.weight, - metadata: edge.metadata - ? JSON.parse(JSON.stringify(edge.metadata)) - : undefined - } - - // Copy connections - for (const [level, connections] of edge.connections.entries()) { - edgeCopy.connections.set(level, new Set(connections)) - } - - return edgeCopy - } - - public async getAllEdges(): Promise { - const edges: Edge[] = [] - - for (const edgeId of this.verbs.keys()) { - const edge = await this.getEdge(edgeId) - if (edge) { - edges.push(edge) - } - } - - return edges - } - - public async getEdgesBySource(sourceId: string): Promise { - const edges: Edge[] = [] - - for (const edge of this.verbs.values()) { - if (edge.sourceId === sourceId) { - const edgeCopy = await this.getEdge(edge.id) - if (edgeCopy) { - edges.push(edgeCopy) - } - } - } - - return edges - } - - public async getEdgesByTarget(targetId: string): Promise { - const edges: Edge[] = [] - - for (const edge of this.verbs.values()) { - if (edge.targetId === targetId) { - const edgeCopy = await this.getEdge(edge.id) - if (edgeCopy) { - edges.push(edgeCopy) - } - } - } - - return edges - } - - public async getEdgesByType(type: string): Promise { - const edges: Edge[] = [] - - for (const edge of this.verbs.values()) { - if (edge.type === type) { - const edgeCopy = await this.getEdge(edge.id) - if (edgeCopy) { - edges.push(edgeCopy) - } - } - } - - return edges - } - - public async deleteEdge(id: string): Promise { - this.verbs.delete(id) - } - - public async clear(): Promise { - // Clear all noun type maps - const nodeTypes = [ - PERSON_DIR, - PLACE_DIR, - THING_DIR, - EVENT_DIR, - CONCEPT_DIR, - CONTENT_DIR, - DEFAULT_DIR - ] - for (const type of nodeTypes) { - const typeMap = this.nouns.get(type)! - typeMap.clear() - } - - this.verbs.clear() - this.metadata.clear() - } - - /** - * Get information about storage usage and capacity - */ - public async getStorageStatus(): Promise<{ - type: string - used: number - quota: number | null - details?: Record - }> { - try { - // Estimate the size of data in memory - let totalSize = 0 - - // Helper function to estimate object size in bytes - const estimateSize = (obj: any): number => { - if (obj === null || obj === undefined) return 0 - - const type = typeof obj - - // Handle primitive types - if (type === 'number') return 8 - if (type === 'string') return obj.length * 2 - if (type === 'boolean') return 4 - - // Handle arrays and objects - if (Array.isArray(obj)) { - return obj.reduce((size, item) => size + estimateSize(item), 0) + return null } - if (type === 'object') { - if (obj instanceof Map) { - let mapSize = 0 - for (const [key, value] of obj.entries()) { - mapSize += estimateSize(key) + estimateSize(value) + // Return a deep copy to avoid reference issues + const nodeCopy: HNSWNode = { + id: node.id, + vector: [...node.vector], + connections: new Map() + } + + // Copy connections + for (const [level, connections] of node.connections.entries()) { + nodeCopy.connections.set(level, new Set(connections)) + } + + return nodeCopy + } + + /** + * Get nodes by noun type + * @param nounType The noun type to filter by + * @returns Promise that resolves to an array of nodes of the specified noun type + */ + public async getNodesByNounType(nounType: string): Promise { + const nodes: HNSWNode[] = [] + + // Iterate through all nodes and filter by noun type using metadata + for (const [nodeId, node] of this.nouns.entries()) { + // Get the metadata to check the noun type + const metadata = await this.getMetadata(nodeId) + + // Include the node if its noun type matches the requested type + if (metadata && metadata.noun === nounType) { + // Return a deep copy to avoid reference issues + const nodeCopy: HNSWNode = { + id: node.id, + vector: [...node.vector], + connections: new Map() + } + + // Copy connections + for (const [level, connections] of node.connections.entries()) { + nodeCopy.connections.set(level, new Set(connections)) + } + + nodes.push(nodeCopy) } - return mapSize - } - - if (obj instanceof Set) { - let setSize = 0 - for (const item of obj) { - setSize += estimateSize(item) - } - return setSize - } - - // Regular object - return Object.entries(obj).reduce( - (size, [key, value]) => size + key.length * 2 + estimateSize(value), - 0 - ) } - return 0 - } - - // Calculate sizes and counts for each noun type - const nounTypeDetails: Record = - {} - const nodeTypes = [ - PERSON_DIR, - PLACE_DIR, - THING_DIR, - EVENT_DIR, - CONCEPT_DIR, - CONTENT_DIR, - DEFAULT_DIR - ] - - let totalNodeCount = 0 - let nodesSize = 0 - - for (const type of nodeTypes) { - const typeMap = this.nouns.get(type)! - let typeSize = 0 - - for (const node of typeMap.values()) { - typeSize += estimateSize(node) - } - - nounTypeDetails[type] = { - size: typeSize, - count: typeMap.size - } - - nodesSize += typeSize - totalNodeCount += typeMap.size - } - - totalSize += nodesSize - - // Estimate size of edges - let edgesSize = 0 - for (const edge of this.verbs.values()) { - edgesSize += estimateSize(edge) - } - totalSize += edgesSize - - // Estimate size of metadata - let metadataSize = 0 - for (const meta of this.metadata.values()) { - metadataSize += estimateSize(meta) - } - totalSize += metadataSize - - // Get memory information if available - let quota = null - let details: Record = { - nodeCount: totalNodeCount, - edgeCount: this.verbs.size, - metadataCount: this.metadata.size, - nounTypes: nounTypeDetails, - nodesSize, - edgesSize, - metadataSize - } - - // Try to get memory information if in a browser environment - if ( - typeof window !== 'undefined' && - (window as any).performance && - (window as any).performance.memory - ) { - const memory = (window as any).performance.memory - quota = memory.jsHeapSizeLimit - details = { - ...details, - totalJSHeapSize: memory.totalJSHeapSize, - usedJSHeapSize: memory.usedJSHeapSize, - jsHeapSizeLimit: memory.jsHeapSizeLimit - } - } - - return { - type: 'memory', - used: totalSize, - quota, - details - } - } catch (error) { - console.error('Failed to get memory storage status:', error) - return { - type: 'memory', - used: 0, - quota: null, - details: { error: String(error) } - } + return nodes + } + + public async getAllNodes(): Promise { + const allNodes: HNSWNode[] = [] + + // Iterate through all nodes in the nouns map + for (const [nodeId, node] of this.nouns.entries()) { + // Return a deep copy to avoid reference issues + const nodeCopy: HNSWNode = { + id: node.id, + vector: [...node.vector], + connections: new Map() + } + + // Copy connections + for (const [level, connections] of node.connections.entries()) { + nodeCopy.connections.set(level, new Set(connections)) + } + + allNodes.push(nodeCopy) + } + + return allNodes + } + + public async deleteNode(id: string): Promise { + // Delete the node directly from the nouns map + this.nouns.delete(id) + } + + public async saveMetadata(id: string, metadata: any): Promise { + this.metadata.set(id, JSON.parse(JSON.stringify(metadata))) + } + + public async getMetadata(id: string): Promise { + const metadata = this.metadata.get(id) + if (!metadata) { + return null + } + + return JSON.parse(JSON.stringify(metadata)) + } + + public async saveEdge(edge: Edge): Promise { + // Create a deep copy to avoid reference issues + const edgeCopy: Edge = { + id: edge.id, + vector: [...edge.vector], + connections: new Map(), + sourceId: edge.sourceId, + targetId: edge.targetId, + type: edge.type, + weight: edge.weight, + metadata: edge.metadata + ? JSON.parse(JSON.stringify(edge.metadata)) + : undefined + } + + // Copy connections + for (const [level, connections] of edge.connections.entries()) { + edgeCopy.connections.set(level, new Set(connections)) + } + + this.verbs.set(edge.id, edgeCopy) + } + + public async getEdge(id: string): Promise { + const edge = this.verbs.get(id) + if (!edge) { + return null + } + + // Return a deep copy to avoid reference issues + const edgeCopy: Edge = { + id: edge.id, + vector: [...edge.vector], + connections: new Map(), + sourceId: edge.sourceId, + targetId: edge.targetId, + type: edge.type, + weight: edge.weight, + metadata: edge.metadata + ? JSON.parse(JSON.stringify(edge.metadata)) + : undefined + } + + // Copy connections + for (const [level, connections] of edge.connections.entries()) { + edgeCopy.connections.set(level, new Set(connections)) + } + + return edgeCopy + } + + public async getAllEdges(): Promise { + const edges: Edge[] = [] + + for (const edgeId of this.verbs.keys()) { + const edge = await this.getEdge(edgeId) + if (edge) { + edges.push(edge) + } + } + + return edges + } + + public async getEdgesBySource(sourceId: string): Promise { + const edges: Edge[] = [] + + for (const edge of this.verbs.values()) { + if (edge.sourceId === sourceId) { + const edgeCopy = await this.getEdge(edge.id) + if (edgeCopy) { + edges.push(edgeCopy) + } + } + } + + return edges + } + + public async getEdgesByTarget(targetId: string): Promise { + const edges: Edge[] = [] + + for (const edge of this.verbs.values()) { + if (edge.targetId === targetId) { + const edgeCopy = await this.getEdge(edge.id) + if (edgeCopy) { + edges.push(edgeCopy) + } + } + } + + return edges + } + + public async getEdgesByType(type: string): Promise { + const edges: Edge[] = [] + + for (const edge of this.verbs.values()) { + if (edge.type === type) { + const edgeCopy = await this.getEdge(edge.id) + if (edgeCopy) { + edges.push(edgeCopy) + } + } + } + + return edges + } + + public async deleteEdge(id: string): Promise { + this.verbs.delete(id) + } + + public async clear(): Promise { + // Clear the single nouns map + this.nouns.clear() + this.verbs.clear() + this.metadata.clear() + } + + /** + * Get information about storage usage and capacity + */ + public async getStorageStatus(): Promise<{ + type: string + used: number + quota: number | null + details?: Record + }> { + try { + // Estimate the size of data in memory + let totalSize = 0 + + // Helper function to estimate object size in bytes + const estimateSize = (obj: any): number => { + if (obj === null || obj === undefined) return 0 + + const type = typeof obj + + // Handle primitive types + if (type === 'number') return 8 + if (type === 'string') return obj.length * 2 + if (type === 'boolean') return 4 + + // Handle arrays and objects + if (Array.isArray(obj)) { + return obj.reduce((size, item) => size + estimateSize(item), 0) + } + + if (type === 'object') { + if (obj instanceof Map) { + let mapSize = 0 + for (const [key, value] of obj.entries()) { + mapSize += estimateSize(key) + estimateSize(value) + } + return mapSize + } + + if (obj instanceof Set) { + let setSize = 0 + for (const item of obj) { + setSize += estimateSize(item) + } + return setSize + } + + // Regular object + return Object.entries(obj).reduce( + (size, [key, value]) => size + key.length * 2 + estimateSize(value), + 0 + ) + } + + return 0 + } + + // Calculate sizes and counts for each noun type + const nounTypeDetails: Record = {} + + // Initialize counts for all noun types + const nodeTypes = [ + 'person', + 'place', + 'thing', + 'event', + 'concept', + 'content', + 'group', + 'list', + 'category', + 'default' + ] + + for (const type of nodeTypes) { + nounTypeDetails[type] = { + size: 0, + count: 0 + } + } + + let totalNodeCount = 0 + let nodesSize = 0 + + // Process all nouns and categorize them by type using metadata + for (const [nodeId, node] of this.nouns.entries()) { + // Get the metadata to determine the noun type + const metadata = this.metadata.get(nodeId) + const nounType = metadata?.noun || 'default' + + // Calculate size of this node + const nodeSize = estimateSize(node) + + // Update counts for this noun type + if (nounTypeDetails[nounType]) { + nounTypeDetails[nounType].size += nodeSize + nounTypeDetails[nounType].count += 1 + } else { + nounTypeDetails.default.size += nodeSize + nounTypeDetails.default.count += 1 + } + + // Update totals + nodesSize += nodeSize + totalNodeCount += 1 + } + + totalSize += nodesSize + + // Estimate size of edges + let edgesSize = 0 + for (const edge of this.verbs.values()) { + edgesSize += estimateSize(edge) + } + totalSize += edgesSize + + // Estimate size of metadata + let metadataSize = 0 + for (const meta of this.metadata.values()) { + metadataSize += estimateSize(meta) + } + totalSize += metadataSize + + // Get memory information if available + let quota = null + let details: Record = { + nodeCount: totalNodeCount, + edgeCount: this.verbs.size, + metadataCount: this.metadata.size, + nounTypes: nounTypeDetails, + nodesSize, + edgesSize, + metadataSize + } + + // Try to get memory information if in a browser environment + if ( + typeof window !== 'undefined' && + (window as any).performance && + (window as any).performance.memory + ) { + const memory = (window as any).performance.memory + quota = memory.jsHeapSizeLimit + details = { + ...details, + totalJSHeapSize: memory.totalJSHeapSize, + usedJSHeapSize: memory.usedJSHeapSize, + jsHeapSizeLimit: memory.jsHeapSizeLimit + } + } + + return { + type: 'memory', + used: totalSize, + quota, + details + } + } catch (error) { + console.error('Failed to get memory storage status:', error) + return { + type: 'memory', + used: 0, + quota: null, + details: {error: String(error)} + } + } } - } } /** @@ -1630,262 +1289,262 @@ export class MemoryStorage implements StorageAdapter { * @returns Promise that resolves to a StorageAdapter instance */ export async function createStorage( - options: { - requestPersistentStorage?: boolean - r2Storage?: { - bucketName?: string - accountId?: string - accessKeyId?: string - secretAccessKey?: string - } - s3Storage?: { - bucketName?: string - accessKeyId?: string - secretAccessKey?: string - region?: string - } - gcsStorage?: { - bucketName?: string - accessKeyId?: string - secretAccessKey?: string - endpoint?: string - } - customS3Storage?: { - bucketName?: string - accessKeyId?: string - secretAccessKey?: string - endpoint?: string - region?: string - } - forceFileSystemStorage?: boolean - forceMemoryStorage?: boolean - } = {} -): Promise { - // Detect environment - const environment = { - isBrowser: typeof window !== 'undefined', - isNode: - typeof process !== 'undefined' && - process.versions != null && - process.versions.node != null, - isServerless: - typeof window === 'undefined' && - (typeof process === 'undefined' || - !process.versions || - !process.versions.node) - } - - // If force memory storage is specified, use MemoryStorage regardless of environment - if (options.forceMemoryStorage) { - console.log('Using in-memory storage (forced by configuration)') - return new MemoryStorage() - } - - // Default empty values for environment variables - const defaultEnvStorage = { - bucketName: undefined, - accountId: undefined, - accessKeyId: undefined, - secretAccessKey: undefined, - region: undefined, - endpoint: undefined - } - - // Try to use cloud storage if configured - if ( - options.r2Storage || - options.s3Storage || - options.gcsStorage || - options.customS3Storage || - (environment.isNode && - (process.env.R2_BUCKET_NAME || - process.env.S3_BUCKET_NAME || - process.env.GCS_BUCKET_NAME)) - ) { - try { - // Only try to access process.env in Node.js environment - const envR2Storage = environment.isNode - ? { - bucketName: process.env.R2_BUCKET_NAME, - accountId: process.env.R2_ACCOUNT_ID, - accessKeyId: process.env.R2_ACCESS_KEY_ID, - secretAccessKey: process.env.R2_SECRET_ACCESS_KEY - } - : defaultEnvStorage - - const envS3Storage = environment.isNode - ? { - bucketName: process.env.S3_BUCKET_NAME, - accessKeyId: - process.env.S3_ACCESS_KEY_ID || process.env.AWS_ACCESS_KEY_ID, - secretAccessKey: - process.env.S3_SECRET_ACCESS_KEY || - process.env.AWS_SECRET_ACCESS_KEY, - region: process.env.S3_REGION || process.env.AWS_REGION - } - : defaultEnvStorage - - const envGCSStorage = environment.isNode - ? { - bucketName: process.env.GCS_BUCKET_NAME, - accessKeyId: process.env.GCS_ACCESS_KEY_ID, - secretAccessKey: process.env.GCS_SECRET_ACCESS_KEY, - endpoint: process.env.GCS_ENDPOINT - } - : defaultEnvStorage - - // Merge environment variables with provided options - const mergedOptions = { - ...options, - r2Storage: options.r2Storage - ? { - ...envR2Storage, - ...options.r2Storage - } - : // Only use environment variables if they have the required fields - envR2Storage.bucketName && - envR2Storage.accessKeyId && - envR2Storage.secretAccessKey - ? envR2Storage - : undefined, - s3Storage: options.s3Storage - ? { - ...envS3Storage, - ...options.s3Storage - } - : // Only use environment variables if they have the required fields - envS3Storage.bucketName && - envS3Storage.accessKeyId && - envS3Storage.secretAccessKey - ? envS3Storage - : undefined, - gcsStorage: options.gcsStorage - ? { - ...envGCSStorage, - ...options.gcsStorage - } - : // Only use environment variables if they have the required fields - envGCSStorage.bucketName && - envGCSStorage.accessKeyId && - envGCSStorage.secretAccessKey - ? envGCSStorage - : undefined - } - - const s3Module = await import('./s3CompatibleStorage.js') - - if ( - mergedOptions.r2Storage && - mergedOptions.r2Storage.bucketName && - mergedOptions.r2Storage.accountId && - mergedOptions.r2Storage.accessKeyId && - mergedOptions.r2Storage.secretAccessKey - ) { - console.log('Using Cloudflare R2 storage') - return new s3Module.R2Storage({ - bucketName: mergedOptions.r2Storage.bucketName, - accountId: mergedOptions.r2Storage.accountId, - accessKeyId: mergedOptions.r2Storage.accessKeyId, - secretAccessKey: mergedOptions.r2Storage.secretAccessKey, - serviceType: 'r2' - }) - } else if ( - mergedOptions.s3Storage && - mergedOptions.s3Storage.bucketName && - mergedOptions.s3Storage.accessKeyId && - mergedOptions.s3Storage.secretAccessKey - ) { - console.log('Using Amazon S3 storage') - return new s3Module.S3CompatibleStorage({ - bucketName: mergedOptions.s3Storage.bucketName, - accessKeyId: mergedOptions.s3Storage.accessKeyId, - secretAccessKey: mergedOptions.s3Storage.secretAccessKey, - region: mergedOptions.s3Storage.region, - serviceType: 's3' - }) - } else if ( - mergedOptions.gcsStorage && - mergedOptions.gcsStorage.bucketName && - mergedOptions.gcsStorage.accessKeyId && - mergedOptions.gcsStorage.secretAccessKey - ) { - console.log('Using Google Cloud Storage') - return new s3Module.S3CompatibleStorage({ - bucketName: mergedOptions.gcsStorage.bucketName, - accessKeyId: mergedOptions.gcsStorage.accessKeyId, - secretAccessKey: mergedOptions.gcsStorage.secretAccessKey, - endpoint: mergedOptions.gcsStorage.endpoint, - serviceType: 'gcs' - }) - } else if ( - mergedOptions.customS3Storage && - mergedOptions.customS3Storage.bucketName && - mergedOptions.customS3Storage.accessKeyId && - mergedOptions.customS3Storage.secretAccessKey - ) { - console.log('Using custom S3-compatible storage') - return new s3Module.S3CompatibleStorage({ - bucketName: mergedOptions.customS3Storage.bucketName, - accessKeyId: mergedOptions.customS3Storage.accessKeyId, - secretAccessKey: mergedOptions.customS3Storage.secretAccessKey, - endpoint: mergedOptions.customS3Storage.endpoint, - region: mergedOptions.customS3Storage.region, - serviceType: 'custom' - }) - } - } catch (error) { - console.warn( - 'Failed to load S3CompatibleStorage, falling back to environment-specific storage:', - error - ) - // Continue to environment-specific storage selection - } - } - - // Environment-specific storage selection - if (environment.isBrowser) { - // In browser environments - if (!options.forceFileSystemStorage) { - try { - // Try OPFS first (Origin Private File System - browser-specific) - const opfsStorage = new OPFSStorage() - - if (opfsStorage.isOPFSAvailable()) { - // Request persistent storage if specified - if (options.requestPersistentStorage) { - const isPersistentGranted = - await opfsStorage.requestPersistentStorage() - if (isPersistentGranted) { - console.log('Persistent storage permission granted') - } else { - console.warn('Persistent storage permission denied') - } - } - console.log('Using Origin Private File System (OPFS) storage') - return opfsStorage + options: { + requestPersistentStorage?: boolean + r2Storage?: { + bucketName?: string + accountId?: string + accessKeyId?: string + secretAccessKey?: string } - } catch (error) { - console.warn('OPFS storage initialization failed:', error) - } + s3Storage?: { + bucketName?: string + accessKeyId?: string + secretAccessKey?: string + region?: string + } + gcsStorage?: { + bucketName?: string + accessKeyId?: string + secretAccessKey?: string + endpoint?: string + } + customS3Storage?: { + bucketName?: string + accessKeyId?: string + secretAccessKey?: string + endpoint?: string + region?: string + } + forceFileSystemStorage?: boolean + forceMemoryStorage?: boolean + } = {} +): Promise { + // Detect environment + const environment = { + isBrowser: typeof window !== 'undefined', + isNode: + typeof process !== 'undefined' && + process.versions != null && + process.versions.node != null, + isServerless: + typeof window === 'undefined' && + (typeof process === 'undefined' || + !process.versions || + !process.versions.node) } - // Fall back to memory storage for browser environments - console.log('Using in-memory storage for browser environment') - return new MemoryStorage() - } else if (environment.isNode) { - // In Node.js environments - try { - const fileSystemModule = await import('./fileSystemStorage.js') - console.log('Using file system storage for Node.js environment') - return new fileSystemModule.FileSystemStorage() - } catch (error) { - console.warn('Failed to load FileSystemStorage:', error) - console.log('Using in-memory storage for Node.js environment') - return new MemoryStorage() + // If force memory storage is specified, use MemoryStorage regardless of environment + if (options.forceMemoryStorage) { + console.log('Using in-memory storage (forced by configuration)') + return new MemoryStorage() + } + + // Default empty values for environment variables + const defaultEnvStorage = { + bucketName: undefined, + accountId: undefined, + accessKeyId: undefined, + secretAccessKey: undefined, + region: undefined, + endpoint: undefined + } + + // Try to use cloud storage if configured + if ( + options.r2Storage || + options.s3Storage || + options.gcsStorage || + options.customS3Storage || + (environment.isNode && + (process.env.R2_BUCKET_NAME || + process.env.S3_BUCKET_NAME || + process.env.GCS_BUCKET_NAME)) + ) { + try { + // Only try to access process.env in Node.js environment + const envR2Storage = environment.isNode + ? { + bucketName: process.env.R2_BUCKET_NAME, + accountId: process.env.R2_ACCOUNT_ID, + accessKeyId: process.env.R2_ACCESS_KEY_ID, + secretAccessKey: process.env.R2_SECRET_ACCESS_KEY + } + : defaultEnvStorage + + const envS3Storage = environment.isNode + ? { + bucketName: process.env.S3_BUCKET_NAME, + accessKeyId: + process.env.S3_ACCESS_KEY_ID || process.env.AWS_ACCESS_KEY_ID, + secretAccessKey: + process.env.S3_SECRET_ACCESS_KEY || + process.env.AWS_SECRET_ACCESS_KEY, + region: process.env.S3_REGION || process.env.AWS_REGION + } + : defaultEnvStorage + + const envGCSStorage = environment.isNode + ? { + bucketName: process.env.GCS_BUCKET_NAME, + accessKeyId: process.env.GCS_ACCESS_KEY_ID, + secretAccessKey: process.env.GCS_SECRET_ACCESS_KEY, + endpoint: process.env.GCS_ENDPOINT + } + : defaultEnvStorage + + // Merge environment variables with provided options + const mergedOptions = { + ...options, + r2Storage: options.r2Storage + ? { + ...envR2Storage, + ...options.r2Storage + } + : // Only use environment variables if they have the required fields + envR2Storage.bucketName && + envR2Storage.accessKeyId && + envR2Storage.secretAccessKey + ? envR2Storage + : undefined, + s3Storage: options.s3Storage + ? { + ...envS3Storage, + ...options.s3Storage + } + : // Only use environment variables if they have the required fields + envS3Storage.bucketName && + envS3Storage.accessKeyId && + envS3Storage.secretAccessKey + ? envS3Storage + : undefined, + gcsStorage: options.gcsStorage + ? { + ...envGCSStorage, + ...options.gcsStorage + } + : // Only use environment variables if they have the required fields + envGCSStorage.bucketName && + envGCSStorage.accessKeyId && + envGCSStorage.secretAccessKey + ? envGCSStorage + : undefined + } + + const s3Module = await import('./s3CompatibleStorage.js') + + if ( + mergedOptions.r2Storage && + mergedOptions.r2Storage.bucketName && + mergedOptions.r2Storage.accountId && + mergedOptions.r2Storage.accessKeyId && + mergedOptions.r2Storage.secretAccessKey + ) { + console.log('Using Cloudflare R2 storage') + return new s3Module.R2Storage({ + bucketName: mergedOptions.r2Storage.bucketName, + accountId: mergedOptions.r2Storage.accountId, + accessKeyId: mergedOptions.r2Storage.accessKeyId, + secretAccessKey: mergedOptions.r2Storage.secretAccessKey, + serviceType: 'r2' + }) + } else if ( + mergedOptions.s3Storage && + mergedOptions.s3Storage.bucketName && + mergedOptions.s3Storage.accessKeyId && + mergedOptions.s3Storage.secretAccessKey + ) { + console.log('Using Amazon S3 storage') + return new s3Module.S3CompatibleStorage({ + bucketName: mergedOptions.s3Storage.bucketName, + accessKeyId: mergedOptions.s3Storage.accessKeyId, + secretAccessKey: mergedOptions.s3Storage.secretAccessKey, + region: mergedOptions.s3Storage.region, + serviceType: 's3' + }) + } else if ( + mergedOptions.gcsStorage && + mergedOptions.gcsStorage.bucketName && + mergedOptions.gcsStorage.accessKeyId && + mergedOptions.gcsStorage.secretAccessKey + ) { + console.log('Using Google Cloud Storage') + return new s3Module.S3CompatibleStorage({ + bucketName: mergedOptions.gcsStorage.bucketName, + accessKeyId: mergedOptions.gcsStorage.accessKeyId, + secretAccessKey: mergedOptions.gcsStorage.secretAccessKey, + endpoint: mergedOptions.gcsStorage.endpoint, + serviceType: 'gcs' + }) + } else if ( + mergedOptions.customS3Storage && + mergedOptions.customS3Storage.bucketName && + mergedOptions.customS3Storage.accessKeyId && + mergedOptions.customS3Storage.secretAccessKey + ) { + console.log('Using custom S3-compatible storage') + return new s3Module.S3CompatibleStorage({ + bucketName: mergedOptions.customS3Storage.bucketName, + accessKeyId: mergedOptions.customS3Storage.accessKeyId, + secretAccessKey: mergedOptions.customS3Storage.secretAccessKey, + endpoint: mergedOptions.customS3Storage.endpoint, + region: mergedOptions.customS3Storage.region, + serviceType: 'custom' + }) + } + } catch (error) { + console.warn( + 'Failed to load S3CompatibleStorage, falling back to environment-specific storage:', + error + ) + // Continue to environment-specific storage selection + } + } + + // Environment-specific storage selection + if (environment.isBrowser) { + // In browser environments + if (!options.forceFileSystemStorage) { + try { + // Try OPFS first (Origin Private File System - browser-specific) + const opfsStorage = new OPFSStorage() + + if (opfsStorage.isOPFSAvailable()) { + // Request persistent storage if specified + if (options.requestPersistentStorage) { + const isPersistentGranted = + await opfsStorage.requestPersistentStorage() + if (isPersistentGranted) { + console.log('Persistent storage permission granted') + } else { + console.warn('Persistent storage permission denied') + } + } + console.log('Using Origin Private File System (OPFS) storage') + return opfsStorage + } + } catch (error) { + console.warn('OPFS storage initialization failed:', error) + } + } + + // Fall back to memory storage for browser environments + console.log('Using in-memory storage for browser environment') + return new MemoryStorage() + } else if (environment.isNode) { + // In Node.js environments + try { + const fileSystemModule = await import('./fileSystemStorage.js') + console.log('Using file system storage for Node.js environment') + return new fileSystemModule.FileSystemStorage() + } catch (error) { + console.warn('Failed to load FileSystemStorage:', error) + console.log('Using in-memory storage for Node.js environment') + return new MemoryStorage() + } + } else { + // In serverless or other environments + console.log('Using in-memory storage for serverless/unknown environment') + return new MemoryStorage() } - } else { - // In serverless or other environments - console.log('Using in-memory storage for serverless/unknown environment') - return new MemoryStorage() - } } diff --git a/src/storage/s3CompatibleStorage.ts b/src/storage/s3CompatibleStorage.ts index 681827ca..291e177d 100644 --- a/src/storage/s3CompatibleStorage.ts +++ b/src/storage/s3CompatibleStorage.ts @@ -13,14 +13,8 @@ const VERBS_PREFIX = 'verbs/' const EDGES_PREFIX = 'verbs/' // Alias for VERBS_PREFIX for edge operations const METADATA_PREFIX = 'metadata/' -// Constants for noun type prefixes -const PERSON_PREFIX = 'nouns/person/' -const PLACE_PREFIX = 'place/' -const THING_PREFIX = 'thing/' -const EVENT_PREFIX = 'event/' -const CONCEPT_PREFIX = 'concept/' -const CONTENT_PREFIX = 'content/' -const DEFAULT_PREFIX = 'default/' // For nodes without a noun type +// All nouns now use the same prefix - no separate directories per noun type +const NOUN_PREFIX = 'nouns/' // Single directory for all noun types /** * S3-compatible storage adapter for server environments @@ -257,45 +251,86 @@ export class S3CompatibleStorage implements StorageAdapter { await this.ensureInitialized() try { - // Get the appropriate prefix based on the node's metadata - const nodePrefix = await this.getNodePrefix(id) - // Import the GetObjectCommand only when needed const { GetObjectCommand } = await import('@aws-sdk/client-s3') - try { - // Try to get the node from S3-compatible storage - const response = await this.s3Client.send( - new GetObjectCommand({ - Bucket: this.bucketName, - Key: `${nodePrefix}${id}.json` - }) - ) + // Try to get the node from the consolidated nouns directory + const response = await this.s3Client.send( + new GetObjectCommand({ + Bucket: this.bucketName, + Key: `${NOUN_PREFIX}${id}.json` + }) + ) - // Convert the response body to a string - const bodyContents = await response.Body.transformToString() - const parsedNode = JSON.parse(bodyContents) + // Convert the response body to a string + const bodyContents = await response.Body.transformToString() + const parsedNode = JSON.parse(bodyContents) - // Convert serialized connections back to Map> - const connections = new Map>() - for (const [level, nodeIds] of Object.entries(parsedNode.connections)) { - connections.set(Number(level), new Set(nodeIds as string[])) - } + // Convert serialized connections back to Map> + const connections = new Map>() + for (const [level, nodeIds] of Object.entries(parsedNode.connections)) { + connections.set(Number(level), new Set(nodeIds as string[])) + } - return { - id: parsedNode.id, - vector: parsedNode.vector, - connections - } - } catch (error) { - // If the node is not found in the expected prefix, try other prefixes - if (nodePrefix !== DEFAULT_PREFIX) { - // Try the default prefix + return { + id: parsedNode.id, + vector: parsedNode.vector, + connections + } + } catch (error) { + // Node not found or other error + return null + } + } + + /** + * Get nodes by noun type + * @param nounType The noun type to filter by + * @returns Promise that resolves to an array of nodes of the specified noun type + */ + public async getNodesByNounType(nounType: string): Promise { + await this.ensureInitialized() + + try { + // Import the ListObjectsV2Command and GetObjectCommand only when needed + const { ListObjectsV2Command, GetObjectCommand } = await import( + '@aws-sdk/client-s3' + ) + + // List all objects in the consolidated nouns directory + const listResponse = await this.s3Client.send( + new ListObjectsV2Command({ + Bucket: this.bucketName, + Prefix: NOUN_PREFIX + }) + ) + + const nodes: HNSWNode[] = [] + + // If there are no objects, return an empty array + if (!listResponse.Contents || listResponse.Contents.length === 0) { + return nodes + } + + // Get each node and filter by noun type + const nodePromises = listResponse.Contents.map( + async (object: { Key: string }) => { try { + // Extract node ID from the key (remove prefix and .json extension) + const nodeId = object.Key.replace(NOUN_PREFIX, '').replace('.json', '') + + // Get the metadata to check the noun type + const metadata = await this.getMetadata(nodeId) + + // Skip if metadata doesn't exist or noun type doesn't match + if (!metadata || metadata.noun !== nounType) { + return null + } + const response = await this.s3Client.send( new GetObjectCommand({ Bucket: this.bucketName, - Key: `${NOUNS_PREFIX}${DEFAULT_PREFIX}${id}.json` + Key: object.Key }) ) @@ -315,103 +350,38 @@ export class S3CompatibleStorage implements StorageAdapter { vector: parsedNode.vector, connections } - } catch { - // If not found in default prefix, try all other prefixes - const prefixes = [ - PERSON_PREFIX, - PLACE_PREFIX, - THING_PREFIX, - EVENT_PREFIX, - CONCEPT_PREFIX, - CONTENT_PREFIX - ] - - for (const prefix of prefixes) { - if (prefix === nodePrefix) continue // Skip the already checked prefix - - try { - const response = await this.s3Client.send( - new GetObjectCommand({ - Bucket: this.bucketName, - Key: `${NOUNS_PREFIX}${prefix}${id}.json` - }) - ) - - const bodyContents = await response.Body.transformToString() - const parsedNode = JSON.parse(bodyContents) - - // Convert serialized connections back to Map> - const connections = new Map>() - for (const [level, nodeIds] of Object.entries( - parsedNode.connections - )) { - connections.set(Number(level), new Set(nodeIds as string[])) - } - - return { - id: parsedNode.id, - vector: parsedNode.vector, - connections - } - } catch { - // Continue to the next prefix - } - } + } catch (error) { + console.error(`Failed to get node from ${object.Key}:`, error) + return null } } + ) - return null // Node not found in any prefix - } + const nodeResults = await Promise.all(nodePromises) + return nodeResults.filter((node): node is HNSWNode => node !== null) } catch (error) { - console.error(`Failed to get node ${id}:`, error) - return null + console.error(`Failed to get nodes for noun type ${nounType}:`, error) + throw new Error(`Failed to get nodes for noun type ${nounType}: ${error}`) } } /** - * Get nodes by noun type - * @param nounType The noun type to filter by - * @returns Promise that resolves to an array of nodes of the specified noun type + * Get all nodes from storage */ - public async getNodesByNounType(nounType: string): Promise { + public async getAllNodes(): Promise { await this.ensureInitialized() try { - // Determine the prefix based on the noun type - let prefix: string - switch (nounType) { - case 'person': - prefix = PERSON_PREFIX - break - case 'place': - prefix = PLACE_PREFIX - break - case 'thing': - prefix = THING_PREFIX - break - case 'event': - prefix = EVENT_PREFIX - break - case 'concept': - prefix = CONCEPT_PREFIX - break - case 'content': - prefix = CONTENT_PREFIX - break - default: - prefix = DEFAULT_PREFIX - } - // Import the ListObjectsV2Command and GetObjectCommand only when needed const { ListObjectsV2Command, GetObjectCommand } = await import( '@aws-sdk/client-s3' ) - // List all objects with the specified prefix + // List all objects in the consolidated nouns directory const listResponse = await this.s3Client.send( new ListObjectsV2Command({ Bucket: this.bucketName, - Prefix: `${NOUNS_PREFIX}${prefix}` + Prefix: NOUN_PREFIX }) ) @@ -458,43 +428,6 @@ export class S3CompatibleStorage implements StorageAdapter { const nodeResults = await Promise.all(nodePromises) return nodeResults.filter((node): node is HNSWNode => node !== null) - } catch (error) { - console.error(`Failed to get nodes for noun type ${nounType}:`, error) - throw new Error(`Failed to get nodes for noun type ${nounType}: ${error}`) - } - } - - /** - * Get all nodes from storage - */ - public async getAllNodes(): Promise { - await this.ensureInitialized() - - try { - // Get all noun types - const nounTypes = [ - 'person', - 'place', - 'thing', - 'event', - 'concept', - 'content', - 'default' - ] - - // Run searches in parallel for all noun types - const nodePromises = nounTypes.map((nounType) => - this.getNodesByNounType(nounType) - ) - const nodeArrays = await Promise.all(nodePromises) - - // Combine all results - const allNodes: HNSWNode[] = [] - for (const nodes of nodeArrays) { - allNodes.push(...nodes) - } - - return allNodes } catch (error) { console.error('Failed to get all nodes:', error) throw new Error(`Failed to get all nodes: ${error}`) @@ -508,90 +441,16 @@ export class S3CompatibleStorage implements StorageAdapter { await this.ensureInitialized() try { - // Get the appropriate prefix based on the node's metadata - const nodePrefix = await this.getNodePrefix(id) - // Import the DeleteObjectCommand only when needed - const { DeleteObjectCommand, GetObjectCommand } = await import( - '@aws-sdk/client-s3' + const { DeleteObjectCommand } = await import('@aws-sdk/client-s3') + + // Delete the node from the consolidated nouns directory + await this.s3Client.send( + new DeleteObjectCommand({ + Bucket: this.bucketName, + Key: `${NOUN_PREFIX}${id}.json` + }) ) - - try { - // Check if the node exists before deleting - await this.s3Client.send( - new GetObjectCommand({ - Bucket: this.bucketName, - Key: `${nodePrefix}${id}.json` - }) - ) - - // Delete the node - await this.s3Client.send( - new DeleteObjectCommand({ - Bucket: this.bucketName, - Key: `${nodePrefix}${id}.json` - }) - ) - return // Node found and deleted - } catch { - // If the node is not found in the expected prefix, try other prefixes - if (nodePrefix !== DEFAULT_PREFIX) { - try { - // Try the default prefix - await this.s3Client.send( - new GetObjectCommand({ - Bucket: this.bucketName, - Key: `${NOUNS_PREFIX}${DEFAULT_PREFIX}${id}.json` - }) - ) - - // Delete the node - await this.s3Client.send( - new DeleteObjectCommand({ - Bucket: this.bucketName, - Key: `${NOUNS_PREFIX}${DEFAULT_PREFIX}${id}.json` - }) - ) - return // Node found and deleted - } catch { - // If not found in default prefix, try all other prefixes - const prefixes = [ - PERSON_PREFIX, - PLACE_PREFIX, - THING_PREFIX, - EVENT_PREFIX, - CONCEPT_PREFIX, - CONTENT_PREFIX - ] - - for (const prefix of prefixes) { - if (prefix === nodePrefix) continue // Skip the already checked prefix - - try { - await this.s3Client.send( - new GetObjectCommand({ - Bucket: this.bucketName, - Key: `${NOUNS_PREFIX}${prefix}${id}.json` - }) - ) - - // Delete the node - await this.s3Client.send( - new DeleteObjectCommand({ - Bucket: this.bucketName, - Key: `${NOUNS_PREFIX}${prefix}${id}.json` - }) - ) - return // Node found and deleted - } catch { - // Continue to the next prefix - } - } - } - } - - return // Node not found in any prefix, nothing to delete - } } catch (error) { console.error(`Failed to delete node ${id}:`, error) throw new Error(`Failed to delete node ${id}: ${error}`) @@ -982,7 +841,7 @@ export class S3CompatibleStorage implements StorageAdapter { } } - // Count nodes by noun type + // Count nodes by noun type by examining metadata const nounTypeCounts: Record = { person: 0, place: 0, @@ -990,29 +849,41 @@ export class S3CompatibleStorage implements StorageAdapter { event: 0, concept: 0, content: 0, + group: 0, + list: 0, + category: 0, default: 0 } - // List objects for each noun type prefix - const nounTypes = [ - { type: 'person', prefix: PERSON_PREFIX }, - { type: 'place', prefix: PLACE_PREFIX }, - { type: 'thing', prefix: THING_PREFIX }, - { type: 'event', prefix: EVENT_PREFIX }, - { type: 'concept', prefix: CONCEPT_PREFIX }, - { type: 'content', prefix: CONTENT_PREFIX }, - { type: 'default', prefix: DEFAULT_PREFIX } - ] + // List all noun objects and count by type using metadata + const nounsListResponse = await this.s3Client.send( + new ListObjectsV2Command({ + Bucket: this.bucketName, + Prefix: NOUN_PREFIX + }) + ) - for (const { type, prefix } of nounTypes) { - const listResponse = await this.s3Client.send( - new ListObjectsV2Command({ - Bucket: this.bucketName, - Prefix: `${NOUNS_PREFIX}${prefix}` - }) - ) - - nounTypeCounts[type] = listResponse.Contents?.length || 0 + if (nounsListResponse.Contents) { + for (const object of nounsListResponse.Contents) { + try { + // Extract node ID from the key + const nodeId = object.Key?.replace(NOUN_PREFIX, '').replace('.json', '') + if (nodeId) { + // Get metadata to determine noun type + const metadata = await this.getMetadata(nodeId) + const nounType = metadata?.noun || 'default' + + if (nounType in nounTypeCounts) { + nounTypeCounts[nounType]++ + } else { + nounTypeCounts.default++ + } + } + } catch (error) { + // If we can't get metadata, count as default + nounTypeCounts.default++ + } + } } return { @@ -1030,6 +901,9 @@ export class S3CompatibleStorage implements StorageAdapter { event: { count: nounTypeCounts.event }, concept: { count: nounTypeCounts.concept }, content: { count: nounTypeCounts.content }, + group: { count: nounTypeCounts.group }, + list: { count: nounTypeCounts.list }, + category: { count: nounTypeCounts.category }, default: { count: nounTypeCounts.default } } } @@ -1055,39 +929,11 @@ export class S3CompatibleStorage implements StorageAdapter { } /** - * Get the appropriate prefix for a node based on its metadata + * Get the appropriate prefix for a node - now all nouns use the same prefix */ private async getNodePrefix(id: string): Promise { - try { - // Try to get the metadata for the node - const metadata = await this.getMetadata(id) - - // If metadata exists and has a noun field, use the corresponding prefix - if (metadata && metadata.noun) { - switch (metadata.noun) { - case 'person': - return PERSON_PREFIX - case 'place': - return PLACE_PREFIX - case 'thing': - return THING_PREFIX - case 'event': - return EVENT_PREFIX - case 'concept': - return CONCEPT_PREFIX - case 'content': - return CONTENT_PREFIX - default: - return DEFAULT_PREFIX - } - } - - // If no metadata or no noun field, use the default prefix - return DEFAULT_PREFIX - } catch (error) { - // If there's an error getting the metadata, use the default prefix - return DEFAULT_PREFIX - } + // All nouns now use the same prefix regardless of type + return NOUN_PREFIX } /** diff --git a/src/storage/storageFactory.ts b/src/storage/storageFactory.ts new file mode 100644 index 00000000..339d31ad --- /dev/null +++ b/src/storage/storageFactory.ts @@ -0,0 +1,365 @@ +/** + * Storage Factory + * Creates the appropriate storage adapter based on the environment and configuration + */ + +import { StorageAdapter } from '../coreTypes.js' +import { MemoryStorage } from './adapters/memoryStorage.js' +import { OPFSStorage } from './adapters/opfsStorage.js' +import { FileSystemStorage } from './adapters/fileSystemStorage.js' +import { S3CompatibleStorage, R2Storage } from './adapters/s3CompatibleStorage.js' + +/** + * Options for creating a storage adapter + */ +export interface StorageOptions { + /** + * The type of storage to use + * - 'auto': Automatically select the best storage adapter based on the environment + * - 'memory': Use in-memory storage + * - 'opfs': Use Origin Private File System storage (browser only) + * - 'filesystem': Use file system storage (Node.js only) + * - 's3': Use Amazon S3 storage + * - 'r2': Use Cloudflare R2 storage + * - 'gcs': Use Google Cloud Storage + */ + type?: 'auto' | 'memory' | 'opfs' | 'filesystem' | 's3' | 'r2' | 'gcs' + + /** + * Force the use of memory storage even if other storage types are available + */ + forceMemoryStorage?: boolean + + /** + * Force the use of file system storage even if other storage types are available + */ + forceFileSystemStorage?: boolean + + /** + * Request persistent storage permission from the user (browser only) + */ + requestPersistentStorage?: boolean + + /** + * Root directory for file system storage (Node.js only) + */ + rootDirectory?: string + + /** + * Configuration for Amazon S3 storage + */ + s3Storage?: { + /** + * S3 bucket name + */ + bucketName: string + + /** + * AWS region (e.g., 'us-east-1') + */ + region?: string + + /** + * AWS access key ID + */ + accessKeyId: string + + /** + * AWS secret access key + */ + secretAccessKey: string + + /** + * AWS session token (optional) + */ + sessionToken?: string + } + + /** + * Configuration for Cloudflare R2 storage + */ + r2Storage?: { + /** + * R2 bucket name + */ + bucketName: string + + /** + * Cloudflare account ID + */ + accountId: string + + /** + * R2 access key ID + */ + accessKeyId: string + + /** + * R2 secret access key + */ + secretAccessKey: string + } + + /** + * Configuration for Google Cloud Storage + */ + gcsStorage?: { + /** + * GCS bucket name + */ + bucketName: string + + /** + * GCS region (e.g., 'us-central1') + */ + region?: string + + /** + * GCS access key ID + */ + accessKeyId: string + + /** + * GCS secret access key + */ + secretAccessKey: string + + /** + * GCS endpoint (e.g., 'https://storage.googleapis.com') + */ + endpoint?: string + } + + /** + * Configuration for custom S3-compatible storage + */ + customS3Storage?: { + /** + * S3-compatible bucket name + */ + bucketName: string + + /** + * S3-compatible region + */ + region?: string + + /** + * S3-compatible endpoint URL + */ + endpoint: string + + /** + * S3-compatible access key ID + */ + accessKeyId: string + + /** + * S3-compatible secret access key + */ + secretAccessKey: string + + /** + * S3-compatible service type (for logging and error messages) + */ + serviceType?: string + } +} + +/** + * Create a storage adapter based on the environment and configuration + * @param options Options for creating the storage adapter + * @returns Promise that resolves to a storage adapter + */ +export async function createStorage( + options: StorageOptions = {} +): Promise { + // If memory storage is forced, use it regardless of other options + if (options.forceMemoryStorage) { + console.log('Using memory storage (forced)') + return new MemoryStorage() + } + + // If file system storage is forced, use it regardless of other options + if (options.forceFileSystemStorage) { + console.log('Using file system storage (forced)') + return new FileSystemStorage(options.rootDirectory || './brainy-data') + } + + // If a specific storage type is specified, use it + if (options.type && options.type !== 'auto') { + switch (options.type) { + case 'memory': + console.log('Using memory storage') + return new MemoryStorage() + + case 'opfs': { + // Check if OPFS is available + const opfsStorage = new OPFSStorage() + if (opfsStorage.isOPFSAvailable()) { + console.log('Using OPFS storage') + await opfsStorage.init() + + // Request persistent storage if specified + if (options.requestPersistentStorage) { + const isPersistent = await opfsStorage.requestPersistentStorage() + console.log(`Persistent storage ${isPersistent ? 'granted' : 'denied'}`) + } + + return opfsStorage + } else { + console.warn('OPFS storage is not available, falling back to memory storage') + return new MemoryStorage() + } + } + + case 'filesystem': + console.log('Using file system storage') + return new FileSystemStorage(options.rootDirectory || './brainy-data') + + case 's3': + if (options.s3Storage) { + console.log('Using Amazon S3 storage') + return new S3CompatibleStorage({ + bucketName: options.s3Storage.bucketName, + region: options.s3Storage.region, + accessKeyId: options.s3Storage.accessKeyId, + secretAccessKey: options.s3Storage.secretAccessKey, + sessionToken: options.s3Storage.sessionToken, + serviceType: 's3' + }) + } else { + console.warn('S3 storage configuration is missing, falling back to memory storage') + return new MemoryStorage() + } + + case 'r2': + if (options.r2Storage) { + console.log('Using Cloudflare R2 storage') + return new R2Storage({ + bucketName: options.r2Storage.bucketName, + accountId: options.r2Storage.accountId, + accessKeyId: options.r2Storage.accessKeyId, + secretAccessKey: options.r2Storage.secretAccessKey, + serviceType: 'r2' + }) + } else { + console.warn('R2 storage configuration is missing, falling back to memory storage') + return new MemoryStorage() + } + + case 'gcs': + if (options.gcsStorage) { + console.log('Using Google Cloud Storage') + return new S3CompatibleStorage({ + bucketName: options.gcsStorage.bucketName, + region: options.gcsStorage.region, + endpoint: options.gcsStorage.endpoint || 'https://storage.googleapis.com', + accessKeyId: options.gcsStorage.accessKeyId, + secretAccessKey: options.gcsStorage.secretAccessKey, + serviceType: 'gcs' + }) + } else { + console.warn('GCS storage configuration is missing, falling back to memory storage') + return new MemoryStorage() + } + + default: + console.warn(`Unknown storage type: ${options.type}, falling back to memory storage`) + return new MemoryStorage() + } + } + + // If custom S3-compatible storage is specified, use it + if (options.customS3Storage) { + console.log(`Using custom S3-compatible storage: ${options.customS3Storage.serviceType || 'custom'}`) + return new S3CompatibleStorage({ + bucketName: options.customS3Storage.bucketName, + region: options.customS3Storage.region, + endpoint: options.customS3Storage.endpoint, + accessKeyId: options.customS3Storage.accessKeyId, + secretAccessKey: options.customS3Storage.secretAccessKey, + serviceType: options.customS3Storage.serviceType || 'custom' + }) + } + + // If R2 storage is specified, use it + if (options.r2Storage) { + console.log('Using Cloudflare R2 storage') + return new R2Storage({ + bucketName: options.r2Storage.bucketName, + accountId: options.r2Storage.accountId, + accessKeyId: options.r2Storage.accessKeyId, + secretAccessKey: options.r2Storage.secretAccessKey, + serviceType: 'r2' + }) + } + + // If S3 storage is specified, use it + if (options.s3Storage) { + console.log('Using Amazon S3 storage') + return new S3CompatibleStorage({ + bucketName: options.s3Storage.bucketName, + region: options.s3Storage.region, + accessKeyId: options.s3Storage.accessKeyId, + secretAccessKey: options.s3Storage.secretAccessKey, + sessionToken: options.s3Storage.sessionToken, + serviceType: 's3' + }) + } + + // If GCS storage is specified, use it + if (options.gcsStorage) { + console.log('Using Google Cloud Storage') + return new S3CompatibleStorage({ + bucketName: options.gcsStorage.bucketName, + region: options.gcsStorage.region, + endpoint: options.gcsStorage.endpoint || 'https://storage.googleapis.com', + accessKeyId: options.gcsStorage.accessKeyId, + secretAccessKey: options.gcsStorage.secretAccessKey, + serviceType: 'gcs' + }) + } + + // Auto-detect the best storage adapter based on the environment + // First, try OPFS (browser only) + const opfsStorage = new OPFSStorage() + if (opfsStorage.isOPFSAvailable()) { + console.log('Using OPFS storage (auto-detected)') + await opfsStorage.init() + + // Request persistent storage if specified + if (options.requestPersistentStorage) { + const isPersistent = await opfsStorage.requestPersistentStorage() + console.log(`Persistent storage ${isPersistent ? 'granted' : 'denied'}`) + } + + return opfsStorage + } + + // Next, try file system storage (Node.js only) + try { + // Check if we're in a Node.js environment + if (typeof process !== 'undefined' && process.versions && process.versions.node) { + console.log('Using file system storage (auto-detected)') + return new FileSystemStorage(options.rootDirectory || './brainy-data') + } + } catch (error) { + // Not in a Node.js environment or file system is not available + } + + // Finally, fall back to memory storage + console.log('Using memory storage (auto-detected)') + return new MemoryStorage() +} + +/** + * Export all storage adapters + */ +export { + MemoryStorage, + OPFSStorage, + FileSystemStorage, + S3CompatibleStorage, + R2Storage +} \ No newline at end of file