import { GraphVerb, HNSWNoun, StorageAdapter } from '../coreTypes.js' // We'll dynamically import Node.js built-in modules let fs: any let path: any // Constants for directory and file names const ROOT_DIR = 'brainy-data' const NOUNS_DIR = 'nouns' const VERBS_DIR = 'verbs' const METADATA_DIR = 'metadata' // Constants for noun type directories const PERSON_DIR = 'person' const PLACE_DIR = 'place' const THING_DIR = 'thing' const EVENT_DIR = 'event' const CONCEPT_DIR = 'concept' const CONTENT_DIR = 'content' const DEFAULT_DIR = 'default' // For nodes without a noun type /** * File system storage adapter for Node.js environments */ export class FileSystemStorage implements StorageAdapter { private rootDir: string private nounsDir: string private verbsDir: string private metadataDir: string private personDir: string private placeDir: string private thingDir: string private eventDir: string private conceptDir: string private contentDir: string private defaultDir: string private isInitialized = false constructor(rootDirectory?: string) { // We'll set the paths in the init method after dynamically importing the modules this.rootDir = rootDirectory || '' this.nounsDir = '' this.verbsDir = '' this.metadataDir = '' this.personDir = '' this.placeDir = '' this.thingDir = '' this.eventDir = '' this.conceptDir = '' this.contentDir = '' this.defaultDir = '' } /** * Initialize the storage adapter */ public async init(): Promise { if (this.isInitialized) { return } try { // Dynamically import Node.js built-in modules try { // Import the modules const fsModule = await import('fs') const pathModule = await import('path') // Assign to our module-level variables fs = fsModule.default || fsModule path = pathModule.default || pathModule // Now set up the directory paths const rootDir = this.rootDir || process.cwd() this.rootDir = path.resolve(rootDir, ROOT_DIR) this.nounsDir = path.join(this.rootDir, NOUNS_DIR) this.verbsDir = path.join(this.rootDir, VERBS_DIR) this.metadataDir = path.join(this.rootDir, METADATA_DIR) // Set up noun type directory paths this.personDir = path.join(this.nounsDir, PERSON_DIR) this.placeDir = path.join(this.nounsDir, PLACE_DIR) this.thingDir = path.join(this.nounsDir, THING_DIR) this.eventDir = path.join(this.nounsDir, EVENT_DIR) this.conceptDir = path.join(this.nounsDir, CONCEPT_DIR) this.contentDir = path.join(this.nounsDir, CONTENT_DIR) this.defaultDir = path.join(this.nounsDir, DEFAULT_DIR) } catch (importError) { throw new Error( `Failed to import Node.js modules: ${importError}. This adapter requires a Node.js environment.` ) } // Create directories if they don't exist await this.ensureDirectoryExists(this.rootDir) await this.ensureDirectoryExists(this.nounsDir) await this.ensureDirectoryExists(this.verbsDir) await this.ensureDirectoryExists(this.metadataDir) // Create noun type directories await this.ensureDirectoryExists(this.personDir) await this.ensureDirectoryExists(this.placeDir) await this.ensureDirectoryExists(this.thingDir) await this.ensureDirectoryExists(this.eventDir) await this.ensureDirectoryExists(this.conceptDir) await this.ensureDirectoryExists(this.contentDir) await this.ensureDirectoryExists(this.defaultDir) this.isInitialized = true } catch (error) { console.error('Failed to initialize file system storage:', error) throw new Error(`Failed to initialize file system storage: ${error}`) } } /** * Save a node to storage */ public async saveNoun(noun: HNSWNoun): Promise { await this.ensureInitialized() try { // Convert connections Map to a serializable format const serializableNode = { ...noun, connections: this.mapToObject(noun.connections, (set) => Array.from(set as Set) ) } // Get the appropriate directory based on the node's metadata const nodeDir = await this.getNodeDirectory(noun.id) const filePath = path.join(nodeDir, `${noun.id}.json`) await fs.promises.writeFile( filePath, JSON.stringify(serializableNode, null, 2), 'utf8' ) } catch (error) { console.error(`Failed to save node ${noun.id}:`, error) throw new Error(`Failed to save node ${noun.id}: ${error}`) } } /** * Get a node from storage */ public async getNoun(id: string): Promise { await this.ensureInitialized() try { // Get the appropriate directory based on the node's metadata const nodeDir = await this.getNodeDirectory(id) const filePath = path.join(nodeDir, `${id}.json`) // Check if a file exists try { await fs.promises.access(filePath) } catch { // If the file doesn't exist in the expected directory, try the default directory if (nodeDir !== this.defaultDir) { const defaultFilePath = path.join(this.defaultDir, `${id}.json`) try { await fs.promises.access(defaultFilePath) // If found in default directory, use that path const data = await fs.promises.readFile(defaultFilePath, 'utf8') const parsedNode = JSON.parse(data) // Convert serialized connections back to Map> const connections = new Map>() for (const [level, nodeIds] of Object.entries( parsedNode.connections )) { connections.set(Number(level), new Set(nodeIds as string[])) } return { id: parsedNode.id, vector: parsedNode.vector, connections } } catch { // If not found in default directory either, try all noun type directories const directories = [ this.personDir, this.placeDir, this.thingDir, this.eventDir, this.conceptDir, this.contentDir ] for (const dir of directories) { if (dir === nodeDir) continue // Skip the already checked directory const dirFilePath = path.join(dir, `${id}.json`) try { await fs.promises.access(dirFilePath) // If found in this directory, use that path const data = await fs.promises.readFile(dirFilePath, 'utf8') const parsedNode = JSON.parse(data) // Convert serialized connections back to Map> const connections = new Map>() for (const [level, nodeIds] of Object.entries( parsedNode.connections )) { connections.set(Number(level), new Set(nodeIds as string[])) } return { id: parsedNode.id, vector: parsedNode.vector, connections } } catch { // Continue to the next directory } } return null // File doesn't exist in any directory } } return null // File doesn't exist } const data = await fs.promises.readFile(filePath, 'utf8') const parsedNode = JSON.parse(data) // Convert serialized connections back to Map> const connections = new Map>() for (const [level, nodeIds] of Object.entries(parsedNode.connections)) { connections.set(Number(level), new Set(nodeIds as string[])) } return { id: parsedNode.id, vector: parsedNode.vector, connections } } catch (error) { console.error(`Failed to get node ${id}:`, error) return null } } /** * Get nodes by noun type * @param nounType The noun type to filter by * @returns Promise that resolves to an array of nodes of the specified noun type */ public async getNounsByNounType(nounType: string): Promise { await this.ensureInitialized() try { // Determine the directory based on the noun type let dir: string switch (nounType) { case 'person': dir = this.personDir break case 'place': dir = this.placeDir break case 'thing': dir = this.thingDir break case 'event': dir = this.eventDir break case 'concept': dir = this.conceptDir break case 'content': dir = this.contentDir break default: dir = this.defaultDir } const nodes: HNSWNoun[] = [] try { const files = await fs.promises.readdir(dir) const nodePromises = files .filter((file: string) => file.endsWith('.json')) .map((file: string) => { // Use the file path directly instead of getNode to avoid redundant searches return this.readNodeFromFile(path.join(dir, file)) }) const dirNodes = await Promise.all(nodePromises) nodes.push( ...dirNodes.filter((node): node is HNSWNoun => node !== null) ) } catch (dirError) { // If directory doesn't exist or can't be read, log a warning console.warn( `Could not read directory for noun type ${nounType}:`, dirError ) } return nodes } catch (error) { console.error(`Failed to get nodes for noun type ${nounType}:`, error) throw new Error(`Failed to get nodes for noun type ${nounType}: ${error}`) } } /** * Get all nodes from storage */ public async getAllNouns(): Promise { await this.ensureInitialized() try { // Get all noun types const nounTypes = [ 'person', 'place', 'thing', 'event', 'concept', 'content', 'default' ] // Run searches in parallel for all noun types const nodePromises = nounTypes.map((nounType) => this.getNounsByNounType(nounType) ) const nodeArrays = await Promise.all(nodePromises) // Combine all results const allNodes: HNSWNoun[] = [] for (const nodes of nodeArrays) { allNodes.push(...nodes) } return allNodes } catch (error) { console.error('Failed to get all nodes:', error) throw new Error(`Failed to get all nodes: ${error}`) } } /** * Read a node from a file */ private async readNodeFromFile(filePath: string): Promise { try { const data = await fs.promises.readFile(filePath, 'utf8') const parsedNode = JSON.parse(data) // Convert serialized connections back to Map> const connections = new Map>() for (const [level, nodeIds] of Object.entries(parsedNode.connections)) { connections.set(Number(level), new Set(nodeIds as string[])) } return { id: parsedNode.id, vector: parsedNode.vector, connections } } catch (error) { console.error(`Failed to read node from file ${filePath}:`, error) return null } } /** * Delete a node from storage */ public async deleteNoun(id: string): Promise { await this.ensureInitialized() try { // Get the appropriate directory based on the node's metadata const nodeDir = await this.getNodeDirectory(id) const filePath = path.join(nodeDir, `${id}.json`) // Check if a file exists before attempting to delete try { await fs.promises.access(filePath) await fs.promises.unlink(filePath) return // File found and deleted } catch { // If the file doesn't exist in the expected directory, try the default directory if (nodeDir !== this.defaultDir) { const defaultFilePath = path.join(this.defaultDir, `${id}.json`) try { await fs.promises.access(defaultFilePath) await fs.promises.unlink(defaultFilePath) return // File found and deleted } catch { // If not found in default directory either, try all noun type directories const directories = [ this.personDir, this.placeDir, this.thingDir, this.eventDir, this.conceptDir, this.contentDir ] for (const dir of directories) { if (dir === nodeDir) continue // Skip the already checked directory const dirFilePath = path.join(dir, `${id}.json`) try { await fs.promises.access(dirFilePath) await fs.promises.unlink(dirFilePath) return // File found and deleted } catch { // Continue to the next directory } } return // File doesn't exist in any directory, nothing to delete } } return // File doesn't exist, nothing to delete } } catch (error) { console.error(`Failed to delete node ${id}:`, error) throw new Error(`Failed to delete node ${id}: ${error}`) } } /** * Save an edge to storage */ public async saveVerb(verb: GraphVerb): Promise { await this.ensureInitialized() try { // Convert connections Map to a serializable format const serializableEdge = { ...verb, connections: this.mapToObject(verb.connections, (set) => Array.from(set as Set) ) } const filePath = path.join(this.verbsDir, `${verb.id}.json`) await fs.promises.writeFile( filePath, JSON.stringify(serializableEdge, null, 2), 'utf8' ) } catch (error) { console.error(`Failed to save edge ${verb.id}:`, error) throw new Error(`Failed to save edge ${verb.id}: ${error}`) } } /** * Get an edge from storage */ public async getVerb(id: string): Promise { await this.ensureInitialized() try { const filePath = path.join(this.verbsDir, `${id}.json`) // Check if a file exists try { await fs.promises.access(filePath) } catch { return null // File doesn't exist } const data = await fs.promises.readFile(filePath, 'utf8') const parsedEdge = JSON.parse(data) // Convert serialized connections back to Map> const connections = new Map>() for (const [level, nodeIds] of Object.entries(parsedEdge.connections)) { connections.set(Number(level), new Set(nodeIds as string[])) } return { id: parsedEdge.id, vector: parsedEdge.vector, connections, sourceId: parsedEdge.sourceId, targetId: parsedEdge.targetId, type: parsedEdge.type, weight: parsedEdge.weight, metadata: parsedEdge.metadata } } catch (error) { console.error(`Failed to get edge ${id}:`, error) return null } } /** * Get all edges from storage */ public async getAllVerbs(): Promise { await this.ensureInitialized() try { const files = await fs.promises.readdir(this.verbsDir) const edgePromises = files .filter((file: string) => file.endsWith('.json')) .map((file: string) => { const id = path.basename(file, '.json') return this.getVerb(id) }) const edges = await Promise.all(edgePromises) return edges.filter((edge): edge is GraphVerb => edge !== null) } catch (error) { console.error('Failed to get all edges:', error) throw new Error(`Failed to get all edges: ${error}`) } } /** * Get edges by source node ID */ public async getVerbsBySource(sourceId: string): Promise { await this.ensureInitialized() try { const allEdges = await this.getAllVerbs() return allEdges.filter((edge) => edge.sourceId === sourceId) } catch (error) { console.error(`Failed to get edges by source ${sourceId}:`, error) throw new Error(`Failed to get edges by source ${sourceId}: ${error}`) } } /** * Get edges by target node ID */ public async getVerbsByTarget(targetId: string): Promise { await this.ensureInitialized() try { const allEdges = await this.getAllVerbs() return allEdges.filter((edge) => edge.targetId === targetId) } catch (error) { console.error(`Failed to get edges by target ${targetId}:`, error) throw new Error(`Failed to get edges by target ${targetId}: ${error}`) } } /** * Get edges by type */ public async getVerbsByType(type: string): Promise { await this.ensureInitialized() try { const allEdges = await this.getAllVerbs() return allEdges.filter((edge) => edge.type === type) } catch (error) { console.error(`Failed to get edges by type ${type}:`, error) throw new Error(`Failed to get edges by type ${type}: ${error}`) } } /** * Delete an edge from storage */ public async deleteVerb(id: string): Promise { await this.ensureInitialized() try { const filePath = path.join(this.verbsDir, `${id}.json`) // Check if a file exists before attempting to delete try { await fs.promises.access(filePath) } catch { return // File doesn't exist, nothing to delete } await fs.promises.unlink(filePath) } catch (error) { console.error(`Failed to delete edge ${id}:`, error) throw new Error(`Failed to delete edge ${id}: ${error}`) } } /** * Save metadata to storage */ public async saveMetadata(id: string, metadata: any): Promise { await this.ensureInitialized() try { const filePath = path.join(this.metadataDir, `${id}.json`) await fs.promises.writeFile( filePath, JSON.stringify(metadata, null, 2), 'utf8' ) } catch (error) { console.error(`Failed to save metadata for ${id}:`, error) throw new Error(`Failed to save metadata for ${id}: ${error}`) } } /** * Get metadata from storage */ public async getMetadata(id: string): Promise { await this.ensureInitialized() try { const filePath = path.join(this.metadataDir, `${id}.json`) // Check if a file exists try { await fs.promises.access(filePath) } catch { return null // File doesn't exist } const data = await fs.promises.readFile(filePath, 'utf8') return JSON.parse(data) } catch (error) { console.error(`Failed to get metadata for ${id}:`, error) return null } } /** * Clear all data from storage */ public async clear(): Promise { await this.ensureInitialized() try { // Delete and recreate the nodes, edges, and metadata directories await this.deleteDirectory(this.nounsDir) await this.deleteDirectory(this.verbsDir) await this.deleteDirectory(this.metadataDir) await this.ensureDirectoryExists(this.nounsDir) await this.ensureDirectoryExists(this.verbsDir) await this.ensureDirectoryExists(this.metadataDir) // Create noun type directories await this.ensureDirectoryExists(this.personDir) await this.ensureDirectoryExists(this.placeDir) await this.ensureDirectoryExists(this.thingDir) await this.ensureDirectoryExists(this.eventDir) await this.ensureDirectoryExists(this.conceptDir) await this.ensureDirectoryExists(this.contentDir) await this.ensureDirectoryExists(this.defaultDir) } catch (error) { console.error('Failed to clear storage:', error) throw new Error(`Failed to clear storage: ${error}`) } } /** * Ensure the storage adapter is initialized */ private async ensureInitialized(): Promise { if (!this.isInitialized) { await this.init() } } /** * Ensure a directory exists, creating it if necessary */ private async ensureDirectoryExists(dirPath: string): Promise { try { await fs.promises.access(dirPath) } catch { // Directory doesn't exist, create it await fs.promises.mkdir(dirPath, { recursive: true }) } } /** * Delete a directory and all its contents recursively */ private async deleteDirectory(dirPath: string): Promise { try { const files = await fs.promises.readdir(dirPath) for (const file of files) { const filePath = path.join(dirPath, file) const stats = await fs.promises.stat(filePath) if (stats.isDirectory()) { // Recursively delete subdirectories await this.deleteDirectory(filePath) } else { // Delete files await fs.promises.unlink(filePath) } } // After all contents are deleted, remove the directory itself await fs.promises.rmdir(dirPath) } catch (error) { // If the directory doesn't exist, that's fine if ((error as NodeJS.ErrnoException).code !== 'ENOENT') { throw error } } } /** * Count the number of JSON files in a directory */ private async countFilesInDirectory(dirPath: string): Promise { try { const files = await fs.promises.readdir(dirPath) return files.filter((file: string) => file.endsWith('.json')).length } catch (error) { // If the directory doesn't exist, return 0 if ((error as NodeJS.ErrnoException).code === 'ENOENT') { return 0 } throw error } } /** * Convert a Map to a plain object for serialization */ private mapToObject( map: Map, valueTransformer: (value: V) => any = (v) => v ): Record { const obj: Record = {} for (const [key, value] of map.entries()) { obj[key.toString()] = valueTransformer(value) } return obj } /** * Get the appropriate directory for a node based on its metadata */ private async getNodeDirectory(id: string): Promise { try { // Try to get the metadata for the node const metadata = await this.getMetadata(id) // If metadata exists and has a noun field, use the corresponding directory if (metadata && metadata.noun) { switch (metadata.noun) { case 'person': return this.personDir case 'place': return this.placeDir case 'thing': return this.thingDir case 'event': return this.eventDir case 'concept': return this.conceptDir case 'content': return this.contentDir default: return this.defaultDir } } // If no metadata or no noun field, use the default directory return this.defaultDir } catch (error) { // If there's an error getting the metadata, use the default directory return this.defaultDir } } /** * Get information about storage usage and capacity */ public async getStorageStatus(): Promise<{ type: string used: number quota: number | null details?: Record }> { await this.ensureInitialized() try { // Calculate the total size of all files in the storage directories let totalSize = 0 // Helper function to calculate directory size const calculateDirSize = async (dirPath: string): Promise => { let size = 0 try { const files = await fs.promises.readdir(dirPath) for (const file of files) { const filePath = path.join(dirPath, file) const stats = await fs.promises.stat(filePath) if (stats.isDirectory()) { size += await calculateDirSize(filePath) } else { size += stats.size } } } catch (error) { console.warn(`Error calculating size for ${dirPath}:`, error) } return size } // Calculate size for each directory const nodesDirSize = await calculateDirSize(this.nounsDir) const edgesDirSize = await calculateDirSize(this.verbsDir) const metadataDirSize = await calculateDirSize(this.metadataDir) // Calculate sizes of noun type directories const personDirSize = await calculateDirSize(this.personDir) const placeDirSize = await calculateDirSize(this.placeDir) const thingDirSize = await calculateDirSize(this.thingDir) const eventDirSize = await calculateDirSize(this.eventDir) const conceptDirSize = await calculateDirSize(this.conceptDir) const contentDirSize = await calculateDirSize(this.contentDir) const defaultDirSize = await calculateDirSize(this.defaultDir) // Note: The noun type directories are subdirectories of the nodes directory, // so their sizes are already included in nodesDirSize. // We don't need to add them again to avoid double counting. totalSize = nodesDirSize + edgesDirSize + metadataDirSize // Get filesystem information let quota = null let details: { nounTypes?: { person: { size: number; count: number } place: { size: number; count: number } thing: { size: number; count: number } event: { size: number; count: number } concept: { size: number; count: number } content: { size: number; count: number } default: { size: number; count: number } } availableSpace?: number totalSpace?: number freePercentage?: number } = { nounTypes: { person: { size: personDirSize, count: await this.countFilesInDirectory(this.personDir) }, place: { size: placeDirSize, count: await this.countFilesInDirectory(this.placeDir) }, thing: { size: thingDirSize, count: await this.countFilesInDirectory(this.thingDir) }, event: { size: eventDirSize, count: await this.countFilesInDirectory(this.eventDir) }, concept: { size: conceptDirSize, count: await this.countFilesInDirectory(this.conceptDir) }, content: { size: contentDirSize, count: await this.countFilesInDirectory(this.contentDir) }, default: { size: defaultDirSize, count: await this.countFilesInDirectory(this.defaultDir) } } } try { // Try to get disk space information const stats = await fs.promises.statfs(this.rootDir) if (stats) { const availableSpace = stats.bavail * stats.bsize const totalSpace = stats.blocks * stats.bsize quota = totalSpace details = { availableSpace, totalSpace, freePercentage: (availableSpace / totalSpace) * 100 } } } catch (error) { console.warn('Unable to get filesystem stats:', error) // If statfs is not available, try to use df command on Unix-like systems try { const { exec } = await import('child_process') const util = await import('util') const execPromise = util.promisify(exec) const { stdout } = await execPromise(`df -k "${this.rootDir}"`) const lines = stdout.trim().split('\n') if (lines.length > 1) { const parts = lines[1].split(/\s+/) if (parts.length >= 4) { const totalKB = parseInt(parts[1], 10) const usedKB = parseInt(parts[2], 10) const availableKB = parseInt(parts[3], 10) quota = totalKB * 1024 details = { availableSpace: availableKB * 1024, totalSpace: totalKB * 1024, freePercentage: (availableKB / totalKB) * 100 } } } } catch (dfError) { console.warn('Unable to get disk space using df command:', dfError) } } return { type: 'filesystem', used: totalSize, quota, details } } catch (error) { console.error('Failed to get storage status:', error) return { type: 'filesystem', used: 0, quota: null, details: { error: String(error) } } } } }