Initial commit
This commit is contained in:
commit
5a8a6c1ba3
81 changed files with 39269 additions and 0 deletions
931
src/storage/fileSystemStorage.ts
Normal file
931
src/storage/fileSystemStorage.ts
Normal file
|
|
@ -0,0 +1,931 @@
|
|||
import { GraphVerb, HNSWNoun, StorageAdapter } from '../coreTypes.js'
|
||||
|
||||
// We'll dynamically import Node.js built-in modules
|
||||
let fs: any
|
||||
let path: any
|
||||
|
||||
// Constants for directory and file names
|
||||
const ROOT_DIR = 'brainy-data'
|
||||
const NOUNS_DIR = 'nouns'
|
||||
const VERBS_DIR = 'verbs'
|
||||
const METADATA_DIR = 'metadata'
|
||||
|
||||
// Constants for noun type directories
|
||||
const PERSON_DIR = 'person'
|
||||
const PLACE_DIR = 'place'
|
||||
const THING_DIR = 'thing'
|
||||
const EVENT_DIR = 'event'
|
||||
const CONCEPT_DIR = 'concept'
|
||||
const CONTENT_DIR = 'content'
|
||||
const DEFAULT_DIR = 'default' // For nodes without a noun type
|
||||
|
||||
/**
|
||||
* File system storage adapter for Node.js environments
|
||||
*/
|
||||
export class FileSystemStorage implements StorageAdapter {
|
||||
private rootDir: string
|
||||
private nounsDir: string
|
||||
private verbsDir: string
|
||||
private metadataDir: string
|
||||
private personDir: string
|
||||
private placeDir: string
|
||||
private thingDir: string
|
||||
private eventDir: string
|
||||
private conceptDir: string
|
||||
private contentDir: string
|
||||
private defaultDir: string
|
||||
private isInitialized = false
|
||||
|
||||
constructor(rootDirectory?: string) {
|
||||
// We'll set the paths in the init method after dynamically importing the modules
|
||||
this.rootDir = rootDirectory || ''
|
||||
this.nounsDir = ''
|
||||
this.verbsDir = ''
|
||||
this.metadataDir = ''
|
||||
this.personDir = ''
|
||||
this.placeDir = ''
|
||||
this.thingDir = ''
|
||||
this.eventDir = ''
|
||||
this.conceptDir = ''
|
||||
this.contentDir = ''
|
||||
this.defaultDir = ''
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the storage adapter
|
||||
*/
|
||||
public async init(): Promise<void> {
|
||||
if (this.isInitialized) {
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
// Dynamically import Node.js built-in modules
|
||||
try {
|
||||
// Import the modules
|
||||
const fsModule = await import('fs')
|
||||
const pathModule = await import('path')
|
||||
|
||||
// Assign to our module-level variables
|
||||
fs = fsModule.default || fsModule
|
||||
path = pathModule.default || pathModule
|
||||
|
||||
// Now set up the directory paths
|
||||
const rootDir = this.rootDir || process.cwd()
|
||||
this.rootDir = path.resolve(rootDir, ROOT_DIR)
|
||||
this.nounsDir = path.join(this.rootDir, NOUNS_DIR)
|
||||
this.verbsDir = path.join(this.rootDir, VERBS_DIR)
|
||||
this.metadataDir = path.join(this.rootDir, METADATA_DIR)
|
||||
|
||||
// Set up noun type directory paths
|
||||
this.personDir = path.join(this.nounsDir, PERSON_DIR)
|
||||
this.placeDir = path.join(this.nounsDir, PLACE_DIR)
|
||||
this.thingDir = path.join(this.nounsDir, THING_DIR)
|
||||
this.eventDir = path.join(this.nounsDir, EVENT_DIR)
|
||||
this.conceptDir = path.join(this.nounsDir, CONCEPT_DIR)
|
||||
this.contentDir = path.join(this.nounsDir, CONTENT_DIR)
|
||||
this.defaultDir = path.join(this.nounsDir, DEFAULT_DIR)
|
||||
} catch (importError) {
|
||||
throw new Error(`Failed to import Node.js modules: ${importError}. This adapter requires a Node.js environment.`)
|
||||
}
|
||||
|
||||
// Create directories if they don't exist
|
||||
await this.ensureDirectoryExists(this.rootDir)
|
||||
await this.ensureDirectoryExists(this.nounsDir)
|
||||
await this.ensureDirectoryExists(this.verbsDir)
|
||||
await this.ensureDirectoryExists(this.metadataDir)
|
||||
|
||||
// Create noun type directories
|
||||
await this.ensureDirectoryExists(this.personDir)
|
||||
await this.ensureDirectoryExists(this.placeDir)
|
||||
await this.ensureDirectoryExists(this.thingDir)
|
||||
await this.ensureDirectoryExists(this.eventDir)
|
||||
await this.ensureDirectoryExists(this.conceptDir)
|
||||
await this.ensureDirectoryExists(this.contentDir)
|
||||
await this.ensureDirectoryExists(this.defaultDir)
|
||||
|
||||
this.isInitialized = true
|
||||
} catch (error) {
|
||||
console.error('Failed to initialize file system storage:', error)
|
||||
throw new Error(`Failed to initialize file system storage: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save a node to storage
|
||||
*/
|
||||
public async saveNoun(noun: HNSWNoun): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Convert connections Map to a serializable format
|
||||
const serializableNode = {
|
||||
...noun,
|
||||
connections: this.mapToObject(noun.connections, (set) => Array.from(set as Set<string>))
|
||||
}
|
||||
|
||||
// Get the appropriate directory based on the node's metadata
|
||||
const nodeDir = await this.getNodeDirectory(noun.id)
|
||||
|
||||
const filePath = path.join(nodeDir, `${noun.id}.json`)
|
||||
await fs.promises.writeFile(
|
||||
filePath,
|
||||
JSON.stringify(serializableNode, null, 2),
|
||||
'utf8'
|
||||
)
|
||||
} catch (error) {
|
||||
console.error(`Failed to save node ${noun.id}:`, error)
|
||||
throw new Error(`Failed to save node ${noun.id}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a node from storage
|
||||
*/
|
||||
public async getNoun(id: string): Promise<HNSWNoun | null> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Get the appropriate directory based on the node's metadata
|
||||
const nodeDir = await this.getNodeDirectory(id)
|
||||
|
||||
const filePath = path.join(nodeDir, `${id}.json`)
|
||||
|
||||
// Check if a file exists
|
||||
try {
|
||||
await fs.promises.access(filePath)
|
||||
} catch {
|
||||
// If the file doesn't exist in the expected directory, try the default directory
|
||||
if (nodeDir !== this.defaultDir) {
|
||||
const defaultFilePath = path.join(this.defaultDir, `${id}.json`)
|
||||
try {
|
||||
await fs.promises.access(defaultFilePath)
|
||||
// If found in default directory, use that path
|
||||
const data = await fs.promises.readFile(defaultFilePath, 'utf8')
|
||||
const parsedNode = JSON.parse(data)
|
||||
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(parsedNode.connections)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
return {
|
||||
id: parsedNode.id,
|
||||
vector: parsedNode.vector,
|
||||
connections
|
||||
}
|
||||
} catch {
|
||||
// If not found in default directory either, try all noun type directories
|
||||
const directories = [
|
||||
this.personDir,
|
||||
this.placeDir,
|
||||
this.thingDir,
|
||||
this.eventDir,
|
||||
this.conceptDir,
|
||||
this.contentDir
|
||||
]
|
||||
|
||||
for (const dir of directories) {
|
||||
if (dir === nodeDir) continue // Skip the already checked directory
|
||||
|
||||
const dirFilePath = path.join(dir, `${id}.json`)
|
||||
try {
|
||||
await fs.promises.access(dirFilePath)
|
||||
// If found in this directory, use that path
|
||||
const data = await fs.promises.readFile(dirFilePath, 'utf8')
|
||||
const parsedNode = JSON.parse(data)
|
||||
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(parsedNode.connections)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
return {
|
||||
id: parsedNode.id,
|
||||
vector: parsedNode.vector,
|
||||
connections
|
||||
}
|
||||
} catch {
|
||||
// Continue to the next directory
|
||||
}
|
||||
}
|
||||
|
||||
return null // File doesn't exist in any directory
|
||||
}
|
||||
}
|
||||
return null // File doesn't exist
|
||||
}
|
||||
|
||||
const data = await fs.promises.readFile(filePath, 'utf8')
|
||||
const parsedNode = JSON.parse(data)
|
||||
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(parsedNode.connections)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
return {
|
||||
id: parsedNode.id,
|
||||
vector: parsedNode.vector,
|
||||
connections
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Failed to get node ${id}:`, error)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get nodes by noun type
|
||||
* @param nounType The noun type to filter by
|
||||
* @returns Promise that resolves to an array of nodes of the specified noun type
|
||||
*/
|
||||
public async getNounsByNounType(nounType: string): Promise<HNSWNoun[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Determine the directory based on the noun type
|
||||
let dir: string
|
||||
switch (nounType) {
|
||||
case 'person':
|
||||
dir = this.personDir
|
||||
break
|
||||
case 'place':
|
||||
dir = this.placeDir
|
||||
break
|
||||
case 'thing':
|
||||
dir = this.thingDir
|
||||
break
|
||||
case 'event':
|
||||
dir = this.eventDir
|
||||
break
|
||||
case 'concept':
|
||||
dir = this.conceptDir
|
||||
break
|
||||
case 'content':
|
||||
dir = this.contentDir
|
||||
break
|
||||
default:
|
||||
dir = this.defaultDir
|
||||
}
|
||||
|
||||
const nodes: HNSWNoun[] = []
|
||||
|
||||
try {
|
||||
const files = await fs.promises.readdir(dir)
|
||||
const nodePromises = files
|
||||
.filter((file: string) => file.endsWith('.json'))
|
||||
.map((file: string) => {
|
||||
// Use the file path directly instead of getNode to avoid redundant searches
|
||||
return this.readNodeFromFile(path.join(dir, file))
|
||||
})
|
||||
|
||||
const dirNodes = await Promise.all(nodePromises)
|
||||
nodes.push(...dirNodes.filter((node): node is HNSWNoun => node !== null))
|
||||
} catch (dirError) {
|
||||
// If directory doesn't exist or can't be read, log a warning
|
||||
console.warn(`Could not read directory for noun type ${nounType}:`, dirError)
|
||||
}
|
||||
|
||||
return nodes
|
||||
} catch (error) {
|
||||
console.error(`Failed to get nodes for noun type ${nounType}:`, error)
|
||||
throw new Error(`Failed to get nodes for noun type ${nounType}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all nodes from storage
|
||||
*/
|
||||
public async getAllNouns(): Promise<HNSWNoun[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Get all noun types
|
||||
const nounTypes = [
|
||||
'person',
|
||||
'place',
|
||||
'thing',
|
||||
'event',
|
||||
'concept',
|
||||
'content',
|
||||
'default'
|
||||
]
|
||||
|
||||
// Run searches in parallel for all noun types
|
||||
const nodePromises = nounTypes.map(nounType => this.getNounsByNounType(nounType))
|
||||
const nodeArrays = await Promise.all(nodePromises)
|
||||
|
||||
// Combine all results
|
||||
const allNodes: HNSWNoun[] = []
|
||||
for (const nodes of nodeArrays) {
|
||||
allNodes.push(...nodes)
|
||||
}
|
||||
|
||||
return allNodes
|
||||
} catch (error) {
|
||||
console.error('Failed to get all nodes:', error)
|
||||
throw new Error(`Failed to get all nodes: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Read a node from a file
|
||||
*/
|
||||
private async readNodeFromFile(filePath: string): Promise<HNSWNoun | null> {
|
||||
try {
|
||||
const data = await fs.promises.readFile(filePath, 'utf8')
|
||||
const parsedNode = JSON.parse(data)
|
||||
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(parsedNode.connections)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
return {
|
||||
id: parsedNode.id,
|
||||
vector: parsedNode.vector,
|
||||
connections
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Failed to read node from file ${filePath}:`, error)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a node from storage
|
||||
*/
|
||||
public async deleteNoun(id: string): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Get the appropriate directory based on the node's metadata
|
||||
const nodeDir = await this.getNodeDirectory(id)
|
||||
|
||||
const filePath = path.join(nodeDir, `${id}.json`)
|
||||
|
||||
// Check if a file exists before attempting to delete
|
||||
try {
|
||||
await fs.promises.access(filePath)
|
||||
await fs.promises.unlink(filePath)
|
||||
return // File found and deleted
|
||||
} catch {
|
||||
// If the file doesn't exist in the expected directory, try the default directory
|
||||
if (nodeDir !== this.defaultDir) {
|
||||
const defaultFilePath = path.join(this.defaultDir, `${id}.json`)
|
||||
try {
|
||||
await fs.promises.access(defaultFilePath)
|
||||
await fs.promises.unlink(defaultFilePath)
|
||||
return // File found and deleted
|
||||
} catch {
|
||||
// If not found in default directory either, try all noun type directories
|
||||
const directories = [
|
||||
this.personDir,
|
||||
this.placeDir,
|
||||
this.thingDir,
|
||||
this.eventDir,
|
||||
this.conceptDir,
|
||||
this.contentDir
|
||||
]
|
||||
|
||||
for (const dir of directories) {
|
||||
if (dir === nodeDir) continue // Skip the already checked directory
|
||||
|
||||
const dirFilePath = path.join(dir, `${id}.json`)
|
||||
try {
|
||||
await fs.promises.access(dirFilePath)
|
||||
await fs.promises.unlink(dirFilePath)
|
||||
return // File found and deleted
|
||||
} catch {
|
||||
// Continue to the next directory
|
||||
}
|
||||
}
|
||||
|
||||
return // File doesn't exist in any directory, nothing to delete
|
||||
}
|
||||
}
|
||||
return // File doesn't exist, nothing to delete
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Failed to delete node ${id}:`, error)
|
||||
throw new Error(`Failed to delete node ${id}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save an edge to storage
|
||||
*/
|
||||
public async saveVerb(verb: GraphVerb): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Convert connections Map to a serializable format
|
||||
const serializableEdge = {
|
||||
...verb,
|
||||
connections: this.mapToObject(verb.connections, (set) => Array.from(set as Set<string>))
|
||||
}
|
||||
|
||||
const filePath = path.join(this.verbsDir, `${verb.id}.json`)
|
||||
await fs.promises.writeFile(
|
||||
filePath,
|
||||
JSON.stringify(serializableEdge, null, 2),
|
||||
'utf8'
|
||||
)
|
||||
} catch (error) {
|
||||
console.error(`Failed to save edge ${verb.id}:`, error)
|
||||
throw new Error(`Failed to save edge ${verb.id}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get an edge from storage
|
||||
*/
|
||||
public async getVerb(id: string): Promise<GraphVerb | null> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
const filePath = path.join(this.verbsDir, `${id}.json`)
|
||||
|
||||
// Check if a file exists
|
||||
try {
|
||||
await fs.promises.access(filePath)
|
||||
} catch {
|
||||
return null // File doesn't exist
|
||||
}
|
||||
|
||||
const data = await fs.promises.readFile(filePath, 'utf8')
|
||||
const parsedEdge = JSON.parse(data)
|
||||
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(parsedEdge.connections)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
return {
|
||||
id: parsedEdge.id,
|
||||
vector: parsedEdge.vector,
|
||||
connections,
|
||||
sourceId: parsedEdge.sourceId,
|
||||
targetId: parsedEdge.targetId,
|
||||
type: parsedEdge.type,
|
||||
weight: parsedEdge.weight,
|
||||
metadata: parsedEdge.metadata
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Failed to get edge ${id}:`, error)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all edges from storage
|
||||
*/
|
||||
public async getAllVerbs(): Promise<GraphVerb[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
const files = await fs.promises.readdir(this.verbsDir)
|
||||
const edgePromises = files
|
||||
.filter((file: string) => file.endsWith('.json'))
|
||||
.map((file: string) => {
|
||||
const id = path.basename(file, '.json')
|
||||
return this.getVerb(id)
|
||||
})
|
||||
|
||||
const edges = await Promise.all(edgePromises)
|
||||
return edges.filter((edge): edge is GraphVerb => edge !== null)
|
||||
} catch (error) {
|
||||
console.error('Failed to get all edges:', error)
|
||||
throw new Error(`Failed to get all edges: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get edges by source node ID
|
||||
*/
|
||||
public async getVerbsBySource(sourceId: string): Promise<GraphVerb[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
const allEdges = await this.getAllVerbs()
|
||||
return allEdges.filter(edge => edge.sourceId === sourceId)
|
||||
} catch (error) {
|
||||
console.error(`Failed to get edges by source ${sourceId}:`, error)
|
||||
throw new Error(`Failed to get edges by source ${sourceId}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get edges by target node ID
|
||||
*/
|
||||
public async getVerbsByTarget(targetId: string): Promise<GraphVerb[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
const allEdges = await this.getAllVerbs()
|
||||
return allEdges.filter(edge => edge.targetId === targetId)
|
||||
} catch (error) {
|
||||
console.error(`Failed to get edges by target ${targetId}:`, error)
|
||||
throw new Error(`Failed to get edges by target ${targetId}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get edges by type
|
||||
*/
|
||||
public async getVerbsByType(type: string): Promise<GraphVerb[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
const allEdges = await this.getAllVerbs()
|
||||
return allEdges.filter(edge => edge.type === type)
|
||||
} catch (error) {
|
||||
console.error(`Failed to get edges by type ${type}:`, error)
|
||||
throw new Error(`Failed to get edges by type ${type}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete an edge from storage
|
||||
*/
|
||||
public async deleteVerb(id: string): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
const filePath = path.join(this.verbsDir, `${id}.json`)
|
||||
|
||||
// Check if a file exists before attempting to delete
|
||||
try {
|
||||
await fs.promises.access(filePath)
|
||||
} catch {
|
||||
return // File doesn't exist, nothing to delete
|
||||
}
|
||||
|
||||
await fs.promises.unlink(filePath)
|
||||
} catch (error) {
|
||||
console.error(`Failed to delete edge ${id}:`, error)
|
||||
throw new Error(`Failed to delete edge ${id}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save metadata to storage
|
||||
*/
|
||||
public async saveMetadata(id: string, metadata: any): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
const filePath = path.join(this.metadataDir, `${id}.json`)
|
||||
await fs.promises.writeFile(
|
||||
filePath,
|
||||
JSON.stringify(metadata, null, 2),
|
||||
'utf8'
|
||||
)
|
||||
} catch (error) {
|
||||
console.error(`Failed to save metadata for ${id}:`, error)
|
||||
throw new Error(`Failed to save metadata for ${id}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get metadata from storage
|
||||
*/
|
||||
public async getMetadata(id: string): Promise<any | null> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
const filePath = path.join(this.metadataDir, `${id}.json`)
|
||||
|
||||
// Check if a file exists
|
||||
try {
|
||||
await fs.promises.access(filePath)
|
||||
} catch {
|
||||
return null // File doesn't exist
|
||||
}
|
||||
|
||||
const data = await fs.promises.readFile(filePath, 'utf8')
|
||||
return JSON.parse(data)
|
||||
} catch (error) {
|
||||
console.error(`Failed to get metadata for ${id}:`, error)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear all data from storage
|
||||
*/
|
||||
public async clear(): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Delete and recreate the nodes, edges, and metadata directories
|
||||
await this.deleteDirectory(this.nounsDir)
|
||||
await this.deleteDirectory(this.verbsDir)
|
||||
await this.deleteDirectory(this.metadataDir)
|
||||
|
||||
await this.ensureDirectoryExists(this.nounsDir)
|
||||
await this.ensureDirectoryExists(this.verbsDir)
|
||||
await this.ensureDirectoryExists(this.metadataDir)
|
||||
|
||||
// Create noun type directories
|
||||
await this.ensureDirectoryExists(this.personDir)
|
||||
await this.ensureDirectoryExists(this.placeDir)
|
||||
await this.ensureDirectoryExists(this.thingDir)
|
||||
await this.ensureDirectoryExists(this.eventDir)
|
||||
await this.ensureDirectoryExists(this.conceptDir)
|
||||
await this.ensureDirectoryExists(this.contentDir)
|
||||
await this.ensureDirectoryExists(this.defaultDir)
|
||||
} catch (error) {
|
||||
console.error('Failed to clear storage:', error)
|
||||
throw new Error(`Failed to clear storage: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure the storage adapter is initialized
|
||||
*/
|
||||
private async ensureInitialized(): Promise<void> {
|
||||
if (!this.isInitialized) {
|
||||
await this.init()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure a directory exists, creating it if necessary
|
||||
*/
|
||||
private async ensureDirectoryExists(dirPath: string): Promise<void> {
|
||||
try {
|
||||
await fs.promises.access(dirPath)
|
||||
} catch {
|
||||
// Directory doesn't exist, create it
|
||||
await fs.promises.mkdir(dirPath, { recursive: true })
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a directory and all its contents recursively
|
||||
*/
|
||||
private async deleteDirectory(dirPath: string): Promise<void> {
|
||||
try {
|
||||
const files = await fs.promises.readdir(dirPath)
|
||||
|
||||
for (const file of files) {
|
||||
const filePath = path.join(dirPath, file)
|
||||
const stats = await fs.promises.stat(filePath)
|
||||
|
||||
if (stats.isDirectory()) {
|
||||
// Recursively delete subdirectories
|
||||
await this.deleteDirectory(filePath)
|
||||
} else {
|
||||
// Delete files
|
||||
await fs.promises.unlink(filePath)
|
||||
}
|
||||
}
|
||||
|
||||
// After all contents are deleted, remove the directory itself
|
||||
await fs.promises.rmdir(dirPath)
|
||||
} catch (error) {
|
||||
// If the directory doesn't exist, that's fine
|
||||
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') {
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Count the number of JSON files in a directory
|
||||
*/
|
||||
private async countFilesInDirectory(dirPath: string): Promise<number> {
|
||||
try {
|
||||
const files = await fs.promises.readdir(dirPath)
|
||||
return files.filter((file: string) => file.endsWith('.json')).length
|
||||
} catch (error) {
|
||||
// If the directory doesn't exist, return 0
|
||||
if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
|
||||
return 0
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a Map to a plain object for serialization
|
||||
*/
|
||||
private mapToObject<K extends string | number, V>(
|
||||
map: Map<K, V>,
|
||||
valueTransformer: (value: V) => any = (v) => v
|
||||
): Record<string, any> {
|
||||
const obj: Record<string, any> = {}
|
||||
for (const [key, value] of map.entries()) {
|
||||
obj[key.toString()] = valueTransformer(value)
|
||||
}
|
||||
return obj
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the appropriate directory for a node based on its metadata
|
||||
*/
|
||||
private async getNodeDirectory(id: string): Promise<string> {
|
||||
try {
|
||||
// Try to get the metadata for the node
|
||||
const metadata = await this.getMetadata(id)
|
||||
|
||||
// If metadata exists and has a noun field, use the corresponding directory
|
||||
if (metadata && metadata.noun) {
|
||||
switch (metadata.noun) {
|
||||
case 'person':
|
||||
return this.personDir
|
||||
case 'place':
|
||||
return this.placeDir
|
||||
case 'thing':
|
||||
return this.thingDir
|
||||
case 'event':
|
||||
return this.eventDir
|
||||
case 'concept':
|
||||
return this.conceptDir
|
||||
case 'content':
|
||||
return this.contentDir
|
||||
default:
|
||||
return this.defaultDir
|
||||
}
|
||||
}
|
||||
|
||||
// If no metadata or no noun field, use the default directory
|
||||
return this.defaultDir
|
||||
} catch (error) {
|
||||
// If there's an error getting the metadata, use the default directory
|
||||
return this.defaultDir
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get information about storage usage and capacity
|
||||
*/
|
||||
public async getStorageStatus(): Promise<{
|
||||
type: string;
|
||||
used: number;
|
||||
quota: number | null;
|
||||
details?: Record<string, any>;
|
||||
}> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Calculate the total size of all files in the storage directories
|
||||
let totalSize = 0
|
||||
|
||||
// Helper function to calculate directory size
|
||||
const calculateDirSize = async (dirPath: string): Promise<number> => {
|
||||
let size = 0
|
||||
try {
|
||||
const files = await fs.promises.readdir(dirPath)
|
||||
|
||||
for (const file of files) {
|
||||
const filePath = path.join(dirPath, file)
|
||||
const stats = await fs.promises.stat(filePath)
|
||||
|
||||
if (stats.isDirectory()) {
|
||||
size += await calculateDirSize(filePath)
|
||||
} else {
|
||||
size += stats.size
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn(`Error calculating size for ${dirPath}:`, error)
|
||||
}
|
||||
return size
|
||||
}
|
||||
|
||||
// Calculate size for each directory
|
||||
const nodesDirSize = await calculateDirSize(this.nounsDir)
|
||||
const edgesDirSize = await calculateDirSize(this.verbsDir)
|
||||
const metadataDirSize = await calculateDirSize(this.metadataDir)
|
||||
|
||||
// Calculate sizes of noun type directories
|
||||
const personDirSize = await calculateDirSize(this.personDir)
|
||||
const placeDirSize = await calculateDirSize(this.placeDir)
|
||||
const thingDirSize = await calculateDirSize(this.thingDir)
|
||||
const eventDirSize = await calculateDirSize(this.eventDir)
|
||||
const conceptDirSize = await calculateDirSize(this.conceptDir)
|
||||
const contentDirSize = await calculateDirSize(this.contentDir)
|
||||
const defaultDirSize = await calculateDirSize(this.defaultDir)
|
||||
|
||||
// Note: The noun type directories are subdirectories of the nodes directory,
|
||||
// so their sizes are already included in nodesDirSize.
|
||||
// We don't need to add them again to avoid double counting.
|
||||
totalSize = nodesDirSize + edgesDirSize + metadataDirSize
|
||||
|
||||
// Get filesystem information
|
||||
let quota = null
|
||||
let details: {
|
||||
nounTypes?: {
|
||||
person: { size: number; count: number };
|
||||
place: { size: number; count: number };
|
||||
thing: { size: number; count: number };
|
||||
event: { size: number; count: number };
|
||||
concept: { size: number; count: number };
|
||||
content: { size: number; count: number };
|
||||
default: { size: number; count: number };
|
||||
};
|
||||
availableSpace?: number;
|
||||
totalSpace?: number;
|
||||
freePercentage?: number;
|
||||
} = {
|
||||
nounTypes: {
|
||||
person: {
|
||||
size: personDirSize,
|
||||
count: await this.countFilesInDirectory(this.personDir)
|
||||
},
|
||||
place: {
|
||||
size: placeDirSize,
|
||||
count: await this.countFilesInDirectory(this.placeDir)
|
||||
},
|
||||
thing: {
|
||||
size: thingDirSize,
|
||||
count: await this.countFilesInDirectory(this.thingDir)
|
||||
},
|
||||
event: {
|
||||
size: eventDirSize,
|
||||
count: await this.countFilesInDirectory(this.eventDir)
|
||||
},
|
||||
concept: {
|
||||
size: conceptDirSize,
|
||||
count: await this.countFilesInDirectory(this.conceptDir)
|
||||
},
|
||||
content: {
|
||||
size: contentDirSize,
|
||||
count: await this.countFilesInDirectory(this.contentDir)
|
||||
},
|
||||
default: {
|
||||
size: defaultDirSize,
|
||||
count: await this.countFilesInDirectory(this.defaultDir)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
// Try to get disk space information
|
||||
const stats = await fs.promises.statfs(this.rootDir)
|
||||
if (stats) {
|
||||
const availableSpace = stats.bavail * stats.bsize
|
||||
const totalSpace = stats.blocks * stats.bsize
|
||||
|
||||
quota = totalSpace
|
||||
details = {
|
||||
availableSpace,
|
||||
totalSpace,
|
||||
freePercentage: (availableSpace / totalSpace) * 100
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn('Unable to get filesystem stats:', error)
|
||||
// If statfs is not available, try to use df command on Unix-like systems
|
||||
try {
|
||||
const { exec } = await import('child_process')
|
||||
const util = await import('util')
|
||||
const execPromise = util.promisify(exec)
|
||||
|
||||
const { stdout } = await execPromise(`df -k "${this.rootDir}"`)
|
||||
const lines = stdout.trim().split('\n')
|
||||
if (lines.length > 1) {
|
||||
const parts = lines[1].split(/\s+/)
|
||||
if (parts.length >= 4) {
|
||||
const totalKB = parseInt(parts[1], 10)
|
||||
const usedKB = parseInt(parts[2], 10)
|
||||
const availableKB = parseInt(parts[3], 10)
|
||||
|
||||
quota = totalKB * 1024
|
||||
details = {
|
||||
availableSpace: availableKB * 1024,
|
||||
totalSpace: totalKB * 1024,
|
||||
freePercentage: (availableKB / totalKB) * 100
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (dfError) {
|
||||
console.warn('Unable to get disk space using df command:', dfError)
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
type: 'filesystem',
|
||||
used: totalSize,
|
||||
quota,
|
||||
details
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Failed to get storage status:', error)
|
||||
return {
|
||||
type: 'filesystem',
|
||||
used: 0,
|
||||
quota: null,
|
||||
details: { error: String(error) }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue