feat(storage): implement base, file system, and memory storage adapters
- Added `BaseStorage` abstract class to provide common functionality for storage adapters. - Implemented `FileSystemStorage` adapter for Node.js environments, supporting persistent file-based storage. - Added directory structure management methods to `FileSystemStorage` (`ensureDirectoryExists`, `clear`, etc.). - Implemented `MemoryStorage` adapter for in-memory storage suitable for non-persistent environments. - Supported CRUD operations for nouns, verbs, and metadata in both `FileSystemStorage` and `MemoryStorage`. - Included utility methods like `mapToObject` for serialization and deserialization of complex data structures. Purpose: Provide extensible, reusable storage solutions with both persistent and non-persistent options for diverse application needs.
This commit is contained in:
parent
398c120757
commit
1127115664
9 changed files with 4549 additions and 2186 deletions
555
src/storage/adapters/fileSystemStorage.ts
Normal file
555
src/storage/adapters/fileSystemStorage.ts
Normal file
|
|
@ -0,0 +1,555 @@
|
|||
/**
|
||||
* File System Storage Adapter
|
||||
* File system storage adapter for Node.js environments
|
||||
*/
|
||||
|
||||
import { GraphVerb, HNSWNoun } from '../../coreTypes.js'
|
||||
import { BaseStorage, NOUNS_DIR, VERBS_DIR, METADATA_DIR, INDEX_DIR } from '../baseStorage.js'
|
||||
|
||||
// Type aliases for better readability
|
||||
type HNSWNode = HNSWNoun
|
||||
type Edge = GraphVerb
|
||||
|
||||
// Node.js modules - dynamically imported to avoid issues in browser environments
|
||||
let fs: any
|
||||
let path: any
|
||||
|
||||
// Try to load Node.js modules
|
||||
try {
|
||||
// Using dynamic imports to avoid issues in browser environments
|
||||
const fsPromise = import('fs')
|
||||
const pathPromise = import('path')
|
||||
|
||||
Promise.all([fsPromise, pathPromise]).then(([fsModule, pathModule]) => {
|
||||
fs = fsModule
|
||||
path = pathModule.default
|
||||
}).catch(error => {
|
||||
console.error('Failed to load Node.js modules:', error)
|
||||
})
|
||||
} catch (error) {
|
||||
console.error(
|
||||
'FileSystemStorage: Failed to load Node.js modules. This adapter is not supported in this environment.',
|
||||
error
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* File system storage adapter for Node.js environments
|
||||
* Uses the file system to store data in the specified directory structure
|
||||
*/
|
||||
export class FileSystemStorage extends BaseStorage {
|
||||
private rootDir: string
|
||||
private nounsDir: string
|
||||
private verbsDir: string
|
||||
private metadataDir: string
|
||||
private indexDir: string
|
||||
|
||||
/**
|
||||
* Initialize the storage adapter
|
||||
* @param rootDirectory The root directory for storage
|
||||
*/
|
||||
constructor(rootDirectory: string) {
|
||||
super()
|
||||
this.rootDir = rootDirectory
|
||||
this.nounsDir = path.join(this.rootDir, NOUNS_DIR)
|
||||
this.verbsDir = path.join(this.rootDir, VERBS_DIR)
|
||||
this.metadataDir = path.join(this.rootDir, METADATA_DIR)
|
||||
this.indexDir = path.join(this.rootDir, INDEX_DIR)
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the storage adapter
|
||||
*/
|
||||
public async init(): Promise<void> {
|
||||
if (this.isInitialized) {
|
||||
return
|
||||
}
|
||||
|
||||
// Check if Node.js modules are available
|
||||
if (!fs || !path) {
|
||||
throw new Error(
|
||||
'FileSystemStorage requires a Node.js environment, but `fs` and `path` modules could not be loaded.'
|
||||
)
|
||||
}
|
||||
|
||||
try {
|
||||
// Create the root directory if it doesn't exist
|
||||
await this.ensureDirectoryExists(this.rootDir)
|
||||
|
||||
// Create the nouns directory if it doesn't exist
|
||||
await this.ensureDirectoryExists(this.nounsDir)
|
||||
|
||||
// Create the verbs directory if it doesn't exist
|
||||
await this.ensureDirectoryExists(this.verbsDir)
|
||||
|
||||
// Create the metadata directory if it doesn't exist
|
||||
await this.ensureDirectoryExists(this.metadataDir)
|
||||
|
||||
// Create the index directory if it doesn't exist
|
||||
await this.ensureDirectoryExists(this.indexDir)
|
||||
|
||||
this.isInitialized = true
|
||||
} catch (error) {
|
||||
console.error('Error initializing FileSystemStorage:', error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure a directory exists, creating it if necessary
|
||||
*/
|
||||
private async ensureDirectoryExists(dirPath: string): Promise<void> {
|
||||
try {
|
||||
await fs.promises.mkdir(dirPath, { recursive: true })
|
||||
} catch (error: any) {
|
||||
// Ignore EEXIST error, which means the directory already exists
|
||||
if (error.code !== 'EEXIST') {
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save a node to storage
|
||||
*/
|
||||
protected async saveNode(node: HNSWNode): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
// Convert connections Map to a serializable format
|
||||
const serializableNode = {
|
||||
...node,
|
||||
connections: this.mapToObject(node.connections, (set) =>
|
||||
Array.from(set as Set<string>)
|
||||
)
|
||||
}
|
||||
|
||||
const filePath = path.join(this.nounsDir, `${node.id}.json`)
|
||||
await fs.promises.writeFile(filePath, JSON.stringify(serializableNode, null, 2))
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a node from storage
|
||||
*/
|
||||
protected async getNode(id: string): Promise<HNSWNode | null> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
const filePath = path.join(this.nounsDir, `${id}.json`)
|
||||
try {
|
||||
const data = await fs.promises.readFile(filePath, 'utf-8')
|
||||
const parsedNode = JSON.parse(data)
|
||||
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(parsedNode.connections)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
return {
|
||||
id: parsedNode.id,
|
||||
vector: parsedNode.vector,
|
||||
connections
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error reading node ${id}:`, error)
|
||||
}
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all nodes from storage
|
||||
*/
|
||||
protected async getAllNodes(): Promise<HNSWNode[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
const allNodes: HNSWNode[] = []
|
||||
try {
|
||||
const files = await fs.promises.readdir(this.nounsDir)
|
||||
for (const file of files) {
|
||||
if (file.endsWith('.json')) {
|
||||
const filePath = path.join(this.nounsDir, file)
|
||||
const data = await fs.promises.readFile(filePath, 'utf-8')
|
||||
const parsedNode = JSON.parse(data)
|
||||
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(parsedNode.connections)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
allNodes.push({
|
||||
id: parsedNode.id,
|
||||
vector: parsedNode.vector,
|
||||
connections
|
||||
})
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error reading directory ${this.nounsDir}:`, error)
|
||||
}
|
||||
}
|
||||
return allNodes
|
||||
}
|
||||
|
||||
/**
|
||||
* Get nodes by noun type
|
||||
* @param nounType The noun type to filter by
|
||||
* @returns Promise that resolves to an array of nodes of the specified noun type
|
||||
*/
|
||||
protected async getNodesByNounType(nounType: string): Promise<HNSWNode[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
const nouns: HNSWNode[] = []
|
||||
try {
|
||||
const files = await fs.promises.readdir(this.nounsDir)
|
||||
for (const file of files) {
|
||||
if (file.endsWith('.json')) {
|
||||
const filePath = path.join(this.nounsDir, file)
|
||||
const data = await fs.promises.readFile(filePath, 'utf-8')
|
||||
const parsedNode = JSON.parse(data)
|
||||
|
||||
// Filter by noun type using metadata
|
||||
const nodeId = parsedNode.id
|
||||
const metadata = await this.getMetadata(nodeId)
|
||||
if (metadata && metadata.noun === nounType) {
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(parsedNode.connections)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
nouns.push({
|
||||
id: parsedNode.id,
|
||||
vector: parsedNode.vector,
|
||||
connections
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error reading directory ${this.nounsDir}:`, error)
|
||||
}
|
||||
}
|
||||
|
||||
return nouns
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a node from storage
|
||||
*/
|
||||
protected async deleteNode(id: string): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
const filePath = path.join(this.nounsDir, `${id}.json`)
|
||||
try {
|
||||
await fs.promises.unlink(filePath)
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error deleting node file ${filePath}:`, error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save an edge to storage
|
||||
*/
|
||||
protected async saveEdge(edge: Edge): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
// Convert connections Map to a serializable format
|
||||
const serializableEdge = {
|
||||
...edge,
|
||||
connections: this.mapToObject(edge.connections, (set) =>
|
||||
Array.from(set as Set<string>)
|
||||
)
|
||||
}
|
||||
|
||||
const filePath = path.join(this.verbsDir, `${edge.id}.json`)
|
||||
await fs.promises.writeFile(filePath, JSON.stringify(serializableEdge, null, 2))
|
||||
}
|
||||
|
||||
/**
|
||||
* Get an edge from storage
|
||||
*/
|
||||
protected async getEdge(id: string): Promise<Edge | null> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
const filePath = path.join(this.verbsDir, `${id}.json`)
|
||||
try {
|
||||
const data = await fs.promises.readFile(filePath, 'utf-8')
|
||||
const parsedEdge = JSON.parse(data)
|
||||
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(parsedEdge.connections)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
return {
|
||||
id: parsedEdge.id,
|
||||
vector: parsedEdge.vector,
|
||||
connections,
|
||||
sourceId: parsedEdge.sourceId,
|
||||
targetId: parsedEdge.targetId,
|
||||
type: parsedEdge.type,
|
||||
weight: parsedEdge.weight,
|
||||
metadata: parsedEdge.metadata
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error reading edge ${id}:`, error)
|
||||
}
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all edges from storage
|
||||
*/
|
||||
protected async getAllEdges(): Promise<Edge[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
const allEdges: Edge[] = []
|
||||
try {
|
||||
const files = await fs.promises.readdir(this.verbsDir)
|
||||
for (const file of files) {
|
||||
if (file.endsWith('.json')) {
|
||||
const filePath = path.join(this.verbsDir, file)
|
||||
const data = await fs.promises.readFile(filePath, 'utf-8')
|
||||
const parsedEdge = JSON.parse(data)
|
||||
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(parsedEdge.connections)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
allEdges.push({
|
||||
id: parsedEdge.id,
|
||||
vector: parsedEdge.vector,
|
||||
connections,
|
||||
sourceId: parsedEdge.sourceId,
|
||||
targetId: parsedEdge.targetId,
|
||||
type: parsedEdge.type,
|
||||
weight: parsedEdge.weight,
|
||||
metadata: parsedEdge.metadata
|
||||
})
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error reading directory ${this.verbsDir}:`, error)
|
||||
}
|
||||
}
|
||||
return allEdges
|
||||
}
|
||||
|
||||
/**
|
||||
* Get edges by source
|
||||
*/
|
||||
protected async getEdgesBySource(sourceId: string): Promise<Edge[]> {
|
||||
const edges = await this.getAllEdges()
|
||||
return edges.filter((edge) => edge.sourceId === sourceId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get edges by target
|
||||
*/
|
||||
protected async getEdgesByTarget(targetId: string): Promise<Edge[]> {
|
||||
const edges = await this.getAllEdges()
|
||||
return edges.filter((edge) => edge.targetId === targetId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get edges by type
|
||||
*/
|
||||
protected async getEdgesByType(type: string): Promise<Edge[]> {
|
||||
const edges = await this.getAllEdges()
|
||||
return edges.filter((edge) => edge.type === type)
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete an edge from storage
|
||||
*/
|
||||
protected async deleteEdge(id: string): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
const filePath = path.join(this.verbsDir, `${id}.json`)
|
||||
try {
|
||||
await fs.promises.unlink(filePath)
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error deleting edge file ${filePath}:`, error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save metadata to storage
|
||||
*/
|
||||
public async saveMetadata(id: string, metadata: any): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
const filePath = path.join(this.metadataDir, `${id}.json`)
|
||||
await fs.promises.writeFile(filePath, JSON.stringify(metadata, null, 2))
|
||||
}
|
||||
|
||||
/**
|
||||
* Get metadata from storage
|
||||
*/
|
||||
public async getMetadata(id: string): Promise<any | null> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
const filePath = path.join(this.metadataDir, `${id}.json`)
|
||||
try {
|
||||
const data = await fs.promises.readFile(filePath, 'utf-8')
|
||||
return JSON.parse(data)
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error reading metadata ${id}:`, error)
|
||||
}
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear all data from storage
|
||||
*/
|
||||
public async clear(): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
// Helper function to remove all files in a directory
|
||||
const removeDirectoryContents = async (dirPath: string): Promise<void> => {
|
||||
try {
|
||||
const files = await fs.promises.readdir(dirPath)
|
||||
for (const file of files) {
|
||||
const filePath = path.join(dirPath, file)
|
||||
const stats = await fs.promises.stat(filePath)
|
||||
if (stats.isDirectory()) {
|
||||
await removeDirectoryContents(filePath)
|
||||
await fs.promises.rmdir(filePath)
|
||||
} else {
|
||||
await fs.promises.unlink(filePath)
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error removing directory contents ${dirPath}:`, error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Remove all files in the nouns directory
|
||||
await removeDirectoryContents(this.nounsDir)
|
||||
|
||||
// Remove all files in the verbs directory
|
||||
await removeDirectoryContents(this.verbsDir)
|
||||
|
||||
// Remove all files in the metadata directory
|
||||
await removeDirectoryContents(this.metadataDir)
|
||||
|
||||
// Remove all files in the index directory
|
||||
await removeDirectoryContents(this.indexDir)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get information about storage usage and capacity
|
||||
*/
|
||||
public async getStorageStatus(): Promise<{
|
||||
type: string
|
||||
used: number
|
||||
quota: number | null
|
||||
details?: Record<string, any>
|
||||
}> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Calculate the total size of all files in the storage directories
|
||||
let totalSize = 0
|
||||
|
||||
// Helper function to calculate directory size
|
||||
const calculateSize = async (dirPath: string): Promise<number> => {
|
||||
let size = 0
|
||||
try {
|
||||
const files = await fs.promises.readdir(dirPath)
|
||||
for (const file of files) {
|
||||
const filePath = path.join(dirPath, file)
|
||||
const stats = await fs.promises.stat(filePath)
|
||||
if (stats.isDirectory()) {
|
||||
size += await calculateSize(filePath)
|
||||
} else {
|
||||
size += stats.size
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error calculating size for directory ${dirPath}:`, error)
|
||||
}
|
||||
}
|
||||
return size
|
||||
}
|
||||
|
||||
// Calculate size for each directory
|
||||
const nounsDirSize = await calculateSize(this.nounsDir)
|
||||
const verbsDirSize = await calculateSize(this.verbsDir)
|
||||
const metadataDirSize = await calculateSize(this.metadataDir)
|
||||
const indexDirSize = await calculateSize(this.indexDir)
|
||||
|
||||
totalSize = nounsDirSize + verbsDirSize + metadataDirSize + indexDirSize
|
||||
|
||||
// Count files in each directory
|
||||
const nounsCount = (await fs.promises.readdir(this.nounsDir)).filter((file: string) => file.endsWith('.json')).length
|
||||
const verbsCount = (await fs.promises.readdir(this.verbsDir)).filter((file: string) => file.endsWith('.json')).length
|
||||
const metadataCount = (await fs.promises.readdir(this.metadataDir)).filter((file: string) => file.endsWith('.json')).length
|
||||
|
||||
// Count nouns by type using metadata
|
||||
const nounTypeCounts: Record<string, number> = {}
|
||||
const metadataFiles = await fs.promises.readdir(this.metadataDir)
|
||||
for (const file of metadataFiles) {
|
||||
if (file.endsWith('.json')) {
|
||||
try {
|
||||
const filePath = path.join(this.metadataDir, file)
|
||||
const data = await fs.promises.readFile(filePath, 'utf-8')
|
||||
const metadata = JSON.parse(data)
|
||||
if (metadata.noun) {
|
||||
nounTypeCounts[metadata.noun] = (nounTypeCounts[metadata.noun] || 0) + 1
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Error reading metadata file ${file}:`, error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
type: 'filesystem',
|
||||
used: totalSize,
|
||||
quota: null, // File system doesn't provide quota information
|
||||
details: {
|
||||
rootDirectory: this.rootDir,
|
||||
nounsCount,
|
||||
verbsCount,
|
||||
metadataCount,
|
||||
nounsDirSize,
|
||||
verbsDirSize,
|
||||
metadataDirSize,
|
||||
indexDirSize,
|
||||
nounTypes: nounTypeCounts
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Failed to get storage status:', error)
|
||||
return {
|
||||
type: 'filesystem',
|
||||
used: 0,
|
||||
quota: null,
|
||||
details: { error: String(error) }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue