feat(storage): implement base, file system, and memory storage adapters
- Added `BaseStorage` abstract class to provide common functionality for storage adapters. - Implemented `FileSystemStorage` adapter for Node.js environments, supporting persistent file-based storage. - Added directory structure management methods to `FileSystemStorage` (`ensureDirectoryExists`, `clear`, etc.). - Implemented `MemoryStorage` adapter for in-memory storage suitable for non-persistent environments. - Supported CRUD operations for nouns, verbs, and metadata in both `FileSystemStorage` and `MemoryStorage`. - Included utility methods like `mapToObject` for serialization and deserialization of complex data structures. Purpose: Provide extensible, reusable storage solutions with both persistent and non-persistent options for diverse application needs.
This commit is contained in:
parent
398c120757
commit
1127115664
9 changed files with 4549 additions and 2186 deletions
555
src/storage/adapters/fileSystemStorage.ts
Normal file
555
src/storage/adapters/fileSystemStorage.ts
Normal file
|
|
@ -0,0 +1,555 @@
|
|||
/**
|
||||
* File System Storage Adapter
|
||||
* File system storage adapter for Node.js environments
|
||||
*/
|
||||
|
||||
import { GraphVerb, HNSWNoun } from '../../coreTypes.js'
|
||||
import { BaseStorage, NOUNS_DIR, VERBS_DIR, METADATA_DIR, INDEX_DIR } from '../baseStorage.js'
|
||||
|
||||
// Type aliases for better readability
|
||||
type HNSWNode = HNSWNoun
|
||||
type Edge = GraphVerb
|
||||
|
||||
// Node.js modules - dynamically imported to avoid issues in browser environments
|
||||
let fs: any
|
||||
let path: any
|
||||
|
||||
// Try to load Node.js modules
|
||||
try {
|
||||
// Using dynamic imports to avoid issues in browser environments
|
||||
const fsPromise = import('fs')
|
||||
const pathPromise = import('path')
|
||||
|
||||
Promise.all([fsPromise, pathPromise]).then(([fsModule, pathModule]) => {
|
||||
fs = fsModule
|
||||
path = pathModule.default
|
||||
}).catch(error => {
|
||||
console.error('Failed to load Node.js modules:', error)
|
||||
})
|
||||
} catch (error) {
|
||||
console.error(
|
||||
'FileSystemStorage: Failed to load Node.js modules. This adapter is not supported in this environment.',
|
||||
error
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* File system storage adapter for Node.js environments
|
||||
* Uses the file system to store data in the specified directory structure
|
||||
*/
|
||||
export class FileSystemStorage extends BaseStorage {
|
||||
private rootDir: string
|
||||
private nounsDir: string
|
||||
private verbsDir: string
|
||||
private metadataDir: string
|
||||
private indexDir: string
|
||||
|
||||
/**
|
||||
* Initialize the storage adapter
|
||||
* @param rootDirectory The root directory for storage
|
||||
*/
|
||||
constructor(rootDirectory: string) {
|
||||
super()
|
||||
this.rootDir = rootDirectory
|
||||
this.nounsDir = path.join(this.rootDir, NOUNS_DIR)
|
||||
this.verbsDir = path.join(this.rootDir, VERBS_DIR)
|
||||
this.metadataDir = path.join(this.rootDir, METADATA_DIR)
|
||||
this.indexDir = path.join(this.rootDir, INDEX_DIR)
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the storage adapter
|
||||
*/
|
||||
public async init(): Promise<void> {
|
||||
if (this.isInitialized) {
|
||||
return
|
||||
}
|
||||
|
||||
// Check if Node.js modules are available
|
||||
if (!fs || !path) {
|
||||
throw new Error(
|
||||
'FileSystemStorage requires a Node.js environment, but `fs` and `path` modules could not be loaded.'
|
||||
)
|
||||
}
|
||||
|
||||
try {
|
||||
// Create the root directory if it doesn't exist
|
||||
await this.ensureDirectoryExists(this.rootDir)
|
||||
|
||||
// Create the nouns directory if it doesn't exist
|
||||
await this.ensureDirectoryExists(this.nounsDir)
|
||||
|
||||
// Create the verbs directory if it doesn't exist
|
||||
await this.ensureDirectoryExists(this.verbsDir)
|
||||
|
||||
// Create the metadata directory if it doesn't exist
|
||||
await this.ensureDirectoryExists(this.metadataDir)
|
||||
|
||||
// Create the index directory if it doesn't exist
|
||||
await this.ensureDirectoryExists(this.indexDir)
|
||||
|
||||
this.isInitialized = true
|
||||
} catch (error) {
|
||||
console.error('Error initializing FileSystemStorage:', error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure a directory exists, creating it if necessary
|
||||
*/
|
||||
private async ensureDirectoryExists(dirPath: string): Promise<void> {
|
||||
try {
|
||||
await fs.promises.mkdir(dirPath, { recursive: true })
|
||||
} catch (error: any) {
|
||||
// Ignore EEXIST error, which means the directory already exists
|
||||
if (error.code !== 'EEXIST') {
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save a node to storage
|
||||
*/
|
||||
protected async saveNode(node: HNSWNode): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
// Convert connections Map to a serializable format
|
||||
const serializableNode = {
|
||||
...node,
|
||||
connections: this.mapToObject(node.connections, (set) =>
|
||||
Array.from(set as Set<string>)
|
||||
)
|
||||
}
|
||||
|
||||
const filePath = path.join(this.nounsDir, `${node.id}.json`)
|
||||
await fs.promises.writeFile(filePath, JSON.stringify(serializableNode, null, 2))
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a node from storage
|
||||
*/
|
||||
protected async getNode(id: string): Promise<HNSWNode | null> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
const filePath = path.join(this.nounsDir, `${id}.json`)
|
||||
try {
|
||||
const data = await fs.promises.readFile(filePath, 'utf-8')
|
||||
const parsedNode = JSON.parse(data)
|
||||
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(parsedNode.connections)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
return {
|
||||
id: parsedNode.id,
|
||||
vector: parsedNode.vector,
|
||||
connections
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error reading node ${id}:`, error)
|
||||
}
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all nodes from storage
|
||||
*/
|
||||
protected async getAllNodes(): Promise<HNSWNode[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
const allNodes: HNSWNode[] = []
|
||||
try {
|
||||
const files = await fs.promises.readdir(this.nounsDir)
|
||||
for (const file of files) {
|
||||
if (file.endsWith('.json')) {
|
||||
const filePath = path.join(this.nounsDir, file)
|
||||
const data = await fs.promises.readFile(filePath, 'utf-8')
|
||||
const parsedNode = JSON.parse(data)
|
||||
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(parsedNode.connections)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
allNodes.push({
|
||||
id: parsedNode.id,
|
||||
vector: parsedNode.vector,
|
||||
connections
|
||||
})
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error reading directory ${this.nounsDir}:`, error)
|
||||
}
|
||||
}
|
||||
return allNodes
|
||||
}
|
||||
|
||||
/**
|
||||
* Get nodes by noun type
|
||||
* @param nounType The noun type to filter by
|
||||
* @returns Promise that resolves to an array of nodes of the specified noun type
|
||||
*/
|
||||
protected async getNodesByNounType(nounType: string): Promise<HNSWNode[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
const nouns: HNSWNode[] = []
|
||||
try {
|
||||
const files = await fs.promises.readdir(this.nounsDir)
|
||||
for (const file of files) {
|
||||
if (file.endsWith('.json')) {
|
||||
const filePath = path.join(this.nounsDir, file)
|
||||
const data = await fs.promises.readFile(filePath, 'utf-8')
|
||||
const parsedNode = JSON.parse(data)
|
||||
|
||||
// Filter by noun type using metadata
|
||||
const nodeId = parsedNode.id
|
||||
const metadata = await this.getMetadata(nodeId)
|
||||
if (metadata && metadata.noun === nounType) {
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(parsedNode.connections)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
nouns.push({
|
||||
id: parsedNode.id,
|
||||
vector: parsedNode.vector,
|
||||
connections
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error reading directory ${this.nounsDir}:`, error)
|
||||
}
|
||||
}
|
||||
|
||||
return nouns
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a node from storage
|
||||
*/
|
||||
protected async deleteNode(id: string): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
const filePath = path.join(this.nounsDir, `${id}.json`)
|
||||
try {
|
||||
await fs.promises.unlink(filePath)
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error deleting node file ${filePath}:`, error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save an edge to storage
|
||||
*/
|
||||
protected async saveEdge(edge: Edge): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
// Convert connections Map to a serializable format
|
||||
const serializableEdge = {
|
||||
...edge,
|
||||
connections: this.mapToObject(edge.connections, (set) =>
|
||||
Array.from(set as Set<string>)
|
||||
)
|
||||
}
|
||||
|
||||
const filePath = path.join(this.verbsDir, `${edge.id}.json`)
|
||||
await fs.promises.writeFile(filePath, JSON.stringify(serializableEdge, null, 2))
|
||||
}
|
||||
|
||||
/**
|
||||
* Get an edge from storage
|
||||
*/
|
||||
protected async getEdge(id: string): Promise<Edge | null> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
const filePath = path.join(this.verbsDir, `${id}.json`)
|
||||
try {
|
||||
const data = await fs.promises.readFile(filePath, 'utf-8')
|
||||
const parsedEdge = JSON.parse(data)
|
||||
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(parsedEdge.connections)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
return {
|
||||
id: parsedEdge.id,
|
||||
vector: parsedEdge.vector,
|
||||
connections,
|
||||
sourceId: parsedEdge.sourceId,
|
||||
targetId: parsedEdge.targetId,
|
||||
type: parsedEdge.type,
|
||||
weight: parsedEdge.weight,
|
||||
metadata: parsedEdge.metadata
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error reading edge ${id}:`, error)
|
||||
}
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all edges from storage
|
||||
*/
|
||||
protected async getAllEdges(): Promise<Edge[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
const allEdges: Edge[] = []
|
||||
try {
|
||||
const files = await fs.promises.readdir(this.verbsDir)
|
||||
for (const file of files) {
|
||||
if (file.endsWith('.json')) {
|
||||
const filePath = path.join(this.verbsDir, file)
|
||||
const data = await fs.promises.readFile(filePath, 'utf-8')
|
||||
const parsedEdge = JSON.parse(data)
|
||||
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(parsedEdge.connections)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
allEdges.push({
|
||||
id: parsedEdge.id,
|
||||
vector: parsedEdge.vector,
|
||||
connections,
|
||||
sourceId: parsedEdge.sourceId,
|
||||
targetId: parsedEdge.targetId,
|
||||
type: parsedEdge.type,
|
||||
weight: parsedEdge.weight,
|
||||
metadata: parsedEdge.metadata
|
||||
})
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error reading directory ${this.verbsDir}:`, error)
|
||||
}
|
||||
}
|
||||
return allEdges
|
||||
}
|
||||
|
||||
/**
|
||||
* Get edges by source
|
||||
*/
|
||||
protected async getEdgesBySource(sourceId: string): Promise<Edge[]> {
|
||||
const edges = await this.getAllEdges()
|
||||
return edges.filter((edge) => edge.sourceId === sourceId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get edges by target
|
||||
*/
|
||||
protected async getEdgesByTarget(targetId: string): Promise<Edge[]> {
|
||||
const edges = await this.getAllEdges()
|
||||
return edges.filter((edge) => edge.targetId === targetId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get edges by type
|
||||
*/
|
||||
protected async getEdgesByType(type: string): Promise<Edge[]> {
|
||||
const edges = await this.getAllEdges()
|
||||
return edges.filter((edge) => edge.type === type)
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete an edge from storage
|
||||
*/
|
||||
protected async deleteEdge(id: string): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
const filePath = path.join(this.verbsDir, `${id}.json`)
|
||||
try {
|
||||
await fs.promises.unlink(filePath)
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error deleting edge file ${filePath}:`, error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save metadata to storage
|
||||
*/
|
||||
public async saveMetadata(id: string, metadata: any): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
const filePath = path.join(this.metadataDir, `${id}.json`)
|
||||
await fs.promises.writeFile(filePath, JSON.stringify(metadata, null, 2))
|
||||
}
|
||||
|
||||
/**
|
||||
* Get metadata from storage
|
||||
*/
|
||||
public async getMetadata(id: string): Promise<any | null> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
const filePath = path.join(this.metadataDir, `${id}.json`)
|
||||
try {
|
||||
const data = await fs.promises.readFile(filePath, 'utf-8')
|
||||
return JSON.parse(data)
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error reading metadata ${id}:`, error)
|
||||
}
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear all data from storage
|
||||
*/
|
||||
public async clear(): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
// Helper function to remove all files in a directory
|
||||
const removeDirectoryContents = async (dirPath: string): Promise<void> => {
|
||||
try {
|
||||
const files = await fs.promises.readdir(dirPath)
|
||||
for (const file of files) {
|
||||
const filePath = path.join(dirPath, file)
|
||||
const stats = await fs.promises.stat(filePath)
|
||||
if (stats.isDirectory()) {
|
||||
await removeDirectoryContents(filePath)
|
||||
await fs.promises.rmdir(filePath)
|
||||
} else {
|
||||
await fs.promises.unlink(filePath)
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error removing directory contents ${dirPath}:`, error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Remove all files in the nouns directory
|
||||
await removeDirectoryContents(this.nounsDir)
|
||||
|
||||
// Remove all files in the verbs directory
|
||||
await removeDirectoryContents(this.verbsDir)
|
||||
|
||||
// Remove all files in the metadata directory
|
||||
await removeDirectoryContents(this.metadataDir)
|
||||
|
||||
// Remove all files in the index directory
|
||||
await removeDirectoryContents(this.indexDir)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get information about storage usage and capacity
|
||||
*/
|
||||
public async getStorageStatus(): Promise<{
|
||||
type: string
|
||||
used: number
|
||||
quota: number | null
|
||||
details?: Record<string, any>
|
||||
}> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Calculate the total size of all files in the storage directories
|
||||
let totalSize = 0
|
||||
|
||||
// Helper function to calculate directory size
|
||||
const calculateSize = async (dirPath: string): Promise<number> => {
|
||||
let size = 0
|
||||
try {
|
||||
const files = await fs.promises.readdir(dirPath)
|
||||
for (const file of files) {
|
||||
const filePath = path.join(dirPath, file)
|
||||
const stats = await fs.promises.stat(filePath)
|
||||
if (stats.isDirectory()) {
|
||||
size += await calculateSize(filePath)
|
||||
} else {
|
||||
size += stats.size
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error calculating size for directory ${dirPath}:`, error)
|
||||
}
|
||||
}
|
||||
return size
|
||||
}
|
||||
|
||||
// Calculate size for each directory
|
||||
const nounsDirSize = await calculateSize(this.nounsDir)
|
||||
const verbsDirSize = await calculateSize(this.verbsDir)
|
||||
const metadataDirSize = await calculateSize(this.metadataDir)
|
||||
const indexDirSize = await calculateSize(this.indexDir)
|
||||
|
||||
totalSize = nounsDirSize + verbsDirSize + metadataDirSize + indexDirSize
|
||||
|
||||
// Count files in each directory
|
||||
const nounsCount = (await fs.promises.readdir(this.nounsDir)).filter((file: string) => file.endsWith('.json')).length
|
||||
const verbsCount = (await fs.promises.readdir(this.verbsDir)).filter((file: string) => file.endsWith('.json')).length
|
||||
const metadataCount = (await fs.promises.readdir(this.metadataDir)).filter((file: string) => file.endsWith('.json')).length
|
||||
|
||||
// Count nouns by type using metadata
|
||||
const nounTypeCounts: Record<string, number> = {}
|
||||
const metadataFiles = await fs.promises.readdir(this.metadataDir)
|
||||
for (const file of metadataFiles) {
|
||||
if (file.endsWith('.json')) {
|
||||
try {
|
||||
const filePath = path.join(this.metadataDir, file)
|
||||
const data = await fs.promises.readFile(filePath, 'utf-8')
|
||||
const metadata = JSON.parse(data)
|
||||
if (metadata.noun) {
|
||||
nounTypeCounts[metadata.noun] = (nounTypeCounts[metadata.noun] || 0) + 1
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Error reading metadata file ${file}:`, error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
type: 'filesystem',
|
||||
used: totalSize,
|
||||
quota: null, // File system doesn't provide quota information
|
||||
details: {
|
||||
rootDirectory: this.rootDir,
|
||||
nounsCount,
|
||||
verbsCount,
|
||||
metadataCount,
|
||||
nounsDirSize,
|
||||
verbsDirSize,
|
||||
metadataDirSize,
|
||||
indexDirSize,
|
||||
nounTypes: nounTypeCounts
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Failed to get storage status:', error)
|
||||
return {
|
||||
type: 'filesystem',
|
||||
used: 0,
|
||||
quota: null,
|
||||
details: { error: String(error) }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
324
src/storage/adapters/memoryStorage.ts
Normal file
324
src/storage/adapters/memoryStorage.ts
Normal file
|
|
@ -0,0 +1,324 @@
|
|||
/**
|
||||
* Memory Storage Adapter
|
||||
* In-memory storage adapter for environments where persistent storage is not available or needed
|
||||
*/
|
||||
|
||||
import { GraphVerb, HNSWNoun } from '../../coreTypes.js'
|
||||
import { BaseStorage } from '../baseStorage.js'
|
||||
|
||||
/**
|
||||
* Type alias for HNSWNoun to make the code more readable
|
||||
*/
|
||||
type HNSWNode = HNSWNoun
|
||||
|
||||
/**
|
||||
* Type alias for GraphVerb to make the code more readable
|
||||
*/
|
||||
type Edge = GraphVerb
|
||||
|
||||
/**
|
||||
* In-memory storage adapter
|
||||
* Uses Maps to store data in memory
|
||||
*/
|
||||
export class MemoryStorage extends BaseStorage {
|
||||
// Single map of noun ID to noun
|
||||
private nouns: Map<string, HNSWNode> = new Map()
|
||||
private verbs: Map<string, Edge> = new Map()
|
||||
private metadata: Map<string, any> = new Map()
|
||||
|
||||
constructor() {
|
||||
super()
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the storage adapter
|
||||
* Nothing to initialize for in-memory storage
|
||||
*/
|
||||
public async init(): Promise<void> {
|
||||
this.isInitialized = true
|
||||
}
|
||||
|
||||
/**
|
||||
* Save a node to storage
|
||||
*/
|
||||
protected async saveNode(node: HNSWNode): Promise<void> {
|
||||
// Create a deep copy to avoid reference issues
|
||||
const nodeCopy: HNSWNode = {
|
||||
id: node.id,
|
||||
vector: [...node.vector],
|
||||
connections: new Map()
|
||||
}
|
||||
|
||||
// Copy connections
|
||||
for (const [level, connections] of node.connections.entries()) {
|
||||
nodeCopy.connections.set(level, new Set(connections))
|
||||
}
|
||||
|
||||
// Save the node directly in the nouns map
|
||||
this.nouns.set(node.id, nodeCopy)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a node from storage
|
||||
*/
|
||||
protected async getNode(id: string): Promise<HNSWNode | null> {
|
||||
// Get the node directly from the nouns map
|
||||
const node = this.nouns.get(id)
|
||||
|
||||
// If not found, return null
|
||||
if (!node) {
|
||||
return null
|
||||
}
|
||||
|
||||
// Return a deep copy to avoid reference issues
|
||||
const nodeCopy: HNSWNode = {
|
||||
id: node.id,
|
||||
vector: [...node.vector],
|
||||
connections: new Map()
|
||||
}
|
||||
|
||||
// Copy connections
|
||||
for (const [level, connections] of node.connections.entries()) {
|
||||
nodeCopy.connections.set(level, new Set(connections))
|
||||
}
|
||||
|
||||
return nodeCopy
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all nodes from storage
|
||||
*/
|
||||
protected async getAllNodes(): Promise<HNSWNode[]> {
|
||||
const allNodes: HNSWNode[] = []
|
||||
|
||||
// Iterate through all nodes in the nouns map
|
||||
for (const [nodeId, node] of this.nouns.entries()) {
|
||||
// Return a deep copy to avoid reference issues
|
||||
const nodeCopy: HNSWNode = {
|
||||
id: node.id,
|
||||
vector: [...node.vector],
|
||||
connections: new Map()
|
||||
}
|
||||
|
||||
// Copy connections
|
||||
for (const [level, connections] of node.connections.entries()) {
|
||||
nodeCopy.connections.set(level, new Set(connections))
|
||||
}
|
||||
|
||||
allNodes.push(nodeCopy)
|
||||
}
|
||||
|
||||
return allNodes
|
||||
}
|
||||
|
||||
/**
|
||||
* Get nodes by noun type
|
||||
* @param nounType The noun type to filter by
|
||||
* @returns Promise that resolves to an array of nodes of the specified noun type
|
||||
*/
|
||||
protected async getNodesByNounType(nounType: string): Promise<HNSWNode[]> {
|
||||
const nodes: HNSWNode[] = []
|
||||
|
||||
// Iterate through all nodes and filter by noun type using metadata
|
||||
for (const [nodeId, node] of this.nouns.entries()) {
|
||||
// Get the metadata to check the noun type
|
||||
const metadata = await this.getMetadata(nodeId)
|
||||
|
||||
// Include the node if its noun type matches the requested type
|
||||
if (metadata && metadata.noun === nounType) {
|
||||
// Return a deep copy to avoid reference issues
|
||||
const nodeCopy: HNSWNode = {
|
||||
id: node.id,
|
||||
vector: [...node.vector],
|
||||
connections: new Map()
|
||||
}
|
||||
|
||||
// Copy connections
|
||||
for (const [level, connections] of node.connections.entries()) {
|
||||
nodeCopy.connections.set(level, new Set(connections))
|
||||
}
|
||||
|
||||
nodes.push(nodeCopy)
|
||||
}
|
||||
}
|
||||
|
||||
return nodes
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a node from storage
|
||||
*/
|
||||
protected async deleteNode(id: string): Promise<void> {
|
||||
// Delete the node directly from the nouns map
|
||||
this.nouns.delete(id)
|
||||
}
|
||||
|
||||
/**
|
||||
* Save an edge to storage
|
||||
*/
|
||||
protected async saveEdge(edge: Edge): Promise<void> {
|
||||
// Create a deep copy to avoid reference issues
|
||||
const edgeCopy: Edge = {
|
||||
id: edge.id,
|
||||
vector: [...edge.vector],
|
||||
connections: new Map(),
|
||||
sourceId: edge.sourceId,
|
||||
targetId: edge.targetId,
|
||||
type: edge.type,
|
||||
weight: edge.weight,
|
||||
metadata: edge.metadata
|
||||
}
|
||||
|
||||
// Copy connections
|
||||
for (const [level, connections] of edge.connections.entries()) {
|
||||
edgeCopy.connections.set(level, new Set(connections))
|
||||
}
|
||||
|
||||
// Save the edge directly in the verbs map
|
||||
this.verbs.set(edge.id, edgeCopy)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get an edge from storage
|
||||
*/
|
||||
protected async getEdge(id: string): Promise<Edge | null> {
|
||||
// Get the edge directly from the verbs map
|
||||
const edge = this.verbs.get(id)
|
||||
|
||||
// If not found, return null
|
||||
if (!edge) {
|
||||
return null
|
||||
}
|
||||
|
||||
// Return a deep copy to avoid reference issues
|
||||
const edgeCopy: Edge = {
|
||||
id: edge.id,
|
||||
vector: [...edge.vector],
|
||||
connections: new Map(),
|
||||
sourceId: edge.sourceId,
|
||||
targetId: edge.targetId,
|
||||
type: edge.type,
|
||||
weight: edge.weight,
|
||||
metadata: edge.metadata
|
||||
}
|
||||
|
||||
// Copy connections
|
||||
for (const [level, connections] of edge.connections.entries()) {
|
||||
edgeCopy.connections.set(level, new Set(connections))
|
||||
}
|
||||
|
||||
return edgeCopy
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all edges from storage
|
||||
*/
|
||||
protected async getAllEdges(): Promise<Edge[]> {
|
||||
const allEdges: Edge[] = []
|
||||
|
||||
// Iterate through all edges in the verbs map
|
||||
for (const [edgeId, edge] of this.verbs.entries()) {
|
||||
// Return a deep copy to avoid reference issues
|
||||
const edgeCopy: Edge = {
|
||||
id: edge.id,
|
||||
vector: [...edge.vector],
|
||||
connections: new Map(),
|
||||
sourceId: edge.sourceId,
|
||||
targetId: edge.targetId,
|
||||
type: edge.type,
|
||||
weight: edge.weight,
|
||||
metadata: edge.metadata
|
||||
}
|
||||
|
||||
// Copy connections
|
||||
for (const [level, connections] of edge.connections.entries()) {
|
||||
edgeCopy.connections.set(level, new Set(connections))
|
||||
}
|
||||
|
||||
allEdges.push(edgeCopy)
|
||||
}
|
||||
|
||||
return allEdges
|
||||
}
|
||||
|
||||
/**
|
||||
* Get edges by source
|
||||
*/
|
||||
protected async getEdgesBySource(sourceId: string): Promise<Edge[]> {
|
||||
const edges = await this.getAllEdges()
|
||||
return edges.filter((edge) => edge.sourceId === sourceId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get edges by target
|
||||
*/
|
||||
protected async getEdgesByTarget(targetId: string): Promise<Edge[]> {
|
||||
const edges = await this.getAllEdges()
|
||||
return edges.filter((edge) => edge.targetId === targetId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get edges by type
|
||||
*/
|
||||
protected async getEdgesByType(type: string): Promise<Edge[]> {
|
||||
const edges = await this.getAllEdges()
|
||||
return edges.filter((edge) => edge.type === type)
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete an edge from storage
|
||||
*/
|
||||
protected async deleteEdge(id: string): Promise<void> {
|
||||
// Delete the edge directly from the verbs map
|
||||
this.verbs.delete(id)
|
||||
}
|
||||
|
||||
/**
|
||||
* Save metadata to storage
|
||||
*/
|
||||
public async saveMetadata(id: string, metadata: any): Promise<void> {
|
||||
this.metadata.set(id, JSON.parse(JSON.stringify(metadata)))
|
||||
}
|
||||
|
||||
/**
|
||||
* Get metadata from storage
|
||||
*/
|
||||
public async getMetadata(id: string): Promise<any | null> {
|
||||
const metadata = this.metadata.get(id)
|
||||
if (!metadata) {
|
||||
return null
|
||||
}
|
||||
|
||||
return JSON.parse(JSON.stringify(metadata))
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear all data from storage
|
||||
*/
|
||||
public async clear(): Promise<void> {
|
||||
this.nouns.clear()
|
||||
this.verbs.clear()
|
||||
this.metadata.clear()
|
||||
}
|
||||
|
||||
/**
|
||||
* Get information about storage usage and capacity
|
||||
*/
|
||||
public async getStorageStatus(): Promise<{
|
||||
type: string
|
||||
used: number
|
||||
quota: number | null
|
||||
details?: Record<string, any>
|
||||
}> {
|
||||
return {
|
||||
type: 'memory',
|
||||
used: 0, // In-memory storage doesn't have a meaningful size
|
||||
quota: null, // In-memory storage doesn't have a quota
|
||||
details: {
|
||||
nodeCount: this.nouns.size,
|
||||
edgeCount: this.verbs.size,
|
||||
metadataCount: this.metadata.size
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
691
src/storage/adapters/opfsStorage.ts
Normal file
691
src/storage/adapters/opfsStorage.ts
Normal file
|
|
@ -0,0 +1,691 @@
|
|||
/**
|
||||
* OPFS (Origin Private File System) Storage Adapter
|
||||
* Provides persistent storage for the vector database using the Origin Private File System API
|
||||
*/
|
||||
|
||||
import {GraphVerb, HNSWNoun} from '../../coreTypes.js'
|
||||
import {BaseStorage, NOUNS_DIR, VERBS_DIR, METADATA_DIR, INDEX_DIR} from '../baseStorage.js'
|
||||
import '../../types/fileSystemTypes.js'
|
||||
|
||||
/**
|
||||
* Helper function to safely get a file from a FileSystemHandle
|
||||
* This is needed because TypeScript doesn't recognize that a FileSystemHandle
|
||||
* can be a FileSystemFileHandle which has the getFile method
|
||||
*/
|
||||
async function safeGetFile(handle: FileSystemHandle): Promise<File> {
|
||||
// Type cast to any to avoid TypeScript error
|
||||
return (handle as any).getFile()
|
||||
}
|
||||
|
||||
// Type aliases for better readability
|
||||
type HNSWNode = HNSWNoun
|
||||
type Edge = GraphVerb
|
||||
|
||||
// Root directory name for OPFS storage
|
||||
const ROOT_DIR = 'opfs-vector-db'
|
||||
|
||||
/**
|
||||
* OPFS storage adapter for browser environments
|
||||
* Uses the Origin Private File System API to store data persistently
|
||||
*/
|
||||
export class OPFSStorage extends BaseStorage {
|
||||
private rootDir: FileSystemDirectoryHandle | null = null
|
||||
private nounsDir: FileSystemDirectoryHandle | null = null
|
||||
private verbsDir: FileSystemDirectoryHandle | null = null
|
||||
private metadataDir: FileSystemDirectoryHandle | null = null
|
||||
private indexDir: FileSystemDirectoryHandle | null = null
|
||||
private isAvailable = false
|
||||
private isPersistentRequested = false
|
||||
private isPersistentGranted = false
|
||||
|
||||
constructor() {
|
||||
super()
|
||||
// Check if OPFS is available
|
||||
this.isAvailable =
|
||||
typeof navigator !== 'undefined' &&
|
||||
'storage' in navigator &&
|
||||
'getDirectory' in navigator.storage
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the storage adapter
|
||||
*/
|
||||
public async init(): Promise<void> {
|
||||
if (this.isInitialized) {
|
||||
return
|
||||
}
|
||||
|
||||
if (!this.isAvailable) {
|
||||
throw new Error(
|
||||
'Origin Private File System is not available in this environment'
|
||||
)
|
||||
}
|
||||
|
||||
try {
|
||||
// Get the root directory
|
||||
const root = await navigator.storage.getDirectory()
|
||||
|
||||
// Create or get our app's root directory
|
||||
this.rootDir = await root.getDirectoryHandle(ROOT_DIR, {create: true})
|
||||
|
||||
// Create or get nouns directory
|
||||
this.nounsDir = await this.rootDir.getDirectoryHandle(NOUNS_DIR, {
|
||||
create: true
|
||||
})
|
||||
|
||||
// Create or get verbs directory
|
||||
this.verbsDir = await this.rootDir.getDirectoryHandle(VERBS_DIR, {
|
||||
create: true
|
||||
})
|
||||
|
||||
// Create or get metadata directory
|
||||
this.metadataDir = await this.rootDir.getDirectoryHandle(METADATA_DIR, {
|
||||
create: true
|
||||
})
|
||||
|
||||
// Create or get index directory
|
||||
this.indexDir = await this.rootDir.getDirectoryHandle(INDEX_DIR, {
|
||||
create: true
|
||||
})
|
||||
|
||||
this.isInitialized = true
|
||||
} catch (error) {
|
||||
console.error('Failed to initialize OPFS storage:', error)
|
||||
throw new Error(`Failed to initialize OPFS storage: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if OPFS is available in the current environment
|
||||
*/
|
||||
public isOPFSAvailable(): boolean {
|
||||
return this.isAvailable
|
||||
}
|
||||
|
||||
/**
|
||||
* Request persistent storage permission from the user
|
||||
* @returns Promise that resolves to true if permission was granted, false otherwise
|
||||
*/
|
||||
public async requestPersistentStorage(): Promise<boolean> {
|
||||
if (!this.isAvailable) {
|
||||
console.warn('Cannot request persistent storage: OPFS is not available')
|
||||
return false
|
||||
}
|
||||
|
||||
try {
|
||||
// Check if persistence is already granted
|
||||
this.isPersistentGranted = await navigator.storage.persisted()
|
||||
|
||||
if (!this.isPersistentGranted) {
|
||||
// Request permission for persistent storage
|
||||
this.isPersistentGranted = await navigator.storage.persist()
|
||||
}
|
||||
|
||||
this.isPersistentRequested = true
|
||||
return this.isPersistentGranted
|
||||
} catch (error) {
|
||||
console.warn('Failed to request persistent storage:', error)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if persistent storage is granted
|
||||
* @returns Promise that resolves to true if persistent storage is granted, false otherwise
|
||||
*/
|
||||
public async isPersistent(): Promise<boolean> {
|
||||
if (!this.isAvailable) {
|
||||
return false
|
||||
}
|
||||
|
||||
try {
|
||||
this.isPersistentGranted = await navigator.storage.persisted()
|
||||
return this.isPersistentGranted
|
||||
} catch (error) {
|
||||
console.warn('Failed to check persistent storage status:', error)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save a node to storage
|
||||
*/
|
||||
protected async saveNode(node: HNSWNode): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Convert connections Map to a serializable format
|
||||
const serializableNode = {
|
||||
...node,
|
||||
connections: this.mapToObject(node.connections, (set) =>
|
||||
Array.from(set as Set<string>)
|
||||
)
|
||||
}
|
||||
|
||||
// Create or get the file for this noun
|
||||
const fileHandle = await this.nounsDir!.getFileHandle(node.id, {
|
||||
create: true
|
||||
})
|
||||
|
||||
// Write the noun data to the file
|
||||
const writable = await fileHandle.createWritable()
|
||||
await writable.write(JSON.stringify(serializableNode))
|
||||
await writable.close()
|
||||
} catch (error) {
|
||||
console.error(`Failed to save node ${node.id}:`, error)
|
||||
throw new Error(`Failed to save node ${node.id}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a node from storage
|
||||
*/
|
||||
protected async getNode(id: string): Promise<HNSWNode | null> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Get the file handle for this node
|
||||
const fileHandle = await this.nounsDir!.getFileHandle(id)
|
||||
|
||||
// Read the node data from the file
|
||||
const file = await fileHandle.getFile()
|
||||
const text = await file.text()
|
||||
const data = JSON.parse(text)
|
||||
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(data.connections)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
return {
|
||||
id: data.id,
|
||||
vector: data.vector,
|
||||
connections
|
||||
}
|
||||
} catch (error) {
|
||||
// Node not found or other error
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all nodes from storage
|
||||
*/
|
||||
protected async getAllNodes(): Promise<HNSWNode[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
const allNodes: HNSWNode[] = []
|
||||
try {
|
||||
// Iterate through all files in the nouns directory
|
||||
for await (const [name, handle] of this.nounsDir!.entries()) {
|
||||
if (handle.kind === 'file') {
|
||||
try {
|
||||
// Read the node data from the file
|
||||
const file = await safeGetFile(handle)
|
||||
const text = await file.text()
|
||||
const data = JSON.parse(text)
|
||||
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(data.connections)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
allNodes.push({
|
||||
id: data.id,
|
||||
vector: data.vector,
|
||||
connections
|
||||
})
|
||||
} catch (error) {
|
||||
console.error(`Error reading node file ${name}:`, error)
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error reading nouns directory:', error)
|
||||
}
|
||||
|
||||
return allNodes
|
||||
}
|
||||
|
||||
/**
|
||||
* Get nodes by noun type
|
||||
* @param nounType The noun type to filter by
|
||||
* @returns Promise that resolves to an array of nodes of the specified noun type
|
||||
*/
|
||||
protected async getNodesByNounType(nounType: string): Promise<HNSWNode[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
const nodes: HNSWNode[] = []
|
||||
|
||||
try {
|
||||
// Iterate through all files in the nouns directory
|
||||
for await (const [name, handle] of this.nounsDir!.entries()) {
|
||||
if (handle.kind === 'file') {
|
||||
try {
|
||||
// Read the node data from the file
|
||||
const file = await safeGetFile(handle)
|
||||
const text = await file.text()
|
||||
const data = JSON.parse(text)
|
||||
|
||||
// Get the metadata to check the noun type
|
||||
const metadata = await this.getMetadata(data.id)
|
||||
|
||||
// Include the node if its noun type matches the requested type
|
||||
if (metadata && metadata.noun === nounType) {
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(data.connections)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
nodes.push({
|
||||
id: data.id,
|
||||
vector: data.vector,
|
||||
connections
|
||||
})
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Error reading node file ${name}:`, error)
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error reading nouns directory:', error)
|
||||
}
|
||||
|
||||
return nodes
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a node from storage
|
||||
*/
|
||||
protected async deleteNode(id: string): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
await this.nounsDir!.removeEntry(id)
|
||||
} catch (error: any) {
|
||||
// Ignore NotFoundError, which means the file doesn't exist
|
||||
if (error.name !== 'NotFoundError') {
|
||||
console.error(`Error deleting node ${id}:`, error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save an edge to storage
|
||||
*/
|
||||
protected async saveEdge(edge: Edge): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Convert connections Map to a serializable format
|
||||
const serializableEdge = {
|
||||
...edge,
|
||||
connections: this.mapToObject(edge.connections, (set) =>
|
||||
Array.from(set as Set<string>)
|
||||
)
|
||||
}
|
||||
|
||||
// Create or get the file for this verb
|
||||
const fileHandle = await this.verbsDir!.getFileHandle(edge.id, {
|
||||
create: true
|
||||
})
|
||||
|
||||
// Write the verb data to the file
|
||||
const writable = await fileHandle.createWritable()
|
||||
await writable.write(JSON.stringify(serializableEdge))
|
||||
await writable.close()
|
||||
} catch (error) {
|
||||
console.error(`Failed to save edge ${edge.id}:`, error)
|
||||
throw new Error(`Failed to save edge ${edge.id}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get an edge from storage
|
||||
*/
|
||||
protected async getEdge(id: string): Promise<Edge | null> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Get the file handle for this edge
|
||||
const fileHandle = await this.verbsDir!.getFileHandle(id)
|
||||
|
||||
// Read the edge data from the file
|
||||
const file = await fileHandle.getFile()
|
||||
const text = await file.text()
|
||||
const data = JSON.parse(text)
|
||||
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(data.connections)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
return {
|
||||
id: data.id,
|
||||
vector: data.vector,
|
||||
connections,
|
||||
sourceId: data.sourceId,
|
||||
targetId: data.targetId,
|
||||
type: data.type,
|
||||
weight: data.weight,
|
||||
metadata: data.metadata
|
||||
}
|
||||
} catch (error) {
|
||||
// Edge not found or other error
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all edges from storage
|
||||
*/
|
||||
protected async getAllEdges(): Promise<Edge[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
const allEdges: Edge[] = []
|
||||
try {
|
||||
// Iterate through all files in the verbs directory
|
||||
for await (const [name, handle] of this.verbsDir!.entries()) {
|
||||
if (handle.kind === 'file') {
|
||||
try {
|
||||
// Read the edge data from the file
|
||||
const file = await safeGetFile(handle)
|
||||
const text = await file.text()
|
||||
const data = JSON.parse(text)
|
||||
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(data.connections)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
allEdges.push({
|
||||
id: data.id,
|
||||
vector: data.vector,
|
||||
connections,
|
||||
sourceId: data.sourceId,
|
||||
targetId: data.targetId,
|
||||
type: data.type,
|
||||
weight: data.weight,
|
||||
metadata: data.metadata
|
||||
})
|
||||
} catch (error) {
|
||||
console.error(`Error reading edge file ${name}:`, error)
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error reading verbs directory:', error)
|
||||
}
|
||||
|
||||
return allEdges
|
||||
}
|
||||
|
||||
/**
|
||||
* Get edges by source
|
||||
*/
|
||||
protected async getEdgesBySource(sourceId: string): Promise<Edge[]> {
|
||||
const edges = await this.getAllEdges()
|
||||
return edges.filter((edge) => edge.sourceId === sourceId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get edges by target
|
||||
*/
|
||||
protected async getEdgesByTarget(targetId: string): Promise<Edge[]> {
|
||||
const edges = await this.getAllEdges()
|
||||
return edges.filter((edge) => edge.targetId === targetId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get edges by type
|
||||
*/
|
||||
protected async getEdgesByType(type: string): Promise<Edge[]> {
|
||||
const edges = await this.getAllEdges()
|
||||
return edges.filter((edge) => edge.type === type)
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete an edge from storage
|
||||
*/
|
||||
protected async deleteEdge(id: string): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
await this.verbsDir!.removeEntry(id)
|
||||
} catch (error: any) {
|
||||
// Ignore NotFoundError, which means the file doesn't exist
|
||||
if (error.name !== 'NotFoundError') {
|
||||
console.error(`Error deleting edge ${id}:`, error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save metadata to storage
|
||||
*/
|
||||
public async saveMetadata(id: string, metadata: any): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Create or get the file for this metadata
|
||||
const fileHandle = await this.metadataDir!.getFileHandle(id, {
|
||||
create: true
|
||||
})
|
||||
|
||||
// Write the metadata to the file
|
||||
const writable = await fileHandle.createWritable()
|
||||
await writable.write(JSON.stringify(metadata))
|
||||
await writable.close()
|
||||
} catch (error) {
|
||||
console.error(`Failed to save metadata ${id}:`, error)
|
||||
throw new Error(`Failed to save metadata ${id}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get metadata from storage
|
||||
*/
|
||||
public async getMetadata(id: string): Promise<any | null> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Get the file handle for this metadata
|
||||
const fileHandle = await this.metadataDir!.getFileHandle(id)
|
||||
|
||||
// Read the metadata from the file
|
||||
const file = await fileHandle.getFile()
|
||||
const text = await file.text()
|
||||
return JSON.parse(text)
|
||||
} catch (error) {
|
||||
// Metadata not found or other error
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear all data from storage
|
||||
*/
|
||||
public async clear(): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
// Helper function to remove all files in a directory
|
||||
const removeDirectoryContents = async (
|
||||
dirHandle: FileSystemDirectoryHandle
|
||||
): Promise<void> => {
|
||||
try {
|
||||
for await (const [name, handle] of dirHandle.entries()) {
|
||||
await dirHandle.removeEntry(name)
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Error removing directory contents:`, error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
// Remove all files in the nouns directory
|
||||
await removeDirectoryContents(this.nounsDir!)
|
||||
|
||||
// Remove all files in the verbs directory
|
||||
await removeDirectoryContents(this.verbsDir!)
|
||||
|
||||
// Remove all files in the metadata directory
|
||||
await removeDirectoryContents(this.metadataDir!)
|
||||
|
||||
// Remove all files in the index directory
|
||||
await removeDirectoryContents(this.indexDir!)
|
||||
} catch (error) {
|
||||
console.error('Error clearing storage:', error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get information about storage usage and capacity
|
||||
*/
|
||||
public async getStorageStatus(): Promise<{
|
||||
type: string
|
||||
used: number
|
||||
quota: number | null
|
||||
details?: Record<string, any>
|
||||
}> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Calculate the total size of all files in the storage directories
|
||||
let totalSize = 0
|
||||
|
||||
// Helper function to calculate directory size
|
||||
const calculateDirSize = async (
|
||||
dirHandle: FileSystemDirectoryHandle
|
||||
): Promise<number> => {
|
||||
let size = 0
|
||||
try {
|
||||
for await (const [name, handle] of dirHandle.entries()) {
|
||||
if (handle.kind === 'file') {
|
||||
const file = await (handle as FileSystemFileHandle).getFile()
|
||||
size += file.size
|
||||
} else if (handle.kind === 'directory') {
|
||||
size += await calculateDirSize(
|
||||
handle as FileSystemDirectoryHandle
|
||||
)
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn(`Error calculating size for directory:`, error)
|
||||
}
|
||||
return size
|
||||
}
|
||||
|
||||
// Helper function to count files in a directory
|
||||
const countFilesInDirectory = async (
|
||||
dirHandle: FileSystemDirectoryHandle
|
||||
): Promise<number> => {
|
||||
let count = 0
|
||||
try {
|
||||
for await (const [name, handle] of dirHandle.entries()) {
|
||||
if (handle.kind === 'file') {
|
||||
count++
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn(`Error counting files in directory:`, error)
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
// Calculate size for each directory
|
||||
if (this.nounsDir) {
|
||||
totalSize += await calculateDirSize(this.nounsDir)
|
||||
}
|
||||
if (this.verbsDir) {
|
||||
totalSize += await calculateDirSize(this.verbsDir)
|
||||
}
|
||||
if (this.metadataDir) {
|
||||
totalSize += await calculateDirSize(this.metadataDir)
|
||||
}
|
||||
if (this.indexDir) {
|
||||
totalSize += await calculateDirSize(this.indexDir)
|
||||
}
|
||||
|
||||
// Get storage quota information using the Storage API
|
||||
let quota = null
|
||||
let details: Record<string, any> = {
|
||||
isPersistent: await this.isPersistent(),
|
||||
nounTypes: {}
|
||||
}
|
||||
|
||||
try {
|
||||
if (navigator.storage && navigator.storage.estimate) {
|
||||
const estimate = await navigator.storage.estimate()
|
||||
quota = estimate.quota || null
|
||||
details = {
|
||||
...details,
|
||||
usage: estimate.usage,
|
||||
quota: estimate.quota,
|
||||
freePercentage: estimate.quota
|
||||
? ((estimate.quota - (estimate.usage || 0)) / estimate.quota) *
|
||||
100
|
||||
: null
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn('Unable to get storage estimate:', error)
|
||||
}
|
||||
|
||||
// Count files in each directory
|
||||
if (this.nounsDir) {
|
||||
details.nounsCount = await countFilesInDirectory(this.nounsDir)
|
||||
}
|
||||
if (this.verbsDir) {
|
||||
details.verbsCount = await countFilesInDirectory(this.verbsDir)
|
||||
}
|
||||
if (this.metadataDir) {
|
||||
details.metadataCount = await countFilesInDirectory(this.metadataDir)
|
||||
}
|
||||
|
||||
// Count nouns by type using metadata
|
||||
const nounTypeCounts: Record<string, number> = {}
|
||||
if (this.metadataDir) {
|
||||
for await (const [name, handle] of this.metadataDir.entries()) {
|
||||
if (handle.kind === 'file') {
|
||||
try {
|
||||
const file = await safeGetFile(handle)
|
||||
const text = await file.text()
|
||||
const metadata = JSON.parse(text)
|
||||
if (metadata.noun) {
|
||||
nounTypeCounts[metadata.noun] = (nounTypeCounts[metadata.noun] || 0) + 1
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Error reading metadata file ${name}:`, error)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
details.nounTypes = nounTypeCounts
|
||||
|
||||
return {
|
||||
type: 'opfs',
|
||||
used: totalSize,
|
||||
quota,
|
||||
details
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Failed to get storage status:', error)
|
||||
return {
|
||||
type: 'opfs',
|
||||
used: 0,
|
||||
quota: null,
|
||||
details: {error: String(error)}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
776
src/storage/adapters/s3CompatibleStorage.ts
Normal file
776
src/storage/adapters/s3CompatibleStorage.ts
Normal file
|
|
@ -0,0 +1,776 @@
|
|||
/**
|
||||
* S3-Compatible Storage Adapter
|
||||
* Uses the AWS S3 client to interact with S3-compatible storage services
|
||||
* including Amazon S3, Cloudflare R2, and Google Cloud Storage
|
||||
*/
|
||||
|
||||
import { GraphVerb, HNSWNoun } from '../../coreTypes.js'
|
||||
import { BaseStorage, NOUNS_DIR, VERBS_DIR, METADATA_DIR, INDEX_DIR } from '../baseStorage.js'
|
||||
|
||||
// Type aliases for better readability
|
||||
type HNSWNode = HNSWNoun
|
||||
type Edge = GraphVerb
|
||||
|
||||
// Export R2Storage as an alias for S3CompatibleStorage
|
||||
export { S3CompatibleStorage as R2Storage }
|
||||
|
||||
// S3 client and command types - dynamically imported to avoid issues in browser environments
|
||||
type S3Client = any
|
||||
type S3Command = any
|
||||
|
||||
/**
|
||||
* S3-compatible storage adapter for server environments
|
||||
* Uses the AWS S3 client to interact with S3-compatible storage services
|
||||
* including Amazon S3, Cloudflare R2, and Google Cloud Storage
|
||||
*
|
||||
* To use this adapter with Amazon S3, you need to provide:
|
||||
* - region: AWS region (e.g., 'us-east-1')
|
||||
* - credentials: AWS credentials (accessKeyId and secretAccessKey)
|
||||
* - bucketName: S3 bucket name
|
||||
*
|
||||
* To use this adapter with Cloudflare R2, you need to provide:
|
||||
* - accountId: Cloudflare account ID
|
||||
* - accessKeyId: R2 access key ID
|
||||
* - secretAccessKey: R2 secret access key
|
||||
* - bucketName: R2 bucket name
|
||||
*
|
||||
* To use this adapter with Google Cloud Storage, you need to provide:
|
||||
* - region: GCS region (e.g., 'us-central1')
|
||||
* - credentials: GCS credentials (accessKeyId and secretAccessKey)
|
||||
* - endpoint: GCS endpoint (e.g., 'https://storage.googleapis.com')
|
||||
* - bucketName: GCS bucket name
|
||||
*/
|
||||
export class S3CompatibleStorage extends BaseStorage {
|
||||
private s3Client: S3Client | null = null
|
||||
private bucketName: string
|
||||
private serviceType: string
|
||||
private region: string
|
||||
private endpoint?: string
|
||||
private accountId?: string
|
||||
private accessKeyId: string
|
||||
private secretAccessKey: string
|
||||
private sessionToken?: string
|
||||
|
||||
// Prefixes for different types of data
|
||||
private nounPrefix: string
|
||||
private verbPrefix: string
|
||||
private metadataPrefix: string
|
||||
private indexPrefix: string
|
||||
|
||||
/**
|
||||
* Initialize the storage adapter
|
||||
* @param options Configuration options for the S3-compatible storage
|
||||
*/
|
||||
constructor(options: {
|
||||
bucketName: string
|
||||
region?: string
|
||||
endpoint?: string
|
||||
accountId?: string
|
||||
accessKeyId: string
|
||||
secretAccessKey: string
|
||||
sessionToken?: string
|
||||
serviceType?: string
|
||||
}) {
|
||||
super()
|
||||
this.bucketName = options.bucketName
|
||||
this.region = options.region || 'auto'
|
||||
this.endpoint = options.endpoint
|
||||
this.accountId = options.accountId
|
||||
this.accessKeyId = options.accessKeyId
|
||||
this.secretAccessKey = options.secretAccessKey
|
||||
this.sessionToken = options.sessionToken
|
||||
this.serviceType = options.serviceType || 's3'
|
||||
|
||||
// Set up prefixes for different types of data
|
||||
this.nounPrefix = `${NOUNS_DIR}/`
|
||||
this.verbPrefix = `${VERBS_DIR}/`
|
||||
this.metadataPrefix = `${METADATA_DIR}/`
|
||||
this.indexPrefix = `${INDEX_DIR}/`
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the storage adapter
|
||||
*/
|
||||
public async init(): Promise<void> {
|
||||
if (this.isInitialized) {
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
// Import AWS SDK modules only when needed
|
||||
const { S3Client } = await import('@aws-sdk/client-s3')
|
||||
|
||||
// Configure the S3 client based on the service type
|
||||
const clientConfig: any = {
|
||||
region: this.region,
|
||||
credentials: {
|
||||
accessKeyId: this.accessKeyId,
|
||||
secretAccessKey: this.secretAccessKey
|
||||
}
|
||||
}
|
||||
|
||||
// Add session token if provided
|
||||
if (this.sessionToken) {
|
||||
clientConfig.credentials.sessionToken = this.sessionToken
|
||||
}
|
||||
|
||||
// Add endpoint if provided (for R2, GCS, etc.)
|
||||
if (this.endpoint) {
|
||||
clientConfig.endpoint = this.endpoint
|
||||
}
|
||||
|
||||
// Special configuration for Cloudflare R2
|
||||
if (this.serviceType === 'r2' && this.accountId) {
|
||||
clientConfig.endpoint = `https://${this.accountId}.r2.cloudflarestorage.com`
|
||||
}
|
||||
|
||||
// Create the S3 client
|
||||
this.s3Client = new S3Client(clientConfig)
|
||||
|
||||
// Ensure the bucket exists and is accessible
|
||||
const { HeadBucketCommand } = await import('@aws-sdk/client-s3')
|
||||
await this.s3Client.send(
|
||||
new HeadBucketCommand({
|
||||
Bucket: this.bucketName
|
||||
})
|
||||
)
|
||||
|
||||
this.isInitialized = true
|
||||
} catch (error) {
|
||||
console.error(`Failed to initialize ${this.serviceType} storage:`, error)
|
||||
throw new Error(
|
||||
`Failed to initialize ${this.serviceType} storage: ${error}`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save a node to storage
|
||||
*/
|
||||
protected async saveNode(node: HNSWNode): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Convert connections Map to a serializable format
|
||||
const serializableNode = {
|
||||
...node,
|
||||
connections: this.mapToObject(node.connections, (set) =>
|
||||
Array.from(set as Set<string>)
|
||||
)
|
||||
}
|
||||
|
||||
// Import the PutObjectCommand only when needed
|
||||
const { PutObjectCommand } = await import('@aws-sdk/client-s3')
|
||||
|
||||
// Save the node to S3-compatible storage
|
||||
await this.s3Client!.send(
|
||||
new PutObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: `${this.nounPrefix}${node.id}.json`,
|
||||
Body: JSON.stringify(serializableNode, null, 2),
|
||||
ContentType: 'application/json'
|
||||
})
|
||||
)
|
||||
} catch (error) {
|
||||
console.error(`Failed to save node ${node.id}:`, error)
|
||||
throw new Error(`Failed to save node ${node.id}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a node from storage
|
||||
*/
|
||||
protected async getNode(id: string): Promise<HNSWNode | null> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Import the GetObjectCommand only when needed
|
||||
const { GetObjectCommand } = await import('@aws-sdk/client-s3')
|
||||
|
||||
// Try to get the node from the nouns directory
|
||||
const response = await this.s3Client!.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: `${this.nounPrefix}${id}.json`
|
||||
})
|
||||
)
|
||||
|
||||
// Convert the response body to a string
|
||||
const bodyContents = await response.Body.transformToString()
|
||||
const parsedNode = JSON.parse(bodyContents)
|
||||
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(parsedNode.connections)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
return {
|
||||
id: parsedNode.id,
|
||||
vector: parsedNode.vector,
|
||||
connections
|
||||
}
|
||||
} catch (error) {
|
||||
// Node not found or other error
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all nodes from storage
|
||||
*/
|
||||
protected async getAllNodes(): Promise<HNSWNode[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Import the ListObjectsV2Command and GetObjectCommand only when needed
|
||||
const { ListObjectsV2Command, GetObjectCommand } = await import(
|
||||
'@aws-sdk/client-s3'
|
||||
)
|
||||
|
||||
// List all objects in the nouns directory
|
||||
const listResponse = await this.s3Client!.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: this.bucketName,
|
||||
Prefix: this.nounPrefix
|
||||
})
|
||||
)
|
||||
|
||||
const nodes: HNSWNode[] = []
|
||||
|
||||
// If there are no objects, return an empty array
|
||||
if (!listResponse.Contents || listResponse.Contents.length === 0) {
|
||||
return nodes
|
||||
}
|
||||
|
||||
// Get each node
|
||||
const nodePromises = listResponse.Contents.map(
|
||||
async (object: { Key: string }) => {
|
||||
try {
|
||||
// Extract node ID from the key (remove prefix and .json extension)
|
||||
const nodeId = object.Key.replace(this.nounPrefix, '').replace('.json', '')
|
||||
|
||||
// Get the node data
|
||||
const response = await this.s3Client!.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: object.Key
|
||||
})
|
||||
)
|
||||
|
||||
// Convert the response body to a string
|
||||
const bodyContents = await response.Body.transformToString()
|
||||
const parsedNode = JSON.parse(bodyContents)
|
||||
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(parsedNode.connections)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
return {
|
||||
id: parsedNode.id,
|
||||
vector: parsedNode.vector,
|
||||
connections
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Error getting node from ${object.Key}:`, error)
|
||||
return null
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
// Wait for all promises to resolve and filter out nulls
|
||||
const resolvedNodes = await Promise.all(nodePromises)
|
||||
return resolvedNodes.filter((node): node is HNSWNode => node !== null)
|
||||
} catch (error) {
|
||||
console.error('Failed to get all nodes:', error)
|
||||
return []
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get nodes by noun type
|
||||
* @param nounType The noun type to filter by
|
||||
* @returns Promise that resolves to an array of nodes of the specified noun type
|
||||
*/
|
||||
protected async getNodesByNounType(nounType: string): Promise<HNSWNode[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Get all nodes
|
||||
const allNodes = await this.getAllNodes()
|
||||
|
||||
// Filter nodes by noun type using metadata
|
||||
const filteredNodes: HNSWNode[] = []
|
||||
for (const node of allNodes) {
|
||||
const metadata = await this.getMetadata(node.id)
|
||||
if (metadata && metadata.noun === nounType) {
|
||||
filteredNodes.push(node)
|
||||
}
|
||||
}
|
||||
|
||||
return filteredNodes
|
||||
} catch (error) {
|
||||
console.error(`Failed to get nodes by noun type ${nounType}:`, error)
|
||||
return []
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a node from storage
|
||||
*/
|
||||
protected async deleteNode(id: string): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Import the DeleteObjectCommand only when needed
|
||||
const { DeleteObjectCommand } = await import('@aws-sdk/client-s3')
|
||||
|
||||
// Delete the node from S3-compatible storage
|
||||
await this.s3Client!.send(
|
||||
new DeleteObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: `${this.nounPrefix}${id}.json`
|
||||
})
|
||||
)
|
||||
} catch (error) {
|
||||
console.error(`Failed to delete node ${id}:`, error)
|
||||
throw new Error(`Failed to delete node ${id}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save an edge to storage
|
||||
*/
|
||||
protected async saveEdge(edge: Edge): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Convert connections Map to a serializable format
|
||||
const serializableEdge = {
|
||||
...edge,
|
||||
connections: this.mapToObject(edge.connections, (set) =>
|
||||
Array.from(set as Set<string>)
|
||||
)
|
||||
}
|
||||
|
||||
// Import the PutObjectCommand only when needed
|
||||
const { PutObjectCommand } = await import('@aws-sdk/client-s3')
|
||||
|
||||
// Save the edge to S3-compatible storage
|
||||
await this.s3Client!.send(
|
||||
new PutObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: `${this.verbPrefix}${edge.id}.json`,
|
||||
Body: JSON.stringify(serializableEdge, null, 2),
|
||||
ContentType: 'application/json'
|
||||
})
|
||||
)
|
||||
} catch (error) {
|
||||
console.error(`Failed to save edge ${edge.id}:`, error)
|
||||
throw new Error(`Failed to save edge ${edge.id}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get an edge from storage
|
||||
*/
|
||||
protected async getEdge(id: string): Promise<Edge | null> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Import the GetObjectCommand only when needed
|
||||
const { GetObjectCommand } = await import('@aws-sdk/client-s3')
|
||||
|
||||
// Try to get the edge from the verbs directory
|
||||
const response = await this.s3Client!.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: `${this.verbPrefix}${id}.json`
|
||||
})
|
||||
)
|
||||
|
||||
// Convert the response body to a string
|
||||
const bodyContents = await response.Body.transformToString()
|
||||
const parsedEdge = JSON.parse(bodyContents)
|
||||
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(parsedEdge.connections)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
return {
|
||||
id: parsedEdge.id,
|
||||
vector: parsedEdge.vector,
|
||||
connections,
|
||||
sourceId: parsedEdge.sourceId,
|
||||
targetId: parsedEdge.targetId,
|
||||
type: parsedEdge.type,
|
||||
weight: parsedEdge.weight,
|
||||
metadata: parsedEdge.metadata
|
||||
}
|
||||
} catch (error) {
|
||||
// Edge not found or other error
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all edges from storage
|
||||
*/
|
||||
protected async getAllEdges(): Promise<Edge[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Import the ListObjectsV2Command and GetObjectCommand only when needed
|
||||
const { ListObjectsV2Command, GetObjectCommand } = await import(
|
||||
'@aws-sdk/client-s3'
|
||||
)
|
||||
|
||||
// List all objects in the verbs directory
|
||||
const listResponse = await this.s3Client!.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: this.bucketName,
|
||||
Prefix: this.verbPrefix
|
||||
})
|
||||
)
|
||||
|
||||
const edges: Edge[] = []
|
||||
|
||||
// If there are no objects, return an empty array
|
||||
if (!listResponse.Contents || listResponse.Contents.length === 0) {
|
||||
return edges
|
||||
}
|
||||
|
||||
// Get each edge
|
||||
const edgePromises = listResponse.Contents.map(
|
||||
async (object: { Key: string }) => {
|
||||
try {
|
||||
// Extract edge ID from the key (remove prefix and .json extension)
|
||||
const edgeId = object.Key.replace(this.verbPrefix, '').replace('.json', '')
|
||||
|
||||
// Get the edge data
|
||||
const response = await this.s3Client!.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: object.Key
|
||||
})
|
||||
)
|
||||
|
||||
// Convert the response body to a string
|
||||
const bodyContents = await response.Body.transformToString()
|
||||
const parsedEdge = JSON.parse(bodyContents)
|
||||
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(parsedEdge.connections)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
return {
|
||||
id: parsedEdge.id,
|
||||
vector: parsedEdge.vector,
|
||||
connections,
|
||||
sourceId: parsedEdge.sourceId,
|
||||
targetId: parsedEdge.targetId,
|
||||
type: parsedEdge.type,
|
||||
weight: parsedEdge.weight,
|
||||
metadata: parsedEdge.metadata
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Error getting edge from ${object.Key}:`, error)
|
||||
return null
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
// Wait for all promises to resolve and filter out nulls
|
||||
const resolvedEdges = await Promise.all(edgePromises)
|
||||
return resolvedEdges.filter((edge): edge is Edge => edge !== null)
|
||||
} catch (error) {
|
||||
console.error('Failed to get all edges:', error)
|
||||
return []
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get edges by source
|
||||
*/
|
||||
protected async getEdgesBySource(sourceId: string): Promise<Edge[]> {
|
||||
const edges = await this.getAllEdges()
|
||||
return edges.filter((edge) => edge.sourceId === sourceId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get edges by target
|
||||
*/
|
||||
protected async getEdgesByTarget(targetId: string): Promise<Edge[]> {
|
||||
const edges = await this.getAllEdges()
|
||||
return edges.filter((edge) => edge.targetId === targetId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get edges by type
|
||||
*/
|
||||
protected async getEdgesByType(type: string): Promise<Edge[]> {
|
||||
const edges = await this.getAllEdges()
|
||||
return edges.filter((edge) => edge.type === type)
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete an edge from storage
|
||||
*/
|
||||
protected async deleteEdge(id: string): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Import the DeleteObjectCommand only when needed
|
||||
const { DeleteObjectCommand } = await import('@aws-sdk/client-s3')
|
||||
|
||||
// Delete the edge from S3-compatible storage
|
||||
await this.s3Client!.send(
|
||||
new DeleteObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: `${this.verbPrefix}${id}.json`
|
||||
})
|
||||
)
|
||||
} catch (error) {
|
||||
console.error(`Failed to delete edge ${id}:`, error)
|
||||
throw new Error(`Failed to delete edge ${id}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save metadata to storage
|
||||
*/
|
||||
public async saveMetadata(id: string, metadata: any): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Import the PutObjectCommand only when needed
|
||||
const { PutObjectCommand } = await import('@aws-sdk/client-s3')
|
||||
|
||||
// Save the metadata to S3-compatible storage
|
||||
await this.s3Client!.send(
|
||||
new PutObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: `${this.metadataPrefix}${id}.json`,
|
||||
Body: JSON.stringify(metadata, null, 2),
|
||||
ContentType: 'application/json'
|
||||
})
|
||||
)
|
||||
} catch (error) {
|
||||
console.error(`Failed to save metadata ${id}:`, error)
|
||||
throw new Error(`Failed to save metadata ${id}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get metadata from storage
|
||||
*/
|
||||
public async getMetadata(id: string): Promise<any | null> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Import the GetObjectCommand only when needed
|
||||
const { GetObjectCommand } = await import('@aws-sdk/client-s3')
|
||||
|
||||
// Try to get the metadata from the metadata directory
|
||||
const response = await this.s3Client!.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: `${this.metadataPrefix}${id}.json`
|
||||
})
|
||||
)
|
||||
|
||||
// Convert the response body to a string
|
||||
const bodyContents = await response.Body.transformToString()
|
||||
return JSON.parse(bodyContents)
|
||||
} catch (error) {
|
||||
// Metadata not found or other error
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear all data from storage
|
||||
*/
|
||||
public async clear(): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Import the ListObjectsV2Command and DeleteObjectCommand only when needed
|
||||
const { ListObjectsV2Command, DeleteObjectCommand } = await import(
|
||||
'@aws-sdk/client-s3'
|
||||
)
|
||||
|
||||
// Helper function to delete all objects with a given prefix
|
||||
const deleteObjectsWithPrefix = async (prefix: string): Promise<void> => {
|
||||
// List all objects with the given prefix
|
||||
const listResponse = await this.s3Client!.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: this.bucketName,
|
||||
Prefix: prefix
|
||||
})
|
||||
)
|
||||
|
||||
// If there are no objects, return
|
||||
if (!listResponse.Contents || listResponse.Contents.length === 0) {
|
||||
return
|
||||
}
|
||||
|
||||
// Delete each object
|
||||
for (const object of listResponse.Contents) {
|
||||
await this.s3Client!.send(
|
||||
new DeleteObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: object.Key
|
||||
})
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// Delete all objects in the nouns directory
|
||||
await deleteObjectsWithPrefix(this.nounPrefix)
|
||||
|
||||
// Delete all objects in the verbs directory
|
||||
await deleteObjectsWithPrefix(this.verbPrefix)
|
||||
|
||||
// Delete all objects in the metadata directory
|
||||
await deleteObjectsWithPrefix(this.metadataPrefix)
|
||||
|
||||
// Delete all objects in the index directory
|
||||
await deleteObjectsWithPrefix(this.indexPrefix)
|
||||
} catch (error) {
|
||||
console.error('Failed to clear storage:', error)
|
||||
throw new Error(`Failed to clear storage: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get information about storage usage and capacity
|
||||
*/
|
||||
public async getStorageStatus(): Promise<{
|
||||
type: string
|
||||
used: number
|
||||
quota: number | null
|
||||
details?: Record<string, any>
|
||||
}> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Import the ListObjectsV2Command only when needed
|
||||
const { ListObjectsV2Command } = await import('@aws-sdk/client-s3')
|
||||
|
||||
// Calculate the total size of all objects in the storage
|
||||
let totalSize = 0
|
||||
let nodeCount = 0
|
||||
let edgeCount = 0
|
||||
let metadataCount = 0
|
||||
|
||||
// Helper function to calculate size and count for a given prefix
|
||||
const calculateSizeAndCount = async (
|
||||
prefix: string
|
||||
): Promise<{ size: number; count: number }> => {
|
||||
let size = 0
|
||||
let count = 0
|
||||
|
||||
// List all objects with the given prefix
|
||||
const listResponse = await this.s3Client!.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: this.bucketName,
|
||||
Prefix: prefix
|
||||
})
|
||||
)
|
||||
|
||||
// If there are no objects, return
|
||||
if (!listResponse.Contents || listResponse.Contents.length === 0) {
|
||||
return { size, count }
|
||||
}
|
||||
|
||||
// Calculate size and count
|
||||
for (const object of listResponse.Contents) {
|
||||
size += object.Size || 0
|
||||
count++
|
||||
}
|
||||
|
||||
return { size, count }
|
||||
}
|
||||
|
||||
// Calculate size and count for each directory
|
||||
const nounsResult = await calculateSizeAndCount(this.nounPrefix)
|
||||
const verbsResult = await calculateSizeAndCount(this.verbPrefix)
|
||||
const metadataResult = await calculateSizeAndCount(this.metadataPrefix)
|
||||
const indexResult = await calculateSizeAndCount(this.indexPrefix)
|
||||
|
||||
totalSize = nounsResult.size + verbsResult.size + metadataResult.size + indexResult.size
|
||||
nodeCount = nounsResult.count
|
||||
edgeCount = verbsResult.count
|
||||
metadataCount = metadataResult.count
|
||||
|
||||
// Count nouns by type using metadata
|
||||
const nounTypeCounts: Record<string, number> = {}
|
||||
|
||||
// List all objects in the metadata directory
|
||||
const metadataListResponse = await this.s3Client!.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: this.bucketName,
|
||||
Prefix: this.metadataPrefix
|
||||
})
|
||||
)
|
||||
|
||||
if (metadataListResponse.Contents) {
|
||||
// Import the GetObjectCommand only when needed
|
||||
const { GetObjectCommand } = await import('@aws-sdk/client-s3')
|
||||
|
||||
for (const object of metadataListResponse.Contents) {
|
||||
try {
|
||||
// Get the metadata
|
||||
const response = await this.s3Client!.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: object.Key
|
||||
})
|
||||
)
|
||||
|
||||
// Convert the response body to a string
|
||||
const bodyContents = await response.Body.transformToString()
|
||||
const metadata = JSON.parse(bodyContents)
|
||||
|
||||
// Count by noun type
|
||||
if (metadata.noun) {
|
||||
nounTypeCounts[metadata.noun] = (nounTypeCounts[metadata.noun] || 0) + 1
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Error getting metadata from ${object.Key}:`, error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
type: this.serviceType,
|
||||
used: totalSize,
|
||||
quota: null, // S3-compatible services typically don't provide quota information through the API
|
||||
details: {
|
||||
bucketName: this.bucketName,
|
||||
region: this.region,
|
||||
endpoint: this.endpoint,
|
||||
nodeCount,
|
||||
edgeCount,
|
||||
metadataCount,
|
||||
nounTypes: nounTypeCounts
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Failed to get storage status:', error)
|
||||
return {
|
||||
type: this.serviceType,
|
||||
used: 0,
|
||||
quota: null,
|
||||
details: { error: String(error) }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue