feat(storage): implement base, file system, and memory storage adapters
- Added `BaseStorage` abstract class to provide common functionality for storage adapters. - Implemented `FileSystemStorage` adapter for Node.js environments, supporting persistent file-based storage. - Added directory structure management methods to `FileSystemStorage` (`ensureDirectoryExists`, `clear`, etc.). - Implemented `MemoryStorage` adapter for in-memory storage suitable for non-persistent environments. - Supported CRUD operations for nouns, verbs, and metadata in both `FileSystemStorage` and `MemoryStorage`. - Included utility methods like `mapToObject` for serialization and deserialization of complex data structures. Purpose: Provide extensible, reusable storage solutions with both persistent and non-persistent options for diverse application needs.
This commit is contained in:
parent
398c120757
commit
1127115664
9 changed files with 4549 additions and 2186 deletions
555
src/storage/adapters/fileSystemStorage.ts
Normal file
555
src/storage/adapters/fileSystemStorage.ts
Normal file
|
|
@ -0,0 +1,555 @@
|
|||
/**
|
||||
* File System Storage Adapter
|
||||
* File system storage adapter for Node.js environments
|
||||
*/
|
||||
|
||||
import { GraphVerb, HNSWNoun } from '../../coreTypes.js'
|
||||
import { BaseStorage, NOUNS_DIR, VERBS_DIR, METADATA_DIR, INDEX_DIR } from '../baseStorage.js'
|
||||
|
||||
// Type aliases for better readability
|
||||
type HNSWNode = HNSWNoun
|
||||
type Edge = GraphVerb
|
||||
|
||||
// Node.js modules - dynamically imported to avoid issues in browser environments
|
||||
let fs: any
|
||||
let path: any
|
||||
|
||||
// Try to load Node.js modules
|
||||
try {
|
||||
// Using dynamic imports to avoid issues in browser environments
|
||||
const fsPromise = import('fs')
|
||||
const pathPromise = import('path')
|
||||
|
||||
Promise.all([fsPromise, pathPromise]).then(([fsModule, pathModule]) => {
|
||||
fs = fsModule
|
||||
path = pathModule.default
|
||||
}).catch(error => {
|
||||
console.error('Failed to load Node.js modules:', error)
|
||||
})
|
||||
} catch (error) {
|
||||
console.error(
|
||||
'FileSystemStorage: Failed to load Node.js modules. This adapter is not supported in this environment.',
|
||||
error
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* File system storage adapter for Node.js environments
|
||||
* Uses the file system to store data in the specified directory structure
|
||||
*/
|
||||
export class FileSystemStorage extends BaseStorage {
|
||||
private rootDir: string
|
||||
private nounsDir: string
|
||||
private verbsDir: string
|
||||
private metadataDir: string
|
||||
private indexDir: string
|
||||
|
||||
/**
|
||||
* Initialize the storage adapter
|
||||
* @param rootDirectory The root directory for storage
|
||||
*/
|
||||
constructor(rootDirectory: string) {
|
||||
super()
|
||||
this.rootDir = rootDirectory
|
||||
this.nounsDir = path.join(this.rootDir, NOUNS_DIR)
|
||||
this.verbsDir = path.join(this.rootDir, VERBS_DIR)
|
||||
this.metadataDir = path.join(this.rootDir, METADATA_DIR)
|
||||
this.indexDir = path.join(this.rootDir, INDEX_DIR)
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the storage adapter
|
||||
*/
|
||||
public async init(): Promise<void> {
|
||||
if (this.isInitialized) {
|
||||
return
|
||||
}
|
||||
|
||||
// Check if Node.js modules are available
|
||||
if (!fs || !path) {
|
||||
throw new Error(
|
||||
'FileSystemStorage requires a Node.js environment, but `fs` and `path` modules could not be loaded.'
|
||||
)
|
||||
}
|
||||
|
||||
try {
|
||||
// Create the root directory if it doesn't exist
|
||||
await this.ensureDirectoryExists(this.rootDir)
|
||||
|
||||
// Create the nouns directory if it doesn't exist
|
||||
await this.ensureDirectoryExists(this.nounsDir)
|
||||
|
||||
// Create the verbs directory if it doesn't exist
|
||||
await this.ensureDirectoryExists(this.verbsDir)
|
||||
|
||||
// Create the metadata directory if it doesn't exist
|
||||
await this.ensureDirectoryExists(this.metadataDir)
|
||||
|
||||
// Create the index directory if it doesn't exist
|
||||
await this.ensureDirectoryExists(this.indexDir)
|
||||
|
||||
this.isInitialized = true
|
||||
} catch (error) {
|
||||
console.error('Error initializing FileSystemStorage:', error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure a directory exists, creating it if necessary
|
||||
*/
|
||||
private async ensureDirectoryExists(dirPath: string): Promise<void> {
|
||||
try {
|
||||
await fs.promises.mkdir(dirPath, { recursive: true })
|
||||
} catch (error: any) {
|
||||
// Ignore EEXIST error, which means the directory already exists
|
||||
if (error.code !== 'EEXIST') {
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save a node to storage
|
||||
*/
|
||||
protected async saveNode(node: HNSWNode): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
// Convert connections Map to a serializable format
|
||||
const serializableNode = {
|
||||
...node,
|
||||
connections: this.mapToObject(node.connections, (set) =>
|
||||
Array.from(set as Set<string>)
|
||||
)
|
||||
}
|
||||
|
||||
const filePath = path.join(this.nounsDir, `${node.id}.json`)
|
||||
await fs.promises.writeFile(filePath, JSON.stringify(serializableNode, null, 2))
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a node from storage
|
||||
*/
|
||||
protected async getNode(id: string): Promise<HNSWNode | null> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
const filePath = path.join(this.nounsDir, `${id}.json`)
|
||||
try {
|
||||
const data = await fs.promises.readFile(filePath, 'utf-8')
|
||||
const parsedNode = JSON.parse(data)
|
||||
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(parsedNode.connections)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
return {
|
||||
id: parsedNode.id,
|
||||
vector: parsedNode.vector,
|
||||
connections
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error reading node ${id}:`, error)
|
||||
}
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all nodes from storage
|
||||
*/
|
||||
protected async getAllNodes(): Promise<HNSWNode[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
const allNodes: HNSWNode[] = []
|
||||
try {
|
||||
const files = await fs.promises.readdir(this.nounsDir)
|
||||
for (const file of files) {
|
||||
if (file.endsWith('.json')) {
|
||||
const filePath = path.join(this.nounsDir, file)
|
||||
const data = await fs.promises.readFile(filePath, 'utf-8')
|
||||
const parsedNode = JSON.parse(data)
|
||||
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(parsedNode.connections)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
allNodes.push({
|
||||
id: parsedNode.id,
|
||||
vector: parsedNode.vector,
|
||||
connections
|
||||
})
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error reading directory ${this.nounsDir}:`, error)
|
||||
}
|
||||
}
|
||||
return allNodes
|
||||
}
|
||||
|
||||
/**
|
||||
* Get nodes by noun type
|
||||
* @param nounType The noun type to filter by
|
||||
* @returns Promise that resolves to an array of nodes of the specified noun type
|
||||
*/
|
||||
protected async getNodesByNounType(nounType: string): Promise<HNSWNode[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
const nouns: HNSWNode[] = []
|
||||
try {
|
||||
const files = await fs.promises.readdir(this.nounsDir)
|
||||
for (const file of files) {
|
||||
if (file.endsWith('.json')) {
|
||||
const filePath = path.join(this.nounsDir, file)
|
||||
const data = await fs.promises.readFile(filePath, 'utf-8')
|
||||
const parsedNode = JSON.parse(data)
|
||||
|
||||
// Filter by noun type using metadata
|
||||
const nodeId = parsedNode.id
|
||||
const metadata = await this.getMetadata(nodeId)
|
||||
if (metadata && metadata.noun === nounType) {
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(parsedNode.connections)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
nouns.push({
|
||||
id: parsedNode.id,
|
||||
vector: parsedNode.vector,
|
||||
connections
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error reading directory ${this.nounsDir}:`, error)
|
||||
}
|
||||
}
|
||||
|
||||
return nouns
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a node from storage
|
||||
*/
|
||||
protected async deleteNode(id: string): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
const filePath = path.join(this.nounsDir, `${id}.json`)
|
||||
try {
|
||||
await fs.promises.unlink(filePath)
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error deleting node file ${filePath}:`, error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save an edge to storage
|
||||
*/
|
||||
protected async saveEdge(edge: Edge): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
// Convert connections Map to a serializable format
|
||||
const serializableEdge = {
|
||||
...edge,
|
||||
connections: this.mapToObject(edge.connections, (set) =>
|
||||
Array.from(set as Set<string>)
|
||||
)
|
||||
}
|
||||
|
||||
const filePath = path.join(this.verbsDir, `${edge.id}.json`)
|
||||
await fs.promises.writeFile(filePath, JSON.stringify(serializableEdge, null, 2))
|
||||
}
|
||||
|
||||
/**
|
||||
* Get an edge from storage
|
||||
*/
|
||||
protected async getEdge(id: string): Promise<Edge | null> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
const filePath = path.join(this.verbsDir, `${id}.json`)
|
||||
try {
|
||||
const data = await fs.promises.readFile(filePath, 'utf-8')
|
||||
const parsedEdge = JSON.parse(data)
|
||||
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(parsedEdge.connections)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
return {
|
||||
id: parsedEdge.id,
|
||||
vector: parsedEdge.vector,
|
||||
connections,
|
||||
sourceId: parsedEdge.sourceId,
|
||||
targetId: parsedEdge.targetId,
|
||||
type: parsedEdge.type,
|
||||
weight: parsedEdge.weight,
|
||||
metadata: parsedEdge.metadata
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error reading edge ${id}:`, error)
|
||||
}
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all edges from storage
|
||||
*/
|
||||
protected async getAllEdges(): Promise<Edge[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
const allEdges: Edge[] = []
|
||||
try {
|
||||
const files = await fs.promises.readdir(this.verbsDir)
|
||||
for (const file of files) {
|
||||
if (file.endsWith('.json')) {
|
||||
const filePath = path.join(this.verbsDir, file)
|
||||
const data = await fs.promises.readFile(filePath, 'utf-8')
|
||||
const parsedEdge = JSON.parse(data)
|
||||
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(parsedEdge.connections)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
allEdges.push({
|
||||
id: parsedEdge.id,
|
||||
vector: parsedEdge.vector,
|
||||
connections,
|
||||
sourceId: parsedEdge.sourceId,
|
||||
targetId: parsedEdge.targetId,
|
||||
type: parsedEdge.type,
|
||||
weight: parsedEdge.weight,
|
||||
metadata: parsedEdge.metadata
|
||||
})
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error reading directory ${this.verbsDir}:`, error)
|
||||
}
|
||||
}
|
||||
return allEdges
|
||||
}
|
||||
|
||||
/**
|
||||
* Get edges by source
|
||||
*/
|
||||
protected async getEdgesBySource(sourceId: string): Promise<Edge[]> {
|
||||
const edges = await this.getAllEdges()
|
||||
return edges.filter((edge) => edge.sourceId === sourceId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get edges by target
|
||||
*/
|
||||
protected async getEdgesByTarget(targetId: string): Promise<Edge[]> {
|
||||
const edges = await this.getAllEdges()
|
||||
return edges.filter((edge) => edge.targetId === targetId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get edges by type
|
||||
*/
|
||||
protected async getEdgesByType(type: string): Promise<Edge[]> {
|
||||
const edges = await this.getAllEdges()
|
||||
return edges.filter((edge) => edge.type === type)
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete an edge from storage
|
||||
*/
|
||||
protected async deleteEdge(id: string): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
const filePath = path.join(this.verbsDir, `${id}.json`)
|
||||
try {
|
||||
await fs.promises.unlink(filePath)
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error deleting edge file ${filePath}:`, error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save metadata to storage
|
||||
*/
|
||||
public async saveMetadata(id: string, metadata: any): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
const filePath = path.join(this.metadataDir, `${id}.json`)
|
||||
await fs.promises.writeFile(filePath, JSON.stringify(metadata, null, 2))
|
||||
}
|
||||
|
||||
/**
|
||||
* Get metadata from storage
|
||||
*/
|
||||
public async getMetadata(id: string): Promise<any | null> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
const filePath = path.join(this.metadataDir, `${id}.json`)
|
||||
try {
|
||||
const data = await fs.promises.readFile(filePath, 'utf-8')
|
||||
return JSON.parse(data)
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error reading metadata ${id}:`, error)
|
||||
}
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear all data from storage
|
||||
*/
|
||||
public async clear(): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
// Helper function to remove all files in a directory
|
||||
const removeDirectoryContents = async (dirPath: string): Promise<void> => {
|
||||
try {
|
||||
const files = await fs.promises.readdir(dirPath)
|
||||
for (const file of files) {
|
||||
const filePath = path.join(dirPath, file)
|
||||
const stats = await fs.promises.stat(filePath)
|
||||
if (stats.isDirectory()) {
|
||||
await removeDirectoryContents(filePath)
|
||||
await fs.promises.rmdir(filePath)
|
||||
} else {
|
||||
await fs.promises.unlink(filePath)
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error removing directory contents ${dirPath}:`, error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Remove all files in the nouns directory
|
||||
await removeDirectoryContents(this.nounsDir)
|
||||
|
||||
// Remove all files in the verbs directory
|
||||
await removeDirectoryContents(this.verbsDir)
|
||||
|
||||
// Remove all files in the metadata directory
|
||||
await removeDirectoryContents(this.metadataDir)
|
||||
|
||||
// Remove all files in the index directory
|
||||
await removeDirectoryContents(this.indexDir)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get information about storage usage and capacity
|
||||
*/
|
||||
public async getStorageStatus(): Promise<{
|
||||
type: string
|
||||
used: number
|
||||
quota: number | null
|
||||
details?: Record<string, any>
|
||||
}> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Calculate the total size of all files in the storage directories
|
||||
let totalSize = 0
|
||||
|
||||
// Helper function to calculate directory size
|
||||
const calculateSize = async (dirPath: string): Promise<number> => {
|
||||
let size = 0
|
||||
try {
|
||||
const files = await fs.promises.readdir(dirPath)
|
||||
for (const file of files) {
|
||||
const filePath = path.join(dirPath, file)
|
||||
const stats = await fs.promises.stat(filePath)
|
||||
if (stats.isDirectory()) {
|
||||
size += await calculateSize(filePath)
|
||||
} else {
|
||||
size += stats.size
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error calculating size for directory ${dirPath}:`, error)
|
||||
}
|
||||
}
|
||||
return size
|
||||
}
|
||||
|
||||
// Calculate size for each directory
|
||||
const nounsDirSize = await calculateSize(this.nounsDir)
|
||||
const verbsDirSize = await calculateSize(this.verbsDir)
|
||||
const metadataDirSize = await calculateSize(this.metadataDir)
|
||||
const indexDirSize = await calculateSize(this.indexDir)
|
||||
|
||||
totalSize = nounsDirSize + verbsDirSize + metadataDirSize + indexDirSize
|
||||
|
||||
// Count files in each directory
|
||||
const nounsCount = (await fs.promises.readdir(this.nounsDir)).filter((file: string) => file.endsWith('.json')).length
|
||||
const verbsCount = (await fs.promises.readdir(this.verbsDir)).filter((file: string) => file.endsWith('.json')).length
|
||||
const metadataCount = (await fs.promises.readdir(this.metadataDir)).filter((file: string) => file.endsWith('.json')).length
|
||||
|
||||
// Count nouns by type using metadata
|
||||
const nounTypeCounts: Record<string, number> = {}
|
||||
const metadataFiles = await fs.promises.readdir(this.metadataDir)
|
||||
for (const file of metadataFiles) {
|
||||
if (file.endsWith('.json')) {
|
||||
try {
|
||||
const filePath = path.join(this.metadataDir, file)
|
||||
const data = await fs.promises.readFile(filePath, 'utf-8')
|
||||
const metadata = JSON.parse(data)
|
||||
if (metadata.noun) {
|
||||
nounTypeCounts[metadata.noun] = (nounTypeCounts[metadata.noun] || 0) + 1
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Error reading metadata file ${file}:`, error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
type: 'filesystem',
|
||||
used: totalSize,
|
||||
quota: null, // File system doesn't provide quota information
|
||||
details: {
|
||||
rootDirectory: this.rootDir,
|
||||
nounsCount,
|
||||
verbsCount,
|
||||
metadataCount,
|
||||
nounsDirSize,
|
||||
verbsDirSize,
|
||||
metadataDirSize,
|
||||
indexDirSize,
|
||||
nounTypes: nounTypeCounts
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Failed to get storage status:', error)
|
||||
return {
|
||||
type: 'filesystem',
|
||||
used: 0,
|
||||
quota: null,
|
||||
details: { error: String(error) }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
324
src/storage/adapters/memoryStorage.ts
Normal file
324
src/storage/adapters/memoryStorage.ts
Normal file
|
|
@ -0,0 +1,324 @@
|
|||
/**
|
||||
* Memory Storage Adapter
|
||||
* In-memory storage adapter for environments where persistent storage is not available or needed
|
||||
*/
|
||||
|
||||
import { GraphVerb, HNSWNoun } from '../../coreTypes.js'
|
||||
import { BaseStorage } from '../baseStorage.js'
|
||||
|
||||
/**
|
||||
* Type alias for HNSWNoun to make the code more readable
|
||||
*/
|
||||
type HNSWNode = HNSWNoun
|
||||
|
||||
/**
|
||||
* Type alias for GraphVerb to make the code more readable
|
||||
*/
|
||||
type Edge = GraphVerb
|
||||
|
||||
/**
|
||||
* In-memory storage adapter
|
||||
* Uses Maps to store data in memory
|
||||
*/
|
||||
export class MemoryStorage extends BaseStorage {
|
||||
// Single map of noun ID to noun
|
||||
private nouns: Map<string, HNSWNode> = new Map()
|
||||
private verbs: Map<string, Edge> = new Map()
|
||||
private metadata: Map<string, any> = new Map()
|
||||
|
||||
constructor() {
|
||||
super()
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the storage adapter
|
||||
* Nothing to initialize for in-memory storage
|
||||
*/
|
||||
public async init(): Promise<void> {
|
||||
this.isInitialized = true
|
||||
}
|
||||
|
||||
/**
|
||||
* Save a node to storage
|
||||
*/
|
||||
protected async saveNode(node: HNSWNode): Promise<void> {
|
||||
// Create a deep copy to avoid reference issues
|
||||
const nodeCopy: HNSWNode = {
|
||||
id: node.id,
|
||||
vector: [...node.vector],
|
||||
connections: new Map()
|
||||
}
|
||||
|
||||
// Copy connections
|
||||
for (const [level, connections] of node.connections.entries()) {
|
||||
nodeCopy.connections.set(level, new Set(connections))
|
||||
}
|
||||
|
||||
// Save the node directly in the nouns map
|
||||
this.nouns.set(node.id, nodeCopy)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a node from storage
|
||||
*/
|
||||
protected async getNode(id: string): Promise<HNSWNode | null> {
|
||||
// Get the node directly from the nouns map
|
||||
const node = this.nouns.get(id)
|
||||
|
||||
// If not found, return null
|
||||
if (!node) {
|
||||
return null
|
||||
}
|
||||
|
||||
// Return a deep copy to avoid reference issues
|
||||
const nodeCopy: HNSWNode = {
|
||||
id: node.id,
|
||||
vector: [...node.vector],
|
||||
connections: new Map()
|
||||
}
|
||||
|
||||
// Copy connections
|
||||
for (const [level, connections] of node.connections.entries()) {
|
||||
nodeCopy.connections.set(level, new Set(connections))
|
||||
}
|
||||
|
||||
return nodeCopy
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all nodes from storage
|
||||
*/
|
||||
protected async getAllNodes(): Promise<HNSWNode[]> {
|
||||
const allNodes: HNSWNode[] = []
|
||||
|
||||
// Iterate through all nodes in the nouns map
|
||||
for (const [nodeId, node] of this.nouns.entries()) {
|
||||
// Return a deep copy to avoid reference issues
|
||||
const nodeCopy: HNSWNode = {
|
||||
id: node.id,
|
||||
vector: [...node.vector],
|
||||
connections: new Map()
|
||||
}
|
||||
|
||||
// Copy connections
|
||||
for (const [level, connections] of node.connections.entries()) {
|
||||
nodeCopy.connections.set(level, new Set(connections))
|
||||
}
|
||||
|
||||
allNodes.push(nodeCopy)
|
||||
}
|
||||
|
||||
return allNodes
|
||||
}
|
||||
|
||||
/**
|
||||
* Get nodes by noun type
|
||||
* @param nounType The noun type to filter by
|
||||
* @returns Promise that resolves to an array of nodes of the specified noun type
|
||||
*/
|
||||
protected async getNodesByNounType(nounType: string): Promise<HNSWNode[]> {
|
||||
const nodes: HNSWNode[] = []
|
||||
|
||||
// Iterate through all nodes and filter by noun type using metadata
|
||||
for (const [nodeId, node] of this.nouns.entries()) {
|
||||
// Get the metadata to check the noun type
|
||||
const metadata = await this.getMetadata(nodeId)
|
||||
|
||||
// Include the node if its noun type matches the requested type
|
||||
if (metadata && metadata.noun === nounType) {
|
||||
// Return a deep copy to avoid reference issues
|
||||
const nodeCopy: HNSWNode = {
|
||||
id: node.id,
|
||||
vector: [...node.vector],
|
||||
connections: new Map()
|
||||
}
|
||||
|
||||
// Copy connections
|
||||
for (const [level, connections] of node.connections.entries()) {
|
||||
nodeCopy.connections.set(level, new Set(connections))
|
||||
}
|
||||
|
||||
nodes.push(nodeCopy)
|
||||
}
|
||||
}
|
||||
|
||||
return nodes
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a node from storage
|
||||
*/
|
||||
protected async deleteNode(id: string): Promise<void> {
|
||||
// Delete the node directly from the nouns map
|
||||
this.nouns.delete(id)
|
||||
}
|
||||
|
||||
/**
|
||||
* Save an edge to storage
|
||||
*/
|
||||
protected async saveEdge(edge: Edge): Promise<void> {
|
||||
// Create a deep copy to avoid reference issues
|
||||
const edgeCopy: Edge = {
|
||||
id: edge.id,
|
||||
vector: [...edge.vector],
|
||||
connections: new Map(),
|
||||
sourceId: edge.sourceId,
|
||||
targetId: edge.targetId,
|
||||
type: edge.type,
|
||||
weight: edge.weight,
|
||||
metadata: edge.metadata
|
||||
}
|
||||
|
||||
// Copy connections
|
||||
for (const [level, connections] of edge.connections.entries()) {
|
||||
edgeCopy.connections.set(level, new Set(connections))
|
||||
}
|
||||
|
||||
// Save the edge directly in the verbs map
|
||||
this.verbs.set(edge.id, edgeCopy)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get an edge from storage
|
||||
*/
|
||||
protected async getEdge(id: string): Promise<Edge | null> {
|
||||
// Get the edge directly from the verbs map
|
||||
const edge = this.verbs.get(id)
|
||||
|
||||
// If not found, return null
|
||||
if (!edge) {
|
||||
return null
|
||||
}
|
||||
|
||||
// Return a deep copy to avoid reference issues
|
||||
const edgeCopy: Edge = {
|
||||
id: edge.id,
|
||||
vector: [...edge.vector],
|
||||
connections: new Map(),
|
||||
sourceId: edge.sourceId,
|
||||
targetId: edge.targetId,
|
||||
type: edge.type,
|
||||
weight: edge.weight,
|
||||
metadata: edge.metadata
|
||||
}
|
||||
|
||||
// Copy connections
|
||||
for (const [level, connections] of edge.connections.entries()) {
|
||||
edgeCopy.connections.set(level, new Set(connections))
|
||||
}
|
||||
|
||||
return edgeCopy
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all edges from storage
|
||||
*/
|
||||
protected async getAllEdges(): Promise<Edge[]> {
|
||||
const allEdges: Edge[] = []
|
||||
|
||||
// Iterate through all edges in the verbs map
|
||||
for (const [edgeId, edge] of this.verbs.entries()) {
|
||||
// Return a deep copy to avoid reference issues
|
||||
const edgeCopy: Edge = {
|
||||
id: edge.id,
|
||||
vector: [...edge.vector],
|
||||
connections: new Map(),
|
||||
sourceId: edge.sourceId,
|
||||
targetId: edge.targetId,
|
||||
type: edge.type,
|
||||
weight: edge.weight,
|
||||
metadata: edge.metadata
|
||||
}
|
||||
|
||||
// Copy connections
|
||||
for (const [level, connections] of edge.connections.entries()) {
|
||||
edgeCopy.connections.set(level, new Set(connections))
|
||||
}
|
||||
|
||||
allEdges.push(edgeCopy)
|
||||
}
|
||||
|
||||
return allEdges
|
||||
}
|
||||
|
||||
/**
|
||||
* Get edges by source
|
||||
*/
|
||||
protected async getEdgesBySource(sourceId: string): Promise<Edge[]> {
|
||||
const edges = await this.getAllEdges()
|
||||
return edges.filter((edge) => edge.sourceId === sourceId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get edges by target
|
||||
*/
|
||||
protected async getEdgesByTarget(targetId: string): Promise<Edge[]> {
|
||||
const edges = await this.getAllEdges()
|
||||
return edges.filter((edge) => edge.targetId === targetId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get edges by type
|
||||
*/
|
||||
protected async getEdgesByType(type: string): Promise<Edge[]> {
|
||||
const edges = await this.getAllEdges()
|
||||
return edges.filter((edge) => edge.type === type)
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete an edge from storage
|
||||
*/
|
||||
protected async deleteEdge(id: string): Promise<void> {
|
||||
// Delete the edge directly from the verbs map
|
||||
this.verbs.delete(id)
|
||||
}
|
||||
|
||||
/**
|
||||
* Save metadata to storage
|
||||
*/
|
||||
public async saveMetadata(id: string, metadata: any): Promise<void> {
|
||||
this.metadata.set(id, JSON.parse(JSON.stringify(metadata)))
|
||||
}
|
||||
|
||||
/**
|
||||
* Get metadata from storage
|
||||
*/
|
||||
public async getMetadata(id: string): Promise<any | null> {
|
||||
const metadata = this.metadata.get(id)
|
||||
if (!metadata) {
|
||||
return null
|
||||
}
|
||||
|
||||
return JSON.parse(JSON.stringify(metadata))
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear all data from storage
|
||||
*/
|
||||
public async clear(): Promise<void> {
|
||||
this.nouns.clear()
|
||||
this.verbs.clear()
|
||||
this.metadata.clear()
|
||||
}
|
||||
|
||||
/**
|
||||
* Get information about storage usage and capacity
|
||||
*/
|
||||
public async getStorageStatus(): Promise<{
|
||||
type: string
|
||||
used: number
|
||||
quota: number | null
|
||||
details?: Record<string, any>
|
||||
}> {
|
||||
return {
|
||||
type: 'memory',
|
||||
used: 0, // In-memory storage doesn't have a meaningful size
|
||||
quota: null, // In-memory storage doesn't have a quota
|
||||
details: {
|
||||
nodeCount: this.nouns.size,
|
||||
edgeCount: this.verbs.size,
|
||||
metadataCount: this.metadata.size
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
691
src/storage/adapters/opfsStorage.ts
Normal file
691
src/storage/adapters/opfsStorage.ts
Normal file
|
|
@ -0,0 +1,691 @@
|
|||
/**
|
||||
* OPFS (Origin Private File System) Storage Adapter
|
||||
* Provides persistent storage for the vector database using the Origin Private File System API
|
||||
*/
|
||||
|
||||
import {GraphVerb, HNSWNoun} from '../../coreTypes.js'
|
||||
import {BaseStorage, NOUNS_DIR, VERBS_DIR, METADATA_DIR, INDEX_DIR} from '../baseStorage.js'
|
||||
import '../../types/fileSystemTypes.js'
|
||||
|
||||
/**
|
||||
* Helper function to safely get a file from a FileSystemHandle
|
||||
* This is needed because TypeScript doesn't recognize that a FileSystemHandle
|
||||
* can be a FileSystemFileHandle which has the getFile method
|
||||
*/
|
||||
async function safeGetFile(handle: FileSystemHandle): Promise<File> {
|
||||
// Type cast to any to avoid TypeScript error
|
||||
return (handle as any).getFile()
|
||||
}
|
||||
|
||||
// Type aliases for better readability
|
||||
type HNSWNode = HNSWNoun
|
||||
type Edge = GraphVerb
|
||||
|
||||
// Root directory name for OPFS storage
|
||||
const ROOT_DIR = 'opfs-vector-db'
|
||||
|
||||
/**
|
||||
* OPFS storage adapter for browser environments
|
||||
* Uses the Origin Private File System API to store data persistently
|
||||
*/
|
||||
export class OPFSStorage extends BaseStorage {
|
||||
private rootDir: FileSystemDirectoryHandle | null = null
|
||||
private nounsDir: FileSystemDirectoryHandle | null = null
|
||||
private verbsDir: FileSystemDirectoryHandle | null = null
|
||||
private metadataDir: FileSystemDirectoryHandle | null = null
|
||||
private indexDir: FileSystemDirectoryHandle | null = null
|
||||
private isAvailable = false
|
||||
private isPersistentRequested = false
|
||||
private isPersistentGranted = false
|
||||
|
||||
constructor() {
|
||||
super()
|
||||
// Check if OPFS is available
|
||||
this.isAvailable =
|
||||
typeof navigator !== 'undefined' &&
|
||||
'storage' in navigator &&
|
||||
'getDirectory' in navigator.storage
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the storage adapter
|
||||
*/
|
||||
public async init(): Promise<void> {
|
||||
if (this.isInitialized) {
|
||||
return
|
||||
}
|
||||
|
||||
if (!this.isAvailable) {
|
||||
throw new Error(
|
||||
'Origin Private File System is not available in this environment'
|
||||
)
|
||||
}
|
||||
|
||||
try {
|
||||
// Get the root directory
|
||||
const root = await navigator.storage.getDirectory()
|
||||
|
||||
// Create or get our app's root directory
|
||||
this.rootDir = await root.getDirectoryHandle(ROOT_DIR, {create: true})
|
||||
|
||||
// Create or get nouns directory
|
||||
this.nounsDir = await this.rootDir.getDirectoryHandle(NOUNS_DIR, {
|
||||
create: true
|
||||
})
|
||||
|
||||
// Create or get verbs directory
|
||||
this.verbsDir = await this.rootDir.getDirectoryHandle(VERBS_DIR, {
|
||||
create: true
|
||||
})
|
||||
|
||||
// Create or get metadata directory
|
||||
this.metadataDir = await this.rootDir.getDirectoryHandle(METADATA_DIR, {
|
||||
create: true
|
||||
})
|
||||
|
||||
// Create or get index directory
|
||||
this.indexDir = await this.rootDir.getDirectoryHandle(INDEX_DIR, {
|
||||
create: true
|
||||
})
|
||||
|
||||
this.isInitialized = true
|
||||
} catch (error) {
|
||||
console.error('Failed to initialize OPFS storage:', error)
|
||||
throw new Error(`Failed to initialize OPFS storage: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if OPFS is available in the current environment
|
||||
*/
|
||||
public isOPFSAvailable(): boolean {
|
||||
return this.isAvailable
|
||||
}
|
||||
|
||||
/**
|
||||
* Request persistent storage permission from the user
|
||||
* @returns Promise that resolves to true if permission was granted, false otherwise
|
||||
*/
|
||||
public async requestPersistentStorage(): Promise<boolean> {
|
||||
if (!this.isAvailable) {
|
||||
console.warn('Cannot request persistent storage: OPFS is not available')
|
||||
return false
|
||||
}
|
||||
|
||||
try {
|
||||
// Check if persistence is already granted
|
||||
this.isPersistentGranted = await navigator.storage.persisted()
|
||||
|
||||
if (!this.isPersistentGranted) {
|
||||
// Request permission for persistent storage
|
||||
this.isPersistentGranted = await navigator.storage.persist()
|
||||
}
|
||||
|
||||
this.isPersistentRequested = true
|
||||
return this.isPersistentGranted
|
||||
} catch (error) {
|
||||
console.warn('Failed to request persistent storage:', error)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if persistent storage is granted
|
||||
* @returns Promise that resolves to true if persistent storage is granted, false otherwise
|
||||
*/
|
||||
public async isPersistent(): Promise<boolean> {
|
||||
if (!this.isAvailable) {
|
||||
return false
|
||||
}
|
||||
|
||||
try {
|
||||
this.isPersistentGranted = await navigator.storage.persisted()
|
||||
return this.isPersistentGranted
|
||||
} catch (error) {
|
||||
console.warn('Failed to check persistent storage status:', error)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save a node to storage
|
||||
*/
|
||||
protected async saveNode(node: HNSWNode): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Convert connections Map to a serializable format
|
||||
const serializableNode = {
|
||||
...node,
|
||||
connections: this.mapToObject(node.connections, (set) =>
|
||||
Array.from(set as Set<string>)
|
||||
)
|
||||
}
|
||||
|
||||
// Create or get the file for this noun
|
||||
const fileHandle = await this.nounsDir!.getFileHandle(node.id, {
|
||||
create: true
|
||||
})
|
||||
|
||||
// Write the noun data to the file
|
||||
const writable = await fileHandle.createWritable()
|
||||
await writable.write(JSON.stringify(serializableNode))
|
||||
await writable.close()
|
||||
} catch (error) {
|
||||
console.error(`Failed to save node ${node.id}:`, error)
|
||||
throw new Error(`Failed to save node ${node.id}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a node from storage
|
||||
*/
|
||||
protected async getNode(id: string): Promise<HNSWNode | null> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Get the file handle for this node
|
||||
const fileHandle = await this.nounsDir!.getFileHandle(id)
|
||||
|
||||
// Read the node data from the file
|
||||
const file = await fileHandle.getFile()
|
||||
const text = await file.text()
|
||||
const data = JSON.parse(text)
|
||||
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(data.connections)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
return {
|
||||
id: data.id,
|
||||
vector: data.vector,
|
||||
connections
|
||||
}
|
||||
} catch (error) {
|
||||
// Node not found or other error
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all nodes from storage
|
||||
*/
|
||||
protected async getAllNodes(): Promise<HNSWNode[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
const allNodes: HNSWNode[] = []
|
||||
try {
|
||||
// Iterate through all files in the nouns directory
|
||||
for await (const [name, handle] of this.nounsDir!.entries()) {
|
||||
if (handle.kind === 'file') {
|
||||
try {
|
||||
// Read the node data from the file
|
||||
const file = await safeGetFile(handle)
|
||||
const text = await file.text()
|
||||
const data = JSON.parse(text)
|
||||
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(data.connections)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
allNodes.push({
|
||||
id: data.id,
|
||||
vector: data.vector,
|
||||
connections
|
||||
})
|
||||
} catch (error) {
|
||||
console.error(`Error reading node file ${name}:`, error)
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error reading nouns directory:', error)
|
||||
}
|
||||
|
||||
return allNodes
|
||||
}
|
||||
|
||||
/**
|
||||
* Get nodes by noun type
|
||||
* @param nounType The noun type to filter by
|
||||
* @returns Promise that resolves to an array of nodes of the specified noun type
|
||||
*/
|
||||
protected async getNodesByNounType(nounType: string): Promise<HNSWNode[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
const nodes: HNSWNode[] = []
|
||||
|
||||
try {
|
||||
// Iterate through all files in the nouns directory
|
||||
for await (const [name, handle] of this.nounsDir!.entries()) {
|
||||
if (handle.kind === 'file') {
|
||||
try {
|
||||
// Read the node data from the file
|
||||
const file = await safeGetFile(handle)
|
||||
const text = await file.text()
|
||||
const data = JSON.parse(text)
|
||||
|
||||
// Get the metadata to check the noun type
|
||||
const metadata = await this.getMetadata(data.id)
|
||||
|
||||
// Include the node if its noun type matches the requested type
|
||||
if (metadata && metadata.noun === nounType) {
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(data.connections)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
nodes.push({
|
||||
id: data.id,
|
||||
vector: data.vector,
|
||||
connections
|
||||
})
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Error reading node file ${name}:`, error)
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error reading nouns directory:', error)
|
||||
}
|
||||
|
||||
return nodes
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a node from storage
|
||||
*/
|
||||
protected async deleteNode(id: string): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
await this.nounsDir!.removeEntry(id)
|
||||
} catch (error: any) {
|
||||
// Ignore NotFoundError, which means the file doesn't exist
|
||||
if (error.name !== 'NotFoundError') {
|
||||
console.error(`Error deleting node ${id}:`, error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save an edge to storage
|
||||
*/
|
||||
protected async saveEdge(edge: Edge): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Convert connections Map to a serializable format
|
||||
const serializableEdge = {
|
||||
...edge,
|
||||
connections: this.mapToObject(edge.connections, (set) =>
|
||||
Array.from(set as Set<string>)
|
||||
)
|
||||
}
|
||||
|
||||
// Create or get the file for this verb
|
||||
const fileHandle = await this.verbsDir!.getFileHandle(edge.id, {
|
||||
create: true
|
||||
})
|
||||
|
||||
// Write the verb data to the file
|
||||
const writable = await fileHandle.createWritable()
|
||||
await writable.write(JSON.stringify(serializableEdge))
|
||||
await writable.close()
|
||||
} catch (error) {
|
||||
console.error(`Failed to save edge ${edge.id}:`, error)
|
||||
throw new Error(`Failed to save edge ${edge.id}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get an edge from storage
|
||||
*/
|
||||
protected async getEdge(id: string): Promise<Edge | null> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Get the file handle for this edge
|
||||
const fileHandle = await this.verbsDir!.getFileHandle(id)
|
||||
|
||||
// Read the edge data from the file
|
||||
const file = await fileHandle.getFile()
|
||||
const text = await file.text()
|
||||
const data = JSON.parse(text)
|
||||
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(data.connections)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
return {
|
||||
id: data.id,
|
||||
vector: data.vector,
|
||||
connections,
|
||||
sourceId: data.sourceId,
|
||||
targetId: data.targetId,
|
||||
type: data.type,
|
||||
weight: data.weight,
|
||||
metadata: data.metadata
|
||||
}
|
||||
} catch (error) {
|
||||
// Edge not found or other error
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all edges from storage
|
||||
*/
|
||||
protected async getAllEdges(): Promise<Edge[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
const allEdges: Edge[] = []
|
||||
try {
|
||||
// Iterate through all files in the verbs directory
|
||||
for await (const [name, handle] of this.verbsDir!.entries()) {
|
||||
if (handle.kind === 'file') {
|
||||
try {
|
||||
// Read the edge data from the file
|
||||
const file = await safeGetFile(handle)
|
||||
const text = await file.text()
|
||||
const data = JSON.parse(text)
|
||||
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(data.connections)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
allEdges.push({
|
||||
id: data.id,
|
||||
vector: data.vector,
|
||||
connections,
|
||||
sourceId: data.sourceId,
|
||||
targetId: data.targetId,
|
||||
type: data.type,
|
||||
weight: data.weight,
|
||||
metadata: data.metadata
|
||||
})
|
||||
} catch (error) {
|
||||
console.error(`Error reading edge file ${name}:`, error)
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error reading verbs directory:', error)
|
||||
}
|
||||
|
||||
return allEdges
|
||||
}
|
||||
|
||||
/**
|
||||
* Get edges by source
|
||||
*/
|
||||
protected async getEdgesBySource(sourceId: string): Promise<Edge[]> {
|
||||
const edges = await this.getAllEdges()
|
||||
return edges.filter((edge) => edge.sourceId === sourceId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get edges by target
|
||||
*/
|
||||
protected async getEdgesByTarget(targetId: string): Promise<Edge[]> {
|
||||
const edges = await this.getAllEdges()
|
||||
return edges.filter((edge) => edge.targetId === targetId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get edges by type
|
||||
*/
|
||||
protected async getEdgesByType(type: string): Promise<Edge[]> {
|
||||
const edges = await this.getAllEdges()
|
||||
return edges.filter((edge) => edge.type === type)
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete an edge from storage
|
||||
*/
|
||||
protected async deleteEdge(id: string): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
await this.verbsDir!.removeEntry(id)
|
||||
} catch (error: any) {
|
||||
// Ignore NotFoundError, which means the file doesn't exist
|
||||
if (error.name !== 'NotFoundError') {
|
||||
console.error(`Error deleting edge ${id}:`, error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save metadata to storage
|
||||
*/
|
||||
public async saveMetadata(id: string, metadata: any): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Create or get the file for this metadata
|
||||
const fileHandle = await this.metadataDir!.getFileHandle(id, {
|
||||
create: true
|
||||
})
|
||||
|
||||
// Write the metadata to the file
|
||||
const writable = await fileHandle.createWritable()
|
||||
await writable.write(JSON.stringify(metadata))
|
||||
await writable.close()
|
||||
} catch (error) {
|
||||
console.error(`Failed to save metadata ${id}:`, error)
|
||||
throw new Error(`Failed to save metadata ${id}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get metadata from storage
|
||||
*/
|
||||
public async getMetadata(id: string): Promise<any | null> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Get the file handle for this metadata
|
||||
const fileHandle = await this.metadataDir!.getFileHandle(id)
|
||||
|
||||
// Read the metadata from the file
|
||||
const file = await fileHandle.getFile()
|
||||
const text = await file.text()
|
||||
return JSON.parse(text)
|
||||
} catch (error) {
|
||||
// Metadata not found or other error
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear all data from storage
|
||||
*/
|
||||
public async clear(): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
// Helper function to remove all files in a directory
|
||||
const removeDirectoryContents = async (
|
||||
dirHandle: FileSystemDirectoryHandle
|
||||
): Promise<void> => {
|
||||
try {
|
||||
for await (const [name, handle] of dirHandle.entries()) {
|
||||
await dirHandle.removeEntry(name)
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Error removing directory contents:`, error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
// Remove all files in the nouns directory
|
||||
await removeDirectoryContents(this.nounsDir!)
|
||||
|
||||
// Remove all files in the verbs directory
|
||||
await removeDirectoryContents(this.verbsDir!)
|
||||
|
||||
// Remove all files in the metadata directory
|
||||
await removeDirectoryContents(this.metadataDir!)
|
||||
|
||||
// Remove all files in the index directory
|
||||
await removeDirectoryContents(this.indexDir!)
|
||||
} catch (error) {
|
||||
console.error('Error clearing storage:', error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get information about storage usage and capacity
|
||||
*/
|
||||
public async getStorageStatus(): Promise<{
|
||||
type: string
|
||||
used: number
|
||||
quota: number | null
|
||||
details?: Record<string, any>
|
||||
}> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Calculate the total size of all files in the storage directories
|
||||
let totalSize = 0
|
||||
|
||||
// Helper function to calculate directory size
|
||||
const calculateDirSize = async (
|
||||
dirHandle: FileSystemDirectoryHandle
|
||||
): Promise<number> => {
|
||||
let size = 0
|
||||
try {
|
||||
for await (const [name, handle] of dirHandle.entries()) {
|
||||
if (handle.kind === 'file') {
|
||||
const file = await (handle as FileSystemFileHandle).getFile()
|
||||
size += file.size
|
||||
} else if (handle.kind === 'directory') {
|
||||
size += await calculateDirSize(
|
||||
handle as FileSystemDirectoryHandle
|
||||
)
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn(`Error calculating size for directory:`, error)
|
||||
}
|
||||
return size
|
||||
}
|
||||
|
||||
// Helper function to count files in a directory
|
||||
const countFilesInDirectory = async (
|
||||
dirHandle: FileSystemDirectoryHandle
|
||||
): Promise<number> => {
|
||||
let count = 0
|
||||
try {
|
||||
for await (const [name, handle] of dirHandle.entries()) {
|
||||
if (handle.kind === 'file') {
|
||||
count++
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn(`Error counting files in directory:`, error)
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
// Calculate size for each directory
|
||||
if (this.nounsDir) {
|
||||
totalSize += await calculateDirSize(this.nounsDir)
|
||||
}
|
||||
if (this.verbsDir) {
|
||||
totalSize += await calculateDirSize(this.verbsDir)
|
||||
}
|
||||
if (this.metadataDir) {
|
||||
totalSize += await calculateDirSize(this.metadataDir)
|
||||
}
|
||||
if (this.indexDir) {
|
||||
totalSize += await calculateDirSize(this.indexDir)
|
||||
}
|
||||
|
||||
// Get storage quota information using the Storage API
|
||||
let quota = null
|
||||
let details: Record<string, any> = {
|
||||
isPersistent: await this.isPersistent(),
|
||||
nounTypes: {}
|
||||
}
|
||||
|
||||
try {
|
||||
if (navigator.storage && navigator.storage.estimate) {
|
||||
const estimate = await navigator.storage.estimate()
|
||||
quota = estimate.quota || null
|
||||
details = {
|
||||
...details,
|
||||
usage: estimate.usage,
|
||||
quota: estimate.quota,
|
||||
freePercentage: estimate.quota
|
||||
? ((estimate.quota - (estimate.usage || 0)) / estimate.quota) *
|
||||
100
|
||||
: null
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn('Unable to get storage estimate:', error)
|
||||
}
|
||||
|
||||
// Count files in each directory
|
||||
if (this.nounsDir) {
|
||||
details.nounsCount = await countFilesInDirectory(this.nounsDir)
|
||||
}
|
||||
if (this.verbsDir) {
|
||||
details.verbsCount = await countFilesInDirectory(this.verbsDir)
|
||||
}
|
||||
if (this.metadataDir) {
|
||||
details.metadataCount = await countFilesInDirectory(this.metadataDir)
|
||||
}
|
||||
|
||||
// Count nouns by type using metadata
|
||||
const nounTypeCounts: Record<string, number> = {}
|
||||
if (this.metadataDir) {
|
||||
for await (const [name, handle] of this.metadataDir.entries()) {
|
||||
if (handle.kind === 'file') {
|
||||
try {
|
||||
const file = await safeGetFile(handle)
|
||||
const text = await file.text()
|
||||
const metadata = JSON.parse(text)
|
||||
if (metadata.noun) {
|
||||
nounTypeCounts[metadata.noun] = (nounTypeCounts[metadata.noun] || 0) + 1
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Error reading metadata file ${name}:`, error)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
details.nounTypes = nounTypeCounts
|
||||
|
||||
return {
|
||||
type: 'opfs',
|
||||
used: totalSize,
|
||||
quota,
|
||||
details
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Failed to get storage status:', error)
|
||||
return {
|
||||
type: 'opfs',
|
||||
used: 0,
|
||||
quota: null,
|
||||
details: {error: String(error)}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
776
src/storage/adapters/s3CompatibleStorage.ts
Normal file
776
src/storage/adapters/s3CompatibleStorage.ts
Normal file
|
|
@ -0,0 +1,776 @@
|
|||
/**
|
||||
* S3-Compatible Storage Adapter
|
||||
* Uses the AWS S3 client to interact with S3-compatible storage services
|
||||
* including Amazon S3, Cloudflare R2, and Google Cloud Storage
|
||||
*/
|
||||
|
||||
import { GraphVerb, HNSWNoun } from '../../coreTypes.js'
|
||||
import { BaseStorage, NOUNS_DIR, VERBS_DIR, METADATA_DIR, INDEX_DIR } from '../baseStorage.js'
|
||||
|
||||
// Type aliases for better readability
|
||||
type HNSWNode = HNSWNoun
|
||||
type Edge = GraphVerb
|
||||
|
||||
// Export R2Storage as an alias for S3CompatibleStorage
|
||||
export { S3CompatibleStorage as R2Storage }
|
||||
|
||||
// S3 client and command types - dynamically imported to avoid issues in browser environments
|
||||
type S3Client = any
|
||||
type S3Command = any
|
||||
|
||||
/**
|
||||
* S3-compatible storage adapter for server environments
|
||||
* Uses the AWS S3 client to interact with S3-compatible storage services
|
||||
* including Amazon S3, Cloudflare R2, and Google Cloud Storage
|
||||
*
|
||||
* To use this adapter with Amazon S3, you need to provide:
|
||||
* - region: AWS region (e.g., 'us-east-1')
|
||||
* - credentials: AWS credentials (accessKeyId and secretAccessKey)
|
||||
* - bucketName: S3 bucket name
|
||||
*
|
||||
* To use this adapter with Cloudflare R2, you need to provide:
|
||||
* - accountId: Cloudflare account ID
|
||||
* - accessKeyId: R2 access key ID
|
||||
* - secretAccessKey: R2 secret access key
|
||||
* - bucketName: R2 bucket name
|
||||
*
|
||||
* To use this adapter with Google Cloud Storage, you need to provide:
|
||||
* - region: GCS region (e.g., 'us-central1')
|
||||
* - credentials: GCS credentials (accessKeyId and secretAccessKey)
|
||||
* - endpoint: GCS endpoint (e.g., 'https://storage.googleapis.com')
|
||||
* - bucketName: GCS bucket name
|
||||
*/
|
||||
export class S3CompatibleStorage extends BaseStorage {
|
||||
private s3Client: S3Client | null = null
|
||||
private bucketName: string
|
||||
private serviceType: string
|
||||
private region: string
|
||||
private endpoint?: string
|
||||
private accountId?: string
|
||||
private accessKeyId: string
|
||||
private secretAccessKey: string
|
||||
private sessionToken?: string
|
||||
|
||||
// Prefixes for different types of data
|
||||
private nounPrefix: string
|
||||
private verbPrefix: string
|
||||
private metadataPrefix: string
|
||||
private indexPrefix: string
|
||||
|
||||
/**
|
||||
* Initialize the storage adapter
|
||||
* @param options Configuration options for the S3-compatible storage
|
||||
*/
|
||||
constructor(options: {
|
||||
bucketName: string
|
||||
region?: string
|
||||
endpoint?: string
|
||||
accountId?: string
|
||||
accessKeyId: string
|
||||
secretAccessKey: string
|
||||
sessionToken?: string
|
||||
serviceType?: string
|
||||
}) {
|
||||
super()
|
||||
this.bucketName = options.bucketName
|
||||
this.region = options.region || 'auto'
|
||||
this.endpoint = options.endpoint
|
||||
this.accountId = options.accountId
|
||||
this.accessKeyId = options.accessKeyId
|
||||
this.secretAccessKey = options.secretAccessKey
|
||||
this.sessionToken = options.sessionToken
|
||||
this.serviceType = options.serviceType || 's3'
|
||||
|
||||
// Set up prefixes for different types of data
|
||||
this.nounPrefix = `${NOUNS_DIR}/`
|
||||
this.verbPrefix = `${VERBS_DIR}/`
|
||||
this.metadataPrefix = `${METADATA_DIR}/`
|
||||
this.indexPrefix = `${INDEX_DIR}/`
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the storage adapter
|
||||
*/
|
||||
public async init(): Promise<void> {
|
||||
if (this.isInitialized) {
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
// Import AWS SDK modules only when needed
|
||||
const { S3Client } = await import('@aws-sdk/client-s3')
|
||||
|
||||
// Configure the S3 client based on the service type
|
||||
const clientConfig: any = {
|
||||
region: this.region,
|
||||
credentials: {
|
||||
accessKeyId: this.accessKeyId,
|
||||
secretAccessKey: this.secretAccessKey
|
||||
}
|
||||
}
|
||||
|
||||
// Add session token if provided
|
||||
if (this.sessionToken) {
|
||||
clientConfig.credentials.sessionToken = this.sessionToken
|
||||
}
|
||||
|
||||
// Add endpoint if provided (for R2, GCS, etc.)
|
||||
if (this.endpoint) {
|
||||
clientConfig.endpoint = this.endpoint
|
||||
}
|
||||
|
||||
// Special configuration for Cloudflare R2
|
||||
if (this.serviceType === 'r2' && this.accountId) {
|
||||
clientConfig.endpoint = `https://${this.accountId}.r2.cloudflarestorage.com`
|
||||
}
|
||||
|
||||
// Create the S3 client
|
||||
this.s3Client = new S3Client(clientConfig)
|
||||
|
||||
// Ensure the bucket exists and is accessible
|
||||
const { HeadBucketCommand } = await import('@aws-sdk/client-s3')
|
||||
await this.s3Client.send(
|
||||
new HeadBucketCommand({
|
||||
Bucket: this.bucketName
|
||||
})
|
||||
)
|
||||
|
||||
this.isInitialized = true
|
||||
} catch (error) {
|
||||
console.error(`Failed to initialize ${this.serviceType} storage:`, error)
|
||||
throw new Error(
|
||||
`Failed to initialize ${this.serviceType} storage: ${error}`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save a node to storage
|
||||
*/
|
||||
protected async saveNode(node: HNSWNode): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Convert connections Map to a serializable format
|
||||
const serializableNode = {
|
||||
...node,
|
||||
connections: this.mapToObject(node.connections, (set) =>
|
||||
Array.from(set as Set<string>)
|
||||
)
|
||||
}
|
||||
|
||||
// Import the PutObjectCommand only when needed
|
||||
const { PutObjectCommand } = await import('@aws-sdk/client-s3')
|
||||
|
||||
// Save the node to S3-compatible storage
|
||||
await this.s3Client!.send(
|
||||
new PutObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: `${this.nounPrefix}${node.id}.json`,
|
||||
Body: JSON.stringify(serializableNode, null, 2),
|
||||
ContentType: 'application/json'
|
||||
})
|
||||
)
|
||||
} catch (error) {
|
||||
console.error(`Failed to save node ${node.id}:`, error)
|
||||
throw new Error(`Failed to save node ${node.id}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a node from storage
|
||||
*/
|
||||
protected async getNode(id: string): Promise<HNSWNode | null> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Import the GetObjectCommand only when needed
|
||||
const { GetObjectCommand } = await import('@aws-sdk/client-s3')
|
||||
|
||||
// Try to get the node from the nouns directory
|
||||
const response = await this.s3Client!.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: `${this.nounPrefix}${id}.json`
|
||||
})
|
||||
)
|
||||
|
||||
// Convert the response body to a string
|
||||
const bodyContents = await response.Body.transformToString()
|
||||
const parsedNode = JSON.parse(bodyContents)
|
||||
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(parsedNode.connections)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
return {
|
||||
id: parsedNode.id,
|
||||
vector: parsedNode.vector,
|
||||
connections
|
||||
}
|
||||
} catch (error) {
|
||||
// Node not found or other error
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all nodes from storage
|
||||
*/
|
||||
protected async getAllNodes(): Promise<HNSWNode[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Import the ListObjectsV2Command and GetObjectCommand only when needed
|
||||
const { ListObjectsV2Command, GetObjectCommand } = await import(
|
||||
'@aws-sdk/client-s3'
|
||||
)
|
||||
|
||||
// List all objects in the nouns directory
|
||||
const listResponse = await this.s3Client!.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: this.bucketName,
|
||||
Prefix: this.nounPrefix
|
||||
})
|
||||
)
|
||||
|
||||
const nodes: HNSWNode[] = []
|
||||
|
||||
// If there are no objects, return an empty array
|
||||
if (!listResponse.Contents || listResponse.Contents.length === 0) {
|
||||
return nodes
|
||||
}
|
||||
|
||||
// Get each node
|
||||
const nodePromises = listResponse.Contents.map(
|
||||
async (object: { Key: string }) => {
|
||||
try {
|
||||
// Extract node ID from the key (remove prefix and .json extension)
|
||||
const nodeId = object.Key.replace(this.nounPrefix, '').replace('.json', '')
|
||||
|
||||
// Get the node data
|
||||
const response = await this.s3Client!.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: object.Key
|
||||
})
|
||||
)
|
||||
|
||||
// Convert the response body to a string
|
||||
const bodyContents = await response.Body.transformToString()
|
||||
const parsedNode = JSON.parse(bodyContents)
|
||||
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(parsedNode.connections)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
return {
|
||||
id: parsedNode.id,
|
||||
vector: parsedNode.vector,
|
||||
connections
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Error getting node from ${object.Key}:`, error)
|
||||
return null
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
// Wait for all promises to resolve and filter out nulls
|
||||
const resolvedNodes = await Promise.all(nodePromises)
|
||||
return resolvedNodes.filter((node): node is HNSWNode => node !== null)
|
||||
} catch (error) {
|
||||
console.error('Failed to get all nodes:', error)
|
||||
return []
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get nodes by noun type
|
||||
* @param nounType The noun type to filter by
|
||||
* @returns Promise that resolves to an array of nodes of the specified noun type
|
||||
*/
|
||||
protected async getNodesByNounType(nounType: string): Promise<HNSWNode[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Get all nodes
|
||||
const allNodes = await this.getAllNodes()
|
||||
|
||||
// Filter nodes by noun type using metadata
|
||||
const filteredNodes: HNSWNode[] = []
|
||||
for (const node of allNodes) {
|
||||
const metadata = await this.getMetadata(node.id)
|
||||
if (metadata && metadata.noun === nounType) {
|
||||
filteredNodes.push(node)
|
||||
}
|
||||
}
|
||||
|
||||
return filteredNodes
|
||||
} catch (error) {
|
||||
console.error(`Failed to get nodes by noun type ${nounType}:`, error)
|
||||
return []
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a node from storage
|
||||
*/
|
||||
protected async deleteNode(id: string): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Import the DeleteObjectCommand only when needed
|
||||
const { DeleteObjectCommand } = await import('@aws-sdk/client-s3')
|
||||
|
||||
// Delete the node from S3-compatible storage
|
||||
await this.s3Client!.send(
|
||||
new DeleteObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: `${this.nounPrefix}${id}.json`
|
||||
})
|
||||
)
|
||||
} catch (error) {
|
||||
console.error(`Failed to delete node ${id}:`, error)
|
||||
throw new Error(`Failed to delete node ${id}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save an edge to storage
|
||||
*/
|
||||
protected async saveEdge(edge: Edge): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Convert connections Map to a serializable format
|
||||
const serializableEdge = {
|
||||
...edge,
|
||||
connections: this.mapToObject(edge.connections, (set) =>
|
||||
Array.from(set as Set<string>)
|
||||
)
|
||||
}
|
||||
|
||||
// Import the PutObjectCommand only when needed
|
||||
const { PutObjectCommand } = await import('@aws-sdk/client-s3')
|
||||
|
||||
// Save the edge to S3-compatible storage
|
||||
await this.s3Client!.send(
|
||||
new PutObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: `${this.verbPrefix}${edge.id}.json`,
|
||||
Body: JSON.stringify(serializableEdge, null, 2),
|
||||
ContentType: 'application/json'
|
||||
})
|
||||
)
|
||||
} catch (error) {
|
||||
console.error(`Failed to save edge ${edge.id}:`, error)
|
||||
throw new Error(`Failed to save edge ${edge.id}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get an edge from storage
|
||||
*/
|
||||
protected async getEdge(id: string): Promise<Edge | null> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Import the GetObjectCommand only when needed
|
||||
const { GetObjectCommand } = await import('@aws-sdk/client-s3')
|
||||
|
||||
// Try to get the edge from the verbs directory
|
||||
const response = await this.s3Client!.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: `${this.verbPrefix}${id}.json`
|
||||
})
|
||||
)
|
||||
|
||||
// Convert the response body to a string
|
||||
const bodyContents = await response.Body.transformToString()
|
||||
const parsedEdge = JSON.parse(bodyContents)
|
||||
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(parsedEdge.connections)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
return {
|
||||
id: parsedEdge.id,
|
||||
vector: parsedEdge.vector,
|
||||
connections,
|
||||
sourceId: parsedEdge.sourceId,
|
||||
targetId: parsedEdge.targetId,
|
||||
type: parsedEdge.type,
|
||||
weight: parsedEdge.weight,
|
||||
metadata: parsedEdge.metadata
|
||||
}
|
||||
} catch (error) {
|
||||
// Edge not found or other error
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all edges from storage
|
||||
*/
|
||||
protected async getAllEdges(): Promise<Edge[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Import the ListObjectsV2Command and GetObjectCommand only when needed
|
||||
const { ListObjectsV2Command, GetObjectCommand } = await import(
|
||||
'@aws-sdk/client-s3'
|
||||
)
|
||||
|
||||
// List all objects in the verbs directory
|
||||
const listResponse = await this.s3Client!.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: this.bucketName,
|
||||
Prefix: this.verbPrefix
|
||||
})
|
||||
)
|
||||
|
||||
const edges: Edge[] = []
|
||||
|
||||
// If there are no objects, return an empty array
|
||||
if (!listResponse.Contents || listResponse.Contents.length === 0) {
|
||||
return edges
|
||||
}
|
||||
|
||||
// Get each edge
|
||||
const edgePromises = listResponse.Contents.map(
|
||||
async (object: { Key: string }) => {
|
||||
try {
|
||||
// Extract edge ID from the key (remove prefix and .json extension)
|
||||
const edgeId = object.Key.replace(this.verbPrefix, '').replace('.json', '')
|
||||
|
||||
// Get the edge data
|
||||
const response = await this.s3Client!.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: object.Key
|
||||
})
|
||||
)
|
||||
|
||||
// Convert the response body to a string
|
||||
const bodyContents = await response.Body.transformToString()
|
||||
const parsedEdge = JSON.parse(bodyContents)
|
||||
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(parsedEdge.connections)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
return {
|
||||
id: parsedEdge.id,
|
||||
vector: parsedEdge.vector,
|
||||
connections,
|
||||
sourceId: parsedEdge.sourceId,
|
||||
targetId: parsedEdge.targetId,
|
||||
type: parsedEdge.type,
|
||||
weight: parsedEdge.weight,
|
||||
metadata: parsedEdge.metadata
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Error getting edge from ${object.Key}:`, error)
|
||||
return null
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
// Wait for all promises to resolve and filter out nulls
|
||||
const resolvedEdges = await Promise.all(edgePromises)
|
||||
return resolvedEdges.filter((edge): edge is Edge => edge !== null)
|
||||
} catch (error) {
|
||||
console.error('Failed to get all edges:', error)
|
||||
return []
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get edges by source
|
||||
*/
|
||||
protected async getEdgesBySource(sourceId: string): Promise<Edge[]> {
|
||||
const edges = await this.getAllEdges()
|
||||
return edges.filter((edge) => edge.sourceId === sourceId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get edges by target
|
||||
*/
|
||||
protected async getEdgesByTarget(targetId: string): Promise<Edge[]> {
|
||||
const edges = await this.getAllEdges()
|
||||
return edges.filter((edge) => edge.targetId === targetId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get edges by type
|
||||
*/
|
||||
protected async getEdgesByType(type: string): Promise<Edge[]> {
|
||||
const edges = await this.getAllEdges()
|
||||
return edges.filter((edge) => edge.type === type)
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete an edge from storage
|
||||
*/
|
||||
protected async deleteEdge(id: string): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Import the DeleteObjectCommand only when needed
|
||||
const { DeleteObjectCommand } = await import('@aws-sdk/client-s3')
|
||||
|
||||
// Delete the edge from S3-compatible storage
|
||||
await this.s3Client!.send(
|
||||
new DeleteObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: `${this.verbPrefix}${id}.json`
|
||||
})
|
||||
)
|
||||
} catch (error) {
|
||||
console.error(`Failed to delete edge ${id}:`, error)
|
||||
throw new Error(`Failed to delete edge ${id}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save metadata to storage
|
||||
*/
|
||||
public async saveMetadata(id: string, metadata: any): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Import the PutObjectCommand only when needed
|
||||
const { PutObjectCommand } = await import('@aws-sdk/client-s3')
|
||||
|
||||
// Save the metadata to S3-compatible storage
|
||||
await this.s3Client!.send(
|
||||
new PutObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: `${this.metadataPrefix}${id}.json`,
|
||||
Body: JSON.stringify(metadata, null, 2),
|
||||
ContentType: 'application/json'
|
||||
})
|
||||
)
|
||||
} catch (error) {
|
||||
console.error(`Failed to save metadata ${id}:`, error)
|
||||
throw new Error(`Failed to save metadata ${id}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get metadata from storage
|
||||
*/
|
||||
public async getMetadata(id: string): Promise<any | null> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Import the GetObjectCommand only when needed
|
||||
const { GetObjectCommand } = await import('@aws-sdk/client-s3')
|
||||
|
||||
// Try to get the metadata from the metadata directory
|
||||
const response = await this.s3Client!.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: `${this.metadataPrefix}${id}.json`
|
||||
})
|
||||
)
|
||||
|
||||
// Convert the response body to a string
|
||||
const bodyContents = await response.Body.transformToString()
|
||||
return JSON.parse(bodyContents)
|
||||
} catch (error) {
|
||||
// Metadata not found or other error
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear all data from storage
|
||||
*/
|
||||
public async clear(): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Import the ListObjectsV2Command and DeleteObjectCommand only when needed
|
||||
const { ListObjectsV2Command, DeleteObjectCommand } = await import(
|
||||
'@aws-sdk/client-s3'
|
||||
)
|
||||
|
||||
// Helper function to delete all objects with a given prefix
|
||||
const deleteObjectsWithPrefix = async (prefix: string): Promise<void> => {
|
||||
// List all objects with the given prefix
|
||||
const listResponse = await this.s3Client!.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: this.bucketName,
|
||||
Prefix: prefix
|
||||
})
|
||||
)
|
||||
|
||||
// If there are no objects, return
|
||||
if (!listResponse.Contents || listResponse.Contents.length === 0) {
|
||||
return
|
||||
}
|
||||
|
||||
// Delete each object
|
||||
for (const object of listResponse.Contents) {
|
||||
await this.s3Client!.send(
|
||||
new DeleteObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: object.Key
|
||||
})
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// Delete all objects in the nouns directory
|
||||
await deleteObjectsWithPrefix(this.nounPrefix)
|
||||
|
||||
// Delete all objects in the verbs directory
|
||||
await deleteObjectsWithPrefix(this.verbPrefix)
|
||||
|
||||
// Delete all objects in the metadata directory
|
||||
await deleteObjectsWithPrefix(this.metadataPrefix)
|
||||
|
||||
// Delete all objects in the index directory
|
||||
await deleteObjectsWithPrefix(this.indexPrefix)
|
||||
} catch (error) {
|
||||
console.error('Failed to clear storage:', error)
|
||||
throw new Error(`Failed to clear storage: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get information about storage usage and capacity
|
||||
*/
|
||||
public async getStorageStatus(): Promise<{
|
||||
type: string
|
||||
used: number
|
||||
quota: number | null
|
||||
details?: Record<string, any>
|
||||
}> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Import the ListObjectsV2Command only when needed
|
||||
const { ListObjectsV2Command } = await import('@aws-sdk/client-s3')
|
||||
|
||||
// Calculate the total size of all objects in the storage
|
||||
let totalSize = 0
|
||||
let nodeCount = 0
|
||||
let edgeCount = 0
|
||||
let metadataCount = 0
|
||||
|
||||
// Helper function to calculate size and count for a given prefix
|
||||
const calculateSizeAndCount = async (
|
||||
prefix: string
|
||||
): Promise<{ size: number; count: number }> => {
|
||||
let size = 0
|
||||
let count = 0
|
||||
|
||||
// List all objects with the given prefix
|
||||
const listResponse = await this.s3Client!.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: this.bucketName,
|
||||
Prefix: prefix
|
||||
})
|
||||
)
|
||||
|
||||
// If there are no objects, return
|
||||
if (!listResponse.Contents || listResponse.Contents.length === 0) {
|
||||
return { size, count }
|
||||
}
|
||||
|
||||
// Calculate size and count
|
||||
for (const object of listResponse.Contents) {
|
||||
size += object.Size || 0
|
||||
count++
|
||||
}
|
||||
|
||||
return { size, count }
|
||||
}
|
||||
|
||||
// Calculate size and count for each directory
|
||||
const nounsResult = await calculateSizeAndCount(this.nounPrefix)
|
||||
const verbsResult = await calculateSizeAndCount(this.verbPrefix)
|
||||
const metadataResult = await calculateSizeAndCount(this.metadataPrefix)
|
||||
const indexResult = await calculateSizeAndCount(this.indexPrefix)
|
||||
|
||||
totalSize = nounsResult.size + verbsResult.size + metadataResult.size + indexResult.size
|
||||
nodeCount = nounsResult.count
|
||||
edgeCount = verbsResult.count
|
||||
metadataCount = metadataResult.count
|
||||
|
||||
// Count nouns by type using metadata
|
||||
const nounTypeCounts: Record<string, number> = {}
|
||||
|
||||
// List all objects in the metadata directory
|
||||
const metadataListResponse = await this.s3Client!.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: this.bucketName,
|
||||
Prefix: this.metadataPrefix
|
||||
})
|
||||
)
|
||||
|
||||
if (metadataListResponse.Contents) {
|
||||
// Import the GetObjectCommand only when needed
|
||||
const { GetObjectCommand } = await import('@aws-sdk/client-s3')
|
||||
|
||||
for (const object of metadataListResponse.Contents) {
|
||||
try {
|
||||
// Get the metadata
|
||||
const response = await this.s3Client!.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: object.Key
|
||||
})
|
||||
)
|
||||
|
||||
// Convert the response body to a string
|
||||
const bodyContents = await response.Body.transformToString()
|
||||
const metadata = JSON.parse(bodyContents)
|
||||
|
||||
// Count by noun type
|
||||
if (metadata.noun) {
|
||||
nounTypeCounts[metadata.noun] = (nounTypeCounts[metadata.noun] || 0) + 1
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Error getting metadata from ${object.Key}:`, error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
type: this.serviceType,
|
||||
used: totalSize,
|
||||
quota: null, // S3-compatible services typically don't provide quota information through the API
|
||||
details: {
|
||||
bucketName: this.bucketName,
|
||||
region: this.region,
|
||||
endpoint: this.endpoint,
|
||||
nodeCount,
|
||||
edgeCount,
|
||||
metadataCount,
|
||||
nounTypes: nounTypeCounts
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Failed to get storage status:', error)
|
||||
return {
|
||||
type: this.serviceType,
|
||||
used: 0,
|
||||
quota: null,
|
||||
details: { error: String(error) }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
248
src/storage/baseStorage.ts
Normal file
248
src/storage/baseStorage.ts
Normal file
|
|
@ -0,0 +1,248 @@
|
|||
/**
|
||||
* Base Storage Adapter
|
||||
* Provides common functionality for all storage adapters
|
||||
*/
|
||||
|
||||
import { GraphVerb, HNSWNoun, StorageAdapter } from '../coreTypes.js'
|
||||
|
||||
// Common directory/prefix names
|
||||
export const NOUNS_DIR = 'nouns'
|
||||
export const VERBS_DIR = 'verbs'
|
||||
export const METADATA_DIR = 'metadata'
|
||||
export const INDEX_DIR = 'index'
|
||||
|
||||
/**
|
||||
* Base storage adapter that implements common functionality
|
||||
* This is an abstract class that should be extended by specific storage adapters
|
||||
*/
|
||||
export abstract class BaseStorage implements StorageAdapter {
|
||||
protected isInitialized = false
|
||||
|
||||
/**
|
||||
* Initialize the storage adapter
|
||||
* This method should be implemented by each specific adapter
|
||||
*/
|
||||
public abstract init(): Promise<void>
|
||||
|
||||
/**
|
||||
* Ensure the storage adapter is initialized
|
||||
*/
|
||||
protected async ensureInitialized(): Promise<void> {
|
||||
if (!this.isInitialized) {
|
||||
await this.init()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save a noun to storage
|
||||
*/
|
||||
public async saveNoun(noun: HNSWNoun): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
return this.saveNode(noun)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a noun from storage
|
||||
*/
|
||||
public async getNoun(id: string): Promise<HNSWNoun | null> {
|
||||
await this.ensureInitialized()
|
||||
return this.getNode(id)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all nouns from storage
|
||||
*/
|
||||
public async getAllNouns(): Promise<HNSWNoun[]> {
|
||||
await this.ensureInitialized()
|
||||
return this.getAllNodes()
|
||||
}
|
||||
|
||||
/**
|
||||
* Get nouns by noun type
|
||||
* @param nounType The noun type to filter by
|
||||
* @returns Promise that resolves to an array of nouns of the specified noun type
|
||||
*/
|
||||
public async getNounsByNounType(nounType: string): Promise<HNSWNoun[]> {
|
||||
await this.ensureInitialized()
|
||||
return this.getNodesByNounType(nounType)
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a noun from storage
|
||||
*/
|
||||
public async deleteNoun(id: string): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
return this.deleteNode(id)
|
||||
}
|
||||
|
||||
/**
|
||||
* Save a verb to storage
|
||||
*/
|
||||
public async saveVerb(verb: GraphVerb): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
return this.saveEdge(verb)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a verb from storage
|
||||
*/
|
||||
public async getVerb(id: string): Promise<GraphVerb | null> {
|
||||
await this.ensureInitialized()
|
||||
return this.getEdge(id)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all verbs from storage
|
||||
*/
|
||||
public async getAllVerbs(): Promise<GraphVerb[]> {
|
||||
await this.ensureInitialized()
|
||||
return this.getAllEdges()
|
||||
}
|
||||
|
||||
/**
|
||||
* Get verbs by source
|
||||
*/
|
||||
public async getVerbsBySource(sourceId: string): Promise<GraphVerb[]> {
|
||||
await this.ensureInitialized()
|
||||
return this.getEdgesBySource(sourceId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get verbs by target
|
||||
*/
|
||||
public async getVerbsByTarget(targetId: string): Promise<GraphVerb[]> {
|
||||
await this.ensureInitialized()
|
||||
return this.getEdgesByTarget(targetId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get verbs by type
|
||||
*/
|
||||
public async getVerbsByType(type: string): Promise<GraphVerb[]> {
|
||||
await this.ensureInitialized()
|
||||
return this.getEdgesByType(type)
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a verb from storage
|
||||
*/
|
||||
public async deleteVerb(id: string): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
return this.deleteEdge(id)
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear all data from storage
|
||||
* This method should be implemented by each specific adapter
|
||||
*/
|
||||
public abstract clear(): Promise<void>
|
||||
|
||||
/**
|
||||
* Get information about storage usage and capacity
|
||||
* This method should be implemented by each specific adapter
|
||||
*/
|
||||
public abstract getStorageStatus(): Promise<{
|
||||
type: string
|
||||
used: number
|
||||
quota: number | null
|
||||
details?: Record<string, any>
|
||||
}>
|
||||
|
||||
/**
|
||||
* Save metadata to storage
|
||||
* This method should be implemented by each specific adapter
|
||||
*/
|
||||
public abstract saveMetadata(id: string, metadata: any): Promise<void>
|
||||
|
||||
/**
|
||||
* Get metadata from storage
|
||||
* This method should be implemented by each specific adapter
|
||||
*/
|
||||
public abstract getMetadata(id: string): Promise<any | null>
|
||||
|
||||
/**
|
||||
* Save a node to storage
|
||||
* This method should be implemented by each specific adapter
|
||||
*/
|
||||
protected abstract saveNode(node: HNSWNoun): Promise<void>
|
||||
|
||||
/**
|
||||
* Get a node from storage
|
||||
* This method should be implemented by each specific adapter
|
||||
*/
|
||||
protected abstract getNode(id: string): Promise<HNSWNoun | null>
|
||||
|
||||
/**
|
||||
* Get all nodes from storage
|
||||
* This method should be implemented by each specific adapter
|
||||
*/
|
||||
protected abstract getAllNodes(): Promise<HNSWNoun[]>
|
||||
|
||||
/**
|
||||
* Get nodes by noun type
|
||||
* This method should be implemented by each specific adapter
|
||||
*/
|
||||
protected abstract getNodesByNounType(nounType: string): Promise<HNSWNoun[]>
|
||||
|
||||
/**
|
||||
* Delete a node from storage
|
||||
* This method should be implemented by each specific adapter
|
||||
*/
|
||||
protected abstract deleteNode(id: string): Promise<void>
|
||||
|
||||
/**
|
||||
* Save an edge to storage
|
||||
* This method should be implemented by each specific adapter
|
||||
*/
|
||||
protected abstract saveEdge(edge: GraphVerb): Promise<void>
|
||||
|
||||
/**
|
||||
* Get an edge from storage
|
||||
* This method should be implemented by each specific adapter
|
||||
*/
|
||||
protected abstract getEdge(id: string): Promise<GraphVerb | null>
|
||||
|
||||
/**
|
||||
* Get all edges from storage
|
||||
* This method should be implemented by each specific adapter
|
||||
*/
|
||||
protected abstract getAllEdges(): Promise<GraphVerb[]>
|
||||
|
||||
/**
|
||||
* Get edges by source
|
||||
* This method should be implemented by each specific adapter
|
||||
*/
|
||||
protected abstract getEdgesBySource(sourceId: string): Promise<GraphVerb[]>
|
||||
|
||||
/**
|
||||
* Get edges by target
|
||||
* This method should be implemented by each specific adapter
|
||||
*/
|
||||
protected abstract getEdgesByTarget(targetId: string): Promise<GraphVerb[]>
|
||||
|
||||
/**
|
||||
* Get edges by type
|
||||
* This method should be implemented by each specific adapter
|
||||
*/
|
||||
protected abstract getEdgesByType(type: string): Promise<GraphVerb[]>
|
||||
|
||||
/**
|
||||
* Delete an edge from storage
|
||||
* This method should be implemented by each specific adapter
|
||||
*/
|
||||
protected abstract deleteEdge(id: string): Promise<void>
|
||||
|
||||
/**
|
||||
* Helper method to convert a Map to a plain object for serialization
|
||||
*/
|
||||
protected mapToObject<K extends string | number, V>(
|
||||
map: Map<K, V>,
|
||||
valueTransformer: (value: V) => any = (v) => v
|
||||
): Record<string, any> {
|
||||
const obj: Record<string, any> = {}
|
||||
for (const [key, value] of map.entries()) {
|
||||
obj[key.toString()] = valueTransformer(value)
|
||||
}
|
||||
return obj
|
||||
}
|
||||
}
|
||||
|
|
@ -49,14 +49,7 @@ const NOUNS_DIR = 'nouns'
|
|||
const VERBS_DIR = 'verbs'
|
||||
const METADATA_DIR = 'metadata'
|
||||
|
||||
// Constants for noun type directories
|
||||
const PERSON_DIR = 'person'
|
||||
const PLACE_DIR = 'place'
|
||||
const THING_DIR = 'thing'
|
||||
const EVENT_DIR = 'event'
|
||||
const CONCEPT_DIR = 'concept'
|
||||
const CONTENT_DIR = 'content'
|
||||
const DEFAULT_DIR = 'default' // For nodes without a noun type
|
||||
// All nouns now use the same directory - no separate directories per noun type
|
||||
|
||||
/**
|
||||
* File system storage adapter for Node.js environments
|
||||
|
|
@ -66,13 +59,6 @@ export class FileSystemStorage implements StorageAdapter {
|
|||
private nounsDir: string
|
||||
private verbsDir: string
|
||||
private metadataDir: string
|
||||
private personDir: string
|
||||
private placeDir: string
|
||||
private thingDir: string
|
||||
private eventDir: string
|
||||
private conceptDir: string
|
||||
private contentDir: string
|
||||
private defaultDir: string
|
||||
private isInitialized = false
|
||||
|
||||
constructor(rootDirectory?: string) {
|
||||
|
|
@ -81,13 +67,6 @@ export class FileSystemStorage implements StorageAdapter {
|
|||
this.nounsDir = ''
|
||||
this.verbsDir = ''
|
||||
this.metadataDir = ''
|
||||
this.personDir = ''
|
||||
this.placeDir = ''
|
||||
this.thingDir = ''
|
||||
this.eventDir = ''
|
||||
this.conceptDir = ''
|
||||
this.contentDir = ''
|
||||
this.defaultDir = ''
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -123,27 +102,11 @@ export class FileSystemStorage implements StorageAdapter {
|
|||
this.verbsDir = path.join(this.rootDir, VERBS_DIR)
|
||||
this.metadataDir = path.join(this.rootDir, METADATA_DIR)
|
||||
|
||||
// Set up noun type directory paths
|
||||
this.personDir = path.join(this.nounsDir, PERSON_DIR)
|
||||
this.placeDir = path.join(this.nounsDir, PLACE_DIR)
|
||||
this.thingDir = path.join(this.nounsDir, THING_DIR)
|
||||
this.eventDir = path.join(this.nounsDir, EVENT_DIR)
|
||||
this.conceptDir = path.join(this.nounsDir, CONCEPT_DIR)
|
||||
this.contentDir = path.join(this.nounsDir, CONTENT_DIR)
|
||||
this.defaultDir = path.join(this.nounsDir, DEFAULT_DIR)
|
||||
|
||||
// Create directories if they don't exist
|
||||
await this.ensureDirectoryExists(this.rootDir)
|
||||
await this.ensureDirectoryExists(this.nounsDir)
|
||||
await this.ensureDirectoryExists(this.verbsDir)
|
||||
await this.ensureDirectoryExists(this.metadataDir)
|
||||
await this.ensureDirectoryExists(this.personDir)
|
||||
await this.ensureDirectoryExists(this.placeDir)
|
||||
await this.ensureDirectoryExists(this.thingDir)
|
||||
await this.ensureDirectoryExists(this.eventDir)
|
||||
await this.ensureDirectoryExists(this.conceptDir)
|
||||
await this.ensureDirectoryExists(this.contentDir)
|
||||
await this.ensureDirectoryExists(this.defaultDir)
|
||||
|
||||
this.isInitialized = true
|
||||
} catch (error: any) {
|
||||
|
|
@ -164,32 +127,8 @@ export class FileSystemStorage implements StorageAdapter {
|
|||
}
|
||||
|
||||
private getNounPath(id: string, nounType?: string): string {
|
||||
let typeDir = this.defaultDir
|
||||
if (nounType) {
|
||||
switch (nounType.toLowerCase()) {
|
||||
case 'person':
|
||||
typeDir = this.personDir
|
||||
break
|
||||
case 'place':
|
||||
typeDir = this.placeDir
|
||||
break
|
||||
case 'thing':
|
||||
typeDir = this.thingDir
|
||||
break
|
||||
case 'event':
|
||||
typeDir = this.eventDir
|
||||
break
|
||||
case 'concept':
|
||||
typeDir = this.conceptDir
|
||||
break
|
||||
case 'content':
|
||||
typeDir = this.contentDir
|
||||
break
|
||||
default:
|
||||
typeDir = this.defaultDir
|
||||
}
|
||||
}
|
||||
return path.join(typeDir, `${id}.json`)
|
||||
// All nouns now use the same directory regardless of type
|
||||
return path.join(this.nounsDir, `${id}.json`)
|
||||
}
|
||||
|
||||
public async saveNoun(
|
||||
|
|
@ -204,27 +143,16 @@ export class FileSystemStorage implements StorageAdapter {
|
|||
|
||||
public async getNoun(id: string): Promise<HNSWNoun | null> {
|
||||
if (!this.isInitialized) await this.init()
|
||||
const nounDirs = [
|
||||
this.personDir,
|
||||
this.placeDir,
|
||||
this.thingDir,
|
||||
this.eventDir,
|
||||
this.conceptDir,
|
||||
this.contentDir,
|
||||
this.defaultDir
|
||||
]
|
||||
for (const dir of nounDirs) {
|
||||
const filePath = path.join(dir, `${id}.json`)
|
||||
try {
|
||||
const data = await fs.promises.readFile(filePath, 'utf-8')
|
||||
return JSON.parse(data)
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error reading noun ${id}:`, error)
|
||||
}
|
||||
const filePath = path.join(this.nounsDir, `${id}.json`)
|
||||
try {
|
||||
const data = await fs.promises.readFile(filePath, 'utf-8')
|
||||
return JSON.parse(data)
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error reading noun ${id}:`, error)
|
||||
}
|
||||
return null
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
public async deleteNoun(id: string): Promise<void> {
|
||||
|
|
@ -247,30 +175,19 @@ export class FileSystemStorage implements StorageAdapter {
|
|||
public async getAllNouns(): Promise<HNSWNoun[]> {
|
||||
if (!this.isInitialized) await this.init()
|
||||
const allNouns: HNSWNoun[] = []
|
||||
const nounDirs = [
|
||||
this.personDir,
|
||||
this.placeDir,
|
||||
this.thingDir,
|
||||
this.eventDir,
|
||||
this.conceptDir,
|
||||
this.contentDir,
|
||||
this.defaultDir
|
||||
]
|
||||
for (const dir of nounDirs) {
|
||||
try {
|
||||
const files = await fs.promises.readdir(dir)
|
||||
for (const file of files) {
|
||||
if (file.endsWith('.json')) {
|
||||
const filePath = path.join(dir, file)
|
||||
const data = await fs.promises.readFile(filePath, 'utf-8')
|
||||
allNouns.push(JSON.parse(data))
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error reading directory ${dir}:`, error)
|
||||
try {
|
||||
const files = await fs.promises.readdir(this.nounsDir)
|
||||
for (const file of files) {
|
||||
if (file.endsWith('.json')) {
|
||||
const filePath = path.join(this.nounsDir, file)
|
||||
const data = await fs.promises.readFile(filePath, 'utf-8')
|
||||
allNouns.push(JSON.parse(data))
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error reading directory ${this.nounsDir}:`, error)
|
||||
}
|
||||
}
|
||||
return allNouns
|
||||
}
|
||||
|
|
@ -283,43 +200,25 @@ export class FileSystemStorage implements StorageAdapter {
|
|||
public async getNounsByNounType(nounType: string): Promise<HNSWNoun[]> {
|
||||
if (!this.isInitialized) await this.init()
|
||||
|
||||
let typeDir: string
|
||||
switch (nounType.toLowerCase()) {
|
||||
case 'person':
|
||||
typeDir = this.personDir
|
||||
break
|
||||
case 'place':
|
||||
typeDir = this.placeDir
|
||||
break
|
||||
case 'thing':
|
||||
typeDir = this.thingDir
|
||||
break
|
||||
case 'event':
|
||||
typeDir = this.eventDir
|
||||
break
|
||||
case 'concept':
|
||||
typeDir = this.conceptDir
|
||||
break
|
||||
case 'content':
|
||||
typeDir = this.contentDir
|
||||
break
|
||||
default:
|
||||
typeDir = this.defaultDir
|
||||
}
|
||||
|
||||
const nouns: HNSWNoun[] = []
|
||||
try {
|
||||
const files = await fs.promises.readdir(typeDir)
|
||||
const files = await fs.promises.readdir(this.nounsDir)
|
||||
for (const file of files) {
|
||||
if (file.endsWith('.json')) {
|
||||
const filePath = path.join(typeDir, file)
|
||||
const filePath = path.join(this.nounsDir, file)
|
||||
const data = await fs.promises.readFile(filePath, 'utf-8')
|
||||
nouns.push(JSON.parse(data))
|
||||
const noun = JSON.parse(data)
|
||||
|
||||
// Filter by noun type using metadata
|
||||
const metadata = await this.getMetadata(noun.id)
|
||||
if (metadata && metadata.noun === nounType) {
|
||||
nouns.push(noun)
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error reading directory ${typeDir}:`, error)
|
||||
console.error(`Error reading directory ${this.nounsDir}:`, error)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -13,14 +13,8 @@ const VERBS_PREFIX = 'verbs/'
|
|||
const EDGES_PREFIX = 'verbs/' // Alias for VERBS_PREFIX for edge operations
|
||||
const METADATA_PREFIX = 'metadata/'
|
||||
|
||||
// Constants for noun type prefixes
|
||||
const PERSON_PREFIX = 'nouns/person/'
|
||||
const PLACE_PREFIX = 'place/'
|
||||
const THING_PREFIX = 'thing/'
|
||||
const EVENT_PREFIX = 'event/'
|
||||
const CONCEPT_PREFIX = 'concept/'
|
||||
const CONTENT_PREFIX = 'content/'
|
||||
const DEFAULT_PREFIX = 'default/' // For nodes without a noun type
|
||||
// All nouns now use the same prefix - no separate directories per noun type
|
||||
const NOUN_PREFIX = 'nouns/' // Single directory for all noun types
|
||||
|
||||
/**
|
||||
* S3-compatible storage adapter for server environments
|
||||
|
|
@ -257,45 +251,86 @@ export class S3CompatibleStorage implements StorageAdapter {
|
|||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Get the appropriate prefix based on the node's metadata
|
||||
const nodePrefix = await this.getNodePrefix(id)
|
||||
|
||||
// Import the GetObjectCommand only when needed
|
||||
const { GetObjectCommand } = await import('@aws-sdk/client-s3')
|
||||
|
||||
try {
|
||||
// Try to get the node from S3-compatible storage
|
||||
const response = await this.s3Client.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: `${nodePrefix}${id}.json`
|
||||
})
|
||||
)
|
||||
// Try to get the node from the consolidated nouns directory
|
||||
const response = await this.s3Client.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: `${NOUN_PREFIX}${id}.json`
|
||||
})
|
||||
)
|
||||
|
||||
// Convert the response body to a string
|
||||
const bodyContents = await response.Body.transformToString()
|
||||
const parsedNode = JSON.parse(bodyContents)
|
||||
// Convert the response body to a string
|
||||
const bodyContents = await response.Body.transformToString()
|
||||
const parsedNode = JSON.parse(bodyContents)
|
||||
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(parsedNode.connections)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(parsedNode.connections)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
return {
|
||||
id: parsedNode.id,
|
||||
vector: parsedNode.vector,
|
||||
connections
|
||||
}
|
||||
} catch (error) {
|
||||
// If the node is not found in the expected prefix, try other prefixes
|
||||
if (nodePrefix !== DEFAULT_PREFIX) {
|
||||
// Try the default prefix
|
||||
return {
|
||||
id: parsedNode.id,
|
||||
vector: parsedNode.vector,
|
||||
connections
|
||||
}
|
||||
} catch (error) {
|
||||
// Node not found or other error
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get nodes by noun type
|
||||
* @param nounType The noun type to filter by
|
||||
* @returns Promise that resolves to an array of nodes of the specified noun type
|
||||
*/
|
||||
public async getNodesByNounType(nounType: string): Promise<HNSWNode[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Import the ListObjectsV2Command and GetObjectCommand only when needed
|
||||
const { ListObjectsV2Command, GetObjectCommand } = await import(
|
||||
'@aws-sdk/client-s3'
|
||||
)
|
||||
|
||||
// List all objects in the consolidated nouns directory
|
||||
const listResponse = await this.s3Client.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: this.bucketName,
|
||||
Prefix: NOUN_PREFIX
|
||||
})
|
||||
)
|
||||
|
||||
const nodes: HNSWNode[] = []
|
||||
|
||||
// If there are no objects, return an empty array
|
||||
if (!listResponse.Contents || listResponse.Contents.length === 0) {
|
||||
return nodes
|
||||
}
|
||||
|
||||
// Get each node and filter by noun type
|
||||
const nodePromises = listResponse.Contents.map(
|
||||
async (object: { Key: string }) => {
|
||||
try {
|
||||
// Extract node ID from the key (remove prefix and .json extension)
|
||||
const nodeId = object.Key.replace(NOUN_PREFIX, '').replace('.json', '')
|
||||
|
||||
// Get the metadata to check the noun type
|
||||
const metadata = await this.getMetadata(nodeId)
|
||||
|
||||
// Skip if metadata doesn't exist or noun type doesn't match
|
||||
if (!metadata || metadata.noun !== nounType) {
|
||||
return null
|
||||
}
|
||||
|
||||
const response = await this.s3Client.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: `${NOUNS_PREFIX}${DEFAULT_PREFIX}${id}.json`
|
||||
Key: object.Key
|
||||
})
|
||||
)
|
||||
|
||||
|
|
@ -315,103 +350,38 @@ export class S3CompatibleStorage implements StorageAdapter {
|
|||
vector: parsedNode.vector,
|
||||
connections
|
||||
}
|
||||
} catch {
|
||||
// If not found in default prefix, try all other prefixes
|
||||
const prefixes = [
|
||||
PERSON_PREFIX,
|
||||
PLACE_PREFIX,
|
||||
THING_PREFIX,
|
||||
EVENT_PREFIX,
|
||||
CONCEPT_PREFIX,
|
||||
CONTENT_PREFIX
|
||||
]
|
||||
|
||||
for (const prefix of prefixes) {
|
||||
if (prefix === nodePrefix) continue // Skip the already checked prefix
|
||||
|
||||
try {
|
||||
const response = await this.s3Client.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: `${NOUNS_PREFIX}${prefix}${id}.json`
|
||||
})
|
||||
)
|
||||
|
||||
const bodyContents = await response.Body.transformToString()
|
||||
const parsedNode = JSON.parse(bodyContents)
|
||||
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(
|
||||
parsedNode.connections
|
||||
)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
return {
|
||||
id: parsedNode.id,
|
||||
vector: parsedNode.vector,
|
||||
connections
|
||||
}
|
||||
} catch {
|
||||
// Continue to the next prefix
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Failed to get node from ${object.Key}:`, error)
|
||||
return null
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
return null // Node not found in any prefix
|
||||
}
|
||||
const nodeResults = await Promise.all(nodePromises)
|
||||
return nodeResults.filter((node): node is HNSWNode => node !== null)
|
||||
} catch (error) {
|
||||
console.error(`Failed to get node ${id}:`, error)
|
||||
return null
|
||||
console.error(`Failed to get nodes for noun type ${nounType}:`, error)
|
||||
throw new Error(`Failed to get nodes for noun type ${nounType}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get nodes by noun type
|
||||
* @param nounType The noun type to filter by
|
||||
* @returns Promise that resolves to an array of nodes of the specified noun type
|
||||
* Get all nodes from storage
|
||||
*/
|
||||
public async getNodesByNounType(nounType: string): Promise<HNSWNode[]> {
|
||||
public async getAllNodes(): Promise<HNSWNode[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Determine the prefix based on the noun type
|
||||
let prefix: string
|
||||
switch (nounType) {
|
||||
case 'person':
|
||||
prefix = PERSON_PREFIX
|
||||
break
|
||||
case 'place':
|
||||
prefix = PLACE_PREFIX
|
||||
break
|
||||
case 'thing':
|
||||
prefix = THING_PREFIX
|
||||
break
|
||||
case 'event':
|
||||
prefix = EVENT_PREFIX
|
||||
break
|
||||
case 'concept':
|
||||
prefix = CONCEPT_PREFIX
|
||||
break
|
||||
case 'content':
|
||||
prefix = CONTENT_PREFIX
|
||||
break
|
||||
default:
|
||||
prefix = DEFAULT_PREFIX
|
||||
}
|
||||
|
||||
// Import the ListObjectsV2Command and GetObjectCommand only when needed
|
||||
const { ListObjectsV2Command, GetObjectCommand } = await import(
|
||||
'@aws-sdk/client-s3'
|
||||
)
|
||||
|
||||
// List all objects with the specified prefix
|
||||
// List all objects in the consolidated nouns directory
|
||||
const listResponse = await this.s3Client.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: this.bucketName,
|
||||
Prefix: `${NOUNS_PREFIX}${prefix}`
|
||||
Prefix: NOUN_PREFIX
|
||||
})
|
||||
)
|
||||
|
||||
|
|
@ -458,43 +428,6 @@ export class S3CompatibleStorage implements StorageAdapter {
|
|||
|
||||
const nodeResults = await Promise.all(nodePromises)
|
||||
return nodeResults.filter((node): node is HNSWNode => node !== null)
|
||||
} catch (error) {
|
||||
console.error(`Failed to get nodes for noun type ${nounType}:`, error)
|
||||
throw new Error(`Failed to get nodes for noun type ${nounType}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all nodes from storage
|
||||
*/
|
||||
public async getAllNodes(): Promise<HNSWNode[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Get all noun types
|
||||
const nounTypes = [
|
||||
'person',
|
||||
'place',
|
||||
'thing',
|
||||
'event',
|
||||
'concept',
|
||||
'content',
|
||||
'default'
|
||||
]
|
||||
|
||||
// Run searches in parallel for all noun types
|
||||
const nodePromises = nounTypes.map((nounType) =>
|
||||
this.getNodesByNounType(nounType)
|
||||
)
|
||||
const nodeArrays = await Promise.all(nodePromises)
|
||||
|
||||
// Combine all results
|
||||
const allNodes: HNSWNode[] = []
|
||||
for (const nodes of nodeArrays) {
|
||||
allNodes.push(...nodes)
|
||||
}
|
||||
|
||||
return allNodes
|
||||
} catch (error) {
|
||||
console.error('Failed to get all nodes:', error)
|
||||
throw new Error(`Failed to get all nodes: ${error}`)
|
||||
|
|
@ -508,90 +441,16 @@ export class S3CompatibleStorage implements StorageAdapter {
|
|||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Get the appropriate prefix based on the node's metadata
|
||||
const nodePrefix = await this.getNodePrefix(id)
|
||||
|
||||
// Import the DeleteObjectCommand only when needed
|
||||
const { DeleteObjectCommand, GetObjectCommand } = await import(
|
||||
'@aws-sdk/client-s3'
|
||||
const { DeleteObjectCommand } = await import('@aws-sdk/client-s3')
|
||||
|
||||
// Delete the node from the consolidated nouns directory
|
||||
await this.s3Client.send(
|
||||
new DeleteObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: `${NOUN_PREFIX}${id}.json`
|
||||
})
|
||||
)
|
||||
|
||||
try {
|
||||
// Check if the node exists before deleting
|
||||
await this.s3Client.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: `${nodePrefix}${id}.json`
|
||||
})
|
||||
)
|
||||
|
||||
// Delete the node
|
||||
await this.s3Client.send(
|
||||
new DeleteObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: `${nodePrefix}${id}.json`
|
||||
})
|
||||
)
|
||||
return // Node found and deleted
|
||||
} catch {
|
||||
// If the node is not found in the expected prefix, try other prefixes
|
||||
if (nodePrefix !== DEFAULT_PREFIX) {
|
||||
try {
|
||||
// Try the default prefix
|
||||
await this.s3Client.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: `${NOUNS_PREFIX}${DEFAULT_PREFIX}${id}.json`
|
||||
})
|
||||
)
|
||||
|
||||
// Delete the node
|
||||
await this.s3Client.send(
|
||||
new DeleteObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: `${NOUNS_PREFIX}${DEFAULT_PREFIX}${id}.json`
|
||||
})
|
||||
)
|
||||
return // Node found and deleted
|
||||
} catch {
|
||||
// If not found in default prefix, try all other prefixes
|
||||
const prefixes = [
|
||||
PERSON_PREFIX,
|
||||
PLACE_PREFIX,
|
||||
THING_PREFIX,
|
||||
EVENT_PREFIX,
|
||||
CONCEPT_PREFIX,
|
||||
CONTENT_PREFIX
|
||||
]
|
||||
|
||||
for (const prefix of prefixes) {
|
||||
if (prefix === nodePrefix) continue // Skip the already checked prefix
|
||||
|
||||
try {
|
||||
await this.s3Client.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: `${NOUNS_PREFIX}${prefix}${id}.json`
|
||||
})
|
||||
)
|
||||
|
||||
// Delete the node
|
||||
await this.s3Client.send(
|
||||
new DeleteObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: `${NOUNS_PREFIX}${prefix}${id}.json`
|
||||
})
|
||||
)
|
||||
return // Node found and deleted
|
||||
} catch {
|
||||
// Continue to the next prefix
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return // Node not found in any prefix, nothing to delete
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Failed to delete node ${id}:`, error)
|
||||
throw new Error(`Failed to delete node ${id}: ${error}`)
|
||||
|
|
@ -982,7 +841,7 @@ export class S3CompatibleStorage implements StorageAdapter {
|
|||
}
|
||||
}
|
||||
|
||||
// Count nodes by noun type
|
||||
// Count nodes by noun type by examining metadata
|
||||
const nounTypeCounts: Record<string, number> = {
|
||||
person: 0,
|
||||
place: 0,
|
||||
|
|
@ -990,29 +849,41 @@ export class S3CompatibleStorage implements StorageAdapter {
|
|||
event: 0,
|
||||
concept: 0,
|
||||
content: 0,
|
||||
group: 0,
|
||||
list: 0,
|
||||
category: 0,
|
||||
default: 0
|
||||
}
|
||||
|
||||
// List objects for each noun type prefix
|
||||
const nounTypes = [
|
||||
{ type: 'person', prefix: PERSON_PREFIX },
|
||||
{ type: 'place', prefix: PLACE_PREFIX },
|
||||
{ type: 'thing', prefix: THING_PREFIX },
|
||||
{ type: 'event', prefix: EVENT_PREFIX },
|
||||
{ type: 'concept', prefix: CONCEPT_PREFIX },
|
||||
{ type: 'content', prefix: CONTENT_PREFIX },
|
||||
{ type: 'default', prefix: DEFAULT_PREFIX }
|
||||
]
|
||||
// List all noun objects and count by type using metadata
|
||||
const nounsListResponse = await this.s3Client.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: this.bucketName,
|
||||
Prefix: NOUN_PREFIX
|
||||
})
|
||||
)
|
||||
|
||||
for (const { type, prefix } of nounTypes) {
|
||||
const listResponse = await this.s3Client.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: this.bucketName,
|
||||
Prefix: `${NOUNS_PREFIX}${prefix}`
|
||||
})
|
||||
)
|
||||
|
||||
nounTypeCounts[type] = listResponse.Contents?.length || 0
|
||||
if (nounsListResponse.Contents) {
|
||||
for (const object of nounsListResponse.Contents) {
|
||||
try {
|
||||
// Extract node ID from the key
|
||||
const nodeId = object.Key?.replace(NOUN_PREFIX, '').replace('.json', '')
|
||||
if (nodeId) {
|
||||
// Get metadata to determine noun type
|
||||
const metadata = await this.getMetadata(nodeId)
|
||||
const nounType = metadata?.noun || 'default'
|
||||
|
||||
if (nounType in nounTypeCounts) {
|
||||
nounTypeCounts[nounType]++
|
||||
} else {
|
||||
nounTypeCounts.default++
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
// If we can't get metadata, count as default
|
||||
nounTypeCounts.default++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
|
|
@ -1030,6 +901,9 @@ export class S3CompatibleStorage implements StorageAdapter {
|
|||
event: { count: nounTypeCounts.event },
|
||||
concept: { count: nounTypeCounts.concept },
|
||||
content: { count: nounTypeCounts.content },
|
||||
group: { count: nounTypeCounts.group },
|
||||
list: { count: nounTypeCounts.list },
|
||||
category: { count: nounTypeCounts.category },
|
||||
default: { count: nounTypeCounts.default }
|
||||
}
|
||||
}
|
||||
|
|
@ -1055,39 +929,11 @@ export class S3CompatibleStorage implements StorageAdapter {
|
|||
}
|
||||
|
||||
/**
|
||||
* Get the appropriate prefix for a node based on its metadata
|
||||
* Get the appropriate prefix for a node - now all nouns use the same prefix
|
||||
*/
|
||||
private async getNodePrefix(id: string): Promise<string> {
|
||||
try {
|
||||
// Try to get the metadata for the node
|
||||
const metadata = await this.getMetadata(id)
|
||||
|
||||
// If metadata exists and has a noun field, use the corresponding prefix
|
||||
if (metadata && metadata.noun) {
|
||||
switch (metadata.noun) {
|
||||
case 'person':
|
||||
return PERSON_PREFIX
|
||||
case 'place':
|
||||
return PLACE_PREFIX
|
||||
case 'thing':
|
||||
return THING_PREFIX
|
||||
case 'event':
|
||||
return EVENT_PREFIX
|
||||
case 'concept':
|
||||
return CONCEPT_PREFIX
|
||||
case 'content':
|
||||
return CONTENT_PREFIX
|
||||
default:
|
||||
return DEFAULT_PREFIX
|
||||
}
|
||||
}
|
||||
|
||||
// If no metadata or no noun field, use the default prefix
|
||||
return DEFAULT_PREFIX
|
||||
} catch (error) {
|
||||
// If there's an error getting the metadata, use the default prefix
|
||||
return DEFAULT_PREFIX
|
||||
}
|
||||
// All nouns now use the same prefix regardless of type
|
||||
return NOUN_PREFIX
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
365
src/storage/storageFactory.ts
Normal file
365
src/storage/storageFactory.ts
Normal file
|
|
@ -0,0 +1,365 @@
|
|||
/**
|
||||
* Storage Factory
|
||||
* Creates the appropriate storage adapter based on the environment and configuration
|
||||
*/
|
||||
|
||||
import { StorageAdapter } from '../coreTypes.js'
|
||||
import { MemoryStorage } from './adapters/memoryStorage.js'
|
||||
import { OPFSStorage } from './adapters/opfsStorage.js'
|
||||
import { FileSystemStorage } from './adapters/fileSystemStorage.js'
|
||||
import { S3CompatibleStorage, R2Storage } from './adapters/s3CompatibleStorage.js'
|
||||
|
||||
/**
|
||||
* Options for creating a storage adapter
|
||||
*/
|
||||
export interface StorageOptions {
|
||||
/**
|
||||
* The type of storage to use
|
||||
* - 'auto': Automatically select the best storage adapter based on the environment
|
||||
* - 'memory': Use in-memory storage
|
||||
* - 'opfs': Use Origin Private File System storage (browser only)
|
||||
* - 'filesystem': Use file system storage (Node.js only)
|
||||
* - 's3': Use Amazon S3 storage
|
||||
* - 'r2': Use Cloudflare R2 storage
|
||||
* - 'gcs': Use Google Cloud Storage
|
||||
*/
|
||||
type?: 'auto' | 'memory' | 'opfs' | 'filesystem' | 's3' | 'r2' | 'gcs'
|
||||
|
||||
/**
|
||||
* Force the use of memory storage even if other storage types are available
|
||||
*/
|
||||
forceMemoryStorage?: boolean
|
||||
|
||||
/**
|
||||
* Force the use of file system storage even if other storage types are available
|
||||
*/
|
||||
forceFileSystemStorage?: boolean
|
||||
|
||||
/**
|
||||
* Request persistent storage permission from the user (browser only)
|
||||
*/
|
||||
requestPersistentStorage?: boolean
|
||||
|
||||
/**
|
||||
* Root directory for file system storage (Node.js only)
|
||||
*/
|
||||
rootDirectory?: string
|
||||
|
||||
/**
|
||||
* Configuration for Amazon S3 storage
|
||||
*/
|
||||
s3Storage?: {
|
||||
/**
|
||||
* S3 bucket name
|
||||
*/
|
||||
bucketName: string
|
||||
|
||||
/**
|
||||
* AWS region (e.g., 'us-east-1')
|
||||
*/
|
||||
region?: string
|
||||
|
||||
/**
|
||||
* AWS access key ID
|
||||
*/
|
||||
accessKeyId: string
|
||||
|
||||
/**
|
||||
* AWS secret access key
|
||||
*/
|
||||
secretAccessKey: string
|
||||
|
||||
/**
|
||||
* AWS session token (optional)
|
||||
*/
|
||||
sessionToken?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Configuration for Cloudflare R2 storage
|
||||
*/
|
||||
r2Storage?: {
|
||||
/**
|
||||
* R2 bucket name
|
||||
*/
|
||||
bucketName: string
|
||||
|
||||
/**
|
||||
* Cloudflare account ID
|
||||
*/
|
||||
accountId: string
|
||||
|
||||
/**
|
||||
* R2 access key ID
|
||||
*/
|
||||
accessKeyId: string
|
||||
|
||||
/**
|
||||
* R2 secret access key
|
||||
*/
|
||||
secretAccessKey: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Configuration for Google Cloud Storage
|
||||
*/
|
||||
gcsStorage?: {
|
||||
/**
|
||||
* GCS bucket name
|
||||
*/
|
||||
bucketName: string
|
||||
|
||||
/**
|
||||
* GCS region (e.g., 'us-central1')
|
||||
*/
|
||||
region?: string
|
||||
|
||||
/**
|
||||
* GCS access key ID
|
||||
*/
|
||||
accessKeyId: string
|
||||
|
||||
/**
|
||||
* GCS secret access key
|
||||
*/
|
||||
secretAccessKey: string
|
||||
|
||||
/**
|
||||
* GCS endpoint (e.g., 'https://storage.googleapis.com')
|
||||
*/
|
||||
endpoint?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Configuration for custom S3-compatible storage
|
||||
*/
|
||||
customS3Storage?: {
|
||||
/**
|
||||
* S3-compatible bucket name
|
||||
*/
|
||||
bucketName: string
|
||||
|
||||
/**
|
||||
* S3-compatible region
|
||||
*/
|
||||
region?: string
|
||||
|
||||
/**
|
||||
* S3-compatible endpoint URL
|
||||
*/
|
||||
endpoint: string
|
||||
|
||||
/**
|
||||
* S3-compatible access key ID
|
||||
*/
|
||||
accessKeyId: string
|
||||
|
||||
/**
|
||||
* S3-compatible secret access key
|
||||
*/
|
||||
secretAccessKey: string
|
||||
|
||||
/**
|
||||
* S3-compatible service type (for logging and error messages)
|
||||
*/
|
||||
serviceType?: string
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a storage adapter based on the environment and configuration
|
||||
* @param options Options for creating the storage adapter
|
||||
* @returns Promise that resolves to a storage adapter
|
||||
*/
|
||||
export async function createStorage(
|
||||
options: StorageOptions = {}
|
||||
): Promise<StorageAdapter> {
|
||||
// If memory storage is forced, use it regardless of other options
|
||||
if (options.forceMemoryStorage) {
|
||||
console.log('Using memory storage (forced)')
|
||||
return new MemoryStorage()
|
||||
}
|
||||
|
||||
// If file system storage is forced, use it regardless of other options
|
||||
if (options.forceFileSystemStorage) {
|
||||
console.log('Using file system storage (forced)')
|
||||
return new FileSystemStorage(options.rootDirectory || './brainy-data')
|
||||
}
|
||||
|
||||
// If a specific storage type is specified, use it
|
||||
if (options.type && options.type !== 'auto') {
|
||||
switch (options.type) {
|
||||
case 'memory':
|
||||
console.log('Using memory storage')
|
||||
return new MemoryStorage()
|
||||
|
||||
case 'opfs': {
|
||||
// Check if OPFS is available
|
||||
const opfsStorage = new OPFSStorage()
|
||||
if (opfsStorage.isOPFSAvailable()) {
|
||||
console.log('Using OPFS storage')
|
||||
await opfsStorage.init()
|
||||
|
||||
// Request persistent storage if specified
|
||||
if (options.requestPersistentStorage) {
|
||||
const isPersistent = await opfsStorage.requestPersistentStorage()
|
||||
console.log(`Persistent storage ${isPersistent ? 'granted' : 'denied'}`)
|
||||
}
|
||||
|
||||
return opfsStorage
|
||||
} else {
|
||||
console.warn('OPFS storage is not available, falling back to memory storage')
|
||||
return new MemoryStorage()
|
||||
}
|
||||
}
|
||||
|
||||
case 'filesystem':
|
||||
console.log('Using file system storage')
|
||||
return new FileSystemStorage(options.rootDirectory || './brainy-data')
|
||||
|
||||
case 's3':
|
||||
if (options.s3Storage) {
|
||||
console.log('Using Amazon S3 storage')
|
||||
return new S3CompatibleStorage({
|
||||
bucketName: options.s3Storage.bucketName,
|
||||
region: options.s3Storage.region,
|
||||
accessKeyId: options.s3Storage.accessKeyId,
|
||||
secretAccessKey: options.s3Storage.secretAccessKey,
|
||||
sessionToken: options.s3Storage.sessionToken,
|
||||
serviceType: 's3'
|
||||
})
|
||||
} else {
|
||||
console.warn('S3 storage configuration is missing, falling back to memory storage')
|
||||
return new MemoryStorage()
|
||||
}
|
||||
|
||||
case 'r2':
|
||||
if (options.r2Storage) {
|
||||
console.log('Using Cloudflare R2 storage')
|
||||
return new R2Storage({
|
||||
bucketName: options.r2Storage.bucketName,
|
||||
accountId: options.r2Storage.accountId,
|
||||
accessKeyId: options.r2Storage.accessKeyId,
|
||||
secretAccessKey: options.r2Storage.secretAccessKey,
|
||||
serviceType: 'r2'
|
||||
})
|
||||
} else {
|
||||
console.warn('R2 storage configuration is missing, falling back to memory storage')
|
||||
return new MemoryStorage()
|
||||
}
|
||||
|
||||
case 'gcs':
|
||||
if (options.gcsStorage) {
|
||||
console.log('Using Google Cloud Storage')
|
||||
return new S3CompatibleStorage({
|
||||
bucketName: options.gcsStorage.bucketName,
|
||||
region: options.gcsStorage.region,
|
||||
endpoint: options.gcsStorage.endpoint || 'https://storage.googleapis.com',
|
||||
accessKeyId: options.gcsStorage.accessKeyId,
|
||||
secretAccessKey: options.gcsStorage.secretAccessKey,
|
||||
serviceType: 'gcs'
|
||||
})
|
||||
} else {
|
||||
console.warn('GCS storage configuration is missing, falling back to memory storage')
|
||||
return new MemoryStorage()
|
||||
}
|
||||
|
||||
default:
|
||||
console.warn(`Unknown storage type: ${options.type}, falling back to memory storage`)
|
||||
return new MemoryStorage()
|
||||
}
|
||||
}
|
||||
|
||||
// If custom S3-compatible storage is specified, use it
|
||||
if (options.customS3Storage) {
|
||||
console.log(`Using custom S3-compatible storage: ${options.customS3Storage.serviceType || 'custom'}`)
|
||||
return new S3CompatibleStorage({
|
||||
bucketName: options.customS3Storage.bucketName,
|
||||
region: options.customS3Storage.region,
|
||||
endpoint: options.customS3Storage.endpoint,
|
||||
accessKeyId: options.customS3Storage.accessKeyId,
|
||||
secretAccessKey: options.customS3Storage.secretAccessKey,
|
||||
serviceType: options.customS3Storage.serviceType || 'custom'
|
||||
})
|
||||
}
|
||||
|
||||
// If R2 storage is specified, use it
|
||||
if (options.r2Storage) {
|
||||
console.log('Using Cloudflare R2 storage')
|
||||
return new R2Storage({
|
||||
bucketName: options.r2Storage.bucketName,
|
||||
accountId: options.r2Storage.accountId,
|
||||
accessKeyId: options.r2Storage.accessKeyId,
|
||||
secretAccessKey: options.r2Storage.secretAccessKey,
|
||||
serviceType: 'r2'
|
||||
})
|
||||
}
|
||||
|
||||
// If S3 storage is specified, use it
|
||||
if (options.s3Storage) {
|
||||
console.log('Using Amazon S3 storage')
|
||||
return new S3CompatibleStorage({
|
||||
bucketName: options.s3Storage.bucketName,
|
||||
region: options.s3Storage.region,
|
||||
accessKeyId: options.s3Storage.accessKeyId,
|
||||
secretAccessKey: options.s3Storage.secretAccessKey,
|
||||
sessionToken: options.s3Storage.sessionToken,
|
||||
serviceType: 's3'
|
||||
})
|
||||
}
|
||||
|
||||
// If GCS storage is specified, use it
|
||||
if (options.gcsStorage) {
|
||||
console.log('Using Google Cloud Storage')
|
||||
return new S3CompatibleStorage({
|
||||
bucketName: options.gcsStorage.bucketName,
|
||||
region: options.gcsStorage.region,
|
||||
endpoint: options.gcsStorage.endpoint || 'https://storage.googleapis.com',
|
||||
accessKeyId: options.gcsStorage.accessKeyId,
|
||||
secretAccessKey: options.gcsStorage.secretAccessKey,
|
||||
serviceType: 'gcs'
|
||||
})
|
||||
}
|
||||
|
||||
// Auto-detect the best storage adapter based on the environment
|
||||
// First, try OPFS (browser only)
|
||||
const opfsStorage = new OPFSStorage()
|
||||
if (opfsStorage.isOPFSAvailable()) {
|
||||
console.log('Using OPFS storage (auto-detected)')
|
||||
await opfsStorage.init()
|
||||
|
||||
// Request persistent storage if specified
|
||||
if (options.requestPersistentStorage) {
|
||||
const isPersistent = await opfsStorage.requestPersistentStorage()
|
||||
console.log(`Persistent storage ${isPersistent ? 'granted' : 'denied'}`)
|
||||
}
|
||||
|
||||
return opfsStorage
|
||||
}
|
||||
|
||||
// Next, try file system storage (Node.js only)
|
||||
try {
|
||||
// Check if we're in a Node.js environment
|
||||
if (typeof process !== 'undefined' && process.versions && process.versions.node) {
|
||||
console.log('Using file system storage (auto-detected)')
|
||||
return new FileSystemStorage(options.rootDirectory || './brainy-data')
|
||||
}
|
||||
} catch (error) {
|
||||
// Not in a Node.js environment or file system is not available
|
||||
}
|
||||
|
||||
// Finally, fall back to memory storage
|
||||
console.log('Using memory storage (auto-detected)')
|
||||
return new MemoryStorage()
|
||||
}
|
||||
|
||||
/**
|
||||
* Export all storage adapters
|
||||
*/
|
||||
export {
|
||||
MemoryStorage,
|
||||
OPFSStorage,
|
||||
FileSystemStorage,
|
||||
S3CompatibleStorage,
|
||||
R2Storage
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue