soulcraft/brainy
2
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

feat(storage): implement base, file system, and memory storage adapters

Browse source 
- Added `BaseStorage` abstract class to provide common functionality for storage adapters.
- Implemented `FileSystemStorage` adapter for Node.js environments, supporting persistent file-based storage.
- Added directory structure management methods to `FileSystemStorage` (`ensureDirectoryExists`, `clear`, etc.).
- Implemented `MemoryStorage` adapter for in-memory storage suitable for non-persistent environments.
- Supported CRUD operations for nouns, verbs, and metadata in both `FileSystemStorage` and `MemoryStorage`.
- Included utility methods like `mapToObject` for serialization and deserialization of complex data structures.

Purpose: Provide extensible, reusable storage solutions with both persistent and non-persistent options for diverse application needs.
This commit is contained in:
David Snelling 2025-07-21 12:46:41 -07:00
parent 398c120757
commit 1127115664
9 changed files with 4549 additions and 2186 deletions
Download patch file Download diff file Expand all files Collapse all files

555
src/storage/adapters/fileSystemStorage.ts Normal file
View file

@ -0,0 +1,555 @@
/**
* File System Storage Adapter
* File system storage adapter for Node.js environments
*/
import { GraphVerb, HNSWNoun } from '../../coreTypes.js'
import { BaseStorage, NOUNS_DIR, VERBS_DIR, METADATA_DIR, INDEX_DIR } from '../baseStorage.js'
// Type aliases for better readability
type HNSWNode = HNSWNoun
type Edge = GraphVerb
// Node.js modules - dynamically imported to avoid issues in browser environments
let fs: any
let path: any
// Try to load Node.js modules
try {
// Using dynamic imports to avoid issues in browser environments
const fsPromise = import('fs')
const pathPromise = import('path')
Promise.all([fsPromise, pathPromise]).then(([fsModule, pathModule]) => {
fs = fsModule
path = pathModule.default
}).catch(error => {
console.error('Failed to load Node.js modules:', error)
})
} catch (error) {
console.error(
'FileSystemStorage: Failed to load Node.js modules. This adapter is not supported in this environment.',
error
)
}
/**
* File system storage adapter for Node.js environments
* Uses the file system to store data in the specified directory structure
*/
export class FileSystemStorage extends BaseStorage {
private rootDir: string
private nounsDir: string
private verbsDir: string
private metadataDir: string
private indexDir: string
/**
* Initialize the storage adapter
* @param rootDirectory The root directory for storage
*/
constructor(rootDirectory: string) {
super()
this.rootDir = rootDirectory
this.nounsDir = path.join(this.rootDir, NOUNS_DIR)
this.verbsDir = path.join(this.rootDir, VERBS_DIR)
this.metadataDir = path.join(this.rootDir, METADATA_DIR)
this.indexDir = path.join(this.rootDir, INDEX_DIR)
}
/**
* Initialize the storage adapter
*/
public async init(): Promise<void> {
if (this.isInitialized) {
return
}
// Check if Node.js modules are available
if (!fs || !path) {
throw new Error(
'FileSystemStorage requires a Node.js environment, but `fs` and `path` modules could not be loaded.'
)
}
try {
// Create the root directory if it doesn't exist
await this.ensureDirectoryExists(this.rootDir)
// Create the nouns directory if it doesn't exist
await this.ensureDirectoryExists(this.nounsDir)
// Create the verbs directory if it doesn't exist
await this.ensureDirectoryExists(this.verbsDir)
// Create the metadata directory if it doesn't exist
await this.ensureDirectoryExists(this.metadataDir)
// Create the index directory if it doesn't exist
await this.ensureDirectoryExists(this.indexDir)
this.isInitialized = true
} catch (error) {
console.error('Error initializing FileSystemStorage:', error)
throw error
}
}
/**
* Ensure a directory exists, creating it if necessary
*/
private async ensureDirectoryExists(dirPath: string): Promise<void> {
try {
await fs.promises.mkdir(dirPath, { recursive: true })
} catch (error: any) {
// Ignore EEXIST error, which means the directory already exists
if (error.code !== 'EEXIST') {
throw error
}
}
}
/**
* Save a node to storage
*/
protected async saveNode(node: HNSWNode): Promise<void> {
await this.ensureInitialized()
// Convert connections Map to a serializable format
const serializableNode = {
...node,
connections: this.mapToObject(node.connections, (set) =>
Array.from(set as Set<string>)
)
}
const filePath = path.join(this.nounsDir, `${node.id}.json`)
await fs.promises.writeFile(filePath, JSON.stringify(serializableNode, null, 2))
}
/**
* Get a node from storage
*/
protected async getNode(id: string): Promise<HNSWNode | null> {
await this.ensureInitialized()
const filePath = path.join(this.nounsDir, `${id}.json`)
try {
const data = await fs.promises.readFile(filePath, 'utf-8')
const parsedNode = JSON.parse(data)
// Convert serialized connections back to Map<number, Set<string>>
const connections = new Map<number, Set<string>>()
for (const [level, nodeIds] of Object.entries(parsedNode.connections)) {
connections.set(Number(level), new Set(nodeIds as string[]))
}
return {
id: parsedNode.id,
vector: parsedNode.vector,
connections
}
} catch (error: any) {
if (error.code !== 'ENOENT') {
console.error(`Error reading node ${id}:`, error)
}
return null
}
}
/**
* Get all nodes from storage
*/
protected async getAllNodes(): Promise<HNSWNode[]> {
await this.ensureInitialized()
const allNodes: HNSWNode[] = []
try {
const files = await fs.promises.readdir(this.nounsDir)
for (const file of files) {
if (file.endsWith('.json')) {
const filePath = path.join(this.nounsDir, file)
const data = await fs.promises.readFile(filePath, 'utf-8')
const parsedNode = JSON.parse(data)
// Convert serialized connections back to Map<number, Set<string>>
const connections = new Map<number, Set<string>>()
for (const [level, nodeIds] of Object.entries(parsedNode.connections)) {
connections.set(Number(level), new Set(nodeIds as string[]))
}
allNodes.push({
id: parsedNode.id,
vector: parsedNode.vector,
connections
})
}
}
} catch (error: any) {
if (error.code !== 'ENOENT') {
console.error(`Error reading directory ${this.nounsDir}:`, error)
}
}
return allNodes
}
/**
* Get nodes by noun type
* @param nounType The noun type to filter by
* @returns Promise that resolves to an array of nodes of the specified noun type
*/
protected async getNodesByNounType(nounType: string): Promise<HNSWNode[]> {
await this.ensureInitialized()
const nouns: HNSWNode[] = []
try {
const files = await fs.promises.readdir(this.nounsDir)
for (const file of files) {
if (file.endsWith('.json')) {
const filePath = path.join(this.nounsDir, file)
const data = await fs.promises.readFile(filePath, 'utf-8')
const parsedNode = JSON.parse(data)
// Filter by noun type using metadata
const nodeId = parsedNode.id
const metadata = await this.getMetadata(nodeId)
if (metadata && metadata.noun === nounType) {
// Convert serialized connections back to Map<number, Set<string>>
const connections = new Map<number, Set<string>>()
for (const [level, nodeIds] of Object.entries(parsedNode.connections)) {
connections.set(Number(level), new Set(nodeIds as string[]))
}
nouns.push({
id: parsedNode.id,
vector: parsedNode.vector,
connections
})
}
}
}
} catch (error: any) {
if (error.code !== 'ENOENT') {
console.error(`Error reading directory ${this.nounsDir}:`, error)
}
}
return nouns
}
/**
* Delete a node from storage
*/
protected async deleteNode(id: string): Promise<void> {
await this.ensureInitialized()
const filePath = path.join(this.nounsDir, `${id}.json`)
try {
await fs.promises.unlink(filePath)
} catch (error: any) {
if (error.code !== 'ENOENT') {
console.error(`Error deleting node file ${filePath}:`, error)
throw error
}
}
}
/**
* Save an edge to storage
*/
protected async saveEdge(edge: Edge): Promise<void> {
await this.ensureInitialized()
// Convert connections Map to a serializable format
const serializableEdge = {
...edge,
connections: this.mapToObject(edge.connections, (set) =>
Array.from(set as Set<string>)
)
}
const filePath = path.join(this.verbsDir, `${edge.id}.json`)
await fs.promises.writeFile(filePath, JSON.stringify(serializableEdge, null, 2))
}
/**
* Get an edge from storage
*/
protected async getEdge(id: string): Promise<Edge | null> {
await this.ensureInitialized()
const filePath = path.join(this.verbsDir, `${id}.json`)
try {
const data = await fs.promises.readFile(filePath, 'utf-8')
const parsedEdge = JSON.parse(data)
// Convert serialized connections back to Map<number, Set<string>>
const connections = new Map<number, Set<string>>()
for (const [level, nodeIds] of Object.entries(parsedEdge.connections)) {
connections.set(Number(level), new Set(nodeIds as string[]))
}
return {
id: parsedEdge.id,
vector: parsedEdge.vector,
connections,
sourceId: parsedEdge.sourceId,
targetId: parsedEdge.targetId,
type: parsedEdge.type,
weight: parsedEdge.weight,
metadata: parsedEdge.metadata
}
} catch (error: any) {
if (error.code !== 'ENOENT') {
console.error(`Error reading edge ${id}:`, error)
}
return null
}
}
/**
* Get all edges from storage
*/
protected async getAllEdges(): Promise<Edge[]> {
await this.ensureInitialized()
const allEdges: Edge[] = []
try {
const files = await fs.promises.readdir(this.verbsDir)
for (const file of files) {
if (file.endsWith('.json')) {
const filePath = path.join(this.verbsDir, file)
const data = await fs.promises.readFile(filePath, 'utf-8')
const parsedEdge = JSON.parse(data)
// Convert serialized connections back to Map<number, Set<string>>
const connections = new Map<number, Set<string>>()
for (const [level, nodeIds] of Object.entries(parsedEdge.connections)) {
connections.set(Number(level), new Set(nodeIds as string[]))
}
allEdges.push({
id: parsedEdge.id,
vector: parsedEdge.vector,
connections,
sourceId: parsedEdge.sourceId,
targetId: parsedEdge.targetId,
type: parsedEdge.type,
weight: parsedEdge.weight,
metadata: parsedEdge.metadata
})
}
}
} catch (error: any) {
if (error.code !== 'ENOENT') {
console.error(`Error reading directory ${this.verbsDir}:`, error)
}
}
return allEdges
}
/**
* Get edges by source
*/
protected async getEdgesBySource(sourceId: string): Promise<Edge[]> {
const edges = await this.getAllEdges()
return edges.filter((edge) => edge.sourceId === sourceId)
}
/**
* Get edges by target
*/
protected async getEdgesByTarget(targetId: string): Promise<Edge[]> {
const edges = await this.getAllEdges()
return edges.filter((edge) => edge.targetId === targetId)
}
/**
* Get edges by type
*/
protected async getEdgesByType(type: string): Promise<Edge[]> {
const edges = await this.getAllEdges()
return edges.filter((edge) => edge.type === type)
}
/**
* Delete an edge from storage
*/
protected async deleteEdge(id: string): Promise<void> {
await this.ensureInitialized()
const filePath = path.join(this.verbsDir, `${id}.json`)
try {
await fs.promises.unlink(filePath)
} catch (error: any) {
if (error.code !== 'ENOENT') {
console.error(`Error deleting edge file ${filePath}:`, error)
throw error
}
}
}
/**
* Save metadata to storage
*/
public async saveMetadata(id: string, metadata: any): Promise<void> {
await this.ensureInitialized()
const filePath = path.join(this.metadataDir, `${id}.json`)
await fs.promises.writeFile(filePath, JSON.stringify(metadata, null, 2))
}
/**
* Get metadata from storage
*/
public async getMetadata(id: string): Promise<any | null> {
await this.ensureInitialized()
const filePath = path.join(this.metadataDir, `${id}.json`)
try {
const data = await fs.promises.readFile(filePath, 'utf-8')
return JSON.parse(data)
} catch (error: any) {
if (error.code !== 'ENOENT') {
console.error(`Error reading metadata ${id}:`, error)
}
return null
}
}
/**
* Clear all data from storage
*/
public async clear(): Promise<void> {
await this.ensureInitialized()
// Helper function to remove all files in a directory
const removeDirectoryContents = async (dirPath: string): Promise<void> => {
try {
const files = await fs.promises.readdir(dirPath)
for (const file of files) {
const filePath = path.join(dirPath, file)
const stats = await fs.promises.stat(filePath)
if (stats.isDirectory()) {
await removeDirectoryContents(filePath)
await fs.promises.rmdir(filePath)
} else {
await fs.promises.unlink(filePath)
}
}
} catch (error: any) {
if (error.code !== 'ENOENT') {
console.error(`Error removing directory contents ${dirPath}:`, error)
throw error
}
}
}
// Remove all files in the nouns directory
await removeDirectoryContents(this.nounsDir)
// Remove all files in the verbs directory
await removeDirectoryContents(this.verbsDir)
// Remove all files in the metadata directory
await removeDirectoryContents(this.metadataDir)
// Remove all files in the index directory
await removeDirectoryContents(this.indexDir)
}
/**
* Get information about storage usage and capacity
*/
public async getStorageStatus(): Promise<{
type: string
used: number
quota: number | null
details?: Record<string, any>
}> {
await this.ensureInitialized()
try {
// Calculate the total size of all files in the storage directories
let totalSize = 0
// Helper function to calculate directory size
const calculateSize = async (dirPath: string): Promise<number> => {
let size = 0
try {
const files = await fs.promises.readdir(dirPath)
for (const file of files) {
const filePath = path.join(dirPath, file)
const stats = await fs.promises.stat(filePath)
if (stats.isDirectory()) {
size += await calculateSize(filePath)
} else {
size += stats.size
}
}
} catch (error: any) {
if (error.code !== 'ENOENT') {
console.error(`Error calculating size for directory ${dirPath}:`, error)
}
}
return size
}
// Calculate size for each directory
const nounsDirSize = await calculateSize(this.nounsDir)
const verbsDirSize = await calculateSize(this.verbsDir)
const metadataDirSize = await calculateSize(this.metadataDir)
const indexDirSize = await calculateSize(this.indexDir)
totalSize = nounsDirSize + verbsDirSize + metadataDirSize + indexDirSize
// Count files in each directory
const nounsCount = (await fs.promises.readdir(this.nounsDir)).filter((file: string) => file.endsWith('.json')).length
const verbsCount = (await fs.promises.readdir(this.verbsDir)).filter((file: string) => file.endsWith('.json')).length
const metadataCount = (await fs.promises.readdir(this.metadataDir)).filter((file: string) => file.endsWith('.json')).length
// Count nouns by type using metadata
const nounTypeCounts: Record<string, number> = {}
const metadataFiles = await fs.promises.readdir(this.metadataDir)
for (const file of metadataFiles) {
if (file.endsWith('.json')) {
try {
const filePath = path.join(this.metadataDir, file)
const data = await fs.promises.readFile(filePath, 'utf-8')
const metadata = JSON.parse(data)
if (metadata.noun) {
nounTypeCounts[metadata.noun] = (nounTypeCounts[metadata.noun] || 0) + 1
}
} catch (error) {
console.error(`Error reading metadata file ${file}:`, error)
}
}
}
return {
type: 'filesystem',
used: totalSize,
quota: null, // File system doesn't provide quota information
details: {
rootDirectory: this.rootDir,
nounsCount,
verbsCount,
metadataCount,
nounsDirSize,
verbsDirSize,
metadataDirSize,
indexDirSize,
nounTypes: nounTypeCounts
}
}
} catch (error) {
console.error('Failed to get storage status:', error)
return {
type: 'filesystem',
used: 0,
quota: null,
details: { error: String(error) }
}
}
}
}

324
src/storage/adapters/memoryStorage.ts Normal file
View file

@ -0,0 +1,324 @@
/**
* Memory Storage Adapter
* In-memory storage adapter for environments where persistent storage is not available or needed
*/
import { GraphVerb, HNSWNoun } from '../../coreTypes.js'
import { BaseStorage } from '../baseStorage.js'
/**
* Type alias for HNSWNoun to make the code more readable
*/
type HNSWNode = HNSWNoun
/**
* Type alias for GraphVerb to make the code more readable
*/
type Edge = GraphVerb
/**
* In-memory storage adapter
* Uses Maps to store data in memory
*/
export class MemoryStorage extends BaseStorage {
// Single map of noun ID to noun
private nouns: Map<string, HNSWNode> = new Map()
private verbs: Map<string, Edge> = new Map()
private metadata: Map<string, any> = new Map()
constructor() {
super()
}
/**
* Initialize the storage adapter
* Nothing to initialize for in-memory storage
*/
public async init(): Promise<void> {
this.isInitialized = true
}
/**
* Save a node to storage
*/
protected async saveNode(node: HNSWNode): Promise<void> {
// Create a deep copy to avoid reference issues
const nodeCopy: HNSWNode = {
id: node.id,
vector: [...node.vector],
connections: new Map()
}
// Copy connections
for (const [level, connections] of node.connections.entries()) {
nodeCopy.connections.set(level, new Set(connections))
}
// Save the node directly in the nouns map
this.nouns.set(node.id, nodeCopy)
}
/**
* Get a node from storage
*/
protected async getNode(id: string): Promise<HNSWNode | null> {
// Get the node directly from the nouns map
const node = this.nouns.get(id)
// If not found, return null
if (!node) {
return null
}
// Return a deep copy to avoid reference issues
const nodeCopy: HNSWNode = {
id: node.id,
vector: [...node.vector],
connections: new Map()
}
// Copy connections
for (const [level, connections] of node.connections.entries()) {
nodeCopy.connections.set(level, new Set(connections))
}
return nodeCopy
}
/**
* Get all nodes from storage
*/
protected async getAllNodes(): Promise<HNSWNode[]> {
const allNodes: HNSWNode[] = []
// Iterate through all nodes in the nouns map
for (const [nodeId, node] of this.nouns.entries()) {
// Return a deep copy to avoid reference issues
const nodeCopy: HNSWNode = {
id: node.id,
vector: [...node.vector],
connections: new Map()
}
// Copy connections
for (const [level, connections] of node.connections.entries()) {
nodeCopy.connections.set(level, new Set(connections))
}
allNodes.push(nodeCopy)
}
return allNodes
}
/**
* Get nodes by noun type
* @param nounType The noun type to filter by
* @returns Promise that resolves to an array of nodes of the specified noun type
*/
protected async getNodesByNounType(nounType: string): Promise<HNSWNode[]> {
const nodes: HNSWNode[] = []
// Iterate through all nodes and filter by noun type using metadata
for (const [nodeId, node] of this.nouns.entries()) {
// Get the metadata to check the noun type
const metadata = await this.getMetadata(nodeId)
// Include the node if its noun type matches the requested type
if (metadata && metadata.noun === nounType) {
// Return a deep copy to avoid reference issues
const nodeCopy: HNSWNode = {
id: node.id,
vector: [...node.vector],
connections: new Map()
}
// Copy connections
for (const [level, connections] of node.connections.entries()) {
nodeCopy.connections.set(level, new Set(connections))
}
nodes.push(nodeCopy)
}
}
return nodes
}
/**
* Delete a node from storage
*/
protected async deleteNode(id: string): Promise<void> {
// Delete the node directly from the nouns map
this.nouns.delete(id)
}
/**
* Save an edge to storage
*/
protected async saveEdge(edge: Edge): Promise<void> {
// Create a deep copy to avoid reference issues
const edgeCopy: Edge = {
id: edge.id,
vector: [...edge.vector],
connections: new Map(),
sourceId: edge.sourceId,
targetId: edge.targetId,
type: edge.type,
weight: edge.weight,
metadata: edge.metadata
}
// Copy connections
for (const [level, connections] of edge.connections.entries()) {
edgeCopy.connections.set(level, new Set(connections))
}
// Save the edge directly in the verbs map
this.verbs.set(edge.id, edgeCopy)
}
/**
* Get an edge from storage
*/
protected async getEdge(id: string): Promise<Edge | null> {
// Get the edge directly from the verbs map
const edge = this.verbs.get(id)
// If not found, return null
if (!edge) {
return null
}
// Return a deep copy to avoid reference issues
const edgeCopy: Edge = {
id: edge.id,
vector: [...edge.vector],
connections: new Map(),
sourceId: edge.sourceId,
targetId: edge.targetId,
type: edge.type,
weight: edge.weight,
metadata: edge.metadata
}
// Copy connections
for (const [level, connections] of edge.connections.entries()) {
edgeCopy.connections.set(level, new Set(connections))
}
return edgeCopy
}
/**
* Get all edges from storage
*/
protected async getAllEdges(): Promise<Edge[]> {
const allEdges: Edge[] = []
// Iterate through all edges in the verbs map
for (const [edgeId, edge] of this.verbs.entries()) {
// Return a deep copy to avoid reference issues
const edgeCopy: Edge = {
id: edge.id,
vector: [...edge.vector],
connections: new Map(),
sourceId: edge.sourceId,
targetId: edge.targetId,
type: edge.type,
weight: edge.weight,
metadata: edge.metadata
}
// Copy connections
for (const [level, connections] of edge.connections.entries()) {
edgeCopy.connections.set(level, new Set(connections))
}
allEdges.push(edgeCopy)
}
return allEdges
}
/**
* Get edges by source
*/
protected async getEdgesBySource(sourceId: string): Promise<Edge[]> {
const edges = await this.getAllEdges()
return edges.filter((edge) => edge.sourceId === sourceId)
}
/**
* Get edges by target
*/
protected async getEdgesByTarget(targetId: string): Promise<Edge[]> {
const edges = await this.getAllEdges()
return edges.filter((edge) => edge.targetId === targetId)
}
/**
* Get edges by type
*/
protected async getEdgesByType(type: string): Promise<Edge[]> {
const edges = await this.getAllEdges()
return edges.filter((edge) => edge.type === type)
}
/**
* Delete an edge from storage
*/
protected async deleteEdge(id: string): Promise<void> {
// Delete the edge directly from the verbs map
this.verbs.delete(id)
}
/**
* Save metadata to storage
*/
public async saveMetadata(id: string, metadata: any): Promise<void> {
this.metadata.set(id, JSON.parse(JSON.stringify(metadata)))
}
/**
* Get metadata from storage
*/
public async getMetadata(id: string): Promise<any | null> {
const metadata = this.metadata.get(id)
if (!metadata) {
return null
}
return JSON.parse(JSON.stringify(metadata))
}
/**
* Clear all data from storage
*/
public async clear(): Promise<void> {
this.nouns.clear()
this.verbs.clear()
this.metadata.clear()
}
/**
* Get information about storage usage and capacity
*/
public async getStorageStatus(): Promise<{
type: string
used: number
quota: number | null
details?: Record<string, any>
}> {
return {
type: 'memory',
used: 0, // In-memory storage doesn't have a meaningful size
quota: null, // In-memory storage doesn't have a quota
details: {
nodeCount: this.nouns.size,
edgeCount: this.verbs.size,
metadataCount: this.metadata.size
}
}
}
}

691
src/storage/adapters/opfsStorage.ts Normal file
View file

@ -0,0 +1,691 @@
/**
* OPFS (Origin Private File System) Storage Adapter
* Provides persistent storage for the vector database using the Origin Private File System API
*/
import {GraphVerb, HNSWNoun} from '../../coreTypes.js'
import {BaseStorage, NOUNS_DIR, VERBS_DIR, METADATA_DIR, INDEX_DIR} from '../baseStorage.js'
import '../../types/fileSystemTypes.js'
/**
* Helper function to safely get a file from a FileSystemHandle
* This is needed because TypeScript doesn't recognize that a FileSystemHandle
* can be a FileSystemFileHandle which has the getFile method
*/
async function safeGetFile(handle: FileSystemHandle): Promise<File> {
// Type cast to any to avoid TypeScript error
return (handle as any).getFile()
}
// Type aliases for better readability
type HNSWNode = HNSWNoun
type Edge = GraphVerb
// Root directory name for OPFS storage
const ROOT_DIR = 'opfs-vector-db'
/**
* OPFS storage adapter for browser environments
* Uses the Origin Private File System API to store data persistently
*/
export class OPFSStorage extends BaseStorage {
private rootDir: FileSystemDirectoryHandle | null = null
private nounsDir: FileSystemDirectoryHandle | null = null
private verbsDir: FileSystemDirectoryHandle | null = null
private metadataDir: FileSystemDirectoryHandle | null = null
private indexDir: FileSystemDirectoryHandle | null = null
private isAvailable = false
private isPersistentRequested = false
private isPersistentGranted = false
constructor() {
super()
// Check if OPFS is available
this.isAvailable =
typeof navigator !== 'undefined' &&
'storage' in navigator &&
'getDirectory' in navigator.storage
}
/**
* Initialize the storage adapter
*/
public async init(): Promise<void> {
if (this.isInitialized) {
return
}
if (!this.isAvailable) {
throw new Error(
'Origin Private File System is not available in this environment'
)
}
try {
// Get the root directory
const root = await navigator.storage.getDirectory()
// Create or get our app's root directory
this.rootDir = await root.getDirectoryHandle(ROOT_DIR, {create: true})
// Create or get nouns directory
this.nounsDir = await this.rootDir.getDirectoryHandle(NOUNS_DIR, {
create: true
})
// Create or get verbs directory
this.verbsDir = await this.rootDir.getDirectoryHandle(VERBS_DIR, {
create: true
})
// Create or get metadata directory
this.metadataDir = await this.rootDir.getDirectoryHandle(METADATA_DIR, {
create: true
})
// Create or get index directory
this.indexDir = await this.rootDir.getDirectoryHandle(INDEX_DIR, {
create: true
})
this.isInitialized = true
} catch (error) {
console.error('Failed to initialize OPFS storage:', error)
throw new Error(`Failed to initialize OPFS storage: ${error}`)
}
}
/**
* Check if OPFS is available in the current environment
*/
public isOPFSAvailable(): boolean {
return this.isAvailable
}
/**
* Request persistent storage permission from the user
* @returns Promise that resolves to true if permission was granted, false otherwise
*/
public async requestPersistentStorage(): Promise<boolean> {
if (!this.isAvailable) {
console.warn('Cannot request persistent storage: OPFS is not available')
return false
}
try {
// Check if persistence is already granted
this.isPersistentGranted = await navigator.storage.persisted()
if (!this.isPersistentGranted) {
// Request permission for persistent storage
this.isPersistentGranted = await navigator.storage.persist()
}
this.isPersistentRequested = true
return this.isPersistentGranted
} catch (error) {
console.warn('Failed to request persistent storage:', error)
return false
}
}
/**
* Check if persistent storage is granted
* @returns Promise that resolves to true if persistent storage is granted, false otherwise
*/
public async isPersistent(): Promise<boolean> {
if (!this.isAvailable) {
return false
}
try {
this.isPersistentGranted = await navigator.storage.persisted()
return this.isPersistentGranted
} catch (error) {
console.warn('Failed to check persistent storage status:', error)
return false
}
}
/**
* Save a node to storage
*/
protected async saveNode(node: HNSWNode): Promise<void> {
await this.ensureInitialized()
try {
// Convert connections Map to a serializable format
const serializableNode = {
...node,
connections: this.mapToObject(node.connections, (set) =>
Array.from(set as Set<string>)
)
}
// Create or get the file for this noun
const fileHandle = await this.nounsDir!.getFileHandle(node.id, {
create: true
})
// Write the noun data to the file
const writable = await fileHandle.createWritable()
await writable.write(JSON.stringify(serializableNode))
await writable.close()
} catch (error) {
console.error(`Failed to save node ${node.id}:`, error)
throw new Error(`Failed to save node ${node.id}: ${error}`)
}
}
/**
* Get a node from storage
*/
protected async getNode(id: string): Promise<HNSWNode | null> {
await this.ensureInitialized()
try {
// Get the file handle for this node
const fileHandle = await this.nounsDir!.getFileHandle(id)
// Read the node data from the file
const file = await fileHandle.getFile()
const text = await file.text()
const data = JSON.parse(text)
// Convert serialized connections back to Map<number, Set<string>>
const connections = new Map<number, Set<string>>()
for (const [level, nodeIds] of Object.entries(data.connections)) {
connections.set(Number(level), new Set(nodeIds as string[]))
}
return {
id: data.id,
vector: data.vector,
connections
}
} catch (error) {
// Node not found or other error
return null
}
}
/**
* Get all nodes from storage
*/
protected async getAllNodes(): Promise<HNSWNode[]> {
await this.ensureInitialized()
const allNodes: HNSWNode[] = []
try {
// Iterate through all files in the nouns directory
for await (const [name, handle] of this.nounsDir!.entries()) {
if (handle.kind === 'file') {
try {
// Read the node data from the file
const file = await safeGetFile(handle)
const text = await file.text()
const data = JSON.parse(text)
// Convert serialized connections back to Map<number, Set<string>>
const connections = new Map<number, Set<string>>()
for (const [level, nodeIds] of Object.entries(data.connections)) {
connections.set(Number(level), new Set(nodeIds as string[]))
}
allNodes.push({
id: data.id,
vector: data.vector,
connections
})
} catch (error) {
console.error(`Error reading node file ${name}:`, error)
}
}
}
} catch (error) {
console.error('Error reading nouns directory:', error)
}
return allNodes
}
/**
* Get nodes by noun type
* @param nounType The noun type to filter by
* @returns Promise that resolves to an array of nodes of the specified noun type
*/
protected async getNodesByNounType(nounType: string): Promise<HNSWNode[]> {
await this.ensureInitialized()
const nodes: HNSWNode[] = []
try {
// Iterate through all files in the nouns directory
for await (const [name, handle] of this.nounsDir!.entries()) {
if (handle.kind === 'file') {
try {
// Read the node data from the file
const file = await safeGetFile(handle)
const text = await file.text()
const data = JSON.parse(text)
// Get the metadata to check the noun type
const metadata = await this.getMetadata(data.id)
// Include the node if its noun type matches the requested type
if (metadata && metadata.noun === nounType) {
// Convert serialized connections back to Map<number, Set<string>>
const connections = new Map<number, Set<string>>()
for (const [level, nodeIds] of Object.entries(data.connections)) {
connections.set(Number(level), new Set(nodeIds as string[]))
}
nodes.push({
id: data.id,
vector: data.vector,
connections
})
}
} catch (error) {
console.error(`Error reading node file ${name}:`, error)
}
}
}
} catch (error) {
console.error('Error reading nouns directory:', error)
}
return nodes
}
/**
* Delete a node from storage
*/
protected async deleteNode(id: string): Promise<void> {
await this.ensureInitialized()
try {
await this.nounsDir!.removeEntry(id)
} catch (error: any) {
// Ignore NotFoundError, which means the file doesn't exist
if (error.name !== 'NotFoundError') {
console.error(`Error deleting node ${id}:`, error)
throw error
}
}
}
/**
* Save an edge to storage
*/
protected async saveEdge(edge: Edge): Promise<void> {
await this.ensureInitialized()
try {
// Convert connections Map to a serializable format
const serializableEdge = {
...edge,
connections: this.mapToObject(edge.connections, (set) =>
Array.from(set as Set<string>)
)
}
// Create or get the file for this verb
const fileHandle = await this.verbsDir!.getFileHandle(edge.id, {
create: true
})
// Write the verb data to the file
const writable = await fileHandle.createWritable()
await writable.write(JSON.stringify(serializableEdge))
await writable.close()
} catch (error) {
console.error(`Failed to save edge ${edge.id}:`, error)
throw new Error(`Failed to save edge ${edge.id}: ${error}`)
}
}
/**
* Get an edge from storage
*/
protected async getEdge(id: string): Promise<Edge | null> {
await this.ensureInitialized()
try {
// Get the file handle for this edge
const fileHandle = await this.verbsDir!.getFileHandle(id)
// Read the edge data from the file
const file = await fileHandle.getFile()
const text = await file.text()
const data = JSON.parse(text)
// Convert serialized connections back to Map<number, Set<string>>
const connections = new Map<number, Set<string>>()
for (const [level, nodeIds] of Object.entries(data.connections)) {
connections.set(Number(level), new Set(nodeIds as string[]))
}
return {
id: data.id,
vector: data.vector,
connections,
sourceId: data.sourceId,
targetId: data.targetId,
type: data.type,
weight: data.weight,
metadata: data.metadata
}
} catch (error) {
// Edge not found or other error
return null
}
}
/**
* Get all edges from storage
*/
protected async getAllEdges(): Promise<Edge[]> {
await this.ensureInitialized()
const allEdges: Edge[] = []
try {
// Iterate through all files in the verbs directory
for await (const [name, handle] of this.verbsDir!.entries()) {
if (handle.kind === 'file') {
try {
// Read the edge data from the file
const file = await safeGetFile(handle)
const text = await file.text()
const data = JSON.parse(text)
// Convert serialized connections back to Map<number, Set<string>>
const connections = new Map<number, Set<string>>()
for (const [level, nodeIds] of Object.entries(data.connections)) {
connections.set(Number(level), new Set(nodeIds as string[]))
}
allEdges.push({
id: data.id,
vector: data.vector,
connections,
sourceId: data.sourceId,
targetId: data.targetId,
type: data.type,
weight: data.weight,
metadata: data.metadata
})
} catch (error) {
console.error(`Error reading edge file ${name}:`, error)
}
}
}
} catch (error) {
console.error('Error reading verbs directory:', error)
}
return allEdges
}
/**
* Get edges by source
*/
protected async getEdgesBySource(sourceId: string): Promise<Edge[]> {
const edges = await this.getAllEdges()
return edges.filter((edge) => edge.sourceId === sourceId)
}
/**
* Get edges by target
*/
protected async getEdgesByTarget(targetId: string): Promise<Edge[]> {
const edges = await this.getAllEdges()
return edges.filter((edge) => edge.targetId === targetId)
}
/**
* Get edges by type
*/
protected async getEdgesByType(type: string): Promise<Edge[]> {
const edges = await this.getAllEdges()
return edges.filter((edge) => edge.type === type)
}
/**
* Delete an edge from storage
*/
protected async deleteEdge(id: string): Promise<void> {
await this.ensureInitialized()
try {
await this.verbsDir!.removeEntry(id)
} catch (error: any) {
// Ignore NotFoundError, which means the file doesn't exist
if (error.name !== 'NotFoundError') {
console.error(`Error deleting edge ${id}:`, error)
throw error
}
}
}
/**
* Save metadata to storage
*/
public async saveMetadata(id: string, metadata: any): Promise<void> {
await this.ensureInitialized()
try {
// Create or get the file for this metadata
const fileHandle = await this.metadataDir!.getFileHandle(id, {
create: true
})
// Write the metadata to the file
const writable = await fileHandle.createWritable()
await writable.write(JSON.stringify(metadata))
await writable.close()
} catch (error) {
console.error(`Failed to save metadata ${id}:`, error)
throw new Error(`Failed to save metadata ${id}: ${error}`)
}
}
/**
* Get metadata from storage
*/
public async getMetadata(id: string): Promise<any | null> {
await this.ensureInitialized()
try {
// Get the file handle for this metadata
const fileHandle = await this.metadataDir!.getFileHandle(id)
// Read the metadata from the file
const file = await fileHandle.getFile()
const text = await file.text()
return JSON.parse(text)
} catch (error) {
// Metadata not found or other error
return null
}
}
/**
* Clear all data from storage
*/
public async clear(): Promise<void> {
await this.ensureInitialized()
// Helper function to remove all files in a directory
const removeDirectoryContents = async (
dirHandle: FileSystemDirectoryHandle
): Promise<void> => {
try {
for await (const [name, handle] of dirHandle.entries()) {
await dirHandle.removeEntry(name)
}
} catch (error) {
console.error(`Error removing directory contents:`, error)
throw error
}
}
try {
// Remove all files in the nouns directory
await removeDirectoryContents(this.nounsDir!)
// Remove all files in the verbs directory
await removeDirectoryContents(this.verbsDir!)
// Remove all files in the metadata directory
await removeDirectoryContents(this.metadataDir!)
// Remove all files in the index directory
await removeDirectoryContents(this.indexDir!)
} catch (error) {
console.error('Error clearing storage:', error)
throw error
}
}
/**
* Get information about storage usage and capacity
*/
public async getStorageStatus(): Promise<{
type: string
used: number
quota: number | null
details?: Record<string, any>
}> {
await this.ensureInitialized()
try {
// Calculate the total size of all files in the storage directories
let totalSize = 0
// Helper function to calculate directory size
const calculateDirSize = async (
dirHandle: FileSystemDirectoryHandle
): Promise<number> => {
let size = 0
try {
for await (const [name, handle] of dirHandle.entries()) {
if (handle.kind === 'file') {
const file = await (handle as FileSystemFileHandle).getFile()
size += file.size
} else if (handle.kind === 'directory') {
size += await calculateDirSize(
handle as FileSystemDirectoryHandle
)
}
}
} catch (error) {
console.warn(`Error calculating size for directory:`, error)
}
return size
}
// Helper function to count files in a directory
const countFilesInDirectory = async (
dirHandle: FileSystemDirectoryHandle
): Promise<number> => {
let count = 0
try {
for await (const [name, handle] of dirHandle.entries()) {
if (handle.kind === 'file') {
count++
}
}
} catch (error) {
console.warn(`Error counting files in directory:`, error)
}
return count
}
// Calculate size for each directory
if (this.nounsDir) {
totalSize += await calculateDirSize(this.nounsDir)
}
if (this.verbsDir) {
totalSize += await calculateDirSize(this.verbsDir)
}
if (this.metadataDir) {
totalSize += await calculateDirSize(this.metadataDir)
}
if (this.indexDir) {
totalSize += await calculateDirSize(this.indexDir)
}
// Get storage quota information using the Storage API
let quota = null
let details: Record<string, any> = {
isPersistent: await this.isPersistent(),
nounTypes: {}
}
try {
if (navigator.storage && navigator.storage.estimate) {
const estimate = await navigator.storage.estimate()
quota = estimate.quota || null
details = {
...details,
usage: estimate.usage,
quota: estimate.quota,
freePercentage: estimate.quota
? ((estimate.quota - (estimate.usage || 0)) / estimate.quota) *
100
: null
}
}
} catch (error) {
console.warn('Unable to get storage estimate:', error)
}
// Count files in each directory
if (this.nounsDir) {
details.nounsCount = await countFilesInDirectory(this.nounsDir)
}
if (this.verbsDir) {
details.verbsCount = await countFilesInDirectory(this.verbsDir)
}
if (this.metadataDir) {
details.metadataCount = await countFilesInDirectory(this.metadataDir)
}
// Count nouns by type using metadata
const nounTypeCounts: Record<string, number> = {}
if (this.metadataDir) {
for await (const [name, handle] of this.metadataDir.entries()) {
if (handle.kind === 'file') {
try {
const file = await safeGetFile(handle)
const text = await file.text()
const metadata = JSON.parse(text)
if (metadata.noun) {
nounTypeCounts[metadata.noun] = (nounTypeCounts[metadata.noun] || 0) + 1
}
} catch (error) {
console.error(`Error reading metadata file ${name}:`, error)
}
}
}
}
details.nounTypes = nounTypeCounts
return {
type: 'opfs',
used: totalSize,
quota,
details
}
} catch (error) {
console.error('Failed to get storage status:', error)
return {
type: 'opfs',
used: 0,
quota: null,
details: {error: String(error)}
}
}
}
}

776
src/storage/adapters/s3CompatibleStorage.ts Normal file
View file

@ -0,0 +1,776 @@
/**
* S3-Compatible Storage Adapter
* Uses the AWS S3 client to interact with S3-compatible storage services
* including Amazon S3, Cloudflare R2, and Google Cloud Storage
*/
import { GraphVerb, HNSWNoun } from '../../coreTypes.js'
import { BaseStorage, NOUNS_DIR, VERBS_DIR, METADATA_DIR, INDEX_DIR } from '../baseStorage.js'
// Type aliases for better readability
type HNSWNode = HNSWNoun
type Edge = GraphVerb
// Export R2Storage as an alias for S3CompatibleStorage
export { S3CompatibleStorage as R2Storage }
// S3 client and command types - dynamically imported to avoid issues in browser environments
type S3Client = any
type S3Command = any
/**
* S3-compatible storage adapter for server environments
* Uses the AWS S3 client to interact with S3-compatible storage services
* including Amazon S3, Cloudflare R2, and Google Cloud Storage
*
* To use this adapter with Amazon S3, you need to provide:
* - region: AWS region (e.g., 'us-east-1')
* - credentials: AWS credentials (accessKeyId and secretAccessKey)
* - bucketName: S3 bucket name
*
* To use this adapter with Cloudflare R2, you need to provide:
* - accountId: Cloudflare account ID
* - accessKeyId: R2 access key ID
* - secretAccessKey: R2 secret access key
* - bucketName: R2 bucket name
*
* To use this adapter with Google Cloud Storage, you need to provide:
* - region: GCS region (e.g., 'us-central1')
* - credentials: GCS credentials (accessKeyId and secretAccessKey)
* - endpoint: GCS endpoint (e.g., 'https://storage.googleapis.com')
* - bucketName: GCS bucket name
*/
export class S3CompatibleStorage extends BaseStorage {
private s3Client: S3Client | null = null
private bucketName: string
private serviceType: string
private region: string
private endpoint?: string
private accountId?: string
private accessKeyId: string
private secretAccessKey: string
private sessionToken?: string
// Prefixes for different types of data
private nounPrefix: string
private verbPrefix: string
private metadataPrefix: string
private indexPrefix: string
/**
* Initialize the storage adapter
* @param options Configuration options for the S3-compatible storage
*/
constructor(options: {
bucketName: string
region?: string
endpoint?: string
accountId?: string
accessKeyId: string
secretAccessKey: string
sessionToken?: string
serviceType?: string
}) {
super()
this.bucketName = options.bucketName
this.region = options.region || 'auto'
this.endpoint = options.endpoint
this.accountId = options.accountId
this.accessKeyId = options.accessKeyId
this.secretAccessKey = options.secretAccessKey
this.sessionToken = options.sessionToken
this.serviceType = options.serviceType || 's3'
// Set up prefixes for different types of data
this.nounPrefix = `${NOUNS_DIR}/`
this.verbPrefix = `${VERBS_DIR}/`
this.metadataPrefix = `${METADATA_DIR}/`
this.indexPrefix = `${INDEX_DIR}/`
}
/**
* Initialize the storage adapter
*/
public async init(): Promise<void> {
if (this.isInitialized) {
return
}
try {
// Import AWS SDK modules only when needed
const { S3Client } = await import('@aws-sdk/client-s3')
// Configure the S3 client based on the service type
const clientConfig: any = {
region: this.region,
credentials: {
accessKeyId: this.accessKeyId,
secretAccessKey: this.secretAccessKey
}
}
// Add session token if provided
if (this.sessionToken) {
clientConfig.credentials.sessionToken = this.sessionToken
}
// Add endpoint if provided (for R2, GCS, etc.)
if (this.endpoint) {
clientConfig.endpoint = this.endpoint
}
// Special configuration for Cloudflare R2
if (this.serviceType === 'r2' && this.accountId) {
clientConfig.endpoint = `https://${this.accountId}.r2.cloudflarestorage.com`
}
// Create the S3 client
this.s3Client = new S3Client(clientConfig)
// Ensure the bucket exists and is accessible
const { HeadBucketCommand } = await import('@aws-sdk/client-s3')
await this.s3Client.send(
new HeadBucketCommand({
Bucket: this.bucketName
})
)
this.isInitialized = true
} catch (error) {
console.error(`Failed to initialize ${this.serviceType} storage:`, error)
throw new Error(
`Failed to initialize ${this.serviceType} storage: ${error}`
)
}
}
/**
* Save a node to storage
*/
protected async saveNode(node: HNSWNode): Promise<void> {
await this.ensureInitialized()
try {
// Convert connections Map to a serializable format
const serializableNode = {
...node,
connections: this.mapToObject(node.connections, (set) =>
Array.from(set as Set<string>)
)
}
// Import the PutObjectCommand only when needed
const { PutObjectCommand } = await import('@aws-sdk/client-s3')
// Save the node to S3-compatible storage
await this.s3Client!.send(
new PutObjectCommand({
Bucket: this.bucketName,
Key: `${this.nounPrefix}${node.id}.json`,
Body: JSON.stringify(serializableNode, null, 2),
ContentType: 'application/json'
})
)
} catch (error) {
console.error(`Failed to save node ${node.id}:`, error)
throw new Error(`Failed to save node ${node.id}: ${error}`)
}
}
/**
* Get a node from storage
*/
protected async getNode(id: string): Promise<HNSWNode | null> {
await this.ensureInitialized()
try {
// Import the GetObjectCommand only when needed
const { GetObjectCommand } = await import('@aws-sdk/client-s3')
// Try to get the node from the nouns directory
const response = await this.s3Client!.send(
new GetObjectCommand({
Bucket: this.bucketName,
Key: `${this.nounPrefix}${id}.json`
})
)
// Convert the response body to a string
const bodyContents = await response.Body.transformToString()
const parsedNode = JSON.parse(bodyContents)
// Convert serialized connections back to Map<number, Set<string>>
const connections = new Map<number, Set<string>>()
for (const [level, nodeIds] of Object.entries(parsedNode.connections)) {
connections.set(Number(level), new Set(nodeIds as string[]))
}
return {
id: parsedNode.id,
vector: parsedNode.vector,
connections
}
} catch (error) {
// Node not found or other error
return null
}
}
/**
* Get all nodes from storage
*/
protected async getAllNodes(): Promise<HNSWNode[]> {
await this.ensureInitialized()
try {
// Import the ListObjectsV2Command and GetObjectCommand only when needed
const { ListObjectsV2Command, GetObjectCommand } = await import(
'@aws-sdk/client-s3'
)
// List all objects in the nouns directory
const listResponse = await this.s3Client!.send(
new ListObjectsV2Command({
Bucket: this.bucketName,
Prefix: this.nounPrefix
})
)
const nodes: HNSWNode[] = []
// If there are no objects, return an empty array
if (!listResponse.Contents || listResponse.Contents.length === 0) {
return nodes
}
// Get each node
const nodePromises = listResponse.Contents.map(
async (object: { Key: string }) => {
try {
// Extract node ID from the key (remove prefix and .json extension)
const nodeId = object.Key.replace(this.nounPrefix, '').replace('.json', '')
// Get the node data
const response = await this.s3Client!.send(
new GetObjectCommand({
Bucket: this.bucketName,
Key: object.Key
})
)
// Convert the response body to a string
const bodyContents = await response.Body.transformToString()
const parsedNode = JSON.parse(bodyContents)
// Convert serialized connections back to Map<number, Set<string>>
const connections = new Map<number, Set<string>>()
for (const [level, nodeIds] of Object.entries(parsedNode.connections)) {
connections.set(Number(level), new Set(nodeIds as string[]))
}
return {
id: parsedNode.id,
vector: parsedNode.vector,
connections
}
} catch (error) {
console.error(`Error getting node from ${object.Key}:`, error)
return null
}
}
)
// Wait for all promises to resolve and filter out nulls
const resolvedNodes = await Promise.all(nodePromises)
return resolvedNodes.filter((node): node is HNSWNode => node !== null)
} catch (error) {
console.error('Failed to get all nodes:', error)
return []
}
}
/**
* Get nodes by noun type
* @param nounType The noun type to filter by
* @returns Promise that resolves to an array of nodes of the specified noun type
*/
protected async getNodesByNounType(nounType: string): Promise<HNSWNode[]> {
await this.ensureInitialized()
try {
// Get all nodes
const allNodes = await this.getAllNodes()
// Filter nodes by noun type using metadata
const filteredNodes: HNSWNode[] = []
for (const node of allNodes) {
const metadata = await this.getMetadata(node.id)
if (metadata && metadata.noun === nounType) {
filteredNodes.push(node)
}
}
return filteredNodes
} catch (error) {
console.error(`Failed to get nodes by noun type ${nounType}:`, error)
return []
}
}
/**
* Delete a node from storage
*/
protected async deleteNode(id: string): Promise<void> {
await this.ensureInitialized()
try {
// Import the DeleteObjectCommand only when needed
const { DeleteObjectCommand } = await import('@aws-sdk/client-s3')
// Delete the node from S3-compatible storage
await this.s3Client!.send(
new DeleteObjectCommand({
Bucket: this.bucketName,
Key: `${this.nounPrefix}${id}.json`
})
)
} catch (error) {
console.error(`Failed to delete node ${id}:`, error)
throw new Error(`Failed to delete node ${id}: ${error}`)
}
}
/**
* Save an edge to storage
*/
protected async saveEdge(edge: Edge): Promise<void> {
await this.ensureInitialized()
try {
// Convert connections Map to a serializable format
const serializableEdge = {
...edge,
connections: this.mapToObject(edge.connections, (set) =>
Array.from(set as Set<string>)
)
}
// Import the PutObjectCommand only when needed
const { PutObjectCommand } = await import('@aws-sdk/client-s3')
// Save the edge to S3-compatible storage
await this.s3Client!.send(
new PutObjectCommand({
Bucket: this.bucketName,
Key: `${this.verbPrefix}${edge.id}.json`,
Body: JSON.stringify(serializableEdge, null, 2),
ContentType: 'application/json'
})
)
} catch (error) {
console.error(`Failed to save edge ${edge.id}:`, error)
throw new Error(`Failed to save edge ${edge.id}: ${error}`)
}
}
/**
* Get an edge from storage
*/
protected async getEdge(id: string): Promise<Edge | null> {
await this.ensureInitialized()
try {
// Import the GetObjectCommand only when needed
const { GetObjectCommand } = await import('@aws-sdk/client-s3')
// Try to get the edge from the verbs directory
const response = await this.s3Client!.send(
new GetObjectCommand({
Bucket: this.bucketName,
Key: `${this.verbPrefix}${id}.json`
})
)
// Convert the response body to a string
const bodyContents = await response.Body.transformToString()
const parsedEdge = JSON.parse(bodyContents)
// Convert serialized connections back to Map<number, Set<string>>
const connections = new Map<number, Set<string>>()
for (const [level, nodeIds] of Object.entries(parsedEdge.connections)) {
connections.set(Number(level), new Set(nodeIds as string[]))
}
return {
id: parsedEdge.id,
vector: parsedEdge.vector,
connections,
sourceId: parsedEdge.sourceId,
targetId: parsedEdge.targetId,
type: parsedEdge.type,
weight: parsedEdge.weight,
metadata: parsedEdge.metadata
}
} catch (error) {
// Edge not found or other error
return null
}
}
/**
* Get all edges from storage
*/
protected async getAllEdges(): Promise<Edge[]> {
await this.ensureInitialized()
try {
// Import the ListObjectsV2Command and GetObjectCommand only when needed
const { ListObjectsV2Command, GetObjectCommand } = await import(
'@aws-sdk/client-s3'
)
// List all objects in the verbs directory
const listResponse = await this.s3Client!.send(
new ListObjectsV2Command({
Bucket: this.bucketName,
Prefix: this.verbPrefix
})
)
const edges: Edge[] = []
// If there are no objects, return an empty array
if (!listResponse.Contents || listResponse.Contents.length === 0) {
return edges
}
// Get each edge
const edgePromises = listResponse.Contents.map(
async (object: { Key: string }) => {
try {
// Extract edge ID from the key (remove prefix and .json extension)
const edgeId = object.Key.replace(this.verbPrefix, '').replace('.json', '')
// Get the edge data
const response = await this.s3Client!.send(
new GetObjectCommand({
Bucket: this.bucketName,
Key: object.Key
})
)
// Convert the response body to a string
const bodyContents = await response.Body.transformToString()
const parsedEdge = JSON.parse(bodyContents)
// Convert serialized connections back to Map<number, Set<string>>
const connections = new Map<number, Set<string>>()
for (const [level, nodeIds] of Object.entries(parsedEdge.connections)) {
connections.set(Number(level), new Set(nodeIds as string[]))
}
return {
id: parsedEdge.id,
vector: parsedEdge.vector,
connections,
sourceId: parsedEdge.sourceId,
targetId: parsedEdge.targetId,
type: parsedEdge.type,
weight: parsedEdge.weight,
metadata: parsedEdge.metadata
}
} catch (error) {
console.error(`Error getting edge from ${object.Key}:`, error)
return null
}
}
)
// Wait for all promises to resolve and filter out nulls
const resolvedEdges = await Promise.all(edgePromises)
return resolvedEdges.filter((edge): edge is Edge => edge !== null)
} catch (error) {
console.error('Failed to get all edges:', error)
return []
}
}
/**
* Get edges by source
*/
protected async getEdgesBySource(sourceId: string): Promise<Edge[]> {
const edges = await this.getAllEdges()
return edges.filter((edge) => edge.sourceId === sourceId)
}
/**
* Get edges by target
*/
protected async getEdgesByTarget(targetId: string): Promise<Edge[]> {
const edges = await this.getAllEdges()
return edges.filter((edge) => edge.targetId === targetId)
}
/**
* Get edges by type
*/
protected async getEdgesByType(type: string): Promise<Edge[]> {
const edges = await this.getAllEdges()
return edges.filter((edge) => edge.type === type)
}
/**
* Delete an edge from storage
*/
protected async deleteEdge(id: string): Promise<void> {
await this.ensureInitialized()
try {
// Import the DeleteObjectCommand only when needed
const { DeleteObjectCommand } = await import('@aws-sdk/client-s3')
// Delete the edge from S3-compatible storage
await this.s3Client!.send(
new DeleteObjectCommand({
Bucket: this.bucketName,
Key: `${this.verbPrefix}${id}.json`
})
)
} catch (error) {
console.error(`Failed to delete edge ${id}:`, error)
throw new Error(`Failed to delete edge ${id}: ${error}`)
}
}
/**
* Save metadata to storage
*/
public async saveMetadata(id: string, metadata: any): Promise<void> {
await this.ensureInitialized()
try {
// Import the PutObjectCommand only when needed
const { PutObjectCommand } = await import('@aws-sdk/client-s3')
// Save the metadata to S3-compatible storage
await this.s3Client!.send(
new PutObjectCommand({
Bucket: this.bucketName,
Key: `${this.metadataPrefix}${id}.json`,
Body: JSON.stringify(metadata, null, 2),
ContentType: 'application/json'
})
)
} catch (error) {
console.error(`Failed to save metadata ${id}:`, error)
throw new Error(`Failed to save metadata ${id}: ${error}`)
}
}
/**
* Get metadata from storage
*/
public async getMetadata(id: string): Promise<any | null> {
await this.ensureInitialized()
try {
// Import the GetObjectCommand only when needed
const { GetObjectCommand } = await import('@aws-sdk/client-s3')
// Try to get the metadata from the metadata directory
const response = await this.s3Client!.send(
new GetObjectCommand({
Bucket: this.bucketName,
Key: `${this.metadataPrefix}${id}.json`
})
)
// Convert the response body to a string
const bodyContents = await response.Body.transformToString()
return JSON.parse(bodyContents)
} catch (error) {
// Metadata not found or other error
return null
}
}
/**
* Clear all data from storage
*/
public async clear(): Promise<void> {
await this.ensureInitialized()
try {
// Import the ListObjectsV2Command and DeleteObjectCommand only when needed
const { ListObjectsV2Command, DeleteObjectCommand } = await import(
'@aws-sdk/client-s3'
)
// Helper function to delete all objects with a given prefix
const deleteObjectsWithPrefix = async (prefix: string): Promise<void> => {
// List all objects with the given prefix
const listResponse = await this.s3Client!.send(
new ListObjectsV2Command({
Bucket: this.bucketName,
Prefix: prefix
})
)
// If there are no objects, return
if (!listResponse.Contents || listResponse.Contents.length === 0) {
return
}
// Delete each object
for (const object of listResponse.Contents) {
await this.s3Client!.send(
new DeleteObjectCommand({
Bucket: this.bucketName,
Key: object.Key
})
)
}
}
// Delete all objects in the nouns directory
await deleteObjectsWithPrefix(this.nounPrefix)
// Delete all objects in the verbs directory
await deleteObjectsWithPrefix(this.verbPrefix)
// Delete all objects in the metadata directory
await deleteObjectsWithPrefix(this.metadataPrefix)
// Delete all objects in the index directory
await deleteObjectsWithPrefix(this.indexPrefix)
} catch (error) {
console.error('Failed to clear storage:', error)
throw new Error(`Failed to clear storage: ${error}`)
}
}
/**
* Get information about storage usage and capacity
*/
public async getStorageStatus(): Promise<{
type: string
used: number
quota: number | null
details?: Record<string, any>
}> {
await this.ensureInitialized()
try {
// Import the ListObjectsV2Command only when needed
const { ListObjectsV2Command } = await import('@aws-sdk/client-s3')
// Calculate the total size of all objects in the storage
let totalSize = 0
let nodeCount = 0
let edgeCount = 0
let metadataCount = 0
// Helper function to calculate size and count for a given prefix
const calculateSizeAndCount = async (
prefix: string
): Promise<{ size: number; count: number }> => {
let size = 0
let count = 0
// List all objects with the given prefix
const listResponse = await this.s3Client!.send(
new ListObjectsV2Command({
Bucket: this.bucketName,
Prefix: prefix
})
)
// If there are no objects, return
if (!listResponse.Contents || listResponse.Contents.length === 0) {
return { size, count }
}
// Calculate size and count
for (const object of listResponse.Contents) {
size += object.Size || 0
count++
}
return { size, count }
}
// Calculate size and count for each directory
const nounsResult = await calculateSizeAndCount(this.nounPrefix)
const verbsResult = await calculateSizeAndCount(this.verbPrefix)
const metadataResult = await calculateSizeAndCount(this.metadataPrefix)
const indexResult = await calculateSizeAndCount(this.indexPrefix)
totalSize = nounsResult.size + verbsResult.size + metadataResult.size + indexResult.size
nodeCount = nounsResult.count
edgeCount = verbsResult.count
metadataCount = metadataResult.count
// Count nouns by type using metadata
const nounTypeCounts: Record<string, number> = {}
// List all objects in the metadata directory
const metadataListResponse = await this.s3Client!.send(
new ListObjectsV2Command({
Bucket: this.bucketName,
Prefix: this.metadataPrefix
})
)
if (metadataListResponse.Contents) {
// Import the GetObjectCommand only when needed
const { GetObjectCommand } = await import('@aws-sdk/client-s3')
for (const object of metadataListResponse.Contents) {
try {
// Get the metadata
const response = await this.s3Client!.send(
new GetObjectCommand({
Bucket: this.bucketName,
Key: object.Key
})
)
// Convert the response body to a string
const bodyContents = await response.Body.transformToString()
const metadata = JSON.parse(bodyContents)
// Count by noun type
if (metadata.noun) {
nounTypeCounts[metadata.noun] = (nounTypeCounts[metadata.noun] || 0) + 1
}
} catch (error) {
console.error(`Error getting metadata from ${object.Key}:`, error)
}
}
}
return {
type: this.serviceType,
used: totalSize,
quota: null, // S3-compatible services typically don't provide quota information through the API
details: {
bucketName: this.bucketName,
region: this.region,
endpoint: this.endpoint,
nodeCount,
edgeCount,
metadataCount,
nounTypes: nounTypeCounts
}
}
} catch (error) {
console.error('Failed to get storage status:', error)
return {
type: this.serviceType,
used: 0,
quota: null,
details: { error: String(error) }
}
}
}
}

248
src/storage/baseStorage.ts Normal file
View file

@ -0,0 +1,248 @@
/**
* Base Storage Adapter
* Provides common functionality for all storage adapters
*/
import { GraphVerb, HNSWNoun, StorageAdapter } from '../coreTypes.js'
// Common directory/prefix names
export const NOUNS_DIR = 'nouns'
export const VERBS_DIR = 'verbs'
export const METADATA_DIR = 'metadata'
export const INDEX_DIR = 'index'
/**
* Base storage adapter that implements common functionality
* This is an abstract class that should be extended by specific storage adapters
*/
export abstract class BaseStorage implements StorageAdapter {
protected isInitialized = false
/**
* Initialize the storage adapter
* This method should be implemented by each specific adapter
*/
public abstract init(): Promise<void>
/**
* Ensure the storage adapter is initialized
*/
protected async ensureInitialized(): Promise<void> {
if (!this.isInitialized) {
await this.init()
}
}
/**
* Save a noun to storage
*/
public async saveNoun(noun: HNSWNoun): Promise<void> {
await this.ensureInitialized()
return this.saveNode(noun)
}
/**
* Get a noun from storage
*/
public async getNoun(id: string): Promise<HNSWNoun | null> {
await this.ensureInitialized()
return this.getNode(id)
}
/**
* Get all nouns from storage
*/
public async getAllNouns(): Promise<HNSWNoun[]> {
await this.ensureInitialized()
return this.getAllNodes()
}
/**
* Get nouns by noun type
* @param nounType The noun type to filter by
* @returns Promise that resolves to an array of nouns of the specified noun type
*/
public async getNounsByNounType(nounType: string): Promise<HNSWNoun[]> {
await this.ensureInitialized()
return this.getNodesByNounType(nounType)
}
/**
* Delete a noun from storage
*/
public async deleteNoun(id: string): Promise<void> {
await this.ensureInitialized()
return this.deleteNode(id)
}
/**
* Save a verb to storage
*/
public async saveVerb(verb: GraphVerb): Promise<void> {
await this.ensureInitialized()
return this.saveEdge(verb)
}
/**
* Get a verb from storage
*/
public async getVerb(id: string): Promise<GraphVerb | null> {
await this.ensureInitialized()
return this.getEdge(id)
}
/**
* Get all verbs from storage
*/
public async getAllVerbs(): Promise<GraphVerb[]> {
await this.ensureInitialized()
return this.getAllEdges()
}
/**
* Get verbs by source
*/
public async getVerbsBySource(sourceId: string): Promise<GraphVerb[]> {
await this.ensureInitialized()
return this.getEdgesBySource(sourceId)
}
/**
* Get verbs by target
*/
public async getVerbsByTarget(targetId: string): Promise<GraphVerb[]> {
await this.ensureInitialized()
return this.getEdgesByTarget(targetId)
}
/**
* Get verbs by type
*/
public async getVerbsByType(type: string): Promise<GraphVerb[]> {
await this.ensureInitialized()
return this.getEdgesByType(type)
}
/**
* Delete a verb from storage
*/
public async deleteVerb(id: string): Promise<void> {
await this.ensureInitialized()
return this.deleteEdge(id)
}
/**
* Clear all data from storage
* This method should be implemented by each specific adapter
*/
public abstract clear(): Promise<void>
/**
* Get information about storage usage and capacity
* This method should be implemented by each specific adapter
*/
public abstract getStorageStatus(): Promise<{
type: string
used: number
quota: number | null
details?: Record<string, any>
}>
/**
* Save metadata to storage
* This method should be implemented by each specific adapter
*/
public abstract saveMetadata(id: string, metadata: any): Promise<void>
/**
* Get metadata from storage
* This method should be implemented by each specific adapter
*/
public abstract getMetadata(id: string): Promise<any | null>
/**
* Save a node to storage
* This method should be implemented by each specific adapter
*/
protected abstract saveNode(node: HNSWNoun): Promise<void>
/**
* Get a node from storage
* This method should be implemented by each specific adapter
*/
protected abstract getNode(id: string): Promise<HNSWNoun | null>
/**
* Get all nodes from storage
* This method should be implemented by each specific adapter
*/
protected abstract getAllNodes(): Promise<HNSWNoun[]>
/**
* Get nodes by noun type
* This method should be implemented by each specific adapter
*/
protected abstract getNodesByNounType(nounType: string): Promise<HNSWNoun[]>
/**
* Delete a node from storage
* This method should be implemented by each specific adapter
*/
protected abstract deleteNode(id: string): Promise<void>
/**
* Save an edge to storage
* This method should be implemented by each specific adapter
*/
protected abstract saveEdge(edge: GraphVerb): Promise<void>
/**
* Get an edge from storage
* This method should be implemented by each specific adapter
*/
protected abstract getEdge(id: string): Promise<GraphVerb | null>
/**
* Get all edges from storage
* This method should be implemented by each specific adapter
*/
protected abstract getAllEdges(): Promise<GraphVerb[]>
/**
* Get edges by source
* This method should be implemented by each specific adapter
*/
protected abstract getEdgesBySource(sourceId: string): Promise<GraphVerb[]>
/**
* Get edges by target
* This method should be implemented by each specific adapter
*/
protected abstract getEdgesByTarget(targetId: string): Promise<GraphVerb[]>
/**
* Get edges by type
* This method should be implemented by each specific adapter
*/
protected abstract getEdgesByType(type: string): Promise<GraphVerb[]>
/**
* Delete an edge from storage
* This method should be implemented by each specific adapter
*/
protected abstract deleteEdge(id: string): Promise<void>
/**
* Helper method to convert a Map to a plain object for serialization
*/
protected mapToObject<K extends string | number, V>(
map: Map<K, V>,
valueTransformer: (value: V) => any = (v) => v
): Record<string, any> {
const obj: Record<string, any> = {}
for (const [key, value] of map.entries()) {
obj[key.toString()] = valueTransformer(value)
}
return obj
}
}

165
src/storage/fileSystemStorage.ts
View file

@ -49,14 +49,7 @@ const NOUNS_DIR = 'nouns'
const VERBS_DIR = 'verbs' const VERBS_DIR = 'verbs'
const METADATA_DIR = 'metadata' const METADATA_DIR = 'metadata'
// Constants for noun type directories // All nouns now use the same directory - no separate directories per noun type
const PERSON_DIR = 'person'
const PLACE_DIR = 'place'
const THING_DIR = 'thing'
const EVENT_DIR = 'event'
const CONCEPT_DIR = 'concept'
const CONTENT_DIR = 'content'
const DEFAULT_DIR = 'default' // For nodes without a noun type
/** /**
* File system storage adapter for Node.js environments * File system storage adapter for Node.js environments
@ -66,13 +59,6 @@ export class FileSystemStorage implements StorageAdapter {
private nounsDir: string private nounsDir: string
private verbsDir: string private verbsDir: string
private metadataDir: string private metadataDir: string
private personDir: string
private placeDir: string
private thingDir: string
private eventDir: string
private conceptDir: string
private contentDir: string
private defaultDir: string
private isInitialized = false private isInitialized = false
constructor(rootDirectory?: string) { constructor(rootDirectory?: string) {
@ -81,13 +67,6 @@ export class FileSystemStorage implements StorageAdapter {
this.nounsDir = '' this.nounsDir = ''
this.verbsDir = '' this.verbsDir = ''
this.metadataDir = '' this.metadataDir = ''
this.personDir = ''
this.placeDir = ''
this.thingDir = ''
this.eventDir = ''
this.conceptDir = ''
this.contentDir = ''
this.defaultDir = ''
} }
/** /**
@ -123,27 +102,11 @@ export class FileSystemStorage implements StorageAdapter {
this.verbsDir = path.join(this.rootDir, VERBS_DIR) this.verbsDir = path.join(this.rootDir, VERBS_DIR)
this.metadataDir = path.join(this.rootDir, METADATA_DIR) this.metadataDir = path.join(this.rootDir, METADATA_DIR)
// Set up noun type directory paths
this.personDir = path.join(this.nounsDir, PERSON_DIR)
this.placeDir = path.join(this.nounsDir, PLACE_DIR)
this.thingDir = path.join(this.nounsDir, THING_DIR)
this.eventDir = path.join(this.nounsDir, EVENT_DIR)
this.conceptDir = path.join(this.nounsDir, CONCEPT_DIR)
this.contentDir = path.join(this.nounsDir, CONTENT_DIR)
this.defaultDir = path.join(this.nounsDir, DEFAULT_DIR)
// Create directories if they don't exist // Create directories if they don't exist
await this.ensureDirectoryExists(this.rootDir) await this.ensureDirectoryExists(this.rootDir)
await this.ensureDirectoryExists(this.nounsDir) await this.ensureDirectoryExists(this.nounsDir)
await this.ensureDirectoryExists(this.verbsDir) await this.ensureDirectoryExists(this.verbsDir)
await this.ensureDirectoryExists(this.metadataDir) await this.ensureDirectoryExists(this.metadataDir)
await this.ensureDirectoryExists(this.personDir)
await this.ensureDirectoryExists(this.placeDir)
await this.ensureDirectoryExists(this.thingDir)
await this.ensureDirectoryExists(this.eventDir)
await this.ensureDirectoryExists(this.conceptDir)
await this.ensureDirectoryExists(this.contentDir)
await this.ensureDirectoryExists(this.defaultDir)
this.isInitialized = true this.isInitialized = true
} catch (error: any) { } catch (error: any) {
@ -164,32 +127,8 @@ export class FileSystemStorage implements StorageAdapter {
} }
private getNounPath(id: string, nounType?: string): string { private getNounPath(id: string, nounType?: string): string {
let typeDir = this.defaultDir // All nouns now use the same directory regardless of type
if (nounType) { return path.join(this.nounsDir, `${id}.json`)
switch (nounType.toLowerCase()) {
case 'person':
typeDir = this.personDir
break
case 'place':
typeDir = this.placeDir
break
case 'thing':
typeDir = this.thingDir
break
case 'event':
typeDir = this.eventDir
break
case 'concept':
typeDir = this.conceptDir
break
case 'content':
typeDir = this.contentDir
break
default:
typeDir = this.defaultDir
}
}
return path.join(typeDir, `${id}.json`)
} }
public async saveNoun( public async saveNoun(
@ -204,27 +143,16 @@ export class FileSystemStorage implements StorageAdapter {
public async getNoun(id: string): Promise<HNSWNoun | null> { public async getNoun(id: string): Promise<HNSWNoun | null> {
if (!this.isInitialized) await this.init() if (!this.isInitialized) await this.init()
const nounDirs = [ const filePath = path.join(this.nounsDir, `${id}.json`)
this.personDir, try {
this.placeDir, const data = await fs.promises.readFile(filePath, 'utf-8')
this.thingDir, return JSON.parse(data)
this.eventDir, } catch (error: any) {
this.conceptDir, if (error.code !== 'ENOENT') {
this.contentDir, console.error(`Error reading noun ${id}:`, error)
this.defaultDir
]
for (const dir of nounDirs) {
const filePath = path.join(dir, `${id}.json`)
try {
const data = await fs.promises.readFile(filePath, 'utf-8')
return JSON.parse(data)
} catch (error: any) {
if (error.code !== 'ENOENT') {
console.error(`Error reading noun ${id}:`, error)
}
} }
return null
} }
return null
} }
public async deleteNoun(id: string): Promise<void> { public async deleteNoun(id: string): Promise<void> {
@ -247,30 +175,19 @@ export class FileSystemStorage implements StorageAdapter {
public async getAllNouns(): Promise<HNSWNoun[]> { public async getAllNouns(): Promise<HNSWNoun[]> {
if (!this.isInitialized) await this.init() if (!this.isInitialized) await this.init()
const allNouns: HNSWNoun[] = [] const allNouns: HNSWNoun[] = []
const nounDirs = [ try {
this.personDir, const files = await fs.promises.readdir(this.nounsDir)
this.placeDir, for (const file of files) {
this.thingDir, if (file.endsWith('.json')) {
this.eventDir, const filePath = path.join(this.nounsDir, file)
this.conceptDir, const data = await fs.promises.readFile(filePath, 'utf-8')
this.contentDir, allNouns.push(JSON.parse(data))
this.defaultDir
]
for (const dir of nounDirs) {
try {
const files = await fs.promises.readdir(dir)
for (const file of files) {
if (file.endsWith('.json')) {
const filePath = path.join(dir, file)
const data = await fs.promises.readFile(filePath, 'utf-8')
allNouns.push(JSON.parse(data))
}
}
} catch (error: any) {
if (error.code !== 'ENOENT') {
console.error(`Error reading directory ${dir}:`, error)
} }
} }
} catch (error: any) {
if (error.code !== 'ENOENT') {
console.error(`Error reading directory ${this.nounsDir}:`, error)
}
} }
return allNouns return allNouns
} }
@ -283,43 +200,25 @@ export class FileSystemStorage implements StorageAdapter {
public async getNounsByNounType(nounType: string): Promise<HNSWNoun[]> { public async getNounsByNounType(nounType: string): Promise<HNSWNoun[]> {
if (!this.isInitialized) await this.init() if (!this.isInitialized) await this.init()
let typeDir: string
switch (nounType.toLowerCase()) {
case 'person':
typeDir = this.personDir
break
case 'place':
typeDir = this.placeDir
break
case 'thing':
typeDir = this.thingDir
break
case 'event':
typeDir = this.eventDir
break
case 'concept':
typeDir = this.conceptDir
break
case 'content':
typeDir = this.contentDir
break
default:
typeDir = this.defaultDir
}
const nouns: HNSWNoun[] = [] const nouns: HNSWNoun[] = []
try { try {
const files = await fs.promises.readdir(typeDir) const files = await fs.promises.readdir(this.nounsDir)
for (const file of files) { for (const file of files) {
if (file.endsWith('.json')) { if (file.endsWith('.json')) {
const filePath = path.join(typeDir, file) const filePath = path.join(this.nounsDir, file)
const data = await fs.promises.readFile(filePath, 'utf-8') const data = await fs.promises.readFile(filePath, 'utf-8')
nouns.push(JSON.parse(data)) const noun = JSON.parse(data)
// Filter by noun type using metadata
const metadata = await this.getMetadata(noun.id)
if (metadata && metadata.noun === nounType) {
nouns.push(noun)
}
} }
} }
} catch (error: any) { } catch (error: any) {
if (error.code !== 'ENOENT') { if (error.code !== 'ENOENT') {
console.error(`Error reading directory ${typeDir}:`, error) console.error(`Error reading directory ${this.nounsDir}:`, error)
} }
} }

3197
src/storage/opfsStorage.ts
View file

File diff suppressed because it is too large Load diff

414
src/storage/s3CompatibleStorage.ts
View file

@ -13,14 +13,8 @@ const VERBS_PREFIX = 'verbs/'
const EDGES_PREFIX = 'verbs/' // Alias for VERBS_PREFIX for edge operations const EDGES_PREFIX = 'verbs/' // Alias for VERBS_PREFIX for edge operations
const METADATA_PREFIX = 'metadata/' const METADATA_PREFIX = 'metadata/'
// Constants for noun type prefixes // All nouns now use the same prefix - no separate directories per noun type
const PERSON_PREFIX = 'nouns/person/' const NOUN_PREFIX = 'nouns/' // Single directory for all noun types
const PLACE_PREFIX = 'place/'
const THING_PREFIX = 'thing/'
const EVENT_PREFIX = 'event/'
const CONCEPT_PREFIX = 'concept/'
const CONTENT_PREFIX = 'content/'
const DEFAULT_PREFIX = 'default/' // For nodes without a noun type
/** /**
* S3-compatible storage adapter for server environments * S3-compatible storage adapter for server environments
@ -257,45 +251,86 @@ export class S3CompatibleStorage implements StorageAdapter {
await this.ensureInitialized() await this.ensureInitialized()
try { try {
// Get the appropriate prefix based on the node's metadata
const nodePrefix = await this.getNodePrefix(id)
// Import the GetObjectCommand only when needed // Import the GetObjectCommand only when needed
const { GetObjectCommand } = await import('@aws-sdk/client-s3') const { GetObjectCommand } = await import('@aws-sdk/client-s3')
try { // Try to get the node from the consolidated nouns directory
// Try to get the node from S3-compatible storage const response = await this.s3Client.send(
const response = await this.s3Client.send( new GetObjectCommand({
new GetObjectCommand({ Bucket: this.bucketName,
Bucket: this.bucketName, Key: `${NOUN_PREFIX}${id}.json`
Key: `${nodePrefix}${id}.json` })
}) )
)
// Convert the response body to a string // Convert the response body to a string
const bodyContents = await response.Body.transformToString() const bodyContents = await response.Body.transformToString()
const parsedNode = JSON.parse(bodyContents) const parsedNode = JSON.parse(bodyContents)
// Convert serialized connections back to Map<number, Set<string>> // Convert serialized connections back to Map<number, Set<string>>
const connections = new Map<number, Set<string>>() const connections = new Map<number, Set<string>>()
for (const [level, nodeIds] of Object.entries(parsedNode.connections)) { for (const [level, nodeIds] of Object.entries(parsedNode.connections)) {
connections.set(Number(level), new Set(nodeIds as string[])) connections.set(Number(level), new Set(nodeIds as string[]))
} }
return { return {
id: parsedNode.id, id: parsedNode.id,
vector: parsedNode.vector, vector: parsedNode.vector,
connections connections
} }
} catch (error) { } catch (error) {
// If the node is not found in the expected prefix, try other prefixes // Node not found or other error
if (nodePrefix !== DEFAULT_PREFIX) { return null
// Try the default prefix }
}
/**
* Get nodes by noun type
* @param nounType The noun type to filter by
* @returns Promise that resolves to an array of nodes of the specified noun type
*/
public async getNodesByNounType(nounType: string): Promise<HNSWNode[]> {
await this.ensureInitialized()
try {
// Import the ListObjectsV2Command and GetObjectCommand only when needed
const { ListObjectsV2Command, GetObjectCommand } = await import(
'@aws-sdk/client-s3'
)
// List all objects in the consolidated nouns directory
const listResponse = await this.s3Client.send(
new ListObjectsV2Command({
Bucket: this.bucketName,
Prefix: NOUN_PREFIX
})
)
const nodes: HNSWNode[] = []
// If there are no objects, return an empty array
if (!listResponse.Contents || listResponse.Contents.length === 0) {
return nodes
}
// Get each node and filter by noun type
const nodePromises = listResponse.Contents.map(
async (object: { Key: string }) => {
try { try {
// Extract node ID from the key (remove prefix and .json extension)
const nodeId = object.Key.replace(NOUN_PREFIX, '').replace('.json', '')
// Get the metadata to check the noun type
const metadata = await this.getMetadata(nodeId)
// Skip if metadata doesn't exist or noun type doesn't match
if (!metadata || metadata.noun !== nounType) {
return null
}
const response = await this.s3Client.send( const response = await this.s3Client.send(
new GetObjectCommand({ new GetObjectCommand({
Bucket: this.bucketName, Bucket: this.bucketName,
Key: `${NOUNS_PREFIX}${DEFAULT_PREFIX}${id}.json` Key: object.Key
}) })
) )
@ -315,103 +350,38 @@ export class S3CompatibleStorage implements StorageAdapter {
vector: parsedNode.vector, vector: parsedNode.vector,
connections connections
} }
} catch { } catch (error) {
// If not found in default prefix, try all other prefixes console.error(`Failed to get node from ${object.Key}:`, error)
const prefixes = [ return null
PERSON_PREFIX,
PLACE_PREFIX,
THING_PREFIX,
EVENT_PREFIX,
CONCEPT_PREFIX,
CONTENT_PREFIX
]
for (const prefix of prefixes) {
if (prefix === nodePrefix) continue // Skip the already checked prefix
try {
const response = await this.s3Client.send(
new GetObjectCommand({
Bucket: this.bucketName,
Key: `${NOUNS_PREFIX}${prefix}${id}.json`
})
)
const bodyContents = await response.Body.transformToString()
const parsedNode = JSON.parse(bodyContents)
// Convert serialized connections back to Map<number, Set<string>>
const connections = new Map<number, Set<string>>()
for (const [level, nodeIds] of Object.entries(
parsedNode.connections
)) {
connections.set(Number(level), new Set(nodeIds as string[]))
}
return {
id: parsedNode.id,
vector: parsedNode.vector,
connections
}
} catch {
// Continue to the next prefix
}
}
} }
} }
)
return null // Node not found in any prefix const nodeResults = await Promise.all(nodePromises)
} return nodeResults.filter((node): node is HNSWNode => node !== null)
} catch (error) { } catch (error) {
console.error(`Failed to get node ${id}:`, error) console.error(`Failed to get nodes for noun type ${nounType}:`, error)
return null throw new Error(`Failed to get nodes for noun type ${nounType}: ${error}`)
} }
} }
/** /**
* Get nodes by noun type * Get all nodes from storage
* @param nounType The noun type to filter by
* @returns Promise that resolves to an array of nodes of the specified noun type
*/ */
public async getNodesByNounType(nounType: string): Promise<HNSWNode[]> { public async getAllNodes(): Promise<HNSWNode[]> {
await this.ensureInitialized() await this.ensureInitialized()
try { try {
// Determine the prefix based on the noun type
let prefix: string
switch (nounType) {
case 'person':
prefix = PERSON_PREFIX
break
case 'place':
prefix = PLACE_PREFIX
break
case 'thing':
prefix = THING_PREFIX
break
case 'event':
prefix = EVENT_PREFIX
break
case 'concept':
prefix = CONCEPT_PREFIX
break
case 'content':
prefix = CONTENT_PREFIX
break
default:
prefix = DEFAULT_PREFIX
}
// Import the ListObjectsV2Command and GetObjectCommand only when needed // Import the ListObjectsV2Command and GetObjectCommand only when needed
const { ListObjectsV2Command, GetObjectCommand } = await import( const { ListObjectsV2Command, GetObjectCommand } = await import(
'@aws-sdk/client-s3' '@aws-sdk/client-s3'
) )
// List all objects with the specified prefix // List all objects in the consolidated nouns directory
const listResponse = await this.s3Client.send( const listResponse = await this.s3Client.send(
new ListObjectsV2Command({ new ListObjectsV2Command({
Bucket: this.bucketName, Bucket: this.bucketName,
Prefix: `${NOUNS_PREFIX}${prefix}` Prefix: NOUN_PREFIX
}) })
) )
@ -458,43 +428,6 @@ export class S3CompatibleStorage implements StorageAdapter {
const nodeResults = await Promise.all(nodePromises) const nodeResults = await Promise.all(nodePromises)
return nodeResults.filter((node): node is HNSWNode => node !== null) return nodeResults.filter((node): node is HNSWNode => node !== null)
} catch (error) {
console.error(`Failed to get nodes for noun type ${nounType}:`, error)
throw new Error(`Failed to get nodes for noun type ${nounType}: ${error}`)
}
}
/**
* Get all nodes from storage
*/
public async getAllNodes(): Promise<HNSWNode[]> {
await this.ensureInitialized()
try {
// Get all noun types
const nounTypes = [
'person',
'place',
'thing',
'event',
'concept',
'content',
'default'
]
// Run searches in parallel for all noun types
const nodePromises = nounTypes.map((nounType) =>
this.getNodesByNounType(nounType)
)
const nodeArrays = await Promise.all(nodePromises)
// Combine all results
const allNodes: HNSWNode[] = []
for (const nodes of nodeArrays) {
allNodes.push(...nodes)
}
return allNodes
} catch (error) { } catch (error) {
console.error('Failed to get all nodes:', error) console.error('Failed to get all nodes:', error)
throw new Error(`Failed to get all nodes: ${error}`) throw new Error(`Failed to get all nodes: ${error}`)
@ -508,90 +441,16 @@ export class S3CompatibleStorage implements StorageAdapter {
await this.ensureInitialized() await this.ensureInitialized()
try { try {
// Get the appropriate prefix based on the node's metadata
const nodePrefix = await this.getNodePrefix(id)
// Import the DeleteObjectCommand only when needed // Import the DeleteObjectCommand only when needed
const { DeleteObjectCommand, GetObjectCommand } = await import( const { DeleteObjectCommand } = await import('@aws-sdk/client-s3')
'@aws-sdk/client-s3'
// Delete the node from the consolidated nouns directory
await this.s3Client.send(
new DeleteObjectCommand({
Bucket: this.bucketName,
Key: `${NOUN_PREFIX}${id}.json`
})
) )
try {
// Check if the node exists before deleting
await this.s3Client.send(
new GetObjectCommand({
Bucket: this.bucketName,
Key: `${nodePrefix}${id}.json`
})
)
// Delete the node
await this.s3Client.send(
new DeleteObjectCommand({
Bucket: this.bucketName,
Key: `${nodePrefix}${id}.json`
})
)
return // Node found and deleted
} catch {
// If the node is not found in the expected prefix, try other prefixes
if (nodePrefix !== DEFAULT_PREFIX) {
try {
// Try the default prefix
await this.s3Client.send(
new GetObjectCommand({
Bucket: this.bucketName,
Key: `${NOUNS_PREFIX}${DEFAULT_PREFIX}${id}.json`
})
)
// Delete the node
await this.s3Client.send(
new DeleteObjectCommand({
Bucket: this.bucketName,
Key: `${NOUNS_PREFIX}${DEFAULT_PREFIX}${id}.json`
})
)
return // Node found and deleted
} catch {
// If not found in default prefix, try all other prefixes
const prefixes = [
PERSON_PREFIX,
PLACE_PREFIX,
THING_PREFIX,
EVENT_PREFIX,
CONCEPT_PREFIX,
CONTENT_PREFIX
]
for (const prefix of prefixes) {
if (prefix === nodePrefix) continue // Skip the already checked prefix
try {
await this.s3Client.send(
new GetObjectCommand({
Bucket: this.bucketName,
Key: `${NOUNS_PREFIX}${prefix}${id}.json`
})
)
// Delete the node
await this.s3Client.send(
new DeleteObjectCommand({
Bucket: this.bucketName,
Key: `${NOUNS_PREFIX}${prefix}${id}.json`
})
)
return // Node found and deleted
} catch {
// Continue to the next prefix
}
}
}
}
return // Node not found in any prefix, nothing to delete
}
} catch (error) { } catch (error) {
console.error(`Failed to delete node ${id}:`, error) console.error(`Failed to delete node ${id}:`, error)
throw new Error(`Failed to delete node ${id}: ${error}`) throw new Error(`Failed to delete node ${id}: ${error}`)
@ -982,7 +841,7 @@ export class S3CompatibleStorage implements StorageAdapter {
} }
} }
// Count nodes by noun type // Count nodes by noun type by examining metadata
const nounTypeCounts: Record<string, number> = { const nounTypeCounts: Record<string, number> = {
person: 0, person: 0,
place: 0, place: 0,
@ -990,29 +849,41 @@ export class S3CompatibleStorage implements StorageAdapter {
event: 0, event: 0,
concept: 0, concept: 0,
content: 0, content: 0,
group: 0,
list: 0,
category: 0,
default: 0 default: 0
} }
// List objects for each noun type prefix // List all noun objects and count by type using metadata
const nounTypes = [ const nounsListResponse = await this.s3Client.send(
{ type: 'person', prefix: PERSON_PREFIX }, new ListObjectsV2Command({
{ type: 'place', prefix: PLACE_PREFIX }, Bucket: this.bucketName,
{ type: 'thing', prefix: THING_PREFIX }, Prefix: NOUN_PREFIX
{ type: 'event', prefix: EVENT_PREFIX }, })
{ type: 'concept', prefix: CONCEPT_PREFIX }, )
{ type: 'content', prefix: CONTENT_PREFIX },
{ type: 'default', prefix: DEFAULT_PREFIX }
]
for (const { type, prefix } of nounTypes) { if (nounsListResponse.Contents) {
const listResponse = await this.s3Client.send( for (const object of nounsListResponse.Contents) {
new ListObjectsV2Command({ try {
Bucket: this.bucketName, // Extract node ID from the key
Prefix: `${NOUNS_PREFIX}${prefix}` const nodeId = object.Key?.replace(NOUN_PREFIX, '').replace('.json', '')
}) if (nodeId) {
) // Get metadata to determine noun type
const metadata = await this.getMetadata(nodeId)
nounTypeCounts[type] = listResponse.Contents?.length || 0 const nounType = metadata?.noun || 'default'
if (nounType in nounTypeCounts) {
nounTypeCounts[nounType]++
} else {
nounTypeCounts.default++
}
}
} catch (error) {
// If we can't get metadata, count as default
nounTypeCounts.default++
}
}
} }
return { return {
@ -1030,6 +901,9 @@ export class S3CompatibleStorage implements StorageAdapter {
event: { count: nounTypeCounts.event }, event: { count: nounTypeCounts.event },
concept: { count: nounTypeCounts.concept }, concept: { count: nounTypeCounts.concept },
content: { count: nounTypeCounts.content }, content: { count: nounTypeCounts.content },
group: { count: nounTypeCounts.group },
list: { count: nounTypeCounts.list },
category: { count: nounTypeCounts.category },
default: { count: nounTypeCounts.default } default: { count: nounTypeCounts.default }
} }
} }
@ -1055,39 +929,11 @@ export class S3CompatibleStorage implements StorageAdapter {
} }
/** /**
* Get the appropriate prefix for a node based on its metadata * Get the appropriate prefix for a node - now all nouns use the same prefix
*/ */
private async getNodePrefix(id: string): Promise<string> { private async getNodePrefix(id: string): Promise<string> {
try { // All nouns now use the same prefix regardless of type
// Try to get the metadata for the node return NOUN_PREFIX
const metadata = await this.getMetadata(id)
// If metadata exists and has a noun field, use the corresponding prefix
if (metadata && metadata.noun) {
switch (metadata.noun) {
case 'person':
return PERSON_PREFIX
case 'place':
return PLACE_PREFIX
case 'thing':
return THING_PREFIX
case 'event':
return EVENT_PREFIX
case 'concept':
return CONCEPT_PREFIX
case 'content':
return CONTENT_PREFIX
default:
return DEFAULT_PREFIX
}
}
// If no metadata or no noun field, use the default prefix
return DEFAULT_PREFIX
} catch (error) {
// If there's an error getting the metadata, use the default prefix
return DEFAULT_PREFIX
}
} }
/** /**

365
src/storage/storageFactory.ts Normal file
View file

@ -0,0 +1,365 @@
/**
* Storage Factory
* Creates the appropriate storage adapter based on the environment and configuration
*/
import { StorageAdapter } from '../coreTypes.js'
import { MemoryStorage } from './adapters/memoryStorage.js'
import { OPFSStorage } from './adapters/opfsStorage.js'
import { FileSystemStorage } from './adapters/fileSystemStorage.js'
import { S3CompatibleStorage, R2Storage } from './adapters/s3CompatibleStorage.js'
/**
* Options for creating a storage adapter
*/
export interface StorageOptions {
/**
* The type of storage to use
* - 'auto': Automatically select the best storage adapter based on the environment
* - 'memory': Use in-memory storage
* - 'opfs': Use Origin Private File System storage (browser only)
* - 'filesystem': Use file system storage (Node.js only)
* - 's3': Use Amazon S3 storage
* - 'r2': Use Cloudflare R2 storage
* - 'gcs': Use Google Cloud Storage
*/
type?: 'auto' | 'memory' | 'opfs' | 'filesystem' | 's3' | 'r2' | 'gcs'
/**
* Force the use of memory storage even if other storage types are available
*/
forceMemoryStorage?: boolean
/**
* Force the use of file system storage even if other storage types are available
*/
forceFileSystemStorage?: boolean
/**
* Request persistent storage permission from the user (browser only)
*/
requestPersistentStorage?: boolean
/**
* Root directory for file system storage (Node.js only)
*/
rootDirectory?: string
/**
* Configuration for Amazon S3 storage
*/
s3Storage?: {
/**
* S3 bucket name
*/
bucketName: string
/**
* AWS region (e.g., 'us-east-1')
*/
region?: string
/**
* AWS access key ID
*/
accessKeyId: string
/**
* AWS secret access key
*/
secretAccessKey: string
/**
* AWS session token (optional)
*/
sessionToken?: string
}
/**
* Configuration for Cloudflare R2 storage
*/
r2Storage?: {
/**
* R2 bucket name
*/
bucketName: string
/**
* Cloudflare account ID
*/
accountId: string
/**
* R2 access key ID
*/
accessKeyId: string
/**
* R2 secret access key
*/
secretAccessKey: string
}
/**
* Configuration for Google Cloud Storage
*/
gcsStorage?: {
/**
* GCS bucket name
*/
bucketName: string
/**
* GCS region (e.g., 'us-central1')
*/
region?: string
/**
* GCS access key ID
*/
accessKeyId: string
/**
* GCS secret access key
*/
secretAccessKey: string
/**
* GCS endpoint (e.g., 'https://storage.googleapis.com')
*/
endpoint?: string
}
/**
* Configuration for custom S3-compatible storage
*/
customS3Storage?: {
/**
* S3-compatible bucket name
*/
bucketName: string
/**
* S3-compatible region
*/
region?: string
/**
* S3-compatible endpoint URL
*/
endpoint: string
/**
* S3-compatible access key ID
*/
accessKeyId: string
/**
* S3-compatible secret access key
*/
secretAccessKey: string
/**
* S3-compatible service type (for logging and error messages)
*/
serviceType?: string
}
}
/**
* Create a storage adapter based on the environment and configuration
* @param options Options for creating the storage adapter
* @returns Promise that resolves to a storage adapter
*/
export async function createStorage(
options: StorageOptions = {}
): Promise<StorageAdapter> {
// If memory storage is forced, use it regardless of other options
if (options.forceMemoryStorage) {
console.log('Using memory storage (forced)')
return new MemoryStorage()
}
// If file system storage is forced, use it regardless of other options
if (options.forceFileSystemStorage) {
console.log('Using file system storage (forced)')
return new FileSystemStorage(options.rootDirectory || './brainy-data')
}
// If a specific storage type is specified, use it
if (options.type && options.type !== 'auto') {
switch (options.type) {
case 'memory':
console.log('Using memory storage')
return new MemoryStorage()
case 'opfs': {
// Check if OPFS is available
const opfsStorage = new OPFSStorage()
if (opfsStorage.isOPFSAvailable()) {
console.log('Using OPFS storage')
await opfsStorage.init()
// Request persistent storage if specified
if (options.requestPersistentStorage) {
const isPersistent = await opfsStorage.requestPersistentStorage()
console.log(`Persistent storage ${isPersistent ? 'granted' : 'denied'}`)
}
return opfsStorage
} else {
console.warn('OPFS storage is not available, falling back to memory storage')
return new MemoryStorage()
}
}
case 'filesystem':
console.log('Using file system storage')
return new FileSystemStorage(options.rootDirectory || './brainy-data')
case 's3':
if (options.s3Storage) {
console.log('Using Amazon S3 storage')
return new S3CompatibleStorage({
bucketName: options.s3Storage.bucketName,
region: options.s3Storage.region,
accessKeyId: options.s3Storage.accessKeyId,
secretAccessKey: options.s3Storage.secretAccessKey,
sessionToken: options.s3Storage.sessionToken,
serviceType: 's3'
})
} else {
console.warn('S3 storage configuration is missing, falling back to memory storage')
return new MemoryStorage()
}
case 'r2':
if (options.r2Storage) {
console.log('Using Cloudflare R2 storage')
return new R2Storage({
bucketName: options.r2Storage.bucketName,
accountId: options.r2Storage.accountId,
accessKeyId: options.r2Storage.accessKeyId,
secretAccessKey: options.r2Storage.secretAccessKey,
serviceType: 'r2'
})
} else {
console.warn('R2 storage configuration is missing, falling back to memory storage')
return new MemoryStorage()
}
case 'gcs':
if (options.gcsStorage) {
console.log('Using Google Cloud Storage')
return new S3CompatibleStorage({
bucketName: options.gcsStorage.bucketName,
region: options.gcsStorage.region,
endpoint: options.gcsStorage.endpoint || 'https://storage.googleapis.com',
accessKeyId: options.gcsStorage.accessKeyId,
secretAccessKey: options.gcsStorage.secretAccessKey,
serviceType: 'gcs'
})
} else {
console.warn('GCS storage configuration is missing, falling back to memory storage')
return new MemoryStorage()
}
default:
console.warn(`Unknown storage type: ${options.type}, falling back to memory storage`)
return new MemoryStorage()
}
}
// If custom S3-compatible storage is specified, use it
if (options.customS3Storage) {
console.log(`Using custom S3-compatible storage: ${options.customS3Storage.serviceType || 'custom'}`)
return new S3CompatibleStorage({
bucketName: options.customS3Storage.bucketName,
region: options.customS3Storage.region,
endpoint: options.customS3Storage.endpoint,
accessKeyId: options.customS3Storage.accessKeyId,
secretAccessKey: options.customS3Storage.secretAccessKey,
serviceType: options.customS3Storage.serviceType || 'custom'
})
}
// If R2 storage is specified, use it
if (options.r2Storage) {
console.log('Using Cloudflare R2 storage')
return new R2Storage({
bucketName: options.r2Storage.bucketName,
accountId: options.r2Storage.accountId,
accessKeyId: options.r2Storage.accessKeyId,
secretAccessKey: options.r2Storage.secretAccessKey,
serviceType: 'r2'
})
}
// If S3 storage is specified, use it
if (options.s3Storage) {
console.log('Using Amazon S3 storage')
return new S3CompatibleStorage({
bucketName: options.s3Storage.bucketName,
region: options.s3Storage.region,
accessKeyId: options.s3Storage.accessKeyId,
secretAccessKey: options.s3Storage.secretAccessKey,
sessionToken: options.s3Storage.sessionToken,
serviceType: 's3'
})
}
// If GCS storage is specified, use it
if (options.gcsStorage) {
console.log('Using Google Cloud Storage')
return new S3CompatibleStorage({
bucketName: options.gcsStorage.bucketName,
region: options.gcsStorage.region,
endpoint: options.gcsStorage.endpoint || 'https://storage.googleapis.com',
accessKeyId: options.gcsStorage.accessKeyId,
secretAccessKey: options.gcsStorage.secretAccessKey,
serviceType: 'gcs'
})
}
// Auto-detect the best storage adapter based on the environment
// First, try OPFS (browser only)
const opfsStorage = new OPFSStorage()
if (opfsStorage.isOPFSAvailable()) {
console.log('Using OPFS storage (auto-detected)')
await opfsStorage.init()
// Request persistent storage if specified
if (options.requestPersistentStorage) {
const isPersistent = await opfsStorage.requestPersistentStorage()
console.log(`Persistent storage ${isPersistent ? 'granted' : 'denied'}`)
}
return opfsStorage
}
// Next, try file system storage (Node.js only)
try {
// Check if we're in a Node.js environment
if (typeof process !== 'undefined' && process.versions && process.versions.node) {
console.log('Using file system storage (auto-detected)')
return new FileSystemStorage(options.rootDirectory || './brainy-data')
}
} catch (error) {
// Not in a Node.js environment or file system is not available
}
// Finally, fall back to memory storage
console.log('Using memory storage (auto-detected)')
return new MemoryStorage()
}
/**
* Export all storage adapters
*/
export {
MemoryStorage,
OPFSStorage,
FileSystemStorage,
S3CompatibleStorage,
R2Storage
}