**feat(storage): remove FileSystemStorage and OPFSStorage implementations**
- **Removal**: Deleted `FileSystemStorage` and `OPFSStorage` adapters from the `/storage` directory. - **Refactor**: Simplifies the codebase by eliminating unused, redundant, or outdated storage implementations. - **Impact**: Ensures maintainability by focusing on actively supported storage solutions. **Purpose**: Streamline the project by removing deprecated or unused storage adapters, reducing complexity and maintenance overhead.
This commit is contained in:
parent
85e5a6bfb9
commit
68247c9a6c
3 changed files with 0 additions and 2953 deletions
|
|
@ -1,451 +0,0 @@
|
|||
import { GraphVerb, HNSWNoun, StorageAdapter } from '../coreTypes.js'
|
||||
|
||||
// Dynamically and asynchronously load Node.js modules at the top level.
|
||||
// This ensures they are available as soon as this module is imported,
|
||||
// preventing race conditions with dependencies like TensorFlow.js.
|
||||
let fs: any
|
||||
let path: any
|
||||
|
||||
const nodeModulesPromise = (async () => {
|
||||
// A reliable check for a Node.js environment.
|
||||
const isNode =
|
||||
typeof process !== 'undefined' &&
|
||||
process.versions != null &&
|
||||
process.versions.node != null
|
||||
|
||||
if (!isNode) {
|
||||
return { fs: null, path: null }
|
||||
}
|
||||
|
||||
try {
|
||||
// Use the 'node:' prefix for unambiguous importing of built-in modules.
|
||||
const fsModule = await import('node:fs')
|
||||
const pathModule = await import('node:path')
|
||||
// Return the modules, preferring the default export if it exists.
|
||||
return {
|
||||
fs: fsModule.default || fsModule,
|
||||
path: pathModule.default || pathModule
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(
|
||||
'FileSystemStorage: Failed to load Node.js modules. This adapter is not supported in this environment.',
|
||||
error
|
||||
)
|
||||
return { fs: null, path: null }
|
||||
}
|
||||
})()
|
||||
|
||||
// Immediately assign the modules once the promise resolves.
|
||||
nodeModulesPromise.then((modules) => {
|
||||
fs = modules.fs
|
||||
path = modules.path
|
||||
})
|
||||
|
||||
// --- End of Refactored Code ---
|
||||
|
||||
// Constants for directory and file names
|
||||
const ROOT_DIR = 'brainy-data'
|
||||
const NOUNS_DIR = 'nouns'
|
||||
const VERBS_DIR = 'verbs'
|
||||
const METADATA_DIR = 'metadata'
|
||||
|
||||
// All nouns now use the same directory - no separate directories per noun type
|
||||
|
||||
/**
|
||||
* File system storage adapter for Node.js environments
|
||||
*/
|
||||
export class FileSystemStorage implements StorageAdapter {
|
||||
private rootDir: string
|
||||
private nounsDir: string
|
||||
private verbsDir: string
|
||||
private metadataDir: string
|
||||
private isInitialized = false
|
||||
|
||||
constructor(rootDirectory?: string) {
|
||||
// We'll set the paths in the init method after dynamically importing the modules
|
||||
this.rootDir = rootDirectory || ''
|
||||
this.nounsDir = ''
|
||||
this.verbsDir = ''
|
||||
this.metadataDir = ''
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the storage adapter
|
||||
*/
|
||||
public async init(): Promise<void> {
|
||||
if (this.isInitialized) {
|
||||
return
|
||||
}
|
||||
|
||||
// Wait for the top-level module loading to complete.
|
||||
await nodeModulesPromise
|
||||
|
||||
// Check if the modules were loaded successfully.
|
||||
if (!fs || !path) {
|
||||
throw new Error(
|
||||
'FileSystemStorage requires a Node.js environment, but `fs` and `path` modules could not be loaded.'
|
||||
)
|
||||
}
|
||||
|
||||
try {
|
||||
// Now set up the directory paths
|
||||
const rootDir = this.rootDir || process.cwd()
|
||||
|
||||
// Check if rootDir already ends with ROOT_DIR to prevent duplication
|
||||
if (rootDir.endsWith(ROOT_DIR)) {
|
||||
this.rootDir = rootDir
|
||||
} else {
|
||||
this.rootDir = path.resolve(rootDir, ROOT_DIR)
|
||||
}
|
||||
|
||||
this.nounsDir = path.join(this.rootDir, NOUNS_DIR)
|
||||
this.verbsDir = path.join(this.rootDir, VERBS_DIR)
|
||||
this.metadataDir = path.join(this.rootDir, METADATA_DIR)
|
||||
|
||||
// Create directories if they don't exist
|
||||
await this.ensureDirectoryExists(this.rootDir)
|
||||
await this.ensureDirectoryExists(this.nounsDir)
|
||||
await this.ensureDirectoryExists(this.verbsDir)
|
||||
await this.ensureDirectoryExists(this.metadataDir)
|
||||
|
||||
this.isInitialized = true
|
||||
} catch (error: any) {
|
||||
console.error('Error initializing FileSystemStorage:', error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
private async ensureDirectoryExists(dirPath: string): Promise<void> {
|
||||
try {
|
||||
await fs.promises.mkdir(dirPath, { recursive: true })
|
||||
} catch (error: any) {
|
||||
// Ignore EEXIST error, which means the directory already exists
|
||||
if (error.code !== 'EEXIST') {
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private getNounPath(id: string, nounType?: string): string {
|
||||
// All nouns now use the same directory regardless of type
|
||||
return path.join(this.nounsDir, `${id}.json`)
|
||||
}
|
||||
|
||||
public async saveNoun(
|
||||
noun: HNSWNoun & { metadata?: { noun?: string } }
|
||||
): Promise<void> {
|
||||
if (!this.isInitialized) await this.init()
|
||||
const nounType = (noun as any).metadata?.noun
|
||||
const filePath = this.getNounPath(noun.id, nounType)
|
||||
await this.ensureDirectoryExists(path.dirname(filePath))
|
||||
await fs.promises.writeFile(filePath, JSON.stringify(noun, null, 2))
|
||||
}
|
||||
|
||||
public async getNoun(id: string): Promise<HNSWNoun | null> {
|
||||
if (!this.isInitialized) await this.init()
|
||||
const filePath = path.join(this.nounsDir, `${id}.json`)
|
||||
try {
|
||||
const data = await fs.promises.readFile(filePath, 'utf-8')
|
||||
return JSON.parse(data)
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error reading noun ${id}:`, error)
|
||||
}
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
public async deleteNoun(id: string): Promise<void> {
|
||||
if (!this.isInitialized) await this.init()
|
||||
const noun = await this.getNoun(id)
|
||||
if (noun) {
|
||||
const nounType = (noun as any).metadata?.noun
|
||||
const filePath = this.getNounPath(id, nounType)
|
||||
try {
|
||||
await fs.promises.unlink(filePath)
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error deleting noun file ${filePath}:`, error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public async getAllNouns(): Promise<HNSWNoun[]> {
|
||||
if (!this.isInitialized) await this.init()
|
||||
const allNouns: HNSWNoun[] = []
|
||||
try {
|
||||
const files = await fs.promises.readdir(this.nounsDir)
|
||||
for (const file of files) {
|
||||
if (file.endsWith('.json')) {
|
||||
const filePath = path.join(this.nounsDir, file)
|
||||
const data = await fs.promises.readFile(filePath, 'utf-8')
|
||||
allNouns.push(JSON.parse(data))
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error reading directory ${this.nounsDir}:`, error)
|
||||
}
|
||||
}
|
||||
return allNouns
|
||||
}
|
||||
|
||||
/**
|
||||
* Get nouns by noun type
|
||||
* @param nounType The noun type to filter by
|
||||
* @returns Promise that resolves to an array of nouns of the specified noun type
|
||||
*/
|
||||
public async getNounsByNounType(nounType: string): Promise<HNSWNoun[]> {
|
||||
if (!this.isInitialized) await this.init()
|
||||
|
||||
const nouns: HNSWNoun[] = []
|
||||
try {
|
||||
const files = await fs.promises.readdir(this.nounsDir)
|
||||
for (const file of files) {
|
||||
if (file.endsWith('.json')) {
|
||||
const filePath = path.join(this.nounsDir, file)
|
||||
const data = await fs.promises.readFile(filePath, 'utf-8')
|
||||
const noun = JSON.parse(data)
|
||||
|
||||
// Filter by noun type using metadata
|
||||
const metadata = await this.getMetadata(noun.id)
|
||||
if (metadata && metadata.noun === nounType) {
|
||||
nouns.push(noun)
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error reading directory ${this.nounsDir}:`, error)
|
||||
}
|
||||
}
|
||||
|
||||
return nouns
|
||||
}
|
||||
|
||||
public async saveVerb(verb: GraphVerb): Promise<void> {
|
||||
if (!this.isInitialized) await this.init()
|
||||
const filePath = path.join(this.verbsDir, `${verb.id}.json`)
|
||||
await fs.promises.writeFile(filePath, JSON.stringify(verb, null, 2))
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a verb by its ID
|
||||
* @param id The ID of the verb to retrieve
|
||||
* @returns Promise that resolves to the verb or null if not found
|
||||
*/
|
||||
public async getVerb(id: string): Promise<GraphVerb | null> {
|
||||
if (!this.isInitialized) await this.init()
|
||||
const filePath = path.join(this.verbsDir, `${id}.json`)
|
||||
try {
|
||||
const data = await fs.promises.readFile(filePath, 'utf-8')
|
||||
return JSON.parse(data)
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error reading verb ${id}:`, error)
|
||||
}
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
public async getVerbsBySource(sourceId: string): Promise<GraphVerb[]> {
|
||||
if (!this.isInitialized) await this.init()
|
||||
const allVerbs = await this.getAllVerbs()
|
||||
return allVerbs.filter((verb) => verb.sourceId === sourceId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get verbs by target ID
|
||||
* @param targetId The target ID to filter by
|
||||
* @returns Promise that resolves to an array of verbs with the specified target ID
|
||||
*/
|
||||
public async getVerbsByTarget(targetId: string): Promise<GraphVerb[]> {
|
||||
if (!this.isInitialized) await this.init()
|
||||
const allVerbs = await this.getAllVerbs()
|
||||
return allVerbs.filter((verb) => verb.targetId === targetId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get verbs by type
|
||||
* @param type The verb type to filter by
|
||||
* @returns Promise that resolves to an array of verbs of the specified type
|
||||
*/
|
||||
public async getVerbsByType(type: string): Promise<GraphVerb[]> {
|
||||
if (!this.isInitialized) await this.init()
|
||||
const allVerbs = await this.getAllVerbs()
|
||||
return allVerbs.filter((verb) => verb.type === type)
|
||||
}
|
||||
|
||||
public async getAllVerbs(): Promise<GraphVerb[]> {
|
||||
if (!this.isInitialized) await this.init()
|
||||
const allVerbs: GraphVerb[] = []
|
||||
try {
|
||||
const files = await fs.promises.readdir(this.verbsDir)
|
||||
for (const file of files) {
|
||||
if (file.endsWith('.json')) {
|
||||
const filePath = path.join(this.verbsDir, file)
|
||||
const data = await fs.promises.readFile(filePath, 'utf-8')
|
||||
allVerbs.push(JSON.parse(data))
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error reading verbs directory ${this.verbsDir}:`, error)
|
||||
}
|
||||
}
|
||||
return allVerbs
|
||||
}
|
||||
|
||||
public async deleteVerb(id: string): Promise<void> {
|
||||
if (!this.isInitialized) await this.init()
|
||||
const filePath = path.join(this.verbsDir, `${id}.json`)
|
||||
try {
|
||||
await fs.promises.unlink(filePath)
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error deleting verb file ${filePath}:`, error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save metadata for an entity
|
||||
* @param id The ID of the entity
|
||||
* @param metadata The metadata to save
|
||||
*/
|
||||
public async saveMetadata(id: string, metadata: any): Promise<void> {
|
||||
if (!this.isInitialized) await this.init()
|
||||
const filePath = path.join(this.metadataDir, `${id}.json`)
|
||||
await this.ensureDirectoryExists(path.dirname(filePath))
|
||||
await fs.promises.writeFile(filePath, JSON.stringify(metadata, null, 2))
|
||||
}
|
||||
|
||||
/**
|
||||
* Get metadata for an entity
|
||||
* @param id The ID of the entity
|
||||
* @returns Promise that resolves to the metadata or null if not found
|
||||
*/
|
||||
public async getMetadata(id: string): Promise<any | null> {
|
||||
if (!this.isInitialized) await this.init()
|
||||
const filePath = path.join(this.metadataDir, `${id}.json`)
|
||||
try {
|
||||
const data = await fs.promises.readFile(filePath, 'utf-8')
|
||||
return JSON.parse(data)
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error reading metadata for ${id}:`, error)
|
||||
}
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
public async clear(): Promise<void> {
|
||||
if (!this.isInitialized) await this.init()
|
||||
|
||||
// Helper function to recursively remove directory contents
|
||||
const removeDirectoryContents = async (dirPath: string): Promise<void> => {
|
||||
try {
|
||||
const files = await fs.promises.readdir(dirPath, { withFileTypes: true })
|
||||
|
||||
for (const file of files) {
|
||||
const fullPath = path.join(dirPath, file.name)
|
||||
|
||||
if (file.isDirectory()) {
|
||||
await removeDirectoryContents(fullPath)
|
||||
// Use fs.promises.rm with recursive option instead of rmdir
|
||||
try {
|
||||
await fs.promises.rm(fullPath, { recursive: true, force: true })
|
||||
} catch (rmError: any) {
|
||||
// Fallback to rmdir if rm fails
|
||||
await fs.promises.rmdir(fullPath)
|
||||
}
|
||||
} else {
|
||||
await fs.promises.unlink(fullPath)
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error removing directory contents ${dirPath}:`, error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
// First try the modern approach
|
||||
await fs.promises.rm(this.rootDir, { recursive: true, force: true })
|
||||
} catch (error: any) {
|
||||
console.warn('Modern rm failed, falling back to manual cleanup:', error)
|
||||
|
||||
// Fallback: manually remove contents then directory
|
||||
try {
|
||||
await removeDirectoryContents(this.rootDir)
|
||||
// Use fs.promises.rm with recursive option instead of rmdir
|
||||
try {
|
||||
await fs.promises.rm(this.rootDir, { recursive: true, force: true })
|
||||
} catch (rmError: any) {
|
||||
// Final fallback to rmdir if rm fails
|
||||
await fs.promises.rmdir(this.rootDir)
|
||||
}
|
||||
} catch (fallbackError: any) {
|
||||
if (fallbackError.code !== 'ENOENT') {
|
||||
console.error('Manual cleanup also failed:', fallbackError)
|
||||
throw fallbackError
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.isInitialized = false // Reset state
|
||||
await this.init() // Re-create directories
|
||||
}
|
||||
|
||||
public async getStorageStatus(): Promise<{
|
||||
type: string
|
||||
used: number
|
||||
quota: number | null
|
||||
details?: Record<string, any>
|
||||
}> {
|
||||
if (!this.isInitialized) await this.init()
|
||||
|
||||
const calculateSize = async (dirPath: string): Promise<number> => {
|
||||
let size = 0
|
||||
try {
|
||||
const files = await fs.promises.readdir(dirPath, {
|
||||
withFileTypes: true
|
||||
})
|
||||
for (const file of files) {
|
||||
const fullPath = path.join(dirPath, file.name)
|
||||
if (file.isDirectory()) {
|
||||
size += await calculateSize(fullPath)
|
||||
} else {
|
||||
const stats = await fs.promises.stat(fullPath)
|
||||
size += stats.size
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Could not calculate size for ${dirPath}:`, error)
|
||||
}
|
||||
}
|
||||
return size
|
||||
}
|
||||
|
||||
const totalSize = await calculateSize(this.rootDir)
|
||||
const nouns = await this.getAllNouns()
|
||||
const verbs = await this.getAllVerbs()
|
||||
|
||||
return {
|
||||
type: 'FileSystem',
|
||||
used: totalSize,
|
||||
quota: null, // File system quota is not easily available from Node.js
|
||||
details: {
|
||||
rootDir: this.rootDir,
|
||||
nounCount: nouns.length,
|
||||
verbCount: verbs.length
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -1,952 +0,0 @@
|
|||
import { GraphVerb, HNSWNoun, StorageAdapter } from '../coreTypes.js'
|
||||
|
||||
// Type aliases for compatibility
|
||||
type HNSWNode = HNSWNoun
|
||||
type Edge = GraphVerb
|
||||
|
||||
// Export R2Storage as an alias for S3CompatibleStorage
|
||||
export { S3CompatibleStorage as R2Storage }
|
||||
|
||||
// Constants for S3 bucket prefixes
|
||||
const NOUNS_PREFIX = 'nouns/'
|
||||
const VERBS_PREFIX = 'verbs/'
|
||||
const EDGES_PREFIX = 'verbs/' // Alias for VERBS_PREFIX for edge operations
|
||||
const METADATA_PREFIX = 'metadata/'
|
||||
|
||||
// All nouns now use the same prefix - no separate directories per noun type
|
||||
const NOUN_PREFIX = 'nouns/' // Single directory for all noun types
|
||||
|
||||
/**
|
||||
* S3-compatible storage adapter for server environments
|
||||
* Uses the AWS S3 client to interact with S3-compatible storage services
|
||||
* including Amazon S3, Cloudflare R2, and Google Cloud Storage
|
||||
*
|
||||
* To use this adapter with Cloudflare R2, you need to provide:
|
||||
* - bucketName: Your bucket name
|
||||
* - accountId: Your Cloudflare account ID
|
||||
* - accessKeyId: R2 access key ID
|
||||
* - secretAccessKey: R2 secret access key
|
||||
* - serviceType: 'r2'
|
||||
*
|
||||
* To use this adapter with Amazon S3, you need to provide:
|
||||
* - bucketName: Your S3 bucket name
|
||||
* - accessKeyId: AWS access key ID
|
||||
* - secretAccessKey: AWS secret access key
|
||||
* - region: AWS region (e.g., 'us-east-1')
|
||||
* - serviceType: 's3'
|
||||
*
|
||||
* To use this adapter with Google Cloud Storage, you need to provide:
|
||||
* - bucketName: Your GCS bucket name
|
||||
* - accessKeyId: HMAC access key
|
||||
* - secretAccessKey: HMAC secret
|
||||
* - endpoint: GCS endpoint (e.g., 'https://storage.googleapis.com')
|
||||
* - serviceType: 'gcs'
|
||||
*
|
||||
* For other S3-compatible services, provide:
|
||||
* - bucketName: Your bucket name
|
||||
* - accessKeyId: Access key ID
|
||||
* - secretAccessKey: Secret access key
|
||||
* - endpoint: Service endpoint URL
|
||||
* - region: Region (if required)
|
||||
* - serviceType: 'custom'
|
||||
*/
|
||||
export class S3CompatibleStorage implements StorageAdapter {
|
||||
private bucketName: string
|
||||
private accessKeyId: string
|
||||
private secretAccessKey: string
|
||||
private endpoint?: string
|
||||
private region?: string
|
||||
private accountId?: string
|
||||
private serviceType: 'r2' | 's3' | 'gcs' | 'custom'
|
||||
private s3Client: any // Will be initialized in init()
|
||||
private isInitialized = false
|
||||
|
||||
// Alias methods to match StorageAdapter interface
|
||||
public async saveNoun(noun: HNSWNoun): Promise<void> {
|
||||
return this.saveNode(noun)
|
||||
}
|
||||
|
||||
public async getNoun(id: string): Promise<HNSWNoun | null> {
|
||||
return this.getNode(id)
|
||||
}
|
||||
|
||||
public async getAllNouns(): Promise<HNSWNoun[]> {
|
||||
return this.getAllNodes()
|
||||
}
|
||||
|
||||
public async getNounsByNounType(nounType: string): Promise<HNSWNoun[]> {
|
||||
return this.getNodesByNounType(nounType)
|
||||
}
|
||||
|
||||
public async deleteNoun(id: string): Promise<void> {
|
||||
return this.deleteNode(id)
|
||||
}
|
||||
|
||||
public async saveVerb(verb: GraphVerb): Promise<void> {
|
||||
return this.saveEdge(verb)
|
||||
}
|
||||
|
||||
public async getVerb(id: string): Promise<GraphVerb | null> {
|
||||
return this.getEdge(id)
|
||||
}
|
||||
|
||||
public async getAllVerbs(): Promise<GraphVerb[]> {
|
||||
return this.getAllEdges()
|
||||
}
|
||||
|
||||
public async getVerbsBySource(sourceId: string): Promise<GraphVerb[]> {
|
||||
return this.getEdgesBySource(sourceId)
|
||||
}
|
||||
|
||||
public async getVerbsByTarget(targetId: string): Promise<GraphVerb[]> {
|
||||
return this.getEdgesByTarget(targetId)
|
||||
}
|
||||
|
||||
public async getVerbsByType(type: string): Promise<GraphVerb[]> {
|
||||
return this.getEdgesByType(type)
|
||||
}
|
||||
|
||||
public async deleteVerb(id: string): Promise<void> {
|
||||
return this.deleteEdge(id)
|
||||
}
|
||||
|
||||
constructor(options: {
|
||||
bucketName: string
|
||||
accessKeyId: string
|
||||
secretAccessKey: string
|
||||
serviceType: 'r2' | 's3' | 'gcs' | 'custom'
|
||||
accountId?: string
|
||||
region?: string
|
||||
endpoint?: string
|
||||
}) {
|
||||
this.bucketName = options.bucketName
|
||||
this.accessKeyId = options.accessKeyId
|
||||
this.secretAccessKey = options.secretAccessKey
|
||||
this.serviceType = options.serviceType
|
||||
this.accountId = options.accountId
|
||||
this.region = options.region
|
||||
this.endpoint = options.endpoint
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the storage adapter
|
||||
*/
|
||||
public async init(): Promise<void> {
|
||||
if (this.isInitialized) {
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
// Dynamically import the AWS SDK to avoid bundling it in browser environments
|
||||
try {
|
||||
const { S3Client } = await import('@aws-sdk/client-s3')
|
||||
|
||||
// Configure the S3 client based on the service type
|
||||
const clientConfig: any = {
|
||||
credentials: {
|
||||
accessKeyId: this.accessKeyId,
|
||||
secretAccessKey: this.secretAccessKey
|
||||
}
|
||||
}
|
||||
|
||||
switch (this.serviceType) {
|
||||
case 'r2':
|
||||
if (!this.accountId) {
|
||||
throw new Error('accountId is required for Cloudflare R2')
|
||||
}
|
||||
clientConfig.region = 'auto' // R2 uses 'auto' as the region
|
||||
clientConfig.endpoint = `https://${this.accountId}.r2.cloudflarestorage.com`
|
||||
break
|
||||
|
||||
case 's3':
|
||||
if (!this.region) {
|
||||
throw new Error('region is required for Amazon S3')
|
||||
}
|
||||
clientConfig.region = this.region
|
||||
// No endpoint needed for standard S3
|
||||
break
|
||||
|
||||
case 'gcs':
|
||||
if (!this.endpoint) {
|
||||
// Default GCS endpoint if not provided
|
||||
this.endpoint = 'https://storage.googleapis.com'
|
||||
}
|
||||
clientConfig.endpoint = this.endpoint
|
||||
clientConfig.region = this.region || 'auto'
|
||||
break
|
||||
|
||||
case 'custom':
|
||||
if (!this.endpoint) {
|
||||
throw new Error(
|
||||
'endpoint is required for custom S3-compatible services'
|
||||
)
|
||||
}
|
||||
clientConfig.endpoint = this.endpoint
|
||||
if (this.region) {
|
||||
clientConfig.region = this.region
|
||||
}
|
||||
break
|
||||
|
||||
default:
|
||||
throw new Error(`Unsupported service type: ${this.serviceType}`)
|
||||
}
|
||||
|
||||
// Initialize the S3 client with the configured options
|
||||
this.s3Client = new S3Client(clientConfig)
|
||||
|
||||
this.isInitialized = true
|
||||
} catch (importError) {
|
||||
throw new Error(
|
||||
`Failed to import AWS SDK: ${importError}. Make sure @aws-sdk/client-s3 is installed.`
|
||||
)
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Failed to initialize ${this.serviceType} storage:`, error)
|
||||
throw new Error(
|
||||
`Failed to initialize ${this.serviceType} storage: ${error}`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save a node to storage
|
||||
*/
|
||||
public async saveNode(node: HNSWNode): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Convert connections Map to a serializable format
|
||||
const serializableNode = {
|
||||
...node,
|
||||
connections: this.mapToObject(node.connections, (set) =>
|
||||
Array.from(set as Set<string>)
|
||||
)
|
||||
}
|
||||
|
||||
// Get the appropriate prefix based on the node's metadata
|
||||
const nodePrefix = await this.getNodePrefix(node.id)
|
||||
|
||||
// Import the PutObjectCommand only when needed
|
||||
const { PutObjectCommand } = await import('@aws-sdk/client-s3')
|
||||
|
||||
// Save the node to S3-compatible storage
|
||||
await this.s3Client.send(
|
||||
new PutObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: `${nodePrefix}${node.id}.json`,
|
||||
Body: JSON.stringify(serializableNode, null, 2),
|
||||
ContentType: 'application/json'
|
||||
})
|
||||
)
|
||||
} catch (error) {
|
||||
console.error(`Failed to save node ${node.id}:`, error)
|
||||
throw new Error(`Failed to save node ${node.id}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a node from storage
|
||||
*/
|
||||
public async getNode(id: string): Promise<HNSWNode | null> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Import the GetObjectCommand only when needed
|
||||
const { GetObjectCommand } = await import('@aws-sdk/client-s3')
|
||||
|
||||
// Try to get the node from the consolidated nouns directory
|
||||
const response = await this.s3Client.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: `${NOUN_PREFIX}${id}.json`
|
||||
})
|
||||
)
|
||||
|
||||
// Convert the response body to a string
|
||||
const bodyContents = await response.Body.transformToString()
|
||||
const parsedNode = JSON.parse(bodyContents)
|
||||
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(parsedNode.connections)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
return {
|
||||
id: parsedNode.id,
|
||||
vector: parsedNode.vector,
|
||||
connections
|
||||
}
|
||||
} catch (error) {
|
||||
// Node not found or other error
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get nodes by noun type
|
||||
* @param nounType The noun type to filter by
|
||||
* @returns Promise that resolves to an array of nodes of the specified noun type
|
||||
*/
|
||||
public async getNodesByNounType(nounType: string): Promise<HNSWNode[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Import the ListObjectsV2Command and GetObjectCommand only when needed
|
||||
const { ListObjectsV2Command, GetObjectCommand } = await import(
|
||||
'@aws-sdk/client-s3'
|
||||
)
|
||||
|
||||
// List all objects in the consolidated nouns directory
|
||||
const listResponse = await this.s3Client.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: this.bucketName,
|
||||
Prefix: NOUN_PREFIX
|
||||
})
|
||||
)
|
||||
|
||||
const nodes: HNSWNode[] = []
|
||||
|
||||
// If there are no objects, return an empty array
|
||||
if (!listResponse.Contents || listResponse.Contents.length === 0) {
|
||||
return nodes
|
||||
}
|
||||
|
||||
// Get each node and filter by noun type
|
||||
const nodePromises = listResponse.Contents.map(
|
||||
async (object: { Key: string }) => {
|
||||
try {
|
||||
// Extract node ID from the key (remove prefix and .json extension)
|
||||
const nodeId = object.Key.replace(NOUN_PREFIX, '').replace('.json', '')
|
||||
|
||||
// Get the metadata to check the noun type
|
||||
const metadata = await this.getMetadata(nodeId)
|
||||
|
||||
// Skip if metadata doesn't exist or noun type doesn't match
|
||||
if (!metadata || metadata.noun !== nounType) {
|
||||
return null
|
||||
}
|
||||
|
||||
const response = await this.s3Client.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: object.Key
|
||||
})
|
||||
)
|
||||
|
||||
const bodyContents = await response.Body.transformToString()
|
||||
const parsedNode = JSON.parse(bodyContents)
|
||||
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(
|
||||
parsedNode.connections
|
||||
)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
return {
|
||||
id: parsedNode.id,
|
||||
vector: parsedNode.vector,
|
||||
connections
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Failed to get node from ${object.Key}:`, error)
|
||||
return null
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
const nodeResults = await Promise.all(nodePromises)
|
||||
return nodeResults.filter((node): node is HNSWNode => node !== null)
|
||||
} catch (error) {
|
||||
console.error(`Failed to get nodes for noun type ${nounType}:`, error)
|
||||
throw new Error(`Failed to get nodes for noun type ${nounType}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all nodes from storage
|
||||
*/
|
||||
public async getAllNodes(): Promise<HNSWNode[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Import the ListObjectsV2Command and GetObjectCommand only when needed
|
||||
const { ListObjectsV2Command, GetObjectCommand } = await import(
|
||||
'@aws-sdk/client-s3'
|
||||
)
|
||||
|
||||
// List all objects in the consolidated nouns directory
|
||||
const listResponse = await this.s3Client.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: this.bucketName,
|
||||
Prefix: NOUN_PREFIX
|
||||
})
|
||||
)
|
||||
|
||||
const nodes: HNSWNode[] = []
|
||||
|
||||
// If there are no objects, return an empty array
|
||||
if (!listResponse.Contents || listResponse.Contents.length === 0) {
|
||||
return nodes
|
||||
}
|
||||
|
||||
// Get each node
|
||||
const nodePromises = listResponse.Contents.map(
|
||||
async (object: { Key: string }) => {
|
||||
try {
|
||||
const response = await this.s3Client.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: object.Key
|
||||
})
|
||||
)
|
||||
|
||||
const bodyContents = await response.Body.transformToString()
|
||||
const parsedNode = JSON.parse(bodyContents)
|
||||
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(
|
||||
parsedNode.connections
|
||||
)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
return {
|
||||
id: parsedNode.id,
|
||||
vector: parsedNode.vector,
|
||||
connections
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Failed to get node from ${object.Key}:`, error)
|
||||
return null
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
const nodeResults = await Promise.all(nodePromises)
|
||||
return nodeResults.filter((node): node is HNSWNode => node !== null)
|
||||
} catch (error) {
|
||||
console.error('Failed to get all nodes:', error)
|
||||
throw new Error(`Failed to get all nodes: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a node from storage
|
||||
*/
|
||||
public async deleteNode(id: string): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Import the DeleteObjectCommand only when needed
|
||||
const { DeleteObjectCommand } = await import('@aws-sdk/client-s3')
|
||||
|
||||
// Delete the node from the consolidated nouns directory
|
||||
await this.s3Client.send(
|
||||
new DeleteObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: `${NOUN_PREFIX}${id}.json`
|
||||
})
|
||||
)
|
||||
} catch (error) {
|
||||
console.error(`Failed to delete node ${id}:`, error)
|
||||
throw new Error(`Failed to delete node ${id}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save an edge to storage
|
||||
*/
|
||||
public async saveEdge(edge: Edge): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Convert connections Map to a serializable format
|
||||
const serializableEdge = {
|
||||
...edge,
|
||||
connections: this.mapToObject(edge.connections, (set) =>
|
||||
Array.from(set as Set<string>)
|
||||
)
|
||||
}
|
||||
|
||||
// Import the PutObjectCommand only when needed
|
||||
const { PutObjectCommand } = await import('@aws-sdk/client-s3')
|
||||
|
||||
// Save the edge to S3-compatible storage
|
||||
await this.s3Client.send(
|
||||
new PutObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: `${VERBS_PREFIX}${edge.id}.json`,
|
||||
Body: JSON.stringify(serializableEdge, null, 2),
|
||||
ContentType: 'application/json'
|
||||
})
|
||||
)
|
||||
} catch (error) {
|
||||
console.error(`Failed to save edge ${edge.id}:`, error)
|
||||
throw new Error(`Failed to save edge ${edge.id}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get an edge from storage
|
||||
*/
|
||||
public async getEdge(id: string): Promise<Edge | null> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Import the GetObjectCommand only when needed
|
||||
const { GetObjectCommand } = await import('@aws-sdk/client-s3')
|
||||
|
||||
try {
|
||||
// Try to get the edge from S3-compatible storage
|
||||
const response = await this.s3Client.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: `${VERBS_PREFIX}${id}.json`
|
||||
})
|
||||
)
|
||||
|
||||
// Convert the response body to a string
|
||||
const bodyContents = await response.Body.transformToString()
|
||||
const parsedEdge = JSON.parse(bodyContents)
|
||||
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(parsedEdge.connections)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
return {
|
||||
id: parsedEdge.id,
|
||||
vector: parsedEdge.vector,
|
||||
connections,
|
||||
sourceId: parsedEdge.sourceId,
|
||||
targetId: parsedEdge.targetId,
|
||||
type: parsedEdge.type,
|
||||
weight: parsedEdge.weight,
|
||||
metadata: parsedEdge.metadata
|
||||
}
|
||||
} catch {
|
||||
return null // Edge not found
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Failed to get edge ${id}:`, error)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all edges from storage
|
||||
*/
|
||||
public async getAllEdges(): Promise<Edge[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Import the ListObjectsV2Command and GetObjectCommand only when needed
|
||||
const { ListObjectsV2Command, GetObjectCommand } = await import(
|
||||
'@aws-sdk/client-s3'
|
||||
)
|
||||
|
||||
// List all objects with the edges prefix
|
||||
const listResponse = await this.s3Client.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: this.bucketName,
|
||||
Prefix: VERBS_PREFIX
|
||||
})
|
||||
)
|
||||
|
||||
const edges: Edge[] = []
|
||||
|
||||
// If there are no objects, return an empty array
|
||||
if (!listResponse.Contents || listResponse.Contents.length === 0) {
|
||||
return edges
|
||||
}
|
||||
|
||||
// Get each edge
|
||||
const edgePromises = listResponse.Contents.map(
|
||||
async (object: { Key: string }) => {
|
||||
try {
|
||||
const response = await this.s3Client.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: object.Key
|
||||
})
|
||||
)
|
||||
|
||||
const bodyContents = await response.Body.transformToString()
|
||||
const parsedEdge = JSON.parse(bodyContents)
|
||||
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(
|
||||
parsedEdge.connections
|
||||
)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
return {
|
||||
id: parsedEdge.id,
|
||||
vector: parsedEdge.vector,
|
||||
connections,
|
||||
sourceId: parsedEdge.sourceId,
|
||||
targetId: parsedEdge.targetId,
|
||||
type: parsedEdge.type,
|
||||
weight: parsedEdge.weight,
|
||||
metadata: parsedEdge.metadata
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Failed to get edge from ${object.Key}:`, error)
|
||||
return null
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
const edgeResults = await Promise.all(edgePromises)
|
||||
return edgeResults.filter((edge): edge is Edge => edge !== null)
|
||||
} catch (error) {
|
||||
console.error('Failed to get all edges:', error)
|
||||
throw new Error(`Failed to get all edges: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get edges by source node ID
|
||||
*/
|
||||
public async getEdgesBySource(sourceId: string): Promise<Edge[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
const allEdges = await this.getAllEdges()
|
||||
return allEdges.filter((edge) => edge.sourceId === sourceId)
|
||||
} catch (error) {
|
||||
console.error(`Failed to get edges by source ${sourceId}:`, error)
|
||||
throw new Error(`Failed to get edges by source ${sourceId}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get edges by target node ID
|
||||
*/
|
||||
public async getEdgesByTarget(targetId: string): Promise<Edge[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
const allEdges = await this.getAllEdges()
|
||||
return allEdges.filter((edge) => edge.targetId === targetId)
|
||||
} catch (error) {
|
||||
console.error(`Failed to get edges by target ${targetId}:`, error)
|
||||
throw new Error(`Failed to get edges by target ${targetId}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get edges by type
|
||||
*/
|
||||
public async getEdgesByType(type: string): Promise<Edge[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
const allEdges = await this.getAllEdges()
|
||||
return allEdges.filter((edge) => edge.type === type)
|
||||
} catch (error) {
|
||||
console.error(`Failed to get edges by type ${type}:`, error)
|
||||
throw new Error(`Failed to get edges by type ${type}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete an edge from storage
|
||||
*/
|
||||
public async deleteEdge(id: string): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Import the DeleteObjectCommand and GetObjectCommand only when needed
|
||||
const { DeleteObjectCommand, GetObjectCommand } = await import(
|
||||
'@aws-sdk/client-s3'
|
||||
)
|
||||
|
||||
try {
|
||||
// Check if the edge exists before deleting
|
||||
await this.s3Client.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: `${EDGES_PREFIX}${id}.json`
|
||||
})
|
||||
)
|
||||
|
||||
// Delete the edge
|
||||
await this.s3Client.send(
|
||||
new DeleteObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: `${EDGES_PREFIX}${id}.json`
|
||||
})
|
||||
)
|
||||
} catch {
|
||||
// Edge not found, nothing to delete
|
||||
return
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Failed to delete edge ${id}:`, error)
|
||||
throw new Error(`Failed to delete edge ${id}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save metadata to storage
|
||||
*/
|
||||
public async saveMetadata(id: string, metadata: any): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Import the PutObjectCommand only when needed
|
||||
const { PutObjectCommand } = await import('@aws-sdk/client-s3')
|
||||
|
||||
// Save the metadata to S3-compatible storage
|
||||
await this.s3Client.send(
|
||||
new PutObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: `${METADATA_PREFIX}${id}.json`,
|
||||
Body: JSON.stringify(metadata, null, 2),
|
||||
ContentType: 'application/json'
|
||||
})
|
||||
)
|
||||
} catch (error) {
|
||||
console.error(`Failed to save metadata for ${id}:`, error)
|
||||
throw new Error(`Failed to save metadata for ${id}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get metadata from storage
|
||||
*/
|
||||
public async getMetadata(id: string): Promise<any | null> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Import the GetObjectCommand only when needed
|
||||
const { GetObjectCommand } = await import('@aws-sdk/client-s3')
|
||||
|
||||
try {
|
||||
// Try to get the metadata from S3-compatible storage
|
||||
const response = await this.s3Client.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: `${METADATA_PREFIX}${id}.json`
|
||||
})
|
||||
)
|
||||
|
||||
// Convert the response body to a string
|
||||
const bodyContents = await response.Body.transformToString()
|
||||
return JSON.parse(bodyContents)
|
||||
} catch {
|
||||
return null // Metadata not found
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Failed to get metadata for ${id}:`, error)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear all data from storage
|
||||
*/
|
||||
public async clear(): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Import the ListObjectsV2Command and DeleteObjectCommand only when needed
|
||||
const { ListObjectsV2Command, DeleteObjectCommand } = await import(
|
||||
'@aws-sdk/client-s3'
|
||||
)
|
||||
|
||||
// List all objects in the bucket
|
||||
const listResponse = await this.s3Client.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: this.bucketName
|
||||
})
|
||||
)
|
||||
|
||||
// If there are no objects, return
|
||||
if (!listResponse.Contents || listResponse.Contents.length === 0) {
|
||||
return
|
||||
}
|
||||
|
||||
// Delete each object
|
||||
const deletePromises = listResponse.Contents.map(
|
||||
async (object: { Key: string }) => {
|
||||
try {
|
||||
await this.s3Client.send(
|
||||
new DeleteObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: object.Key
|
||||
})
|
||||
)
|
||||
} catch (error) {
|
||||
console.error(`Failed to delete object ${object.Key}:`, error)
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
await Promise.all(deletePromises)
|
||||
} catch (error) {
|
||||
console.error('Failed to clear storage:', error)
|
||||
throw new Error(`Failed to clear storage: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get information about storage usage and capacity
|
||||
*/
|
||||
public async getStorageStatus(): Promise<{
|
||||
type: string
|
||||
used: number
|
||||
quota: number | null
|
||||
details?: Record<string, any>
|
||||
}> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Import the ListObjectsV2Command only when needed
|
||||
const { ListObjectsV2Command } = await import('@aws-sdk/client-s3')
|
||||
|
||||
// List all objects in the bucket to calculate total size
|
||||
const listResponse = await this.s3Client.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: this.bucketName
|
||||
})
|
||||
)
|
||||
|
||||
let totalSize = 0
|
||||
let nodeCount = 0
|
||||
let edgeCount = 0
|
||||
let metadataCount = 0
|
||||
|
||||
// Calculate total size and counts
|
||||
if (listResponse.Contents) {
|
||||
for (const object of listResponse.Contents) {
|
||||
totalSize += object.Size || 0
|
||||
|
||||
const key = object.Key || ''
|
||||
if (key.startsWith(NOUNS_PREFIX)) {
|
||||
nodeCount++
|
||||
} else if (key.startsWith(VERBS_PREFIX)) {
|
||||
edgeCount++
|
||||
} else if (key.startsWith(METADATA_PREFIX)) {
|
||||
metadataCount++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Count nodes by noun type by examining metadata
|
||||
const nounTypeCounts: Record<string, number> = {
|
||||
person: 0,
|
||||
place: 0,
|
||||
thing: 0,
|
||||
event: 0,
|
||||
concept: 0,
|
||||
content: 0,
|
||||
group: 0,
|
||||
list: 0,
|
||||
category: 0,
|
||||
default: 0
|
||||
}
|
||||
|
||||
// List all noun objects and count by type using metadata
|
||||
const nounsListResponse = await this.s3Client.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: this.bucketName,
|
||||
Prefix: NOUN_PREFIX
|
||||
})
|
||||
)
|
||||
|
||||
if (nounsListResponse.Contents) {
|
||||
for (const object of nounsListResponse.Contents) {
|
||||
try {
|
||||
// Extract node ID from the key
|
||||
const nodeId = object.Key?.replace(NOUN_PREFIX, '').replace('.json', '')
|
||||
if (nodeId) {
|
||||
// Get metadata to determine noun type
|
||||
const metadata = await this.getMetadata(nodeId)
|
||||
const nounType = metadata?.noun || 'default'
|
||||
|
||||
if (nounType in nounTypeCounts) {
|
||||
nounTypeCounts[nounType]++
|
||||
} else {
|
||||
nounTypeCounts.default++
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
// If we can't get metadata, count as default
|
||||
nounTypeCounts.default++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
type: this.serviceType,
|
||||
used: totalSize,
|
||||
quota: null, // S3-compatible services typically don't provide quota information through the API
|
||||
details: {
|
||||
nodeCount,
|
||||
edgeCount,
|
||||
metadataCount,
|
||||
nounTypes: {
|
||||
person: { count: nounTypeCounts.person },
|
||||
place: { count: nounTypeCounts.place },
|
||||
thing: { count: nounTypeCounts.thing },
|
||||
event: { count: nounTypeCounts.event },
|
||||
concept: { count: nounTypeCounts.concept },
|
||||
content: { count: nounTypeCounts.content },
|
||||
group: { count: nounTypeCounts.group },
|
||||
list: { count: nounTypeCounts.list },
|
||||
category: { count: nounTypeCounts.category },
|
||||
default: { count: nounTypeCounts.default }
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Failed to get storage status:', error)
|
||||
return {
|
||||
type: this.serviceType,
|
||||
used: 0,
|
||||
quota: null,
|
||||
details: { error: String(error) }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure the storage adapter is initialized
|
||||
*/
|
||||
private async ensureInitialized(): Promise<void> {
|
||||
if (!this.isInitialized) {
|
||||
await this.init()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the appropriate prefix for a node - now all nouns use the same prefix
|
||||
*/
|
||||
private async getNodePrefix(id: string): Promise<string> {
|
||||
// All nouns now use the same prefix regardless of type
|
||||
return NOUN_PREFIX
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a Map to a plain object for serialization
|
||||
*/
|
||||
private mapToObject<K extends string | number, V>(
|
||||
map: Map<K, V>,
|
||||
valueTransformer: (value: V) => any = (v) => v
|
||||
): Record<string, any> {
|
||||
const obj: Record<string, any> = {}
|
||||
for (const [key, value] of map.entries()) {
|
||||
obj[key.toString()] = valueTransformer(value)
|
||||
}
|
||||
return obj
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue