brainy/src/storage/opfsStorage.ts
David Snelling 5f0d9f4b0a **refactor: rename directories and variables for semantic clarity in OPFS storage**
### Changes:
- Renamed `nodes` to `nouns` and `edges` to `verbs` across `opfsStorage.ts`, improving code semantics:
  - Directory constants (e.g., `NODES_DIR` → `NOUNS_DIR`).
  - Class properties, method names, and variables updated accordingly.
- Introduced global type augmentation for `FileSystemDirectoryHandle` to support `entries()` method, improving compatibility.
- Updated directory management logic:
  - Adjusted initialization, recreation, and metadata retrieval methods to reflect new nomenclature.
  - Enhanced handling for directory entries in both `nouns` and `verbs` contexts.
- Refined comments, formatting, and TypeScript annotations for consistency.

### Purpose:
Improved the code's readability and semantic clarity by aligning directory and variable names with their conceptual purpose (`nouns` for entities, `verbs` for relationships). Enhanced maintainability and developer understanding of the storage implementation.
2025-06-19 15:03:06 -07:00

1872 lines
53 KiB
TypeScript

/**
* OPFS (Origin Private File System) Storage Adapter
* Provides persistent storage for the vector database using the Origin Private File System API
*/
import { GraphVerb, HNSWNoun, StorageAdapter } from '../coreTypes.js'
import '../types/fileSystemTypes.js'
// Make sure TypeScript recognizes the FileSystemDirectoryHandle interface extension
declare global {
interface FileSystemDirectoryHandle {
entries(): AsyncIterableIterator<[string, FileSystemHandle]>
}
}
// Type aliases for compatibility
type HNSWNode = HNSWNoun
type Edge = GraphVerb
// Directory and file names
const ROOT_DIR = 'opfs-vector-db'
const NOUNS_DIR = 'nouns'
const VERBS_DIR = 'verbs'
const METADATA_DIR = 'metadata'
const DB_INFO_FILE = 'db-info.json'
// Constants for noun type directories
const PERSON_DIR = 'person'
const PLACE_DIR = 'place'
const THING_DIR = 'thing'
const EVENT_DIR = 'event'
const CONCEPT_DIR = 'concept'
const CONTENT_DIR = 'content'
const DEFAULT_DIR = 'default' // For nodes without a noun type
export class OPFSStorage implements StorageAdapter {
private rootDir: FileSystemDirectoryHandle | null = null
private nounsDir: FileSystemDirectoryHandle | null = null
private verbsDir: FileSystemDirectoryHandle | null = null
private metadataDir: FileSystemDirectoryHandle | null = null
private personDir: FileSystemDirectoryHandle | null = null
private placeDir: FileSystemDirectoryHandle | null = null
private thingDir: FileSystemDirectoryHandle | null = null
private eventDir: FileSystemDirectoryHandle | null = null
private conceptDir: FileSystemDirectoryHandle | null = null
private contentDir: FileSystemDirectoryHandle | null = null
private defaultDir: FileSystemDirectoryHandle | null = null
private isInitialized = false
private isAvailable = false
private isPersistentRequested = false
private isPersistentGranted = false
constructor() {
// Check if OPFS is available
this.isAvailable =
typeof navigator !== 'undefined' &&
'storage' in navigator &&
'getDirectory' in navigator.storage
}
/**
* Initialize the storage adapter
*/
public async init(): Promise<void> {
if (this.isInitialized) {
return
}
if (!this.isAvailable) {
throw new Error(
'Origin Private File System is not available in this environment'
)
}
try {
// Get the root directory
const root = await navigator.storage.getDirectory()
// Create or get our app's root directory
this.rootDir = await root.getDirectoryHandle(ROOT_DIR, { create: true })
// Create or get nouns directory
this.nounsDir = await this.rootDir.getDirectoryHandle(NOUNS_DIR, {
create: true
})
// Create or get verbs directory
this.verbsDir = await this.rootDir.getDirectoryHandle(VERBS_DIR, {
create: true
})
// Create or get metadata directory
this.metadataDir = await this.rootDir.getDirectoryHandle(METADATA_DIR, {
create: true
})
// Create or get noun type directories
this.personDir = await this.nounsDir.getDirectoryHandle(PERSON_DIR, {
create: true
})
this.placeDir = await this.nounsDir.getDirectoryHandle(PLACE_DIR, {
create: true
})
this.thingDir = await this.nounsDir.getDirectoryHandle(THING_DIR, {
create: true
})
this.eventDir = await this.nounsDir.getDirectoryHandle(EVENT_DIR, {
create: true
})
this.conceptDir = await this.nounsDir.getDirectoryHandle(CONCEPT_DIR, {
create: true
})
this.contentDir = await this.nounsDir.getDirectoryHandle(CONTENT_DIR, {
create: true
})
this.defaultDir = await this.nounsDir.getDirectoryHandle(DEFAULT_DIR, {
create: true
})
this.isInitialized = true
} catch (error) {
console.error('Failed to initialize OPFS storage:', error)
throw new Error(`Failed to initialize OPFS storage: ${error}`)
}
}
/**
* Check if OPFS is available in the current environment
*/
public isOPFSAvailable(): boolean {
return this.isAvailable
}
/**
* Request persistent storage permission from the user
* @returns Promise that resolves to true if permission was granted, false otherwise
*/
public async requestPersistentStorage(): Promise<boolean> {
if (!this.isAvailable) {
console.warn('Cannot request persistent storage: OPFS is not available')
return false
}
try {
// Check if persistence is already granted
this.isPersistentGranted = await navigator.storage.persisted()
if (!this.isPersistentGranted) {
// Request permission for persistent storage
this.isPersistentGranted = await navigator.storage.persist()
}
this.isPersistentRequested = true
return this.isPersistentGranted
} catch (error) {
console.warn('Failed to request persistent storage:', error)
return false
}
}
/**
* Check if persistent storage is granted
* @returns Promise that resolves to true if persistent storage is granted, false otherwise
*/
public async isPersistent(): Promise<boolean> {
if (!this.isAvailable) {
return false
}
try {
this.isPersistentGranted = await navigator.storage.persisted()
return this.isPersistentGranted
} catch (error) {
console.warn('Failed to check persistent storage status:', error)
return false
}
}
/**
* Save a noun to storage
*/
public async saveNoun(noun: HNSWNoun): Promise<void> {
await this.ensureInitialized()
try {
// Convert connections Map to a serializable format
const serializableNoun = {
...noun,
connections: this.mapToObject(noun.connections, (set) =>
Array.from(set as Set<string>)
)
}
// Get the appropriate directory based on the noun's metadata
const nounDir = await this.getNodeDirectory(noun.id)
// Create or get the file for this noun
const fileHandle = await nounDir.getFileHandle(noun.id, {
create: true
})
// Write the noun data to the file
const writable = await fileHandle.createWritable()
await writable.write(JSON.stringify(serializableNoun))
await writable.close()
} catch (error) {
console.error(`Failed to save noun ${noun.id}:`, error)
throw new Error(`Failed to save noun ${noun.id}: ${error}`)
}
}
/**
* Get a noun from storage
*/
public async getNoun(id: string): Promise<HNSWNoun | null> {
await this.ensureInitialized()
try {
// Get the appropriate directory based on the node's metadata
const nodeDir = await this.getNodeDirectory(id)
try {
// Get the file handle for this node
const fileHandle = await nodeDir.getFileHandle(id)
// Read the node data from the file
const file = await fileHandle.getFile()
const text = await file.text()
const data = JSON.parse(text)
// Convert serialized connections back to Map<number, Set<string>>
const connections = new Map<number, Set<string>>()
for (const [level, nodeIds] of Object.entries(data.connections)) {
connections.set(Number(level), new Set(nodeIds as string[]))
}
return {
id: data.id,
vector: data.vector,
connections
}
} catch (dirError) {
// If the file doesn't exist in the expected directory, try the default directory
if (nodeDir !== this.defaultDir) {
try {
// Get the file handle from the default directory
const fileHandle = await this.defaultDir!.getFileHandle(id)
// Read the node data from the file
const file = await fileHandle.getFile()
const text = await file.text()
const data = JSON.parse(text)
// Convert serialized connections back to Map<number, Set<string>>
const connections = new Map<number, Set<string>>()
for (const [level, nodeIds] of Object.entries(data.connections)) {
connections.set(Number(level), new Set(nodeIds as string[]))
}
return {
id: data.id,
vector: data.vector,
connections
}
} catch (defaultDirError) {
// If not found in default directory either, try all noun type directories
const directories = [
this.personDir!,
this.placeDir!,
this.thingDir!,
this.eventDir!,
this.conceptDir!,
this.contentDir!
]
for (const dir of directories) {
if (dir === nodeDir) continue // Skip the already checked directory
try {
// Get the file handle from this directory
const fileHandle = await dir.getFileHandle(id)
// Read the node data from the file
const file = await fileHandle.getFile()
const text = await file.text()
const data = JSON.parse(text)
// Convert serialized connections back to Map<number, Set<string>>
const connections = new Map<number, Set<string>>()
for (const [level, nodeIds] of Object.entries(
data.connections
)) {
connections.set(Number(level), new Set(nodeIds as string[]))
}
return {
id: data.id,
vector: data.vector,
connections
}
} catch {
// Continue to the next directory
}
}
return null // File doesn't exist in any directory
}
}
return null // File doesn't exist
}
} catch (error) {
console.error(`Failed to get node ${id}:`, error)
throw new Error(`Failed to get node ${id}: ${error}`)
}
}
/**
* Get nouns by noun type
* @param nounType The noun type to filter by
* @returns Promise that resolves to an array of nouns of the specified noun type
*/
public async getNounsByNounType(nounType: string): Promise<HNSWNoun[]> {
await this.ensureInitialized()
try {
const nouns: HNSWNoun[] = []
// Determine the directory based on the noun type
let dir: FileSystemDirectoryHandle
switch (nounType) {
case 'person':
dir = this.personDir!
break
case 'place':
dir = this.placeDir!
break
case 'thing':
dir = this.thingDir!
break
case 'event':
dir = this.eventDir!
break
case 'concept':
dir = this.conceptDir!
break
case 'content':
dir = this.contentDir!
break
default:
dir = this.defaultDir!
}
try {
// Get all entries (filename and handle pairs) in this directory
const entries = dir.entries()
// Iterate through all entries and get the corresponding nodes
for await (const [name, handle] of entries) {
try {
// The handle is already a FileSystemHandle, but we need to ensure it's a file
if (handle.kind !== 'file') continue
// Cast to FileSystemFileHandle
const fileHandle = handle as FileSystemFileHandle
// Read the node data from the file
const file = await fileHandle.getFile()
const text = await file.text()
const data = JSON.parse(text)
// Convert serialized connections back to Map<number, Set<string>>
const connections = new Map<number, Set<string>>()
for (const [level, nodeIds] of Object.entries(data.connections)) {
connections.set(Number(level), new Set(nodeIds as string[]))
}
nouns.push({
id: data.id,
vector: data.vector,
connections
})
} catch (nodeError) {
console.warn(
`Failed to read node ${name} from directory:`,
nodeError
)
// Continue to the next node
}
}
} catch (dirError) {
console.warn(
`Failed to read directory for noun type ${nounType}:`,
dirError
)
}
return nouns
} catch (error) {
console.error(`Failed to get nouns for noun type ${nounType}:`, error)
throw new Error(`Failed to get nouns for noun type ${nounType}: ${error}`)
}
}
/**
* Get all nouns from storage
*/
public async getAllNouns(): Promise<HNSWNoun[]> {
await this.ensureInitialized()
try {
// Get all noun types
const nounTypes = [
'person',
'place',
'thing',
'event',
'concept',
'content',
'default'
]
// Run searches in parallel for all noun types
const nounPromises = nounTypes.map((nounType) =>
this.getNounsByNounType(nounType)
)
const nounArrays = await Promise.all(nounPromises)
// Combine all results
const allNouns: HNSWNoun[] = []
for (const nouns of nounArrays) {
allNouns.push(...nouns)
}
return allNouns
} catch (error) {
console.error('Failed to get all nouns:', error)
throw new Error(`Failed to get all nouns: ${error}`)
}
}
/**
* Delete a noun from storage
*/
public async deleteNoun(id: string): Promise<void> {
await this.ensureInitialized()
try {
// Get the appropriate directory based on the noun's metadata
const nounDir = await this.getNodeDirectory(id)
try {
// Try to delete the noun from the appropriate directory
await nounDir.removeEntry(id)
return // Noun deleted successfully
} catch (dirError) {
// If the file doesn't exist in the expected directory, try the default directory
if (nounDir !== this.defaultDir) {
try {
await this.defaultDir!.removeEntry(id)
return // Noun deleted successfully
} catch (defaultDirError) {
// If not found in default directory either, try all noun type directories
const directories = [
this.personDir!,
this.placeDir!,
this.thingDir!,
this.eventDir!,
this.conceptDir!,
this.contentDir!
]
for (const dir of directories) {
if (dir === nounDir) continue // Skip the already checked directory
try {
await dir.removeEntry(id)
return // Node deleted successfully
} catch {
// Continue to the next directory
}
}
// If we get here, the node wasn't found in any directory
return
}
}
// If the file doesn't exist, that's fine
return
}
} catch (error) {
console.error(`Failed to delete noun ${id}:`, error)
throw new Error(`Failed to delete noun ${id}: ${error}`)
}
}
/**
* Save a verb to storage
*/
public async saveVerb(verb: GraphVerb): Promise<void> {
await this.ensureInitialized()
try {
// Convert connections Map to a serializable format
const serializableVerb = {
...verb,
connections: this.mapToObject(verb.connections, (set) =>
Array.from(set as Set<string>)
)
}
// Create or get the file for this verb
const fileHandle = await this.verbsDir!.getFileHandle(verb.id, {
create: true
})
// Write the verb data to the file
const writable = await fileHandle.createWritable()
await writable.write(JSON.stringify(serializableVerb))
await writable.close()
} catch (error) {
console.error(`Failed to save verb ${verb.id}:`, error)
throw new Error(`Failed to save verb ${verb.id}: ${error}`)
}
}
/**
* Get a verb from storage
*/
public async getVerb(id: string): Promise<GraphVerb | null> {
await this.ensureInitialized()
try {
// Get the file handle for this verb
const fileHandle = await this.verbsDir!.getFileHandle(id)
// Read the verb data from the file
const file = await fileHandle.getFile()
const text = await file.text()
const data = JSON.parse(text)
// Convert serialized connections back to Map<number, Set<string>>
const connections = new Map<number, Set<string>>()
for (const [level, nodeIds] of Object.entries(data.connections)) {
connections.set(Number(level), new Set(nodeIds as string[]))
}
return {
id: data.id,
vector: data.vector,
connections,
sourceId: data.sourceId,
targetId: data.targetId,
type: data.type,
weight: data.weight,
metadata: data.metadata
}
} catch (error) {
// If the file doesn't exist, return null
if ((error as any).name === 'NotFoundError') {
return null
}
console.error(`Failed to get verb ${id}:`, error)
throw new Error(`Failed to get verb ${id}: ${error}`)
}
}
/**
* Get all edges from storage
*/
public async getAllEdges(): Promise<Edge[]> {
await this.ensureInitialized()
try {
const edges: Edge[] = []
// Get all entries (filename and handle pairs) in the verbs directory
const entries = this.verbsDir!.entries()
// Iterate through all entries and get the corresponding edges
for await (const [name, handle] of entries) {
// Skip if not a file
if (handle.kind !== 'file') continue
const edge = await this.getVerb(name)
if (edge) {
edges.push(edge)
}
}
return edges
} catch (error) {
console.error('Failed to get all edges:', error)
throw new Error(`Failed to get all edges: ${error}`)
}
}
/**
* Get all verbs from storage (alias for getAllEdges)
*/
public async getAllVerbs(): Promise<GraphVerb[]> {
return this.getAllEdges()
}
/**
* Delete an edge from storage
*/
public async deleteEdge(id: string): Promise<void> {
await this.ensureInitialized()
try {
await this.verbsDir!.removeEntry(id)
} catch (error) {
// Ignore if the file doesn't exist
if ((error as any).name !== 'NotFoundError') {
console.error(`Failed to delete edge ${id}:`, error)
throw new Error(`Failed to delete edge ${id}: ${error}`)
}
}
}
/**
* Delete a verb from storage (alias for deleteEdge)
*/
public async deleteVerb(id: string): Promise<void> {
return this.deleteEdge(id)
}
/**
* Get edges by source node ID
*/
public async getEdgesBySource(sourceId: string): Promise<Edge[]> {
await this.ensureInitialized()
try {
const allEdges = await this.getAllEdges()
return allEdges.filter((edge) => edge.sourceId === sourceId)
} catch (error) {
console.error(`Failed to get edges by source ${sourceId}:`, error)
throw new Error(`Failed to get edges by source ${sourceId}: ${error}`)
}
}
/**
* Get verbs by source node ID (alias for getEdgesBySource)
*/
public async getVerbsBySource(sourceId: string): Promise<GraphVerb[]> {
return this.getEdgesBySource(sourceId)
}
/**
* Get edges by target node ID
*/
public async getEdgesByTarget(targetId: string): Promise<Edge[]> {
await this.ensureInitialized()
try {
const allEdges = await this.getAllEdges()
return allEdges.filter((edge) => edge.targetId === targetId)
} catch (error) {
console.error(`Failed to get edges by target ${targetId}:`, error)
throw new Error(`Failed to get edges by target ${targetId}: ${error}`)
}
}
/**
* Get verbs by target node ID (alias for getEdgesByTarget)
*/
public async getVerbsByTarget(targetId: string): Promise<GraphVerb[]> {
return this.getEdgesByTarget(targetId)
}
/**
* Get edges by type
*/
public async getEdgesByType(type: string): Promise<Edge[]> {
await this.ensureInitialized()
try {
const allEdges = await this.getAllEdges()
return allEdges.filter((edge) => edge.type === type)
} catch (error) {
console.error(`Failed to get edges by type ${type}:`, error)
throw new Error(`Failed to get edges by type ${type}: ${error}`)
}
}
/**
* Get verbs by type (alias for getEdgesByType)
*/
public async getVerbsByType(type: string): Promise<GraphVerb[]> {
return this.getEdgesByType(type)
}
/**
* Save metadata for a node
*/
public async saveMetadata(id: string, metadata: any): Promise<void> {
await this.ensureInitialized()
try {
// Create or get the file for this metadata
const fileHandle = await this.metadataDir!.getFileHandle(id, {
create: true
})
// Write the metadata to the file
const writable = await fileHandle.createWritable()
await writable.write(JSON.stringify(metadata))
await writable.close()
} catch (error) {
console.error(`Failed to save metadata for ${id}:`, error)
throw new Error(`Failed to save metadata for ${id}: ${error}`)
}
}
/**
* Get metadata for a node
*/
public async getMetadata(id: string): Promise<any | null> {
await this.ensureInitialized()
try {
// Get the file handle for this metadata
const fileHandle = await this.metadataDir!.getFileHandle(id)
// Read the metadata from the file
const file = await fileHandle.getFile()
const text = await file.text()
return JSON.parse(text)
} catch (error) {
// If the file doesn't exist, return null
if ((error as any).name === 'NotFoundError') {
return null
}
console.error(`Failed to get metadata for ${id}:`, error)
throw new Error(`Failed to get metadata for ${id}: ${error}`)
}
}
/**
* Clear all data from storage
*/
public async clear(): Promise<void> {
await this.ensureInitialized()
try {
// Delete and recreate the nouns directory
await this.rootDir!.removeEntry(NOUNS_DIR, { recursive: true })
this.nounsDir = await this.rootDir!.getDirectoryHandle(NOUNS_DIR, {
create: true
})
// Create noun type directories
this.personDir = await this.nounsDir.getDirectoryHandle(PERSON_DIR, {
create: true
})
this.placeDir = await this.nounsDir.getDirectoryHandle(PLACE_DIR, {
create: true
})
this.thingDir = await this.nounsDir.getDirectoryHandle(THING_DIR, {
create: true
})
this.eventDir = await this.nounsDir.getDirectoryHandle(EVENT_DIR, {
create: true
})
this.conceptDir = await this.nounsDir.getDirectoryHandle(CONCEPT_DIR, {
create: true
})
this.contentDir = await this.nounsDir.getDirectoryHandle(CONTENT_DIR, {
create: true
})
this.defaultDir = await this.nounsDir.getDirectoryHandle(DEFAULT_DIR, {
create: true
})
// Delete and recreate the verbs directory
await this.rootDir!.removeEntry(VERBS_DIR, { recursive: true })
this.verbsDir = await this.rootDir!.getDirectoryHandle(VERBS_DIR, {
create: true
})
// Delete and recreate the metadata directory
await this.rootDir!.removeEntry(METADATA_DIR, { recursive: true })
this.metadataDir = await this.rootDir!.getDirectoryHandle(METADATA_DIR, {
create: true
})
} catch (error) {
console.error('Failed to clear storage:', error)
throw new Error(`Failed to clear storage: ${error}`)
}
}
/**
* Ensure the storage adapter is initialized
*/
private async ensureInitialized(): Promise<void> {
if (!this.isInitialized) {
await this.init()
}
}
/**
* Convert a Map to a plain object for serialization
*/
private mapToObject<K extends string | number, V>(
map: Map<K, V>,
valueTransformer: (value: V) => any = (v) => v
): Record<string, any> {
const obj: Record<string, any> = {}
for (const [key, value] of map.entries()) {
obj[key.toString()] = valueTransformer(value)
}
return obj
}
/**
* Get the appropriate directory for a node based on its metadata
*/
private async getNodeDirectory(
id: string
): Promise<FileSystemDirectoryHandle> {
try {
// Try to get the metadata for the node
const metadata = await this.getMetadata(id)
// If metadata exists and has a noun field, use the corresponding directory
if (metadata && metadata.noun) {
switch (metadata.noun) {
case 'person':
return this.personDir!
case 'place':
return this.placeDir!
case 'thing':
return this.thingDir!
case 'event':
return this.eventDir!
case 'concept':
return this.conceptDir!
case 'content':
return this.contentDir!
default:
return this.defaultDir!
}
}
// If no metadata or no noun field, use the default directory
return this.defaultDir!
} catch (error) {
// If there's an error getting the metadata, use the default directory
return this.defaultDir!
}
}
/**
* Get information about storage usage and capacity
*/
public async getStorageStatus(): Promise<{
type: string
used: number
quota: number | null
details?: Record<string, any>
}> {
await this.ensureInitialized()
try {
// Calculate the total size of all files in the storage directories
let totalSize = 0
// Helper function to calculate directory size
const calculateDirSize = async (
dirHandle: FileSystemDirectoryHandle
): Promise<number> => {
let size = 0
try {
for await (const [name, handle] of dirHandle.entries()) {
if (handle.kind === 'file') {
const file = await (handle as FileSystemFileHandle).getFile()
size += file.size
} else if (handle.kind === 'directory') {
size += await calculateDirSize(
handle as FileSystemDirectoryHandle
)
}
}
} catch (error) {
console.warn(`Error calculating size for directory:`, error)
}
return size
}
// Helper function to count files in a directory
const countFilesInDirectory = async (
dirHandle: FileSystemDirectoryHandle
): Promise<number> => {
let count = 0
try {
for await (const [name, handle] of dirHandle.entries()) {
if (handle.kind === 'file') {
count++
}
}
} catch (error) {
console.warn(`Error counting files in directory:`, error)
}
return count
}
// Calculate size for each directory
if (this.nounsDir) {
totalSize += await calculateDirSize(this.nounsDir)
}
if (this.verbsDir) {
totalSize += await calculateDirSize(this.verbsDir)
}
if (this.metadataDir) {
totalSize += await calculateDirSize(this.metadataDir)
}
// Calculate sizes of noun type directories
const personDirSize = this.personDir
? await calculateDirSize(this.personDir)
: 0
const placeDirSize = this.placeDir
? await calculateDirSize(this.placeDir)
: 0
const thingDirSize = this.thingDir
? await calculateDirSize(this.thingDir)
: 0
const eventDirSize = this.eventDir
? await calculateDirSize(this.eventDir)
: 0
const conceptDirSize = this.conceptDir
? await calculateDirSize(this.conceptDir)
: 0
const contentDirSize = this.contentDir
? await calculateDirSize(this.contentDir)
: 0
const defaultDirSize = this.defaultDir
? await calculateDirSize(this.defaultDir)
: 0
// Get storage quota information using the Storage API
let quota = null
let details: Record<string, any> = {
isPersistent: await this.isPersistent(),
nounTypes: {
person: {
size: personDirSize,
count: this.personDir
? await countFilesInDirectory(this.personDir)
: 0
},
place: {
size: placeDirSize,
count: this.placeDir
? await countFilesInDirectory(this.placeDir)
: 0
},
thing: {
size: thingDirSize,
count: this.thingDir
? await countFilesInDirectory(this.thingDir)
: 0
},
event: {
size: eventDirSize,
count: this.eventDir
? await countFilesInDirectory(this.eventDir)
: 0
},
concept: {
size: conceptDirSize,
count: this.conceptDir
? await countFilesInDirectory(this.conceptDir)
: 0
},
content: {
size: contentDirSize,
count: this.contentDir
? await countFilesInDirectory(this.contentDir)
: 0
},
default: {
size: defaultDirSize,
count: this.defaultDir
? await countFilesInDirectory(this.defaultDir)
: 0
}
}
}
try {
if (navigator.storage && navigator.storage.estimate) {
const estimate = await navigator.storage.estimate()
quota = estimate.quota || null
details = {
...details,
usage: estimate.usage,
quota: estimate.quota,
freePercentage: estimate.quota
? ((estimate.quota - (estimate.usage || 0)) / estimate.quota) *
100
: null
}
}
} catch (error) {
console.warn('Unable to get storage estimate:', error)
}
return {
type: 'opfs',
used: totalSize,
quota,
details
}
} catch (error) {
console.error('Failed to get storage status:', error)
return {
type: 'opfs',
used: 0,
quota: null,
details: { error: String(error) }
}
}
}
}
/**
* In-memory storage adapter for environments where OPFS is not available
*/
export class MemoryStorage implements StorageAdapter {
// Map of noun type to Map of noun ID to noun
private nouns: Map<string, Map<string, HNSWNode>> = new Map()
private verbs: Map<string, Edge> = new Map()
private metadata: Map<string, any> = new Map()
// Initialize maps for each noun type
constructor() {
this.nouns.set(PERSON_DIR, new Map())
this.nouns.set(PLACE_DIR, new Map())
this.nouns.set(THING_DIR, new Map())
this.nouns.set(EVENT_DIR, new Map())
this.nouns.set(CONCEPT_DIR, new Map())
this.nouns.set(CONTENT_DIR, new Map())
this.nouns.set(DEFAULT_DIR, new Map())
}
// Alias methods to match StorageAdapter interface
public async saveNoun(noun: HNSWNoun): Promise<void> {
return this.saveNode(noun)
}
public async getNoun(id: string): Promise<HNSWNoun | null> {
return this.getNode(id)
}
public async getAllNouns(): Promise<HNSWNoun[]> {
return this.getAllNodes()
}
public async getNounsByNounType(nounType: string): Promise<HNSWNoun[]> {
return this.getNodesByNounType(nounType)
}
public async deleteNoun(id: string): Promise<void> {
return this.deleteNode(id)
}
public async saveVerb(verb: GraphVerb): Promise<void> {
return this.saveEdge(verb)
}
public async getVerb(id: string): Promise<GraphVerb | null> {
return this.getEdge(id)
}
public async getAllVerbs(): Promise<GraphVerb[]> {
return this.getAllEdges()
}
public async getVerbsBySource(sourceId: string): Promise<GraphVerb[]> {
return this.getEdgesBySource(sourceId)
}
public async getVerbsByTarget(targetId: string): Promise<GraphVerb[]> {
return this.getEdgesByTarget(targetId)
}
public async getVerbsByType(type: string): Promise<GraphVerb[]> {
return this.getEdgesByType(type)
}
public async deleteVerb(id: string): Promise<void> {
return this.deleteEdge(id)
}
public async init(): Promise<void> {
// Nothing to initialize for in-memory storage
}
/**
* Get the appropriate node type for a node based on its metadata
*/
private async getNodeType(id: string): Promise<string> {
try {
// Try to get the metadata for the node
const metadata = await this.getMetadata(id)
// If metadata exists and has a noun field, use the corresponding type
if (metadata && metadata.noun) {
switch (metadata.noun) {
case 'person':
return PERSON_DIR
case 'place':
return PLACE_DIR
case 'thing':
return THING_DIR
case 'event':
return EVENT_DIR
case 'concept':
return CONCEPT_DIR
case 'content':
return CONTENT_DIR
default:
return DEFAULT_DIR
}
}
// If no metadata or no noun field, use the default type
return DEFAULT_DIR
} catch (error) {
// If there's an error getting the metadata, use the default type
return DEFAULT_DIR
}
}
public async saveNode(node: HNSWNode): Promise<void> {
// Create a deep copy to avoid reference issues
const nodeCopy: HNSWNode = {
id: node.id,
vector: [...node.vector],
connections: new Map()
}
// Copy connections
for (const [level, connections] of node.connections.entries()) {
nodeCopy.connections.set(level, new Set(connections))
}
// Get the appropriate node type based on the node's metadata
const nodeType = await this.getNodeType(node.id)
// Get the map for this node type
const nodeMap = this.nouns.get(nodeType)!
// Save the node in the appropriate map
nodeMap.set(node.id, nodeCopy)
}
public async getNode(id: string): Promise<HNSWNode | null> {
// Get the appropriate node type based on the node's metadata
const nodeType = await this.getNodeType(id)
// Get the map for this node type
const nodeMap = this.nouns.get(nodeType)!
// Try to get the node from the appropriate map
let node = nodeMap.get(id)
// If not found in the expected map, try other maps
if (!node) {
// If the node type is not the default type, try the default map
if (nodeType !== DEFAULT_DIR) {
const defaultMap = this.nouns.get(DEFAULT_DIR)!
node = defaultMap.get(id)
// If still not found, try all other maps
if (!node) {
const nodeTypes = [
PERSON_DIR,
PLACE_DIR,
THING_DIR,
EVENT_DIR,
CONCEPT_DIR,
CONTENT_DIR
]
for (const type of nodeTypes) {
if (type === nodeType) continue // Skip the already checked map
const typeMap = this.nouns.get(type)!
node = typeMap.get(id)
if (node) break // Found the node, exit the loop
}
}
}
// If still not found, return null
if (!node) {
return null
}
}
// Return a deep copy to avoid reference issues
const nodeCopy: HNSWNode = {
id: node.id,
vector: [...node.vector],
connections: new Map()
}
// Copy connections
for (const [level, connections] of node.connections.entries()) {
nodeCopy.connections.set(level, new Set(connections))
}
return nodeCopy
}
/**
* Get nodes by noun type
* @param nounType The noun type to filter by
* @returns Promise that resolves to an array of nodes of the specified noun type
*/
public async getNodesByNounType(nounType: string): Promise<HNSWNode[]> {
const nodes: HNSWNode[] = []
// Get the map for this noun type
let typeMap: Map<string, HNSWNode>
switch (nounType) {
case 'person':
typeMap = this.nouns.get(PERSON_DIR)!
break
case 'place':
typeMap = this.nouns.get(PLACE_DIR)!
break
case 'thing':
typeMap = this.nouns.get(THING_DIR)!
break
case 'event':
typeMap = this.nouns.get(EVENT_DIR)!
break
case 'concept':
typeMap = this.nouns.get(CONCEPT_DIR)!
break
case 'content':
typeMap = this.nouns.get(CONTENT_DIR)!
break
default:
typeMap = this.nouns.get(DEFAULT_DIR)!
}
// Get all nodes from this map
for (const nodeId of typeMap.keys()) {
const node = await this.getNode(nodeId)
if (node) {
nodes.push(node)
}
}
return nodes
}
public async getAllNodes(): Promise<HNSWNode[]> {
// Get all noun types
const nounTypes = [
'person',
'place',
'thing',
'event',
'concept',
'content',
'default'
]
// Run searches in parallel for all noun types
const nodePromises = nounTypes.map((nounType) =>
this.getNodesByNounType(nounType)
)
const nodeArrays = await Promise.all(nodePromises)
// Combine all results
const allNodes: HNSWNode[] = []
for (const nodes of nodeArrays) {
allNodes.push(...nodes)
}
return allNodes
}
public async deleteNode(id: string): Promise<void> {
// Get the appropriate node type based on the node's metadata
const nodeType = await this.getNodeType(id)
// Get the map for this node type
const nodeMap = this.nouns.get(nodeType)!
// Try to delete the node from the appropriate map
const deleted = nodeMap.delete(id)
// If not found in the expected map, try other maps
if (!deleted) {
// If the node type is not the default type, try the default map
if (nodeType !== DEFAULT_DIR) {
const defaultMap = this.nouns.get(DEFAULT_DIR)!
const deletedFromDefault = defaultMap.delete(id)
// If still not found, try all other maps
if (!deletedFromDefault) {
const nodeTypes = [
PERSON_DIR,
PLACE_DIR,
THING_DIR,
EVENT_DIR,
CONCEPT_DIR,
CONTENT_DIR
]
for (const type of nodeTypes) {
if (type === nodeType) continue // Skip the already checked map
const typeMap = this.nouns.get(type)!
const deletedFromType = typeMap.delete(id)
if (deletedFromType) break // Node deleted, exit the loop
}
}
}
}
}
public async saveMetadata(id: string, metadata: any): Promise<void> {
this.metadata.set(id, JSON.parse(JSON.stringify(metadata)))
}
public async getMetadata(id: string): Promise<any | null> {
const metadata = this.metadata.get(id)
if (!metadata) {
return null
}
return JSON.parse(JSON.stringify(metadata))
}
public async saveEdge(edge: Edge): Promise<void> {
// Create a deep copy to avoid reference issues
const edgeCopy: Edge = {
id: edge.id,
vector: [...edge.vector],
connections: new Map(),
sourceId: edge.sourceId,
targetId: edge.targetId,
type: edge.type,
weight: edge.weight,
metadata: edge.metadata
? JSON.parse(JSON.stringify(edge.metadata))
: undefined
}
// Copy connections
for (const [level, connections] of edge.connections.entries()) {
edgeCopy.connections.set(level, new Set(connections))
}
this.verbs.set(edge.id, edgeCopy)
}
public async getEdge(id: string): Promise<Edge | null> {
const edge = this.verbs.get(id)
if (!edge) {
return null
}
// Return a deep copy to avoid reference issues
const edgeCopy: Edge = {
id: edge.id,
vector: [...edge.vector],
connections: new Map(),
sourceId: edge.sourceId,
targetId: edge.targetId,
type: edge.type,
weight: edge.weight,
metadata: edge.metadata
? JSON.parse(JSON.stringify(edge.metadata))
: undefined
}
// Copy connections
for (const [level, connections] of edge.connections.entries()) {
edgeCopy.connections.set(level, new Set(connections))
}
return edgeCopy
}
public async getAllEdges(): Promise<Edge[]> {
const edges: Edge[] = []
for (const edgeId of this.verbs.keys()) {
const edge = await this.getEdge(edgeId)
if (edge) {
edges.push(edge)
}
}
return edges
}
public async getEdgesBySource(sourceId: string): Promise<Edge[]> {
const edges: Edge[] = []
for (const edge of this.verbs.values()) {
if (edge.sourceId === sourceId) {
const edgeCopy = await this.getEdge(edge.id)
if (edgeCopy) {
edges.push(edgeCopy)
}
}
}
return edges
}
public async getEdgesByTarget(targetId: string): Promise<Edge[]> {
const edges: Edge[] = []
for (const edge of this.verbs.values()) {
if (edge.targetId === targetId) {
const edgeCopy = await this.getEdge(edge.id)
if (edgeCopy) {
edges.push(edgeCopy)
}
}
}
return edges
}
public async getEdgesByType(type: string): Promise<Edge[]> {
const edges: Edge[] = []
for (const edge of this.verbs.values()) {
if (edge.type === type) {
const edgeCopy = await this.getEdge(edge.id)
if (edgeCopy) {
edges.push(edgeCopy)
}
}
}
return edges
}
public async deleteEdge(id: string): Promise<void> {
this.verbs.delete(id)
}
public async clear(): Promise<void> {
// Clear all noun type maps
const nodeTypes = [
PERSON_DIR,
PLACE_DIR,
THING_DIR,
EVENT_DIR,
CONCEPT_DIR,
CONTENT_DIR,
DEFAULT_DIR
]
for (const type of nodeTypes) {
const typeMap = this.nouns.get(type)!
typeMap.clear()
}
this.verbs.clear()
this.metadata.clear()
}
/**
* Get information about storage usage and capacity
*/
public async getStorageStatus(): Promise<{
type: string
used: number
quota: number | null
details?: Record<string, any>
}> {
try {
// Estimate the size of data in memory
let totalSize = 0
// Helper function to estimate object size in bytes
const estimateSize = (obj: any): number => {
if (obj === null || obj === undefined) return 0
const type = typeof obj
// Handle primitive types
if (type === 'number') return 8
if (type === 'string') return obj.length * 2
if (type === 'boolean') return 4
// Handle arrays and objects
if (Array.isArray(obj)) {
return obj.reduce((size, item) => size + estimateSize(item), 0)
}
if (type === 'object') {
if (obj instanceof Map) {
let mapSize = 0
for (const [key, value] of obj.entries()) {
mapSize += estimateSize(key) + estimateSize(value)
}
return mapSize
}
if (obj instanceof Set) {
let setSize = 0
for (const item of obj) {
setSize += estimateSize(item)
}
return setSize
}
// Regular object
return Object.entries(obj).reduce(
(size, [key, value]) => size + key.length * 2 + estimateSize(value),
0
)
}
return 0
}
// Calculate sizes and counts for each noun type
const nounTypeDetails: Record<string, { size: number; count: number }> =
{}
const nodeTypes = [
PERSON_DIR,
PLACE_DIR,
THING_DIR,
EVENT_DIR,
CONCEPT_DIR,
CONTENT_DIR,
DEFAULT_DIR
]
let totalNodeCount = 0
let nodesSize = 0
for (const type of nodeTypes) {
const typeMap = this.nouns.get(type)!
let typeSize = 0
for (const node of typeMap.values()) {
typeSize += estimateSize(node)
}
nounTypeDetails[type] = {
size: typeSize,
count: typeMap.size
}
nodesSize += typeSize
totalNodeCount += typeMap.size
}
totalSize += nodesSize
// Estimate size of edges
let edgesSize = 0
for (const edge of this.verbs.values()) {
edgesSize += estimateSize(edge)
}
totalSize += edgesSize
// Estimate size of metadata
let metadataSize = 0
for (const meta of this.metadata.values()) {
metadataSize += estimateSize(meta)
}
totalSize += metadataSize
// Get memory information if available
let quota = null
let details: Record<string, any> = {
nodeCount: totalNodeCount,
edgeCount: this.verbs.size,
metadataCount: this.metadata.size,
nounTypes: nounTypeDetails,
nodesSize,
edgesSize,
metadataSize
}
// Try to get memory information if in a browser environment
if (
typeof window !== 'undefined' &&
(window as any).performance &&
(window as any).performance.memory
) {
const memory = (window as any).performance.memory
quota = memory.jsHeapSizeLimit
details = {
...details,
totalJSHeapSize: memory.totalJSHeapSize,
usedJSHeapSize: memory.usedJSHeapSize,
jsHeapSizeLimit: memory.jsHeapSizeLimit
}
}
return {
type: 'memory',
used: totalSize,
quota,
details
}
} catch (error) {
console.error('Failed to get memory storage status:', error)
return {
type: 'memory',
used: 0,
quota: null,
details: { error: String(error) }
}
}
}
}
/**
* Factory function to create the appropriate storage adapter based on the environment
* @param options Configuration options for storage
* @returns Promise that resolves to a StorageAdapter instance
*/
export async function createStorage(
options: {
requestPersistentStorage?: boolean
r2Storage?: {
bucketName?: string
accountId?: string
accessKeyId?: string
secretAccessKey?: string
}
s3Storage?: {
bucketName?: string
accessKeyId?: string
secretAccessKey?: string
region?: string
}
gcsStorage?: {
bucketName?: string
accessKeyId?: string
secretAccessKey?: string
endpoint?: string
}
customS3Storage?: {
bucketName?: string
accessKeyId?: string
secretAccessKey?: string
endpoint?: string
region?: string
}
forceFileSystemStorage?: boolean
forceMemoryStorage?: boolean
} = {}
): Promise<StorageAdapter> {
// Check for environment variables for cloud storage (only in Node.js environment)
const isNode =
typeof process !== 'undefined' &&
process.versions != null &&
process.versions.node != null
// Default empty values
const defaultEnvStorage = {
bucketName: undefined,
accountId: undefined,
accessKeyId: undefined,
secretAccessKey: undefined,
region: undefined,
endpoint: undefined
}
// Only try to access process.env in Node.js environment
const envR2Storage = isNode
? {
bucketName: process.env.R2_BUCKET_NAME,
accountId: process.env.R2_ACCOUNT_ID,
accessKeyId: process.env.R2_ACCESS_KEY_ID,
secretAccessKey: process.env.R2_SECRET_ACCESS_KEY
}
: defaultEnvStorage
const envS3Storage = isNode
? {
bucketName: process.env.S3_BUCKET_NAME,
accessKeyId:
process.env.S3_ACCESS_KEY_ID || process.env.AWS_ACCESS_KEY_ID,
secretAccessKey:
process.env.S3_SECRET_ACCESS_KEY || process.env.AWS_SECRET_ACCESS_KEY,
region: process.env.S3_REGION || process.env.AWS_REGION
}
: defaultEnvStorage
const envGCSStorage = isNode
? {
bucketName: process.env.GCS_BUCKET_NAME,
accessKeyId: process.env.GCS_ACCESS_KEY_ID,
secretAccessKey: process.env.GCS_SECRET_ACCESS_KEY,
endpoint: process.env.GCS_ENDPOINT
}
: defaultEnvStorage
// Merge environment variables with provided options
const mergedOptions = {
...options,
r2Storage: options.r2Storage
? {
...envR2Storage,
...options.r2Storage
}
: // Only use environment variables if they have the required fields
envR2Storage.bucketName &&
envR2Storage.accessKeyId &&
envR2Storage.secretAccessKey
? envR2Storage
: undefined,
s3Storage: options.s3Storage
? {
...envS3Storage,
...options.s3Storage
}
: // Only use environment variables if they have the required fields
envS3Storage.bucketName &&
envS3Storage.accessKeyId &&
envS3Storage.secretAccessKey
? envS3Storage
: undefined,
gcsStorage: options.gcsStorage
? {
...envGCSStorage,
...options.gcsStorage
}
: // Only use environment variables if they have the required fields
envGCSStorage.bucketName &&
envGCSStorage.accessKeyId &&
envGCSStorage.secretAccessKey
? envGCSStorage
: undefined
}
// If any S3-compatible storage options are provided, use S3CompatibleStorage
if (
mergedOptions.r2Storage ||
mergedOptions.s3Storage ||
mergedOptions.gcsStorage ||
mergedOptions.customS3Storage
) {
try {
const s3Module = await import('./s3CompatibleStorage.js')
if (
mergedOptions.r2Storage &&
mergedOptions.r2Storage.bucketName &&
mergedOptions.r2Storage.accountId &&
mergedOptions.r2Storage.accessKeyId &&
mergedOptions.r2Storage.secretAccessKey
) {
return new s3Module.R2Storage({
bucketName: mergedOptions.r2Storage.bucketName,
accountId: mergedOptions.r2Storage.accountId,
accessKeyId: mergedOptions.r2Storage.accessKeyId,
secretAccessKey: mergedOptions.r2Storage.secretAccessKey
})
} else if (
mergedOptions.s3Storage &&
mergedOptions.s3Storage.bucketName &&
mergedOptions.s3Storage.accessKeyId &&
mergedOptions.s3Storage.secretAccessKey
) {
return new s3Module.S3CompatibleStorage({
bucketName: mergedOptions.s3Storage.bucketName,
accessKeyId: mergedOptions.s3Storage.accessKeyId,
secretAccessKey: mergedOptions.s3Storage.secretAccessKey,
region: mergedOptions.s3Storage.region,
serviceType: 's3'
})
} else if (
mergedOptions.gcsStorage &&
mergedOptions.gcsStorage.bucketName &&
mergedOptions.gcsStorage.accessKeyId &&
mergedOptions.gcsStorage.secretAccessKey
) {
return new s3Module.S3CompatibleStorage({
bucketName: mergedOptions.gcsStorage.bucketName,
accessKeyId: mergedOptions.gcsStorage.accessKeyId,
secretAccessKey: mergedOptions.gcsStorage.secretAccessKey,
endpoint: mergedOptions.gcsStorage.endpoint,
serviceType: 'gcs'
})
} else if (
mergedOptions.customS3Storage &&
mergedOptions.customS3Storage.bucketName &&
mergedOptions.customS3Storage.accessKeyId &&
mergedOptions.customS3Storage.secretAccessKey
) {
return new s3Module.S3CompatibleStorage({
bucketName: mergedOptions.customS3Storage.bucketName,
accessKeyId: mergedOptions.customS3Storage.accessKeyId,
secretAccessKey: mergedOptions.customS3Storage.secretAccessKey,
endpoint: mergedOptions.customS3Storage.endpoint,
region: mergedOptions.customS3Storage.region,
serviceType: 'custom'
})
}
} catch (error) {
console.warn(
'Failed to load S3CompatibleStorage, falling back to default storage:',
error
)
// Continue to default storage selection
}
}
// If force memory storage is specified, use MemoryStorage
if (options.forceMemoryStorage) {
return new MemoryStorage()
}
// Reuse the isNode variable from above
if (isNode) {
// In Node.js, use FileSystemStorage first, then fall back to memory
try {
const fileSystemModule = await import('./fileSystemStorage.js')
return new fileSystemModule.FileSystemStorage()
} catch (error) {
console.warn(
'Failed to load FileSystemStorage, falling back to in-memory storage:',
error
)
return new MemoryStorage()
}
} else {
// In browser, try OPFS first (unless force FileSystem is specified)
if (!options.forceFileSystemStorage) {
const opfsStorage = new OPFSStorage()
if (opfsStorage.isOPFSAvailable()) {
// Request persistent storage if specified
if (options.requestPersistentStorage) {
const isPersistentGranted =
await opfsStorage.requestPersistentStorage()
if (isPersistentGranted) {
console.log('Persistent storage permission granted')
} else {
console.warn('Persistent storage permission denied')
}
}
return opfsStorage
}
}
// OPFS is not available or force FileSystem is specified, try to use FileSystem API if available
try {
// Try to load FileSystemStorage for browser environments
// Note: This will likely fail as FileSystemStorage is designed for Node.js
const fileSystemModule = await import('./fileSystemStorage.js')
return new fileSystemModule.FileSystemStorage()
} catch (error) {
console.warn(
'FileSystem storage is not available, falling back to in-memory storage'
)
return new MemoryStorage()
}
}
}