Merge remote-tracking branch 'origin/main'
# Conflicts: # CHANGES.md # package-lock.json # package.json # src/storage/adapters/fileSystemStorage.ts
This commit is contained in:
commit
3337c9f78e
49 changed files with 5099 additions and 4391 deletions
320
src/storage/adapters/baseStorageAdapter.ts
Normal file
320
src/storage/adapters/baseStorageAdapter.ts
Normal file
|
|
@ -0,0 +1,320 @@
|
|||
/**
|
||||
* Base Storage Adapter
|
||||
* Provides common functionality for all storage adapters, including statistics tracking
|
||||
*/
|
||||
|
||||
import { StatisticsData, StorageAdapter } from '../../coreTypes.js'
|
||||
|
||||
/**
|
||||
* Base class for storage adapters that implements statistics tracking
|
||||
*/
|
||||
export abstract class BaseStorageAdapter implements StorageAdapter {
|
||||
// Abstract methods that must be implemented by subclasses
|
||||
abstract init(): Promise<void>
|
||||
abstract saveNoun(noun: any): Promise<void>
|
||||
abstract getNoun(id: string): Promise<any | null>
|
||||
abstract getAllNouns(): Promise<any[]>
|
||||
abstract getNounsByNounType(nounType: string): Promise<any[]>
|
||||
abstract deleteNoun(id: string): Promise<void>
|
||||
abstract saveVerb(verb: any): Promise<void>
|
||||
abstract getVerb(id: string): Promise<any | null>
|
||||
abstract getAllVerbs(): Promise<any[]>
|
||||
abstract getVerbsBySource(sourceId: string): Promise<any[]>
|
||||
abstract getVerbsByTarget(targetId: string): Promise<any[]>
|
||||
abstract getVerbsByType(type: string): Promise<any[]>
|
||||
abstract deleteVerb(id: string): Promise<void>
|
||||
abstract saveMetadata(id: string, metadata: any): Promise<void>
|
||||
abstract getMetadata(id: string): Promise<any | null>
|
||||
abstract clear(): Promise<void>
|
||||
abstract getStorageStatus(): Promise<{
|
||||
type: string
|
||||
used: number
|
||||
quota: number | null
|
||||
details?: Record<string, any>
|
||||
}>
|
||||
|
||||
// Statistics cache
|
||||
protected statisticsCache: StatisticsData | null = null
|
||||
|
||||
// Batch update timer ID
|
||||
protected statisticsBatchUpdateTimerId: NodeJS.Timeout | null = null
|
||||
|
||||
// Flag to indicate if statistics have been modified since last save
|
||||
protected statisticsModified = false
|
||||
|
||||
// Time of last statistics flush to storage
|
||||
protected lastStatisticsFlushTime = 0
|
||||
|
||||
// Minimum time between statistics flushes (5 seconds)
|
||||
protected readonly MIN_FLUSH_INTERVAL_MS = 5000
|
||||
|
||||
// Maximum time to wait before flushing statistics (30 seconds)
|
||||
protected readonly MAX_FLUSH_DELAY_MS = 30000
|
||||
|
||||
// Statistics-specific methods that must be implemented by subclasses
|
||||
protected abstract saveStatisticsData(statistics: StatisticsData): Promise<void>
|
||||
protected abstract getStatisticsData(): Promise<StatisticsData | null>
|
||||
|
||||
/**
|
||||
* Save statistics data
|
||||
* @param statistics The statistics data to save
|
||||
*/
|
||||
async saveStatistics(statistics: StatisticsData): Promise<void> {
|
||||
// Update the cache with a deep copy to avoid reference issues
|
||||
this.statisticsCache = {
|
||||
nounCount: {...statistics.nounCount},
|
||||
verbCount: {...statistics.verbCount},
|
||||
metadataCount: {...statistics.metadataCount},
|
||||
hnswIndexSize: statistics.hnswIndexSize,
|
||||
lastUpdated: statistics.lastUpdated
|
||||
}
|
||||
|
||||
// Schedule a batch update instead of saving immediately
|
||||
this.scheduleBatchUpdate()
|
||||
}
|
||||
|
||||
/**
|
||||
* Get statistics data
|
||||
* @returns Promise that resolves to the statistics data
|
||||
*/
|
||||
async getStatistics(): Promise<StatisticsData | null> {
|
||||
// If we have cached statistics, return a deep copy
|
||||
if (this.statisticsCache) {
|
||||
return {
|
||||
nounCount: {...this.statisticsCache.nounCount},
|
||||
verbCount: {...this.statisticsCache.verbCount},
|
||||
metadataCount: {...this.statisticsCache.metadataCount},
|
||||
hnswIndexSize: this.statisticsCache.hnswIndexSize,
|
||||
lastUpdated: this.statisticsCache.lastUpdated
|
||||
}
|
||||
}
|
||||
|
||||
// Otherwise, get from storage
|
||||
const statistics = await this.getStatisticsData()
|
||||
|
||||
// If we found statistics, update the cache
|
||||
if (statistics) {
|
||||
// Update the cache with a deep copy
|
||||
this.statisticsCache = {
|
||||
nounCount: {...statistics.nounCount},
|
||||
verbCount: {...statistics.verbCount},
|
||||
metadataCount: {...statistics.metadataCount},
|
||||
hnswIndexSize: statistics.hnswIndexSize,
|
||||
lastUpdated: statistics.lastUpdated
|
||||
}
|
||||
}
|
||||
|
||||
return statistics
|
||||
}
|
||||
|
||||
/**
|
||||
* Schedule a batch update of statistics
|
||||
*/
|
||||
protected scheduleBatchUpdate(): void {
|
||||
// Mark statistics as modified
|
||||
this.statisticsModified = true
|
||||
|
||||
// If a timer is already set, don't set another one
|
||||
if (this.statisticsBatchUpdateTimerId !== null) {
|
||||
return
|
||||
}
|
||||
|
||||
// Calculate time since last flush
|
||||
const now = Date.now()
|
||||
const timeSinceLastFlush = now - this.lastStatisticsFlushTime
|
||||
|
||||
// If we've recently flushed, wait longer before the next flush
|
||||
const delayMs = timeSinceLastFlush < this.MIN_FLUSH_INTERVAL_MS
|
||||
? this.MAX_FLUSH_DELAY_MS
|
||||
: this.MIN_FLUSH_INTERVAL_MS
|
||||
|
||||
// Schedule the batch update
|
||||
this.statisticsBatchUpdateTimerId = setTimeout(() => {
|
||||
this.flushStatistics()
|
||||
}, delayMs)
|
||||
}
|
||||
|
||||
/**
|
||||
* Flush statistics to storage
|
||||
*/
|
||||
protected async flushStatistics(): Promise<void> {
|
||||
// Clear the timer
|
||||
if (this.statisticsBatchUpdateTimerId !== null) {
|
||||
clearTimeout(this.statisticsBatchUpdateTimerId)
|
||||
this.statisticsBatchUpdateTimerId = null
|
||||
}
|
||||
|
||||
// If statistics haven't been modified, no need to flush
|
||||
if (!this.statisticsModified || !this.statisticsCache) {
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
// Save the statistics to storage
|
||||
await this.saveStatisticsData(this.statisticsCache)
|
||||
|
||||
// Update the last flush time
|
||||
this.lastStatisticsFlushTime = Date.now()
|
||||
// Reset the modified flag
|
||||
this.statisticsModified = false
|
||||
} catch (error) {
|
||||
console.error('Failed to flush statistics data:', error)
|
||||
// Mark as still modified so we'll try again later
|
||||
this.statisticsModified = true
|
||||
// Don't throw the error to avoid disrupting the application
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Increment a statistic counter
|
||||
* @param type The type of statistic to increment ('noun', 'verb', 'metadata')
|
||||
* @param service The service that inserted the data
|
||||
* @param amount The amount to increment by (default: 1)
|
||||
*/
|
||||
async incrementStatistic(
|
||||
type: 'noun' | 'verb' | 'metadata',
|
||||
service: string,
|
||||
amount: number = 1
|
||||
): Promise<void> {
|
||||
// Get current statistics from cache or storage
|
||||
let statistics = this.statisticsCache
|
||||
if (!statistics) {
|
||||
statistics = await this.getStatisticsData()
|
||||
if (!statistics) {
|
||||
statistics = this.createDefaultStatistics()
|
||||
}
|
||||
|
||||
// Update the cache
|
||||
this.statisticsCache = {
|
||||
nounCount: {...statistics.nounCount},
|
||||
verbCount: {...statistics.verbCount},
|
||||
metadataCount: {...statistics.metadataCount},
|
||||
hnswIndexSize: statistics.hnswIndexSize,
|
||||
lastUpdated: statistics.lastUpdated
|
||||
}
|
||||
}
|
||||
|
||||
// Increment the appropriate counter
|
||||
const counterMap = {
|
||||
noun: this.statisticsCache!.nounCount,
|
||||
verb: this.statisticsCache!.verbCount,
|
||||
metadata: this.statisticsCache!.metadataCount
|
||||
}
|
||||
|
||||
const counter = counterMap[type]
|
||||
counter[service] = (counter[service] || 0) + amount
|
||||
|
||||
// Update timestamp
|
||||
this.statisticsCache!.lastUpdated = new Date().toISOString()
|
||||
|
||||
// Schedule a batch update instead of saving immediately
|
||||
this.scheduleBatchUpdate()
|
||||
}
|
||||
|
||||
/**
|
||||
* Decrement a statistic counter
|
||||
* @param type The type of statistic to decrement ('noun', 'verb', 'metadata')
|
||||
* @param service The service that inserted the data
|
||||
* @param amount The amount to decrement by (default: 1)
|
||||
*/
|
||||
async decrementStatistic(
|
||||
type: 'noun' | 'verb' | 'metadata',
|
||||
service: string,
|
||||
amount: number = 1
|
||||
): Promise<void> {
|
||||
// Get current statistics from cache or storage
|
||||
let statistics = this.statisticsCache
|
||||
if (!statistics) {
|
||||
statistics = await this.getStatisticsData()
|
||||
if (!statistics) {
|
||||
statistics = this.createDefaultStatistics()
|
||||
}
|
||||
|
||||
// Update the cache
|
||||
this.statisticsCache = {
|
||||
nounCount: {...statistics.nounCount},
|
||||
verbCount: {...statistics.verbCount},
|
||||
metadataCount: {...statistics.metadataCount},
|
||||
hnswIndexSize: statistics.hnswIndexSize,
|
||||
lastUpdated: statistics.lastUpdated
|
||||
}
|
||||
}
|
||||
|
||||
// Decrement the appropriate counter
|
||||
const counterMap = {
|
||||
noun: this.statisticsCache!.nounCount,
|
||||
verb: this.statisticsCache!.verbCount,
|
||||
metadata: this.statisticsCache!.metadataCount
|
||||
}
|
||||
|
||||
const counter = counterMap[type]
|
||||
counter[service] = Math.max(0, (counter[service] || 0) - amount)
|
||||
|
||||
// Update timestamp
|
||||
this.statisticsCache!.lastUpdated = new Date().toISOString()
|
||||
|
||||
// Schedule a batch update instead of saving immediately
|
||||
this.scheduleBatchUpdate()
|
||||
}
|
||||
|
||||
/**
|
||||
* Update the HNSW index size statistic
|
||||
* @param size The new size of the HNSW index
|
||||
*/
|
||||
async updateHnswIndexSize(size: number): Promise<void> {
|
||||
// Get current statistics from cache or storage
|
||||
let statistics = this.statisticsCache
|
||||
if (!statistics) {
|
||||
statistics = await this.getStatisticsData()
|
||||
if (!statistics) {
|
||||
statistics = this.createDefaultStatistics()
|
||||
}
|
||||
|
||||
// Update the cache
|
||||
this.statisticsCache = {
|
||||
nounCount: {...statistics.nounCount},
|
||||
verbCount: {...statistics.verbCount},
|
||||
metadataCount: {...statistics.metadataCount},
|
||||
hnswIndexSize: statistics.hnswIndexSize,
|
||||
lastUpdated: statistics.lastUpdated
|
||||
}
|
||||
}
|
||||
|
||||
// Update HNSW index size
|
||||
this.statisticsCache!.hnswIndexSize = size
|
||||
|
||||
// Update timestamp
|
||||
this.statisticsCache!.lastUpdated = new Date().toISOString()
|
||||
|
||||
// Schedule a batch update instead of saving immediately
|
||||
this.scheduleBatchUpdate()
|
||||
}
|
||||
|
||||
/**
|
||||
* Force an immediate flush of statistics to storage
|
||||
* This ensures that any pending statistics updates are written to persistent storage
|
||||
*/
|
||||
async flushStatisticsToStorage(): Promise<void> {
|
||||
// If there are no statistics in cache or they haven't been modified, nothing to flush
|
||||
if (!this.statisticsCache || !this.statisticsModified) {
|
||||
return
|
||||
}
|
||||
|
||||
// Call the protected flushStatistics method to immediately write to storage
|
||||
await this.flushStatistics()
|
||||
}
|
||||
|
||||
/**
|
||||
* Create default statistics data
|
||||
* @returns Default statistics data
|
||||
*/
|
||||
protected createDefaultStatistics(): StatisticsData {
|
||||
return {
|
||||
nounCount: {},
|
||||
verbCount: {},
|
||||
metadataCount: {},
|
||||
hnswIndexSize: 0,
|
||||
lastUpdated: new Date().toISOString()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -3,18 +3,10 @@
|
|||
* In-memory storage adapter for environments where persistent storage is not available or needed
|
||||
*/
|
||||
|
||||
import { GraphVerb, HNSWNoun } from '../../coreTypes.js'
|
||||
import { BaseStorage } from '../baseStorage.js'
|
||||
import { GraphVerb, HNSWNoun, StatisticsData } from '../../coreTypes.js'
|
||||
import { BaseStorage, STATISTICS_KEY } from '../baseStorage.js'
|
||||
|
||||
/**
|
||||
* Type alias for HNSWNoun to make the code more readable
|
||||
*/
|
||||
type HNSWNode = HNSWNoun
|
||||
|
||||
/**
|
||||
* Type alias for GraphVerb to make the code more readable
|
||||
*/
|
||||
type Edge = GraphVerb
|
||||
// No type aliases needed - using the original types directly
|
||||
|
||||
/**
|
||||
* In-memory storage adapter
|
||||
|
|
@ -22,9 +14,10 @@ type Edge = GraphVerb
|
|||
*/
|
||||
export class MemoryStorage extends BaseStorage {
|
||||
// Single map of noun ID to noun
|
||||
private nouns: Map<string, HNSWNode> = new Map()
|
||||
private verbs: Map<string, Edge> = new Map()
|
||||
private nouns: Map<string, HNSWNoun> = new Map()
|
||||
private verbs: Map<string, GraphVerb> = new Map()
|
||||
private metadata: Map<string, any> = new Map()
|
||||
private statistics: StatisticsData | null = null
|
||||
|
||||
constructor() {
|
||||
super()
|
||||
|
|
@ -39,237 +32,270 @@ export class MemoryStorage extends BaseStorage {
|
|||
}
|
||||
|
||||
/**
|
||||
* Save a node to storage
|
||||
* Save a noun to storage
|
||||
*/
|
||||
protected async saveNode(node: HNSWNode): Promise<void> {
|
||||
protected async saveNoun_internal(noun: HNSWNoun): Promise<void> {
|
||||
// Create a deep copy to avoid reference issues
|
||||
const nodeCopy: HNSWNode = {
|
||||
id: node.id,
|
||||
vector: [...node.vector],
|
||||
const nounCopy: HNSWNoun = {
|
||||
id: noun.id,
|
||||
vector: [...noun.vector],
|
||||
connections: new Map()
|
||||
}
|
||||
|
||||
// Copy connections
|
||||
for (const [level, connections] of node.connections.entries()) {
|
||||
nodeCopy.connections.set(level, new Set(connections))
|
||||
for (const [level, connections] of noun.connections.entries()) {
|
||||
nounCopy.connections.set(level, new Set(connections))
|
||||
}
|
||||
|
||||
// Save the node directly in the nouns map
|
||||
this.nouns.set(node.id, nodeCopy)
|
||||
// Save the noun directly in the nouns map
|
||||
this.nouns.set(noun.id, nounCopy)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a node from storage
|
||||
* Get a noun from storage
|
||||
*/
|
||||
protected async getNode(id: string): Promise<HNSWNode | null> {
|
||||
// Get the node directly from the nouns map
|
||||
const node = this.nouns.get(id)
|
||||
protected async getNoun_internal(id: string): Promise<HNSWNoun | null> {
|
||||
// Get the noun directly from the nouns map
|
||||
const noun = this.nouns.get(id)
|
||||
|
||||
// If not found, return null
|
||||
if (!node) {
|
||||
if (!noun) {
|
||||
return null
|
||||
}
|
||||
|
||||
// Return a deep copy to avoid reference issues
|
||||
const nodeCopy: HNSWNode = {
|
||||
id: node.id,
|
||||
vector: [...node.vector],
|
||||
const nounCopy: HNSWNoun = {
|
||||
id: noun.id,
|
||||
vector: [...noun.vector],
|
||||
connections: new Map()
|
||||
}
|
||||
|
||||
// Copy connections
|
||||
for (const [level, connections] of node.connections.entries()) {
|
||||
nodeCopy.connections.set(level, new Set(connections))
|
||||
for (const [level, connections] of noun.connections.entries()) {
|
||||
nounCopy.connections.set(level, new Set(connections))
|
||||
}
|
||||
|
||||
return nodeCopy
|
||||
return nounCopy
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all nodes from storage
|
||||
* Get all nouns from storage
|
||||
*/
|
||||
protected async getAllNodes(): Promise<HNSWNode[]> {
|
||||
const allNodes: HNSWNode[] = []
|
||||
protected async getAllNouns_internal(): Promise<HNSWNoun[]> {
|
||||
const allNouns: HNSWNoun[] = []
|
||||
|
||||
// Iterate through all nodes in the nouns map
|
||||
for (const [nodeId, node] of this.nouns.entries()) {
|
||||
// Iterate through all nouns in the nouns map
|
||||
for (const [nounId, noun] of this.nouns.entries()) {
|
||||
// Return a deep copy to avoid reference issues
|
||||
const nodeCopy: HNSWNode = {
|
||||
id: node.id,
|
||||
vector: [...node.vector],
|
||||
const nounCopy: HNSWNoun = {
|
||||
id: noun.id,
|
||||
vector: [...noun.vector],
|
||||
connections: new Map()
|
||||
}
|
||||
|
||||
// Copy connections
|
||||
for (const [level, connections] of node.connections.entries()) {
|
||||
nodeCopy.connections.set(level, new Set(connections))
|
||||
for (const [level, connections] of noun.connections.entries()) {
|
||||
nounCopy.connections.set(level, new Set(connections))
|
||||
}
|
||||
|
||||
allNodes.push(nodeCopy)
|
||||
allNouns.push(nounCopy)
|
||||
}
|
||||
|
||||
return allNodes
|
||||
return allNouns
|
||||
}
|
||||
|
||||
/**
|
||||
* Get nodes by noun type
|
||||
* Get nouns by noun type
|
||||
* @param nounType The noun type to filter by
|
||||
* @returns Promise that resolves to an array of nodes of the specified noun type
|
||||
* @returns Promise that resolves to an array of nouns of the specified noun type
|
||||
*/
|
||||
protected async getNodesByNounType(nounType: string): Promise<HNSWNode[]> {
|
||||
const nodes: HNSWNode[] = []
|
||||
protected async getNounsByNounType_internal(nounType: string): Promise<HNSWNoun[]> {
|
||||
const nouns: HNSWNoun[] = []
|
||||
|
||||
// Iterate through all nodes and filter by noun type using metadata
|
||||
for (const [nodeId, node] of this.nouns.entries()) {
|
||||
// Iterate through all nouns and filter by noun type using metadata
|
||||
for (const [nounId, noun] of this.nouns.entries()) {
|
||||
// Get the metadata to check the noun type
|
||||
const metadata = await this.getMetadata(nodeId)
|
||||
const metadata = await this.getMetadata(nounId)
|
||||
|
||||
// Include the node if its noun type matches the requested type
|
||||
// Include the noun if its noun type matches the requested type
|
||||
if (metadata && metadata.noun === nounType) {
|
||||
// Return a deep copy to avoid reference issues
|
||||
const nodeCopy: HNSWNode = {
|
||||
id: node.id,
|
||||
vector: [...node.vector],
|
||||
const nounCopy: HNSWNoun = {
|
||||
id: noun.id,
|
||||
vector: [...noun.vector],
|
||||
connections: new Map()
|
||||
}
|
||||
|
||||
// Copy connections
|
||||
for (const [level, connections] of node.connections.entries()) {
|
||||
nodeCopy.connections.set(level, new Set(connections))
|
||||
for (const [level, connections] of noun.connections.entries()) {
|
||||
nounCopy.connections.set(level, new Set(connections))
|
||||
}
|
||||
|
||||
nodes.push(nodeCopy)
|
||||
nouns.push(nounCopy)
|
||||
}
|
||||
}
|
||||
|
||||
return nodes
|
||||
return nouns
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a node from storage
|
||||
* Delete a noun from storage
|
||||
*/
|
||||
protected async deleteNode(id: string): Promise<void> {
|
||||
// Delete the node directly from the nouns map
|
||||
protected async deleteNoun_internal(id: string): Promise<void> {
|
||||
this.nouns.delete(id)
|
||||
}
|
||||
|
||||
/**
|
||||
* Save an edge to storage
|
||||
* Save a verb to storage
|
||||
*/
|
||||
protected async saveEdge(edge: Edge): Promise<void> {
|
||||
protected async saveVerb_internal(verb: GraphVerb): Promise<void> {
|
||||
// Create a deep copy to avoid reference issues
|
||||
const edgeCopy: Edge = {
|
||||
id: edge.id,
|
||||
vector: [...edge.vector],
|
||||
const verbCopy: GraphVerb = {
|
||||
id: verb.id,
|
||||
vector: [...verb.vector],
|
||||
connections: new Map(),
|
||||
sourceId: edge.sourceId,
|
||||
targetId: edge.targetId,
|
||||
type: edge.type,
|
||||
weight: edge.weight,
|
||||
metadata: edge.metadata
|
||||
sourceId: verb.sourceId,
|
||||
targetId: verb.targetId,
|
||||
type: verb.type,
|
||||
weight: verb.weight,
|
||||
metadata: verb.metadata
|
||||
}
|
||||
|
||||
// Copy connections
|
||||
for (const [level, connections] of edge.connections.entries()) {
|
||||
edgeCopy.connections.set(level, new Set(connections))
|
||||
for (const [level, connections] of verb.connections.entries()) {
|
||||
verbCopy.connections.set(level, new Set(connections))
|
||||
}
|
||||
|
||||
// Save the edge directly in the verbs map
|
||||
this.verbs.set(edge.id, edgeCopy)
|
||||
// Save the verb directly in the verbs map
|
||||
this.verbs.set(verb.id, verbCopy)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get an edge from storage
|
||||
* Get a verb from storage
|
||||
*/
|
||||
protected async getEdge(id: string): Promise<Edge | null> {
|
||||
// Get the edge directly from the verbs map
|
||||
const edge = this.verbs.get(id)
|
||||
protected async getVerb_internal(id: string): Promise<GraphVerb | null> {
|
||||
// Get the verb directly from the verbs map
|
||||
const verb = this.verbs.get(id)
|
||||
|
||||
// If not found, return null
|
||||
if (!edge) {
|
||||
if (!verb) {
|
||||
return null
|
||||
}
|
||||
|
||||
// Create default timestamp if not present
|
||||
const defaultTimestamp = {
|
||||
seconds: Math.floor(Date.now() / 1000),
|
||||
nanoseconds: (Date.now() % 1000) * 1000000
|
||||
}
|
||||
|
||||
// Create default createdBy if not present
|
||||
const defaultCreatedBy = {
|
||||
augmentation: 'unknown',
|
||||
version: '1.0'
|
||||
}
|
||||
|
||||
// Return a deep copy to avoid reference issues
|
||||
const edgeCopy: Edge = {
|
||||
id: edge.id,
|
||||
vector: [...edge.vector],
|
||||
const verbCopy: GraphVerb = {
|
||||
id: verb.id,
|
||||
vector: [...verb.vector],
|
||||
connections: new Map(),
|
||||
sourceId: edge.sourceId,
|
||||
targetId: edge.targetId,
|
||||
type: edge.type,
|
||||
weight: edge.weight,
|
||||
metadata: edge.metadata
|
||||
sourceId: (verb.sourceId || verb.source || ""),
|
||||
targetId: (verb.targetId || verb.target || ""),
|
||||
source: (verb.sourceId || verb.source || ""),
|
||||
target: (verb.targetId || verb.target || ""),
|
||||
verb: verb.type || verb.verb,
|
||||
weight: verb.weight,
|
||||
metadata: verb.metadata,
|
||||
createdAt: verb.createdAt || defaultTimestamp,
|
||||
updatedAt: verb.updatedAt || defaultTimestamp,
|
||||
createdBy: verb.createdBy || defaultCreatedBy
|
||||
}
|
||||
|
||||
// Copy connections
|
||||
for (const [level, connections] of edge.connections.entries()) {
|
||||
edgeCopy.connections.set(level, new Set(connections))
|
||||
for (const [level, connections] of verb.connections.entries()) {
|
||||
verbCopy.connections.set(level, new Set(connections))
|
||||
}
|
||||
|
||||
return edgeCopy
|
||||
return verbCopy
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all edges from storage
|
||||
* Get all verbs from storage
|
||||
*/
|
||||
protected async getAllEdges(): Promise<Edge[]> {
|
||||
const allEdges: Edge[] = []
|
||||
protected async getAllVerbs_internal(): Promise<GraphVerb[]> {
|
||||
const allVerbs: GraphVerb[] = []
|
||||
|
||||
// Iterate through all verbs in the verbs map
|
||||
for (const [verbId, verb] of this.verbs.entries()) {
|
||||
// Create default timestamp if not present
|
||||
const defaultTimestamp = {
|
||||
seconds: Math.floor(Date.now() / 1000),
|
||||
nanoseconds: (Date.now() % 1000) * 1000000
|
||||
}
|
||||
|
||||
// Create default createdBy if not present
|
||||
const defaultCreatedBy = {
|
||||
augmentation: 'unknown',
|
||||
version: '1.0'
|
||||
}
|
||||
|
||||
// Iterate through all edges in the verbs map
|
||||
for (const [edgeId, edge] of this.verbs.entries()) {
|
||||
// Return a deep copy to avoid reference issues
|
||||
const edgeCopy: Edge = {
|
||||
id: edge.id,
|
||||
vector: [...edge.vector],
|
||||
const verbCopy: GraphVerb = {
|
||||
id: verb.id,
|
||||
vector: [...verb.vector],
|
||||
connections: new Map(),
|
||||
sourceId: edge.sourceId,
|
||||
targetId: edge.targetId,
|
||||
type: edge.type,
|
||||
weight: edge.weight,
|
||||
metadata: edge.metadata
|
||||
sourceId: (verb.sourceId || verb.source || ""),
|
||||
targetId: (verb.targetId || verb.target || ""),
|
||||
source: (verb.sourceId || verb.source || ""),
|
||||
target: (verb.targetId || verb.target || ""),
|
||||
verb: verb.type || verb.verb,
|
||||
weight: verb.weight,
|
||||
metadata: verb.metadata,
|
||||
createdAt: verb.createdAt || defaultTimestamp,
|
||||
updatedAt: verb.updatedAt || defaultTimestamp,
|
||||
createdBy: verb.createdBy || defaultCreatedBy
|
||||
}
|
||||
|
||||
// Copy connections
|
||||
for (const [level, connections] of edge.connections.entries()) {
|
||||
edgeCopy.connections.set(level, new Set(connections))
|
||||
for (const [level, connections] of verb.connections.entries()) {
|
||||
verbCopy.connections.set(level, new Set(connections))
|
||||
}
|
||||
|
||||
allEdges.push(edgeCopy)
|
||||
allVerbs.push(verbCopy)
|
||||
}
|
||||
|
||||
return allEdges
|
||||
return allVerbs
|
||||
}
|
||||
|
||||
/**
|
||||
* Get edges by source
|
||||
* Get verbs by source
|
||||
*/
|
||||
protected async getEdgesBySource(sourceId: string): Promise<Edge[]> {
|
||||
const edges = await this.getAllEdges()
|
||||
return edges.filter((edge) => edge.sourceId === sourceId)
|
||||
protected async getVerbsBySource_internal(sourceId: string): Promise<GraphVerb[]> {
|
||||
const allVerbs = await this.getAllVerbs_internal()
|
||||
return allVerbs.filter((verb: GraphVerb) => (verb.sourceId || verb.source) === sourceId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get edges by target
|
||||
* Get verbs by target
|
||||
*/
|
||||
protected async getEdgesByTarget(targetId: string): Promise<Edge[]> {
|
||||
const edges = await this.getAllEdges()
|
||||
return edges.filter((edge) => edge.targetId === targetId)
|
||||
protected async getVerbsByTarget_internal(targetId: string): Promise<GraphVerb[]> {
|
||||
const allVerbs = await this.getAllVerbs_internal()
|
||||
return allVerbs.filter((verb: GraphVerb) => (verb.targetId || verb.target) === targetId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get edges by type
|
||||
* Get verbs by type
|
||||
*/
|
||||
protected async getEdgesByType(type: string): Promise<Edge[]> {
|
||||
const edges = await this.getAllEdges()
|
||||
return edges.filter((edge) => edge.type === type)
|
||||
protected async getVerbsByType_internal(type: string): Promise<GraphVerb[]> {
|
||||
const allVerbs = await this.getAllVerbs_internal()
|
||||
return allVerbs.filter((verb: GraphVerb) => (verb.type || verb.verb) === type)
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete an edge from storage
|
||||
* Delete a verb from storage
|
||||
*/
|
||||
protected async deleteEdge(id: string): Promise<void> {
|
||||
// Delete the edge directly from the verbs map
|
||||
protected async deleteVerb_internal(id: string): Promise<void> {
|
||||
// Delete the verb directly from the verbs map
|
||||
this.verbs.delete(id)
|
||||
}
|
||||
|
||||
|
|
@ -321,4 +347,45 @@ export class MemoryStorage extends BaseStorage {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save statistics data to storage
|
||||
* @param statistics The statistics data to save
|
||||
*/
|
||||
protected async saveStatisticsData(statistics: StatisticsData): Promise<void> {
|
||||
// For memory storage, we just need to store the statistics in memory
|
||||
// Create a deep copy to avoid reference issues
|
||||
this.statistics = {
|
||||
nounCount: {...statistics.nounCount},
|
||||
verbCount: {...statistics.verbCount},
|
||||
metadataCount: {...statistics.metadataCount},
|
||||
hnswIndexSize: statistics.hnswIndexSize,
|
||||
lastUpdated: statistics.lastUpdated
|
||||
}
|
||||
|
||||
// Since this is in-memory, there's no need for time-based partitioning
|
||||
// or legacy file handling
|
||||
}
|
||||
|
||||
/**
|
||||
* Get statistics data from storage
|
||||
* @returns Promise that resolves to the statistics data or null if not found
|
||||
*/
|
||||
protected async getStatisticsData(): Promise<StatisticsData | null> {
|
||||
if (!this.statistics) {
|
||||
return null
|
||||
}
|
||||
|
||||
// Return a deep copy to avoid reference issues
|
||||
return {
|
||||
nounCount: {...this.statistics.nounCount},
|
||||
verbCount: {...this.statistics.verbCount},
|
||||
metadataCount: {...this.statistics.metadataCount},
|
||||
hnswIndexSize: this.statistics.hnswIndexSize,
|
||||
lastUpdated: this.statistics.lastUpdated
|
||||
}
|
||||
|
||||
// Since this is in-memory, there's no need for fallback mechanisms
|
||||
// to check multiple storage locations
|
||||
}
|
||||
}
|
||||
|
|
@ -3,10 +3,18 @@
|
|||
* Provides persistent storage for the vector database using the Origin Private File System API
|
||||
*/
|
||||
|
||||
import {GraphVerb, HNSWNoun} from '../../coreTypes.js'
|
||||
import {BaseStorage, NOUNS_DIR, VERBS_DIR, METADATA_DIR, INDEX_DIR} from '../baseStorage.js'
|
||||
import {GraphVerb, HNSWNoun, StatisticsData} from '../../coreTypes.js'
|
||||
import {BaseStorage, NOUNS_DIR, VERBS_DIR, METADATA_DIR, INDEX_DIR, STATISTICS_KEY} from '../baseStorage.js'
|
||||
import '../../types/fileSystemTypes.js'
|
||||
|
||||
// Type alias for HNSWNode
|
||||
type HNSWNode = HNSWNoun
|
||||
|
||||
/**
|
||||
* Type alias for GraphVerb to make the code more readable
|
||||
*/
|
||||
type Edge = GraphVerb
|
||||
|
||||
/**
|
||||
* Helper function to safely get a file from a FileSystemHandle
|
||||
* This is needed because TypeScript doesn't recognize that a FileSystemHandle
|
||||
|
|
@ -18,8 +26,8 @@ async function safeGetFile(handle: FileSystemHandle): Promise<File> {
|
|||
}
|
||||
|
||||
// Type aliases for better readability
|
||||
type HNSWNode = HNSWNoun
|
||||
type Edge = GraphVerb
|
||||
type HNSWNoun_internal = HNSWNoun
|
||||
type Verb = GraphVerb
|
||||
|
||||
// Root directory name for OPFS storage
|
||||
const ROOT_DIR = 'opfs-vector-db'
|
||||
|
|
@ -37,6 +45,7 @@ export class OPFSStorage extends BaseStorage {
|
|||
private isAvailable = false
|
||||
private isPersistentRequested = false
|
||||
private isPersistentGranted = false
|
||||
private statistics: StatisticsData | null = null
|
||||
|
||||
constructor() {
|
||||
super()
|
||||
|
|
@ -148,54 +157,54 @@ export class OPFSStorage extends BaseStorage {
|
|||
}
|
||||
|
||||
/**
|
||||
* Save a node to storage
|
||||
* Save a noun to storage
|
||||
*/
|
||||
protected async saveNode(node: HNSWNode): Promise<void> {
|
||||
protected async saveNoun_internal(noun: HNSWNoun_internal): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Convert connections Map to a serializable format
|
||||
const serializableNode = {
|
||||
...node,
|
||||
connections: this.mapToObject(node.connections, (set) =>
|
||||
const serializableNoun = {
|
||||
...noun,
|
||||
connections: this.mapToObject(noun.connections, (set) =>
|
||||
Array.from(set as Set<string>)
|
||||
)
|
||||
}
|
||||
|
||||
// Create or get the file for this noun
|
||||
const fileHandle = await this.nounsDir!.getFileHandle(node.id, {
|
||||
const fileHandle = await this.nounsDir!.getFileHandle(noun.id, {
|
||||
create: true
|
||||
})
|
||||
|
||||
// Write the noun data to the file
|
||||
const writable = await fileHandle.createWritable()
|
||||
await writable.write(JSON.stringify(serializableNode))
|
||||
await writable.write(JSON.stringify(serializableNoun))
|
||||
await writable.close()
|
||||
} catch (error) {
|
||||
console.error(`Failed to save node ${node.id}:`, error)
|
||||
throw new Error(`Failed to save node ${node.id}: ${error}`)
|
||||
console.error(`Failed to save noun ${noun.id}:`, error)
|
||||
throw new Error(`Failed to save noun ${noun.id}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a node from storage
|
||||
* Get a noun from storage
|
||||
*/
|
||||
protected async getNode(id: string): Promise<HNSWNode | null> {
|
||||
protected async getNoun_internal(id: string): Promise<HNSWNoun_internal | null> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Get the file handle for this node
|
||||
// Get the file handle for this noun
|
||||
const fileHandle = await this.nounsDir!.getFileHandle(id)
|
||||
|
||||
// Read the node data from the file
|
||||
// Read the noun data from the file
|
||||
const file = await fileHandle.getFile()
|
||||
const text = await file.text()
|
||||
const data = JSON.parse(text)
|
||||
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(data.connections)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
for (const [level, nounIds] of Object.entries(data.connections)) {
|
||||
connections.set(Number(level), new Set(nounIds as string[]))
|
||||
}
|
||||
|
||||
return {
|
||||
|
|
@ -204,41 +213,41 @@ export class OPFSStorage extends BaseStorage {
|
|||
connections
|
||||
}
|
||||
} catch (error) {
|
||||
// Node not found or other error
|
||||
// Noun not found or other error
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all nodes from storage
|
||||
* Get all nouns from storage
|
||||
*/
|
||||
protected async getAllNodes(): Promise<HNSWNode[]> {
|
||||
protected async getAllNouns_internal(): Promise<HNSWNoun_internal[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
const allNodes: HNSWNode[] = []
|
||||
const allNouns: HNSWNoun_internal[] = []
|
||||
try {
|
||||
// Iterate through all files in the nouns directory
|
||||
for await (const [name, handle] of this.nounsDir!.entries()) {
|
||||
if (handle.kind === 'file') {
|
||||
try {
|
||||
// Read the node data from the file
|
||||
// Read the noun data from the file
|
||||
const file = await safeGetFile(handle)
|
||||
const text = await file.text()
|
||||
const data = JSON.parse(text)
|
||||
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(data.connections)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
for (const [level, nounIds] of Object.entries(data.connections)) {
|
||||
connections.set(Number(level), new Set(nounIds as string[]))
|
||||
}
|
||||
|
||||
allNodes.push({
|
||||
allNouns.push({
|
||||
id: data.id,
|
||||
vector: data.vector,
|
||||
connections
|
||||
})
|
||||
} catch (error) {
|
||||
console.error(`Error reading node file ${name}:`, error)
|
||||
console.error(`Error reading noun file ${name}:`, error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -246,7 +255,16 @@ export class OPFSStorage extends BaseStorage {
|
|||
console.error('Error reading nouns directory:', error)
|
||||
}
|
||||
|
||||
return allNodes
|
||||
return allNouns
|
||||
}
|
||||
|
||||
/**
|
||||
* Get nouns by noun type (internal implementation)
|
||||
* @param nounType The noun type to filter by
|
||||
* @returns Promise that resolves to an array of nouns of the specified noun type
|
||||
*/
|
||||
protected async getNounsByNounType_internal(nounType: string): Promise<HNSWNoun[]> {
|
||||
return this.getNodesByNounType(nounType)
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -298,6 +316,13 @@ export class OPFSStorage extends BaseStorage {
|
|||
return nodes
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a noun from storage (internal implementation)
|
||||
*/
|
||||
protected async deleteNoun_internal(id: string): Promise<void> {
|
||||
return this.deleteNode(id)
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a node from storage
|
||||
*/
|
||||
|
|
@ -315,6 +340,13 @@ export class OPFSStorage extends BaseStorage {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save a verb to storage (internal implementation)
|
||||
*/
|
||||
protected async saveVerb_internal(verb: GraphVerb): Promise<void> {
|
||||
return this.saveEdge(verb)
|
||||
}
|
||||
|
||||
/**
|
||||
* Save an edge to storage
|
||||
*/
|
||||
|
|
@ -345,6 +377,13 @@ export class OPFSStorage extends BaseStorage {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a verb from storage (internal implementation)
|
||||
*/
|
||||
protected async getVerb_internal(id: string): Promise<GraphVerb | null> {
|
||||
return this.getEdge(id)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get an edge from storage
|
||||
*/
|
||||
|
|
@ -366,15 +405,32 @@ export class OPFSStorage extends BaseStorage {
|
|||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
// Create default timestamp if not present
|
||||
const defaultTimestamp = {
|
||||
seconds: Math.floor(Date.now() / 1000),
|
||||
nanoseconds: (Date.now() % 1000) * 1000000
|
||||
}
|
||||
|
||||
// Create default createdBy if not present
|
||||
const defaultCreatedBy = {
|
||||
augmentation: 'unknown',
|
||||
version: '1.0'
|
||||
}
|
||||
|
||||
return {
|
||||
id: data.id,
|
||||
vector: data.vector,
|
||||
connections,
|
||||
sourceId: data.sourceId,
|
||||
targetId: data.targetId,
|
||||
type: data.type,
|
||||
sourceId: data.sourceId || data.source,
|
||||
targetId: data.targetId || data.target,
|
||||
source: data.sourceId || data.source,
|
||||
target: data.targetId || data.target,
|
||||
verb: data.type || data.verb,
|
||||
weight: data.weight,
|
||||
metadata: data.metadata
|
||||
metadata: data.metadata,
|
||||
createdAt: data.createdAt || defaultTimestamp,
|
||||
updatedAt: data.updatedAt || defaultTimestamp,
|
||||
createdBy: data.createdBy || defaultCreatedBy
|
||||
}
|
||||
} catch (error) {
|
||||
// Edge not found or other error
|
||||
|
|
@ -382,6 +438,13 @@ export class OPFSStorage extends BaseStorage {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all verbs from storage (internal implementation)
|
||||
*/
|
||||
protected async getAllVerbs_internal(): Promise<GraphVerb[]> {
|
||||
return this.getAllEdges()
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all edges from storage
|
||||
*/
|
||||
|
|
@ -405,15 +468,32 @@ export class OPFSStorage extends BaseStorage {
|
|||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
// Create default timestamp if not present
|
||||
const defaultTimestamp = {
|
||||
seconds: Math.floor(Date.now() / 1000),
|
||||
nanoseconds: (Date.now() % 1000) * 1000000
|
||||
}
|
||||
|
||||
// Create default createdBy if not present
|
||||
const defaultCreatedBy = {
|
||||
augmentation: 'unknown',
|
||||
version: '1.0'
|
||||
}
|
||||
|
||||
allEdges.push({
|
||||
id: data.id,
|
||||
vector: data.vector,
|
||||
connections,
|
||||
sourceId: data.sourceId,
|
||||
targetId: data.targetId,
|
||||
type: data.type,
|
||||
sourceId: data.sourceId || data.source,
|
||||
targetId: data.targetId || data.target,
|
||||
source: data.sourceId || data.source,
|
||||
target: data.targetId || data.target,
|
||||
verb: data.type || data.verb,
|
||||
weight: data.weight,
|
||||
metadata: data.metadata
|
||||
metadata: data.metadata,
|
||||
createdAt: data.createdAt || defaultTimestamp,
|
||||
updatedAt: data.updatedAt || defaultTimestamp,
|
||||
createdBy: data.createdBy || defaultCreatedBy
|
||||
})
|
||||
} catch (error) {
|
||||
console.error(`Error reading edge file ${name}:`, error)
|
||||
|
|
@ -427,12 +507,26 @@ export class OPFSStorage extends BaseStorage {
|
|||
return allEdges
|
||||
}
|
||||
|
||||
/**
|
||||
* Get verbs by source (internal implementation)
|
||||
*/
|
||||
protected async getVerbsBySource_internal(sourceId: string): Promise<GraphVerb[]> {
|
||||
return this.getEdgesBySource(sourceId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get edges by source
|
||||
*/
|
||||
protected async getEdgesBySource(sourceId: string): Promise<Edge[]> {
|
||||
const edges = await this.getAllEdges()
|
||||
return edges.filter((edge) => edge.sourceId === sourceId)
|
||||
return edges.filter((edge) => (edge.sourceId || edge.source) === sourceId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get verbs by target (internal implementation)
|
||||
*/
|
||||
protected async getVerbsByTarget_internal(targetId: string): Promise<GraphVerb[]> {
|
||||
return this.getEdgesByTarget(targetId)
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -440,7 +534,14 @@ export class OPFSStorage extends BaseStorage {
|
|||
*/
|
||||
protected async getEdgesByTarget(targetId: string): Promise<Edge[]> {
|
||||
const edges = await this.getAllEdges()
|
||||
return edges.filter((edge) => edge.targetId === targetId)
|
||||
return edges.filter((edge) => (edge.targetId || edge.target) === targetId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get verbs by type (internal implementation)
|
||||
*/
|
||||
protected async getVerbsByType_internal(type: string): Promise<GraphVerb[]> {
|
||||
return this.getEdgesByType(type)
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -448,7 +549,14 @@ export class OPFSStorage extends BaseStorage {
|
|||
*/
|
||||
protected async getEdgesByType(type: string): Promise<Edge[]> {
|
||||
const edges = await this.getAllEdges()
|
||||
return edges.filter((edge) => edge.type === type)
|
||||
return edges.filter((edge) => (edge.type || edge.verb) === type)
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a verb from storage (internal implementation)
|
||||
*/
|
||||
protected async deleteVerb_internal(id: string): Promise<void> {
|
||||
return this.deleteEdge(id)
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -688,4 +796,190 @@ export class OPFSStorage extends BaseStorage {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the statistics key for a specific date
|
||||
* @param date The date to get the key for
|
||||
* @returns The statistics key for the specified date
|
||||
*/
|
||||
private getStatisticsKeyForDate(date: Date): string {
|
||||
const year = date.getUTCFullYear()
|
||||
const month = String(date.getUTCMonth() + 1).padStart(2, '0')
|
||||
const day = String(date.getUTCDate()).padStart(2, '0')
|
||||
return `statistics_${year}${month}${day}.json`
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the current statistics key
|
||||
* @returns The current statistics key
|
||||
*/
|
||||
private getCurrentStatisticsKey(): string {
|
||||
return this.getStatisticsKeyForDate(new Date())
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the legacy statistics key (for backward compatibility)
|
||||
* @returns The legacy statistics key
|
||||
*/
|
||||
private getLegacyStatisticsKey(): string {
|
||||
return 'statistics.json'
|
||||
}
|
||||
|
||||
/**
|
||||
* Save statistics data to storage
|
||||
* @param statistics The statistics data to save
|
||||
*/
|
||||
protected async saveStatisticsData(statistics: StatisticsData): Promise<void> {
|
||||
// Create a deep copy to avoid reference issues
|
||||
this.statistics = {
|
||||
nounCount: {...statistics.nounCount},
|
||||
verbCount: {...statistics.verbCount},
|
||||
metadataCount: {...statistics.metadataCount},
|
||||
hnswIndexSize: statistics.hnswIndexSize,
|
||||
lastUpdated: statistics.lastUpdated
|
||||
}
|
||||
|
||||
try {
|
||||
// Ensure the root directory is initialized
|
||||
await this.ensureInitialized()
|
||||
|
||||
// Get or create the index directory
|
||||
if (!this.indexDir) {
|
||||
throw new Error('Index directory not initialized')
|
||||
}
|
||||
|
||||
// Get the current statistics key
|
||||
const currentKey = this.getCurrentStatisticsKey()
|
||||
|
||||
// Create a file for the statistics data
|
||||
const fileHandle = await this.indexDir.getFileHandle(currentKey, {
|
||||
create: true
|
||||
})
|
||||
|
||||
// Create a writable stream
|
||||
const writable = await fileHandle.createWritable()
|
||||
|
||||
// Write the statistics data to the file
|
||||
await writable.write(JSON.stringify(this.statistics, null, 2))
|
||||
|
||||
// Close the stream
|
||||
await writable.close()
|
||||
|
||||
// Also update the legacy key for backward compatibility, but less frequently
|
||||
if (Math.random() < 0.1) {
|
||||
const legacyKey = this.getLegacyStatisticsKey()
|
||||
const legacyFileHandle = await this.indexDir.getFileHandle(legacyKey, {
|
||||
create: true
|
||||
})
|
||||
const legacyWritable = await legacyFileHandle.createWritable()
|
||||
await legacyWritable.write(JSON.stringify(this.statistics, null, 2))
|
||||
await legacyWritable.close()
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Failed to save statistics data:', error)
|
||||
throw new Error(`Failed to save statistics data: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get statistics data from storage
|
||||
* @returns Promise that resolves to the statistics data or null if not found
|
||||
*/
|
||||
protected async getStatisticsData(): Promise<StatisticsData | null> {
|
||||
// If we have cached statistics, return a deep copy
|
||||
if (this.statistics) {
|
||||
return {
|
||||
nounCount: {...this.statistics.nounCount},
|
||||
verbCount: {...this.statistics.verbCount},
|
||||
metadataCount: {...this.statistics.metadataCount},
|
||||
hnswIndexSize: this.statistics.hnswIndexSize,
|
||||
lastUpdated: this.statistics.lastUpdated
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
// Ensure the root directory is initialized
|
||||
await this.ensureInitialized()
|
||||
|
||||
if (!this.indexDir) {
|
||||
throw new Error('Index directory not initialized')
|
||||
}
|
||||
|
||||
// First try to get statistics from today's file
|
||||
const currentKey = this.getCurrentStatisticsKey()
|
||||
try {
|
||||
const fileHandle = await this.indexDir.getFileHandle(currentKey, {
|
||||
create: false
|
||||
})
|
||||
const file = await fileHandle.getFile()
|
||||
const text = await file.text()
|
||||
this.statistics = JSON.parse(text)
|
||||
|
||||
if (this.statistics) {
|
||||
return {
|
||||
nounCount: {...this.statistics.nounCount},
|
||||
verbCount: {...this.statistics.verbCount},
|
||||
metadataCount: {...this.statistics.metadataCount},
|
||||
hnswIndexSize: this.statistics.hnswIndexSize,
|
||||
lastUpdated: this.statistics.lastUpdated
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
// If today's file doesn't exist, try yesterday's file
|
||||
const yesterday = new Date()
|
||||
yesterday.setDate(yesterday.getDate() - 1)
|
||||
const yesterdayKey = this.getStatisticsKeyForDate(yesterday)
|
||||
|
||||
try {
|
||||
const fileHandle = await this.indexDir.getFileHandle(yesterdayKey, {
|
||||
create: false
|
||||
})
|
||||
const file = await fileHandle.getFile()
|
||||
const text = await file.text()
|
||||
this.statistics = JSON.parse(text)
|
||||
|
||||
if (this.statistics) {
|
||||
return {
|
||||
nounCount: {...this.statistics.nounCount},
|
||||
verbCount: {...this.statistics.verbCount},
|
||||
metadataCount: {...this.statistics.metadataCount},
|
||||
hnswIndexSize: this.statistics.hnswIndexSize,
|
||||
lastUpdated: this.statistics.lastUpdated
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
// If yesterday's file doesn't exist, try the legacy file
|
||||
const legacyKey = this.getLegacyStatisticsKey()
|
||||
|
||||
try {
|
||||
const fileHandle = await this.indexDir.getFileHandle(legacyKey, {
|
||||
create: false
|
||||
})
|
||||
const file = await fileHandle.getFile()
|
||||
const text = await file.text()
|
||||
this.statistics = JSON.parse(text)
|
||||
|
||||
if (this.statistics) {
|
||||
return {
|
||||
nounCount: {...this.statistics.nounCount},
|
||||
verbCount: {...this.statistics.verbCount},
|
||||
metadataCount: {...this.statistics.metadataCount},
|
||||
hnswIndexSize: this.statistics.hnswIndexSize,
|
||||
lastUpdated: this.statistics.lastUpdated
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
// If the legacy file doesn't exist either, return null
|
||||
return null
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If we get here and statistics is null, return default statistics
|
||||
return this.statistics ? this.statistics : null
|
||||
} catch (error) {
|
||||
console.error('Failed to get statistics data:', error)
|
||||
throw new Error(`Failed to get statistics data: ${error}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -3,19 +3,21 @@
|
|||
* Provides common functionality for all storage adapters
|
||||
*/
|
||||
|
||||
import { GraphVerb, HNSWNoun, StorageAdapter } from '../coreTypes.js'
|
||||
import { GraphVerb, HNSWNoun, StatisticsData } from '../coreTypes.js'
|
||||
import { BaseStorageAdapter } from './adapters/baseStorageAdapter.js'
|
||||
|
||||
// Common directory/prefix names
|
||||
export const NOUNS_DIR = 'nouns'
|
||||
export const VERBS_DIR = 'verbs'
|
||||
export const METADATA_DIR = 'metadata'
|
||||
export const INDEX_DIR = 'index'
|
||||
export const STATISTICS_KEY = 'statistics'
|
||||
|
||||
/**
|
||||
* Base storage adapter that implements common functionality
|
||||
* This is an abstract class that should be extended by specific storage adapters
|
||||
*/
|
||||
export abstract class BaseStorage implements StorageAdapter {
|
||||
export abstract class BaseStorage extends BaseStorageAdapter {
|
||||
protected isInitialized = false
|
||||
|
||||
/**
|
||||
|
|
@ -38,7 +40,7 @@ export abstract class BaseStorage implements StorageAdapter {
|
|||
*/
|
||||
public async saveNoun(noun: HNSWNoun): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
return this.saveNode(noun)
|
||||
return this.saveNoun_internal(noun)
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -46,7 +48,7 @@ export abstract class BaseStorage implements StorageAdapter {
|
|||
*/
|
||||
public async getNoun(id: string): Promise<HNSWNoun | null> {
|
||||
await this.ensureInitialized()
|
||||
return this.getNode(id)
|
||||
return this.getNoun_internal(id)
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -54,7 +56,7 @@ export abstract class BaseStorage implements StorageAdapter {
|
|||
*/
|
||||
public async getAllNouns(): Promise<HNSWNoun[]> {
|
||||
await this.ensureInitialized()
|
||||
return this.getAllNodes()
|
||||
return this.getAllNouns_internal()
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -64,7 +66,7 @@ export abstract class BaseStorage implements StorageAdapter {
|
|||
*/
|
||||
public async getNounsByNounType(nounType: string): Promise<HNSWNoun[]> {
|
||||
await this.ensureInitialized()
|
||||
return this.getNodesByNounType(nounType)
|
||||
return this.getNounsByNounType_internal(nounType)
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -72,7 +74,7 @@ export abstract class BaseStorage implements StorageAdapter {
|
|||
*/
|
||||
public async deleteNoun(id: string): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
return this.deleteNode(id)
|
||||
return this.deleteNoun_internal(id)
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -80,7 +82,7 @@ export abstract class BaseStorage implements StorageAdapter {
|
|||
*/
|
||||
public async saveVerb(verb: GraphVerb): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
return this.saveEdge(verb)
|
||||
return this.saveVerb_internal(verb)
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -88,7 +90,7 @@ export abstract class BaseStorage implements StorageAdapter {
|
|||
*/
|
||||
public async getVerb(id: string): Promise<GraphVerb | null> {
|
||||
await this.ensureInitialized()
|
||||
return this.getEdge(id)
|
||||
return this.getVerb_internal(id)
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -96,7 +98,7 @@ export abstract class BaseStorage implements StorageAdapter {
|
|||
*/
|
||||
public async getAllVerbs(): Promise<GraphVerb[]> {
|
||||
await this.ensureInitialized()
|
||||
return this.getAllEdges()
|
||||
return this.getAllVerbs_internal()
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -104,7 +106,7 @@ export abstract class BaseStorage implements StorageAdapter {
|
|||
*/
|
||||
public async getVerbsBySource(sourceId: string): Promise<GraphVerb[]> {
|
||||
await this.ensureInitialized()
|
||||
return this.getEdgesBySource(sourceId)
|
||||
return this.getVerbsBySource_internal(sourceId)
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -112,7 +114,7 @@ export abstract class BaseStorage implements StorageAdapter {
|
|||
*/
|
||||
public async getVerbsByTarget(targetId: string): Promise<GraphVerb[]> {
|
||||
await this.ensureInitialized()
|
||||
return this.getEdgesByTarget(targetId)
|
||||
return this.getVerbsByTarget_internal(targetId)
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -120,7 +122,7 @@ export abstract class BaseStorage implements StorageAdapter {
|
|||
*/
|
||||
public async getVerbsByType(type: string): Promise<GraphVerb[]> {
|
||||
await this.ensureInitialized()
|
||||
return this.getEdgesByType(type)
|
||||
return this.getVerbsByType_internal(type)
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -128,7 +130,7 @@ export abstract class BaseStorage implements StorageAdapter {
|
|||
*/
|
||||
public async deleteVerb(id: string): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
return this.deleteEdge(id)
|
||||
return this.deleteVerb_internal(id)
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -161,76 +163,76 @@ export abstract class BaseStorage implements StorageAdapter {
|
|||
public abstract getMetadata(id: string): Promise<any | null>
|
||||
|
||||
/**
|
||||
* Save a node to storage
|
||||
* Save a noun to storage
|
||||
* This method should be implemented by each specific adapter
|
||||
*/
|
||||
protected abstract saveNode(node: HNSWNoun): Promise<void>
|
||||
protected abstract saveNoun_internal(noun: HNSWNoun): Promise<void>
|
||||
|
||||
/**
|
||||
* Get a node from storage
|
||||
* Get a noun from storage
|
||||
* This method should be implemented by each specific adapter
|
||||
*/
|
||||
protected abstract getNode(id: string): Promise<HNSWNoun | null>
|
||||
protected abstract getNoun_internal(id: string): Promise<HNSWNoun | null>
|
||||
|
||||
/**
|
||||
* Get all nodes from storage
|
||||
* Get all nouns from storage
|
||||
* This method should be implemented by each specific adapter
|
||||
*/
|
||||
protected abstract getAllNodes(): Promise<HNSWNoun[]>
|
||||
protected abstract getAllNouns_internal(): Promise<HNSWNoun[]>
|
||||
|
||||
/**
|
||||
* Get nodes by noun type
|
||||
* Get nouns by noun type
|
||||
* This method should be implemented by each specific adapter
|
||||
*/
|
||||
protected abstract getNodesByNounType(nounType: string): Promise<HNSWNoun[]>
|
||||
protected abstract getNounsByNounType_internal(nounType: string): Promise<HNSWNoun[]>
|
||||
|
||||
/**
|
||||
* Delete a node from storage
|
||||
* Delete a noun from storage
|
||||
* This method should be implemented by each specific adapter
|
||||
*/
|
||||
protected abstract deleteNode(id: string): Promise<void>
|
||||
protected abstract deleteNoun_internal(id: string): Promise<void>
|
||||
|
||||
/**
|
||||
* Save an edge to storage
|
||||
* Save a verb to storage
|
||||
* This method should be implemented by each specific adapter
|
||||
*/
|
||||
protected abstract saveEdge(edge: GraphVerb): Promise<void>
|
||||
protected abstract saveVerb_internal(verb: GraphVerb): Promise<void>
|
||||
|
||||
/**
|
||||
* Get an edge from storage
|
||||
* Get a verb from storage
|
||||
* This method should be implemented by each specific adapter
|
||||
*/
|
||||
protected abstract getEdge(id: string): Promise<GraphVerb | null>
|
||||
protected abstract getVerb_internal(id: string): Promise<GraphVerb | null>
|
||||
|
||||
/**
|
||||
* Get all edges from storage
|
||||
* Get all verbs from storage
|
||||
* This method should be implemented by each specific adapter
|
||||
*/
|
||||
protected abstract getAllEdges(): Promise<GraphVerb[]>
|
||||
protected abstract getAllVerbs_internal(): Promise<GraphVerb[]>
|
||||
|
||||
/**
|
||||
* Get edges by source
|
||||
* Get verbs by source
|
||||
* This method should be implemented by each specific adapter
|
||||
*/
|
||||
protected abstract getEdgesBySource(sourceId: string): Promise<GraphVerb[]>
|
||||
protected abstract getVerbsBySource_internal(sourceId: string): Promise<GraphVerb[]>
|
||||
|
||||
/**
|
||||
* Get edges by target
|
||||
* Get verbs by target
|
||||
* This method should be implemented by each specific adapter
|
||||
*/
|
||||
protected abstract getEdgesByTarget(targetId: string): Promise<GraphVerb[]>
|
||||
protected abstract getVerbsByTarget_internal(targetId: string): Promise<GraphVerb[]>
|
||||
|
||||
/**
|
||||
* Get edges by type
|
||||
* Get verbs by type
|
||||
* This method should be implemented by each specific adapter
|
||||
*/
|
||||
protected abstract getEdgesByType(type: string): Promise<GraphVerb[]>
|
||||
protected abstract getVerbsByType_internal(type: string): Promise<GraphVerb[]>
|
||||
|
||||
/**
|
||||
* Delete an edge from storage
|
||||
* Delete a verb from storage
|
||||
* This method should be implemented by each specific adapter
|
||||
*/
|
||||
protected abstract deleteEdge(id: string): Promise<void>
|
||||
protected abstract deleteVerb_internal(id: string): Promise<void>
|
||||
|
||||
/**
|
||||
* Helper method to convert a Map to a plain object for serialization
|
||||
|
|
@ -245,4 +247,18 @@ export abstract class BaseStorage implements StorageAdapter {
|
|||
}
|
||||
return obj
|
||||
}
|
||||
|
||||
/**
|
||||
* Save statistics data to storage
|
||||
* This method should be implemented by each specific adapter
|
||||
* @param statistics The statistics data to save
|
||||
*/
|
||||
protected abstract saveStatisticsData(statistics: StatisticsData): Promise<void>
|
||||
|
||||
/**
|
||||
* Get statistics data from storage
|
||||
* This method should be implemented by each specific adapter
|
||||
* @returns Promise that resolves to the statistics data or null if not found
|
||||
*/
|
||||
protected abstract getStatisticsData(): Promise<StatisticsData | null>
|
||||
}
|
||||
|
|
@ -1,451 +0,0 @@
|
|||
import { GraphVerb, HNSWNoun, StorageAdapter } from '../coreTypes.js'
|
||||
|
||||
// Dynamically and asynchronously load Node.js modules at the top level.
|
||||
// This ensures they are available as soon as this module is imported,
|
||||
// preventing race conditions with dependencies like TensorFlow.js.
|
||||
let fs: any
|
||||
let path: any
|
||||
|
||||
const nodeModulesPromise = (async () => {
|
||||
// A reliable check for a Node.js environment.
|
||||
const isNode =
|
||||
typeof process !== 'undefined' &&
|
||||
process.versions != null &&
|
||||
process.versions.node != null
|
||||
|
||||
if (!isNode) {
|
||||
return { fs: null, path: null }
|
||||
}
|
||||
|
||||
try {
|
||||
// Use the 'node:' prefix for unambiguous importing of built-in modules.
|
||||
const fsModule = await import('node:fs')
|
||||
const pathModule = await import('node:path')
|
||||
// Return the modules, preferring the default export if it exists.
|
||||
return {
|
||||
fs: fsModule.default || fsModule,
|
||||
path: pathModule.default || pathModule
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(
|
||||
'FileSystemStorage: Failed to load Node.js modules. This adapter is not supported in this environment.',
|
||||
error
|
||||
)
|
||||
return { fs: null, path: null }
|
||||
}
|
||||
})()
|
||||
|
||||
// Immediately assign the modules once the promise resolves.
|
||||
nodeModulesPromise.then((modules) => {
|
||||
fs = modules.fs
|
||||
path = modules.path
|
||||
})
|
||||
|
||||
// --- End of Refactored Code ---
|
||||
|
||||
// Constants for directory and file names
|
||||
const ROOT_DIR = 'brainy-data'
|
||||
const NOUNS_DIR = 'nouns'
|
||||
const VERBS_DIR = 'verbs'
|
||||
const METADATA_DIR = 'metadata'
|
||||
|
||||
// All nouns now use the same directory - no separate directories per noun type
|
||||
|
||||
/**
|
||||
* File system storage adapter for Node.js environments
|
||||
*/
|
||||
export class FileSystemStorage implements StorageAdapter {
|
||||
private rootDir: string
|
||||
private nounsDir: string
|
||||
private verbsDir: string
|
||||
private metadataDir: string
|
||||
private isInitialized = false
|
||||
|
||||
constructor(rootDirectory?: string) {
|
||||
// We'll set the paths in the init method after dynamically importing the modules
|
||||
this.rootDir = rootDirectory || ''
|
||||
this.nounsDir = ''
|
||||
this.verbsDir = ''
|
||||
this.metadataDir = ''
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the storage adapter
|
||||
*/
|
||||
public async init(): Promise<void> {
|
||||
if (this.isInitialized) {
|
||||
return
|
||||
}
|
||||
|
||||
// Wait for the top-level module loading to complete.
|
||||
await nodeModulesPromise
|
||||
|
||||
// Check if the modules were loaded successfully.
|
||||
if (!fs || !path) {
|
||||
throw new Error(
|
||||
'FileSystemStorage requires a Node.js environment, but `fs` and `path` modules could not be loaded.'
|
||||
)
|
||||
}
|
||||
|
||||
try {
|
||||
// Now set up the directory paths
|
||||
const rootDir = this.rootDir || process.cwd()
|
||||
|
||||
// Check if rootDir already ends with ROOT_DIR to prevent duplication
|
||||
if (rootDir.endsWith(ROOT_DIR)) {
|
||||
this.rootDir = rootDir
|
||||
} else {
|
||||
this.rootDir = path.resolve(rootDir, ROOT_DIR)
|
||||
}
|
||||
|
||||
this.nounsDir = path.join(this.rootDir, NOUNS_DIR)
|
||||
this.verbsDir = path.join(this.rootDir, VERBS_DIR)
|
||||
this.metadataDir = path.join(this.rootDir, METADATA_DIR)
|
||||
|
||||
// Create directories if they don't exist
|
||||
await this.ensureDirectoryExists(this.rootDir)
|
||||
await this.ensureDirectoryExists(this.nounsDir)
|
||||
await this.ensureDirectoryExists(this.verbsDir)
|
||||
await this.ensureDirectoryExists(this.metadataDir)
|
||||
|
||||
this.isInitialized = true
|
||||
} catch (error: any) {
|
||||
console.error('Error initializing FileSystemStorage:', error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
private async ensureDirectoryExists(dirPath: string): Promise<void> {
|
||||
try {
|
||||
await fs.promises.mkdir(dirPath, { recursive: true })
|
||||
} catch (error: any) {
|
||||
// Ignore EEXIST error, which means the directory already exists
|
||||
if (error.code !== 'EEXIST') {
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private getNounPath(id: string, nounType?: string): string {
|
||||
// All nouns now use the same directory regardless of type
|
||||
return path.join(this.nounsDir, `${id}.json`)
|
||||
}
|
||||
|
||||
public async saveNoun(
|
||||
noun: HNSWNoun & { metadata?: { noun?: string } }
|
||||
): Promise<void> {
|
||||
if (!this.isInitialized) await this.init()
|
||||
const nounType = (noun as any).metadata?.noun
|
||||
const filePath = this.getNounPath(noun.id, nounType)
|
||||
await this.ensureDirectoryExists(path.dirname(filePath))
|
||||
await fs.promises.writeFile(filePath, JSON.stringify(noun, null, 2))
|
||||
}
|
||||
|
||||
public async getNoun(id: string): Promise<HNSWNoun | null> {
|
||||
if (!this.isInitialized) await this.init()
|
||||
const filePath = path.join(this.nounsDir, `${id}.json`)
|
||||
try {
|
||||
const data = await fs.promises.readFile(filePath, 'utf-8')
|
||||
return JSON.parse(data)
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error reading noun ${id}:`, error)
|
||||
}
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
public async deleteNoun(id: string): Promise<void> {
|
||||
if (!this.isInitialized) await this.init()
|
||||
const noun = await this.getNoun(id)
|
||||
if (noun) {
|
||||
const nounType = (noun as any).metadata?.noun
|
||||
const filePath = this.getNounPath(id, nounType)
|
||||
try {
|
||||
await fs.promises.unlink(filePath)
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error deleting noun file ${filePath}:`, error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public async getAllNouns(): Promise<HNSWNoun[]> {
|
||||
if (!this.isInitialized) await this.init()
|
||||
const allNouns: HNSWNoun[] = []
|
||||
try {
|
||||
const files = await fs.promises.readdir(this.nounsDir)
|
||||
for (const file of files) {
|
||||
if (file.endsWith('.json')) {
|
||||
const filePath = path.join(this.nounsDir, file)
|
||||
const data = await fs.promises.readFile(filePath, 'utf-8')
|
||||
allNouns.push(JSON.parse(data))
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error reading directory ${this.nounsDir}:`, error)
|
||||
}
|
||||
}
|
||||
return allNouns
|
||||
}
|
||||
|
||||
/**
|
||||
* Get nouns by noun type
|
||||
* @param nounType The noun type to filter by
|
||||
* @returns Promise that resolves to an array of nouns of the specified noun type
|
||||
*/
|
||||
public async getNounsByNounType(nounType: string): Promise<HNSWNoun[]> {
|
||||
if (!this.isInitialized) await this.init()
|
||||
|
||||
const nouns: HNSWNoun[] = []
|
||||
try {
|
||||
const files = await fs.promises.readdir(this.nounsDir)
|
||||
for (const file of files) {
|
||||
if (file.endsWith('.json')) {
|
||||
const filePath = path.join(this.nounsDir, file)
|
||||
const data = await fs.promises.readFile(filePath, 'utf-8')
|
||||
const noun = JSON.parse(data)
|
||||
|
||||
// Filter by noun type using metadata
|
||||
const metadata = await this.getMetadata(noun.id)
|
||||
if (metadata && metadata.noun === nounType) {
|
||||
nouns.push(noun)
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error reading directory ${this.nounsDir}:`, error)
|
||||
}
|
||||
}
|
||||
|
||||
return nouns
|
||||
}
|
||||
|
||||
public async saveVerb(verb: GraphVerb): Promise<void> {
|
||||
if (!this.isInitialized) await this.init()
|
||||
const filePath = path.join(this.verbsDir, `${verb.id}.json`)
|
||||
await fs.promises.writeFile(filePath, JSON.stringify(verb, null, 2))
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a verb by its ID
|
||||
* @param id The ID of the verb to retrieve
|
||||
* @returns Promise that resolves to the verb or null if not found
|
||||
*/
|
||||
public async getVerb(id: string): Promise<GraphVerb | null> {
|
||||
if (!this.isInitialized) await this.init()
|
||||
const filePath = path.join(this.verbsDir, `${id}.json`)
|
||||
try {
|
||||
const data = await fs.promises.readFile(filePath, 'utf-8')
|
||||
return JSON.parse(data)
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error reading verb ${id}:`, error)
|
||||
}
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
public async getVerbsBySource(sourceId: string): Promise<GraphVerb[]> {
|
||||
if (!this.isInitialized) await this.init()
|
||||
const allVerbs = await this.getAllVerbs()
|
||||
return allVerbs.filter((verb) => verb.sourceId === sourceId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get verbs by target ID
|
||||
* @param targetId The target ID to filter by
|
||||
* @returns Promise that resolves to an array of verbs with the specified target ID
|
||||
*/
|
||||
public async getVerbsByTarget(targetId: string): Promise<GraphVerb[]> {
|
||||
if (!this.isInitialized) await this.init()
|
||||
const allVerbs = await this.getAllVerbs()
|
||||
return allVerbs.filter((verb) => verb.targetId === targetId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get verbs by type
|
||||
* @param type The verb type to filter by
|
||||
* @returns Promise that resolves to an array of verbs of the specified type
|
||||
*/
|
||||
public async getVerbsByType(type: string): Promise<GraphVerb[]> {
|
||||
if (!this.isInitialized) await this.init()
|
||||
const allVerbs = await this.getAllVerbs()
|
||||
return allVerbs.filter((verb) => verb.type === type)
|
||||
}
|
||||
|
||||
public async getAllVerbs(): Promise<GraphVerb[]> {
|
||||
if (!this.isInitialized) await this.init()
|
||||
const allVerbs: GraphVerb[] = []
|
||||
try {
|
||||
const files = await fs.promises.readdir(this.verbsDir)
|
||||
for (const file of files) {
|
||||
if (file.endsWith('.json')) {
|
||||
const filePath = path.join(this.verbsDir, file)
|
||||
const data = await fs.promises.readFile(filePath, 'utf-8')
|
||||
allVerbs.push(JSON.parse(data))
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error reading verbs directory ${this.verbsDir}:`, error)
|
||||
}
|
||||
}
|
||||
return allVerbs
|
||||
}
|
||||
|
||||
public async deleteVerb(id: string): Promise<void> {
|
||||
if (!this.isInitialized) await this.init()
|
||||
const filePath = path.join(this.verbsDir, `${id}.json`)
|
||||
try {
|
||||
await fs.promises.unlink(filePath)
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error deleting verb file ${filePath}:`, error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save metadata for an entity
|
||||
* @param id The ID of the entity
|
||||
* @param metadata The metadata to save
|
||||
*/
|
||||
public async saveMetadata(id: string, metadata: any): Promise<void> {
|
||||
if (!this.isInitialized) await this.init()
|
||||
const filePath = path.join(this.metadataDir, `${id}.json`)
|
||||
await this.ensureDirectoryExists(path.dirname(filePath))
|
||||
await fs.promises.writeFile(filePath, JSON.stringify(metadata, null, 2))
|
||||
}
|
||||
|
||||
/**
|
||||
* Get metadata for an entity
|
||||
* @param id The ID of the entity
|
||||
* @returns Promise that resolves to the metadata or null if not found
|
||||
*/
|
||||
public async getMetadata(id: string): Promise<any | null> {
|
||||
if (!this.isInitialized) await this.init()
|
||||
const filePath = path.join(this.metadataDir, `${id}.json`)
|
||||
try {
|
||||
const data = await fs.promises.readFile(filePath, 'utf-8')
|
||||
return JSON.parse(data)
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error reading metadata for ${id}:`, error)
|
||||
}
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
public async clear(): Promise<void> {
|
||||
if (!this.isInitialized) await this.init()
|
||||
|
||||
// Helper function to recursively remove directory contents
|
||||
const removeDirectoryContents = async (dirPath: string): Promise<void> => {
|
||||
try {
|
||||
const files = await fs.promises.readdir(dirPath, { withFileTypes: true })
|
||||
|
||||
for (const file of files) {
|
||||
const fullPath = path.join(dirPath, file.name)
|
||||
|
||||
if (file.isDirectory()) {
|
||||
await removeDirectoryContents(fullPath)
|
||||
// Use fs.promises.rm with recursive option instead of rmdir
|
||||
try {
|
||||
await fs.promises.rm(fullPath, { recursive: true, force: true })
|
||||
} catch (rmError: any) {
|
||||
// Fallback to rmdir if rm fails
|
||||
await fs.promises.rmdir(fullPath)
|
||||
}
|
||||
} else {
|
||||
await fs.promises.unlink(fullPath)
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error removing directory contents ${dirPath}:`, error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
// First try the modern approach
|
||||
await fs.promises.rm(this.rootDir, { recursive: true, force: true })
|
||||
} catch (error: any) {
|
||||
console.warn('Modern rm failed, falling back to manual cleanup:', error)
|
||||
|
||||
// Fallback: manually remove contents then directory
|
||||
try {
|
||||
await removeDirectoryContents(this.rootDir)
|
||||
// Use fs.promises.rm with recursive option instead of rmdir
|
||||
try {
|
||||
await fs.promises.rm(this.rootDir, { recursive: true, force: true })
|
||||
} catch (rmError: any) {
|
||||
// Final fallback to rmdir if rm fails
|
||||
await fs.promises.rmdir(this.rootDir)
|
||||
}
|
||||
} catch (fallbackError: any) {
|
||||
if (fallbackError.code !== 'ENOENT') {
|
||||
console.error('Manual cleanup also failed:', fallbackError)
|
||||
throw fallbackError
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.isInitialized = false // Reset state
|
||||
await this.init() // Re-create directories
|
||||
}
|
||||
|
||||
public async getStorageStatus(): Promise<{
|
||||
type: string
|
||||
used: number
|
||||
quota: number | null
|
||||
details?: Record<string, any>
|
||||
}> {
|
||||
if (!this.isInitialized) await this.init()
|
||||
|
||||
const calculateSize = async (dirPath: string): Promise<number> => {
|
||||
let size = 0
|
||||
try {
|
||||
const files = await fs.promises.readdir(dirPath, {
|
||||
withFileTypes: true
|
||||
})
|
||||
for (const file of files) {
|
||||
const fullPath = path.join(dirPath, file.name)
|
||||
if (file.isDirectory()) {
|
||||
size += await calculateSize(fullPath)
|
||||
} else {
|
||||
const stats = await fs.promises.stat(fullPath)
|
||||
size += stats.size
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Could not calculate size for ${dirPath}:`, error)
|
||||
}
|
||||
}
|
||||
return size
|
||||
}
|
||||
|
||||
const totalSize = await calculateSize(this.rootDir)
|
||||
const nouns = await this.getAllNouns()
|
||||
const verbs = await this.getAllVerbs()
|
||||
|
||||
return {
|
||||
type: 'FileSystem',
|
||||
used: totalSize,
|
||||
quota: null, // File system quota is not easily available from Node.js
|
||||
details: {
|
||||
rootDir: this.rootDir,
|
||||
nounCount: nouns.length,
|
||||
verbCount: verbs.length
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -1,952 +0,0 @@
|
|||
import { GraphVerb, HNSWNoun, StorageAdapter } from '../coreTypes.js'
|
||||
|
||||
// Type aliases for compatibility
|
||||
type HNSWNode = HNSWNoun
|
||||
type Edge = GraphVerb
|
||||
|
||||
// Export R2Storage as an alias for S3CompatibleStorage
|
||||
export { S3CompatibleStorage as R2Storage }
|
||||
|
||||
// Constants for S3 bucket prefixes
|
||||
const NOUNS_PREFIX = 'nouns/'
|
||||
const VERBS_PREFIX = 'verbs/'
|
||||
const EDGES_PREFIX = 'verbs/' // Alias for VERBS_PREFIX for edge operations
|
||||
const METADATA_PREFIX = 'metadata/'
|
||||
|
||||
// All nouns now use the same prefix - no separate directories per noun type
|
||||
const NOUN_PREFIX = 'nouns/' // Single directory for all noun types
|
||||
|
||||
/**
|
||||
* S3-compatible storage adapter for server environments
|
||||
* Uses the AWS S3 client to interact with S3-compatible storage services
|
||||
* including Amazon S3, Cloudflare R2, and Google Cloud Storage
|
||||
*
|
||||
* To use this adapter with Cloudflare R2, you need to provide:
|
||||
* - bucketName: Your bucket name
|
||||
* - accountId: Your Cloudflare account ID
|
||||
* - accessKeyId: R2 access key ID
|
||||
* - secretAccessKey: R2 secret access key
|
||||
* - serviceType: 'r2'
|
||||
*
|
||||
* To use this adapter with Amazon S3, you need to provide:
|
||||
* - bucketName: Your S3 bucket name
|
||||
* - accessKeyId: AWS access key ID
|
||||
* - secretAccessKey: AWS secret access key
|
||||
* - region: AWS region (e.g., 'us-east-1')
|
||||
* - serviceType: 's3'
|
||||
*
|
||||
* To use this adapter with Google Cloud Storage, you need to provide:
|
||||
* - bucketName: Your GCS bucket name
|
||||
* - accessKeyId: HMAC access key
|
||||
* - secretAccessKey: HMAC secret
|
||||
* - endpoint: GCS endpoint (e.g., 'https://storage.googleapis.com')
|
||||
* - serviceType: 'gcs'
|
||||
*
|
||||
* For other S3-compatible services, provide:
|
||||
* - bucketName: Your bucket name
|
||||
* - accessKeyId: Access key ID
|
||||
* - secretAccessKey: Secret access key
|
||||
* - endpoint: Service endpoint URL
|
||||
* - region: Region (if required)
|
||||
* - serviceType: 'custom'
|
||||
*/
|
||||
export class S3CompatibleStorage implements StorageAdapter {
|
||||
private bucketName: string
|
||||
private accessKeyId: string
|
||||
private secretAccessKey: string
|
||||
private endpoint?: string
|
||||
private region?: string
|
||||
private accountId?: string
|
||||
private serviceType: 'r2' | 's3' | 'gcs' | 'custom'
|
||||
private s3Client: any // Will be initialized in init()
|
||||
private isInitialized = false
|
||||
|
||||
// Alias methods to match StorageAdapter interface
|
||||
public async saveNoun(noun: HNSWNoun): Promise<void> {
|
||||
return this.saveNode(noun)
|
||||
}
|
||||
|
||||
public async getNoun(id: string): Promise<HNSWNoun | null> {
|
||||
return this.getNode(id)
|
||||
}
|
||||
|
||||
public async getAllNouns(): Promise<HNSWNoun[]> {
|
||||
return this.getAllNodes()
|
||||
}
|
||||
|
||||
public async getNounsByNounType(nounType: string): Promise<HNSWNoun[]> {
|
||||
return this.getNodesByNounType(nounType)
|
||||
}
|
||||
|
||||
public async deleteNoun(id: string): Promise<void> {
|
||||
return this.deleteNode(id)
|
||||
}
|
||||
|
||||
public async saveVerb(verb: GraphVerb): Promise<void> {
|
||||
return this.saveEdge(verb)
|
||||
}
|
||||
|
||||
public async getVerb(id: string): Promise<GraphVerb | null> {
|
||||
return this.getEdge(id)
|
||||
}
|
||||
|
||||
public async getAllVerbs(): Promise<GraphVerb[]> {
|
||||
return this.getAllEdges()
|
||||
}
|
||||
|
||||
public async getVerbsBySource(sourceId: string): Promise<GraphVerb[]> {
|
||||
return this.getEdgesBySource(sourceId)
|
||||
}
|
||||
|
||||
public async getVerbsByTarget(targetId: string): Promise<GraphVerb[]> {
|
||||
return this.getEdgesByTarget(targetId)
|
||||
}
|
||||
|
||||
public async getVerbsByType(type: string): Promise<GraphVerb[]> {
|
||||
return this.getEdgesByType(type)
|
||||
}
|
||||
|
||||
public async deleteVerb(id: string): Promise<void> {
|
||||
return this.deleteEdge(id)
|
||||
}
|
||||
|
||||
constructor(options: {
|
||||
bucketName: string
|
||||
accessKeyId: string
|
||||
secretAccessKey: string
|
||||
serviceType: 'r2' | 's3' | 'gcs' | 'custom'
|
||||
accountId?: string
|
||||
region?: string
|
||||
endpoint?: string
|
||||
}) {
|
||||
this.bucketName = options.bucketName
|
||||
this.accessKeyId = options.accessKeyId
|
||||
this.secretAccessKey = options.secretAccessKey
|
||||
this.serviceType = options.serviceType
|
||||
this.accountId = options.accountId
|
||||
this.region = options.region
|
||||
this.endpoint = options.endpoint
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the storage adapter
|
||||
*/
|
||||
public async init(): Promise<void> {
|
||||
if (this.isInitialized) {
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
// Dynamically import the AWS SDK to avoid bundling it in browser environments
|
||||
try {
|
||||
const { S3Client } = await import('@aws-sdk/client-s3')
|
||||
|
||||
// Configure the S3 client based on the service type
|
||||
const clientConfig: any = {
|
||||
credentials: {
|
||||
accessKeyId: this.accessKeyId,
|
||||
secretAccessKey: this.secretAccessKey
|
||||
}
|
||||
}
|
||||
|
||||
switch (this.serviceType) {
|
||||
case 'r2':
|
||||
if (!this.accountId) {
|
||||
throw new Error('accountId is required for Cloudflare R2')
|
||||
}
|
||||
clientConfig.region = 'auto' // R2 uses 'auto' as the region
|
||||
clientConfig.endpoint = `https://${this.accountId}.r2.cloudflarestorage.com`
|
||||
break
|
||||
|
||||
case 's3':
|
||||
if (!this.region) {
|
||||
throw new Error('region is required for Amazon S3')
|
||||
}
|
||||
clientConfig.region = this.region
|
||||
// No endpoint needed for standard S3
|
||||
break
|
||||
|
||||
case 'gcs':
|
||||
if (!this.endpoint) {
|
||||
// Default GCS endpoint if not provided
|
||||
this.endpoint = 'https://storage.googleapis.com'
|
||||
}
|
||||
clientConfig.endpoint = this.endpoint
|
||||
clientConfig.region = this.region || 'auto'
|
||||
break
|
||||
|
||||
case 'custom':
|
||||
if (!this.endpoint) {
|
||||
throw new Error(
|
||||
'endpoint is required for custom S3-compatible services'
|
||||
)
|
||||
}
|
||||
clientConfig.endpoint = this.endpoint
|
||||
if (this.region) {
|
||||
clientConfig.region = this.region
|
||||
}
|
||||
break
|
||||
|
||||
default:
|
||||
throw new Error(`Unsupported service type: ${this.serviceType}`)
|
||||
}
|
||||
|
||||
// Initialize the S3 client with the configured options
|
||||
this.s3Client = new S3Client(clientConfig)
|
||||
|
||||
this.isInitialized = true
|
||||
} catch (importError) {
|
||||
throw new Error(
|
||||
`Failed to import AWS SDK: ${importError}. Make sure @aws-sdk/client-s3 is installed.`
|
||||
)
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Failed to initialize ${this.serviceType} storage:`, error)
|
||||
throw new Error(
|
||||
`Failed to initialize ${this.serviceType} storage: ${error}`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save a node to storage
|
||||
*/
|
||||
public async saveNode(node: HNSWNode): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Convert connections Map to a serializable format
|
||||
const serializableNode = {
|
||||
...node,
|
||||
connections: this.mapToObject(node.connections, (set) =>
|
||||
Array.from(set as Set<string>)
|
||||
)
|
||||
}
|
||||
|
||||
// Get the appropriate prefix based on the node's metadata
|
||||
const nodePrefix = await this.getNodePrefix(node.id)
|
||||
|
||||
// Import the PutObjectCommand only when needed
|
||||
const { PutObjectCommand } = await import('@aws-sdk/client-s3')
|
||||
|
||||
// Save the node to S3-compatible storage
|
||||
await this.s3Client.send(
|
||||
new PutObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: `${nodePrefix}${node.id}.json`,
|
||||
Body: JSON.stringify(serializableNode, null, 2),
|
||||
ContentType: 'application/json'
|
||||
})
|
||||
)
|
||||
} catch (error) {
|
||||
console.error(`Failed to save node ${node.id}:`, error)
|
||||
throw new Error(`Failed to save node ${node.id}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a node from storage
|
||||
*/
|
||||
public async getNode(id: string): Promise<HNSWNode | null> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Import the GetObjectCommand only when needed
|
||||
const { GetObjectCommand } = await import('@aws-sdk/client-s3')
|
||||
|
||||
// Try to get the node from the consolidated nouns directory
|
||||
const response = await this.s3Client.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: `${NOUN_PREFIX}${id}.json`
|
||||
})
|
||||
)
|
||||
|
||||
// Convert the response body to a string
|
||||
const bodyContents = await response.Body.transformToString()
|
||||
const parsedNode = JSON.parse(bodyContents)
|
||||
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(parsedNode.connections)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
return {
|
||||
id: parsedNode.id,
|
||||
vector: parsedNode.vector,
|
||||
connections
|
||||
}
|
||||
} catch (error) {
|
||||
// Node not found or other error
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get nodes by noun type
|
||||
* @param nounType The noun type to filter by
|
||||
* @returns Promise that resolves to an array of nodes of the specified noun type
|
||||
*/
|
||||
public async getNodesByNounType(nounType: string): Promise<HNSWNode[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Import the ListObjectsV2Command and GetObjectCommand only when needed
|
||||
const { ListObjectsV2Command, GetObjectCommand } = await import(
|
||||
'@aws-sdk/client-s3'
|
||||
)
|
||||
|
||||
// List all objects in the consolidated nouns directory
|
||||
const listResponse = await this.s3Client.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: this.bucketName,
|
||||
Prefix: NOUN_PREFIX
|
||||
})
|
||||
)
|
||||
|
||||
const nodes: HNSWNode[] = []
|
||||
|
||||
// If there are no objects, return an empty array
|
||||
if (!listResponse.Contents || listResponse.Contents.length === 0) {
|
||||
return nodes
|
||||
}
|
||||
|
||||
// Get each node and filter by noun type
|
||||
const nodePromises = listResponse.Contents.map(
|
||||
async (object: { Key: string }) => {
|
||||
try {
|
||||
// Extract node ID from the key (remove prefix and .json extension)
|
||||
const nodeId = object.Key.replace(NOUN_PREFIX, '').replace('.json', '')
|
||||
|
||||
// Get the metadata to check the noun type
|
||||
const metadata = await this.getMetadata(nodeId)
|
||||
|
||||
// Skip if metadata doesn't exist or noun type doesn't match
|
||||
if (!metadata || metadata.noun !== nounType) {
|
||||
return null
|
||||
}
|
||||
|
||||
const response = await this.s3Client.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: object.Key
|
||||
})
|
||||
)
|
||||
|
||||
const bodyContents = await response.Body.transformToString()
|
||||
const parsedNode = JSON.parse(bodyContents)
|
||||
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(
|
||||
parsedNode.connections
|
||||
)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
return {
|
||||
id: parsedNode.id,
|
||||
vector: parsedNode.vector,
|
||||
connections
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Failed to get node from ${object.Key}:`, error)
|
||||
return null
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
const nodeResults = await Promise.all(nodePromises)
|
||||
return nodeResults.filter((node): node is HNSWNode => node !== null)
|
||||
} catch (error) {
|
||||
console.error(`Failed to get nodes for noun type ${nounType}:`, error)
|
||||
throw new Error(`Failed to get nodes for noun type ${nounType}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all nodes from storage
|
||||
*/
|
||||
public async getAllNodes(): Promise<HNSWNode[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Import the ListObjectsV2Command and GetObjectCommand only when needed
|
||||
const { ListObjectsV2Command, GetObjectCommand } = await import(
|
||||
'@aws-sdk/client-s3'
|
||||
)
|
||||
|
||||
// List all objects in the consolidated nouns directory
|
||||
const listResponse = await this.s3Client.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: this.bucketName,
|
||||
Prefix: NOUN_PREFIX
|
||||
})
|
||||
)
|
||||
|
||||
const nodes: HNSWNode[] = []
|
||||
|
||||
// If there are no objects, return an empty array
|
||||
if (!listResponse.Contents || listResponse.Contents.length === 0) {
|
||||
return nodes
|
||||
}
|
||||
|
||||
// Get each node
|
||||
const nodePromises = listResponse.Contents.map(
|
||||
async (object: { Key: string }) => {
|
||||
try {
|
||||
const response = await this.s3Client.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: object.Key
|
||||
})
|
||||
)
|
||||
|
||||
const bodyContents = await response.Body.transformToString()
|
||||
const parsedNode = JSON.parse(bodyContents)
|
||||
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(
|
||||
parsedNode.connections
|
||||
)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
return {
|
||||
id: parsedNode.id,
|
||||
vector: parsedNode.vector,
|
||||
connections
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Failed to get node from ${object.Key}:`, error)
|
||||
return null
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
const nodeResults = await Promise.all(nodePromises)
|
||||
return nodeResults.filter((node): node is HNSWNode => node !== null)
|
||||
} catch (error) {
|
||||
console.error('Failed to get all nodes:', error)
|
||||
throw new Error(`Failed to get all nodes: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a node from storage
|
||||
*/
|
||||
public async deleteNode(id: string): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Import the DeleteObjectCommand only when needed
|
||||
const { DeleteObjectCommand } = await import('@aws-sdk/client-s3')
|
||||
|
||||
// Delete the node from the consolidated nouns directory
|
||||
await this.s3Client.send(
|
||||
new DeleteObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: `${NOUN_PREFIX}${id}.json`
|
||||
})
|
||||
)
|
||||
} catch (error) {
|
||||
console.error(`Failed to delete node ${id}:`, error)
|
||||
throw new Error(`Failed to delete node ${id}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save an edge to storage
|
||||
*/
|
||||
public async saveEdge(edge: Edge): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Convert connections Map to a serializable format
|
||||
const serializableEdge = {
|
||||
...edge,
|
||||
connections: this.mapToObject(edge.connections, (set) =>
|
||||
Array.from(set as Set<string>)
|
||||
)
|
||||
}
|
||||
|
||||
// Import the PutObjectCommand only when needed
|
||||
const { PutObjectCommand } = await import('@aws-sdk/client-s3')
|
||||
|
||||
// Save the edge to S3-compatible storage
|
||||
await this.s3Client.send(
|
||||
new PutObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: `${VERBS_PREFIX}${edge.id}.json`,
|
||||
Body: JSON.stringify(serializableEdge, null, 2),
|
||||
ContentType: 'application/json'
|
||||
})
|
||||
)
|
||||
} catch (error) {
|
||||
console.error(`Failed to save edge ${edge.id}:`, error)
|
||||
throw new Error(`Failed to save edge ${edge.id}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get an edge from storage
|
||||
*/
|
||||
public async getEdge(id: string): Promise<Edge | null> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Import the GetObjectCommand only when needed
|
||||
const { GetObjectCommand } = await import('@aws-sdk/client-s3')
|
||||
|
||||
try {
|
||||
// Try to get the edge from S3-compatible storage
|
||||
const response = await this.s3Client.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: `${VERBS_PREFIX}${id}.json`
|
||||
})
|
||||
)
|
||||
|
||||
// Convert the response body to a string
|
||||
const bodyContents = await response.Body.transformToString()
|
||||
const parsedEdge = JSON.parse(bodyContents)
|
||||
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(parsedEdge.connections)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
return {
|
||||
id: parsedEdge.id,
|
||||
vector: parsedEdge.vector,
|
||||
connections,
|
||||
sourceId: parsedEdge.sourceId,
|
||||
targetId: parsedEdge.targetId,
|
||||
type: parsedEdge.type,
|
||||
weight: parsedEdge.weight,
|
||||
metadata: parsedEdge.metadata
|
||||
}
|
||||
} catch {
|
||||
return null // Edge not found
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Failed to get edge ${id}:`, error)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all edges from storage
|
||||
*/
|
||||
public async getAllEdges(): Promise<Edge[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Import the ListObjectsV2Command and GetObjectCommand only when needed
|
||||
const { ListObjectsV2Command, GetObjectCommand } = await import(
|
||||
'@aws-sdk/client-s3'
|
||||
)
|
||||
|
||||
// List all objects with the edges prefix
|
||||
const listResponse = await this.s3Client.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: this.bucketName,
|
||||
Prefix: VERBS_PREFIX
|
||||
})
|
||||
)
|
||||
|
||||
const edges: Edge[] = []
|
||||
|
||||
// If there are no objects, return an empty array
|
||||
if (!listResponse.Contents || listResponse.Contents.length === 0) {
|
||||
return edges
|
||||
}
|
||||
|
||||
// Get each edge
|
||||
const edgePromises = listResponse.Contents.map(
|
||||
async (object: { Key: string }) => {
|
||||
try {
|
||||
const response = await this.s3Client.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: object.Key
|
||||
})
|
||||
)
|
||||
|
||||
const bodyContents = await response.Body.transformToString()
|
||||
const parsedEdge = JSON.parse(bodyContents)
|
||||
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(
|
||||
parsedEdge.connections
|
||||
)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
return {
|
||||
id: parsedEdge.id,
|
||||
vector: parsedEdge.vector,
|
||||
connections,
|
||||
sourceId: parsedEdge.sourceId,
|
||||
targetId: parsedEdge.targetId,
|
||||
type: parsedEdge.type,
|
||||
weight: parsedEdge.weight,
|
||||
metadata: parsedEdge.metadata
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Failed to get edge from ${object.Key}:`, error)
|
||||
return null
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
const edgeResults = await Promise.all(edgePromises)
|
||||
return edgeResults.filter((edge): edge is Edge => edge !== null)
|
||||
} catch (error) {
|
||||
console.error('Failed to get all edges:', error)
|
||||
throw new Error(`Failed to get all edges: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get edges by source node ID
|
||||
*/
|
||||
public async getEdgesBySource(sourceId: string): Promise<Edge[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
const allEdges = await this.getAllEdges()
|
||||
return allEdges.filter((edge) => edge.sourceId === sourceId)
|
||||
} catch (error) {
|
||||
console.error(`Failed to get edges by source ${sourceId}:`, error)
|
||||
throw new Error(`Failed to get edges by source ${sourceId}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get edges by target node ID
|
||||
*/
|
||||
public async getEdgesByTarget(targetId: string): Promise<Edge[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
const allEdges = await this.getAllEdges()
|
||||
return allEdges.filter((edge) => edge.targetId === targetId)
|
||||
} catch (error) {
|
||||
console.error(`Failed to get edges by target ${targetId}:`, error)
|
||||
throw new Error(`Failed to get edges by target ${targetId}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get edges by type
|
||||
*/
|
||||
public async getEdgesByType(type: string): Promise<Edge[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
const allEdges = await this.getAllEdges()
|
||||
return allEdges.filter((edge) => edge.type === type)
|
||||
} catch (error) {
|
||||
console.error(`Failed to get edges by type ${type}:`, error)
|
||||
throw new Error(`Failed to get edges by type ${type}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete an edge from storage
|
||||
*/
|
||||
public async deleteEdge(id: string): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Import the DeleteObjectCommand and GetObjectCommand only when needed
|
||||
const { DeleteObjectCommand, GetObjectCommand } = await import(
|
||||
'@aws-sdk/client-s3'
|
||||
)
|
||||
|
||||
try {
|
||||
// Check if the edge exists before deleting
|
||||
await this.s3Client.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: `${EDGES_PREFIX}${id}.json`
|
||||
})
|
||||
)
|
||||
|
||||
// Delete the edge
|
||||
await this.s3Client.send(
|
||||
new DeleteObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: `${EDGES_PREFIX}${id}.json`
|
||||
})
|
||||
)
|
||||
} catch {
|
||||
// Edge not found, nothing to delete
|
||||
return
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Failed to delete edge ${id}:`, error)
|
||||
throw new Error(`Failed to delete edge ${id}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save metadata to storage
|
||||
*/
|
||||
public async saveMetadata(id: string, metadata: any): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Import the PutObjectCommand only when needed
|
||||
const { PutObjectCommand } = await import('@aws-sdk/client-s3')
|
||||
|
||||
// Save the metadata to S3-compatible storage
|
||||
await this.s3Client.send(
|
||||
new PutObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: `${METADATA_PREFIX}${id}.json`,
|
||||
Body: JSON.stringify(metadata, null, 2),
|
||||
ContentType: 'application/json'
|
||||
})
|
||||
)
|
||||
} catch (error) {
|
||||
console.error(`Failed to save metadata for ${id}:`, error)
|
||||
throw new Error(`Failed to save metadata for ${id}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get metadata from storage
|
||||
*/
|
||||
public async getMetadata(id: string): Promise<any | null> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Import the GetObjectCommand only when needed
|
||||
const { GetObjectCommand } = await import('@aws-sdk/client-s3')
|
||||
|
||||
try {
|
||||
// Try to get the metadata from S3-compatible storage
|
||||
const response = await this.s3Client.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: `${METADATA_PREFIX}${id}.json`
|
||||
})
|
||||
)
|
||||
|
||||
// Convert the response body to a string
|
||||
const bodyContents = await response.Body.transformToString()
|
||||
return JSON.parse(bodyContents)
|
||||
} catch {
|
||||
return null // Metadata not found
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Failed to get metadata for ${id}:`, error)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear all data from storage
|
||||
*/
|
||||
public async clear(): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Import the ListObjectsV2Command and DeleteObjectCommand only when needed
|
||||
const { ListObjectsV2Command, DeleteObjectCommand } = await import(
|
||||
'@aws-sdk/client-s3'
|
||||
)
|
||||
|
||||
// List all objects in the bucket
|
||||
const listResponse = await this.s3Client.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: this.bucketName
|
||||
})
|
||||
)
|
||||
|
||||
// If there are no objects, return
|
||||
if (!listResponse.Contents || listResponse.Contents.length === 0) {
|
||||
return
|
||||
}
|
||||
|
||||
// Delete each object
|
||||
const deletePromises = listResponse.Contents.map(
|
||||
async (object: { Key: string }) => {
|
||||
try {
|
||||
await this.s3Client.send(
|
||||
new DeleteObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: object.Key
|
||||
})
|
||||
)
|
||||
} catch (error) {
|
||||
console.error(`Failed to delete object ${object.Key}:`, error)
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
await Promise.all(deletePromises)
|
||||
} catch (error) {
|
||||
console.error('Failed to clear storage:', error)
|
||||
throw new Error(`Failed to clear storage: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get information about storage usage and capacity
|
||||
*/
|
||||
public async getStorageStatus(): Promise<{
|
||||
type: string
|
||||
used: number
|
||||
quota: number | null
|
||||
details?: Record<string, any>
|
||||
}> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Import the ListObjectsV2Command only when needed
|
||||
const { ListObjectsV2Command } = await import('@aws-sdk/client-s3')
|
||||
|
||||
// List all objects in the bucket to calculate total size
|
||||
const listResponse = await this.s3Client.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: this.bucketName
|
||||
})
|
||||
)
|
||||
|
||||
let totalSize = 0
|
||||
let nodeCount = 0
|
||||
let edgeCount = 0
|
||||
let metadataCount = 0
|
||||
|
||||
// Calculate total size and counts
|
||||
if (listResponse.Contents) {
|
||||
for (const object of listResponse.Contents) {
|
||||
totalSize += object.Size || 0
|
||||
|
||||
const key = object.Key || ''
|
||||
if (key.startsWith(NOUNS_PREFIX)) {
|
||||
nodeCount++
|
||||
} else if (key.startsWith(VERBS_PREFIX)) {
|
||||
edgeCount++
|
||||
} else if (key.startsWith(METADATA_PREFIX)) {
|
||||
metadataCount++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Count nodes by noun type by examining metadata
|
||||
const nounTypeCounts: Record<string, number> = {
|
||||
person: 0,
|
||||
place: 0,
|
||||
thing: 0,
|
||||
event: 0,
|
||||
concept: 0,
|
||||
content: 0,
|
||||
group: 0,
|
||||
list: 0,
|
||||
category: 0,
|
||||
default: 0
|
||||
}
|
||||
|
||||
// List all noun objects and count by type using metadata
|
||||
const nounsListResponse = await this.s3Client.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: this.bucketName,
|
||||
Prefix: NOUN_PREFIX
|
||||
})
|
||||
)
|
||||
|
||||
if (nounsListResponse.Contents) {
|
||||
for (const object of nounsListResponse.Contents) {
|
||||
try {
|
||||
// Extract node ID from the key
|
||||
const nodeId = object.Key?.replace(NOUN_PREFIX, '').replace('.json', '')
|
||||
if (nodeId) {
|
||||
// Get metadata to determine noun type
|
||||
const metadata = await this.getMetadata(nodeId)
|
||||
const nounType = metadata?.noun || 'default'
|
||||
|
||||
if (nounType in nounTypeCounts) {
|
||||
nounTypeCounts[nounType]++
|
||||
} else {
|
||||
nounTypeCounts.default++
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
// If we can't get metadata, count as default
|
||||
nounTypeCounts.default++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
type: this.serviceType,
|
||||
used: totalSize,
|
||||
quota: null, // S3-compatible services typically don't provide quota information through the API
|
||||
details: {
|
||||
nodeCount,
|
||||
edgeCount,
|
||||
metadataCount,
|
||||
nounTypes: {
|
||||
person: { count: nounTypeCounts.person },
|
||||
place: { count: nounTypeCounts.place },
|
||||
thing: { count: nounTypeCounts.thing },
|
||||
event: { count: nounTypeCounts.event },
|
||||
concept: { count: nounTypeCounts.concept },
|
||||
content: { count: nounTypeCounts.content },
|
||||
group: { count: nounTypeCounts.group },
|
||||
list: { count: nounTypeCounts.list },
|
||||
category: { count: nounTypeCounts.category },
|
||||
default: { count: nounTypeCounts.default }
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Failed to get storage status:', error)
|
||||
return {
|
||||
type: this.serviceType,
|
||||
used: 0,
|
||||
quota: null,
|
||||
details: { error: String(error) }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure the storage adapter is initialized
|
||||
*/
|
||||
private async ensureInitialized(): Promise<void> {
|
||||
if (!this.isInitialized) {
|
||||
await this.init()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the appropriate prefix for a node - now all nouns use the same prefix
|
||||
*/
|
||||
private async getNodePrefix(id: string): Promise<string> {
|
||||
// All nouns now use the same prefix regardless of type
|
||||
return NOUN_PREFIX
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a Map to a plain object for serialization
|
||||
*/
|
||||
private mapToObject<K extends string | number, V>(
|
||||
map: Map<K, V>,
|
||||
valueTransformer: (value: V) => any = (v) => v
|
||||
): Record<string, any> {
|
||||
const obj: Record<string, any> = {}
|
||||
for (const [key, value] of map.entries()) {
|
||||
obj[key.toString()] = valueTransformer(value)
|
||||
}
|
||||
return obj
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue