Merge remote-tracking branch 'origin/main'
# Conflicts: # CHANGES.md # package-lock.json # package.json # src/storage/adapters/fileSystemStorage.ts
This commit is contained in:
commit
3337c9f78e
49 changed files with 5099 additions and 4391 deletions
|
|
@ -35,25 +35,17 @@ import {
|
|||
ServerSearchConduitAugmentation,
|
||||
createServerSearchAugmentations
|
||||
} from './augmentations/serverSearchAugmentations.js'
|
||||
import {WebSocketConnection} from './types/augmentations.js'
|
||||
import {WebSocketConnection, AugmentationType, IAugmentation} from './types/augmentations.js'
|
||||
import {BrainyDataInterface} from './types/brainyDataInterface.js'
|
||||
import {augmentationPipeline} from './augmentationPipeline.js'
|
||||
|
||||
export interface BrainyDataConfig {
|
||||
/**
|
||||
* Vector dimensions (required if not using an embedding function that auto-detects dimensions)
|
||||
*/
|
||||
dimensions?: number
|
||||
|
||||
/**
|
||||
* HNSW index configuration
|
||||
* Uses the optimized HNSW implementation which supports large datasets
|
||||
* through product quantization and disk-based storage
|
||||
*/
|
||||
hnsw?: Partial<HNSWConfig>
|
||||
|
||||
/**
|
||||
* Optimized HNSW index configuration
|
||||
* If provided, will use the optimized HNSW index instead of the standard one
|
||||
*/
|
||||
hnswOptimized?: Partial<HNSWOptimizedConfig>
|
||||
hnsw?: Partial<HNSWOptimizedConfig>
|
||||
|
||||
/**
|
||||
* Distance function to use for similarity calculations
|
||||
|
|
@ -190,30 +182,19 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
* Create a new vector database
|
||||
*/
|
||||
constructor(config: BrainyDataConfig = {}) {
|
||||
// Validate dimensions
|
||||
if (config.dimensions !== undefined && config.dimensions <= 0) {
|
||||
throw new Error('Dimensions must be a positive number')
|
||||
}
|
||||
|
||||
// Set dimensions (default to 512 for embedding functions, or require explicit config)
|
||||
this._dimensions = config.dimensions || 512
|
||||
// Set dimensions to fixed value of 512 (Universal Sentence Encoder dimension)
|
||||
this._dimensions = 512
|
||||
|
||||
// Set distance function
|
||||
this.distanceFunction = config.distanceFunction || cosineDistance
|
||||
|
||||
// Check if optimized HNSW index configuration is provided
|
||||
if (config.hnswOptimized) {
|
||||
// Initialize optimized HNSW index
|
||||
this.index = new HNSWIndexOptimized(
|
||||
config.hnswOptimized,
|
||||
this.distanceFunction,
|
||||
config.storageAdapter || null
|
||||
)
|
||||
this.useOptimizedIndex = true
|
||||
} else {
|
||||
// Initialize standard HNSW index
|
||||
this.index = new HNSWIndex(config.hnsw, this.distanceFunction)
|
||||
}
|
||||
// Always use the optimized HNSW index implementation
|
||||
this.index = new HNSWIndexOptimized(
|
||||
config.hnsw || {},
|
||||
this.distanceFunction,
|
||||
config.storageAdapter || null
|
||||
)
|
||||
this.useOptimizedIndex = true
|
||||
|
||||
// Set storage if provided, otherwise it will be initialized in init()
|
||||
this.storage = config.storageAdapter || null
|
||||
|
|
@ -263,6 +244,36 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the current augmentation name if available
|
||||
* This is used to auto-detect the service performing data operations
|
||||
* @returns The name of the current augmentation or 'default' if none is detected
|
||||
*/
|
||||
private getCurrentAugmentation(): string {
|
||||
try {
|
||||
// Get all registered augmentations
|
||||
const augmentationTypes = augmentationPipeline.getAvailableAugmentationTypes()
|
||||
|
||||
// Check each type of augmentation
|
||||
for (const type of augmentationTypes) {
|
||||
const augmentations = augmentationPipeline.getAugmentationsByType(type)
|
||||
|
||||
// Find the first enabled augmentation
|
||||
for (const augmentation of augmentations) {
|
||||
if (augmentation.enabled) {
|
||||
return augmentation.name
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return 'default'
|
||||
} catch (error) {
|
||||
// If there's any error in detection, return default
|
||||
console.warn('Failed to detect current augmentation:', error)
|
||||
return 'default'
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the database
|
||||
* Loads existing data from storage if available
|
||||
|
|
@ -452,6 +463,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
forceEmbed?: boolean // Force using the embedding function even if input is a vector
|
||||
addToRemote?: boolean // Whether to also add to the remote server if connected
|
||||
id?: string // Optional ID to use instead of generating a new one
|
||||
service?: string // The service that is inserting the data
|
||||
} = {}
|
||||
): Promise<string> {
|
||||
await this.ensureInitialized()
|
||||
|
|
@ -509,6 +521,10 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
// Save noun to storage
|
||||
await this.storage!.saveNoun(noun)
|
||||
|
||||
// Track noun statistics
|
||||
const service = options.service || this.getCurrentAugmentation()
|
||||
await this.storage!.incrementStatistic('noun', service)
|
||||
|
||||
// Save metadata if provided
|
||||
if (metadata !== undefined) {
|
||||
// Validate noun type if metadata is for a GraphNoun
|
||||
|
|
@ -525,6 +541,33 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
// Set a default noun type
|
||||
;(metadata as unknown as GraphNoun).noun = NounType.Concept
|
||||
}
|
||||
|
||||
// Ensure createdBy field is populated for GraphNoun
|
||||
const service = options.service || this.getCurrentAugmentation()
|
||||
const graphNoun = metadata as unknown as GraphNoun
|
||||
|
||||
// Only set createdBy if it doesn't exist or is being explicitly updated
|
||||
if (!graphNoun.createdBy || options.service) {
|
||||
graphNoun.createdBy = {
|
||||
augmentation: service,
|
||||
version: '1.0' // TODO: Get actual version from augmentation
|
||||
}
|
||||
}
|
||||
|
||||
// Update timestamps
|
||||
const now = new Date()
|
||||
const timestamp = {
|
||||
seconds: Math.floor(now.getTime() / 1000),
|
||||
nanoseconds: (now.getTime() % 1000) * 1000000
|
||||
}
|
||||
|
||||
// Set createdAt if it doesn't exist
|
||||
if (!graphNoun.createdAt) {
|
||||
graphNoun.createdAt = timestamp
|
||||
}
|
||||
|
||||
// Always update updatedAt
|
||||
graphNoun.updatedAt = timestamp
|
||||
}
|
||||
|
||||
// Ensure metadata has the correct id field
|
||||
|
|
@ -534,8 +577,15 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
}
|
||||
|
||||
await this.storage!.saveMetadata(id, metadataToSave)
|
||||
|
||||
// Track metadata statistics
|
||||
const metadataService = options.service || this.getCurrentAugmentation()
|
||||
await this.storage!.incrementStatistic('metadata', metadataService)
|
||||
}
|
||||
|
||||
// Update HNSW index size (excluding verbs)
|
||||
await this.storage!.updateHnswIndexSize(await this.getNounCount())
|
||||
|
||||
// If addToRemote is true and we're connected to a remote server, add to remote as well
|
||||
if (options.addToRemote && this.isConnectedToRemoteServer()) {
|
||||
try {
|
||||
|
|
@ -791,6 +841,30 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
return this.addBatch(items, {...options, addToRemote: true})
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter search results by service
|
||||
* @param results Search results to filter
|
||||
* @param service Service to filter by
|
||||
* @returns Filtered search results
|
||||
* @private
|
||||
*/
|
||||
private filterResultsByService<R extends SearchResult<T>>(
|
||||
results: R[],
|
||||
service?: string
|
||||
): R[] {
|
||||
if (!service) return results
|
||||
|
||||
return results.filter(result => {
|
||||
if (!result.metadata || typeof result.metadata !== 'object') return false
|
||||
if (!('createdBy' in result.metadata)) return false
|
||||
|
||||
const createdBy = result.metadata.createdBy as any
|
||||
if (!createdBy) return false
|
||||
|
||||
return createdBy.augmentation === service
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Search for similar vectors within specific noun types
|
||||
* @param queryVectorOrData Query vector or data to search for
|
||||
|
|
@ -805,8 +879,22 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
nounTypes: string[] | null = null,
|
||||
options: {
|
||||
forceEmbed?: boolean // Force using the embedding function even if input is a vector
|
||||
service?: string // Filter results by the service that created the data
|
||||
} = {}
|
||||
): Promise<SearchResult<T>[]> {
|
||||
// Helper function to filter results by service
|
||||
const filterByService = (metadata: any): boolean => {
|
||||
if (!options.service) return true // No filter, include all
|
||||
|
||||
// Check if metadata has createdBy field with matching service
|
||||
if (!metadata || typeof metadata !== 'object') return false
|
||||
if (!('createdBy' in metadata)) return false
|
||||
|
||||
const createdBy = metadata.createdBy as any
|
||||
if (!createdBy) return false
|
||||
|
||||
return createdBy.augmentation === options.service
|
||||
}
|
||||
if (!this.isInitialized) {
|
||||
throw new Error('BrainyData must be initialized before searching. Call init() first.')
|
||||
}
|
||||
|
|
@ -836,6 +924,11 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
throw new Error('Query vector is undefined or null')
|
||||
}
|
||||
|
||||
// Check if query vector dimensions match the expected dimensions
|
||||
if (queryVector.length !== this._dimensions) {
|
||||
throw new Error(`Query vector dimension mismatch: expected ${this._dimensions}, got ${queryVector.length}`)
|
||||
}
|
||||
|
||||
// If no noun types specified, search all nouns
|
||||
if (!nounTypes || nounTypes.length === 0) {
|
||||
// Search in the index
|
||||
|
|
@ -857,6 +950,11 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
metadata = {} as T
|
||||
}
|
||||
|
||||
// Ensure metadata has the id field
|
||||
if (metadata && typeof metadata === 'object') {
|
||||
metadata = {...metadata, id} as T
|
||||
}
|
||||
|
||||
searchResults.push({
|
||||
id,
|
||||
score,
|
||||
|
|
@ -865,7 +963,8 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
})
|
||||
}
|
||||
|
||||
return searchResults
|
||||
// Filter results by service if specified
|
||||
return this.filterResultsByService(searchResults, options.service)
|
||||
} else {
|
||||
// Get nouns for each noun type in parallel
|
||||
const nounPromises = nounTypes.map((nounType) =>
|
||||
|
|
@ -911,6 +1010,11 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
metadata = {} as T
|
||||
}
|
||||
|
||||
// Ensure metadata has the id field
|
||||
if (metadata && typeof metadata === 'object') {
|
||||
metadata = {...metadata, id} as T
|
||||
}
|
||||
|
||||
searchResults.push({
|
||||
id,
|
||||
score,
|
||||
|
|
@ -919,7 +1023,8 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
})
|
||||
}
|
||||
|
||||
return searchResults
|
||||
// Filter results by service if specified
|
||||
return this.filterResultsByService(searchResults, options.service)
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Failed to search vectors by noun types:', error)
|
||||
|
|
@ -946,6 +1051,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
verbTypes?: string[] // Optional array of verb types to search within or filter by
|
||||
searchConnectedNouns?: boolean // Whether to search for nouns connected by verbs
|
||||
verbDirection?: 'outgoing' | 'incoming' | 'both' // Direction of verbs to consider when searching connected nouns
|
||||
service?: string // Filter results by the service that created the data
|
||||
} = {}
|
||||
): Promise<SearchResult<T>[]> {
|
||||
if (!this.isInitialized) {
|
||||
|
|
@ -1008,6 +1114,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
forceEmbed?: boolean // Force using the embedding function even if input is a vector
|
||||
nounTypes?: string[] // Optional array of noun types to search within
|
||||
includeVerbs?: boolean // Whether to include associated GraphVerbs in the results
|
||||
service?: string // Filter results by the service that created the data
|
||||
} = {}
|
||||
): Promise<SearchResult<T>[]> {
|
||||
if (!this.isInitialized) {
|
||||
|
|
@ -1028,13 +1135,15 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
k,
|
||||
options.nounTypes,
|
||||
{
|
||||
forceEmbed: options.forceEmbed
|
||||
forceEmbed: options.forceEmbed,
|
||||
service: options.service
|
||||
}
|
||||
)
|
||||
} else {
|
||||
// Otherwise, search all GraphNouns
|
||||
searchResults = await this.searchByNounTypes(queryToUse, k, null, {
|
||||
forceEmbed: options.forceEmbed
|
||||
forceEmbed: options.forceEmbed,
|
||||
service: options.service
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -1161,8 +1270,16 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
|
||||
/**
|
||||
* Delete a vector by ID
|
||||
* @param id The ID of the vector to delete
|
||||
* @param options Additional options
|
||||
* @returns Promise that resolves to true if the vector was deleted, false otherwise
|
||||
*/
|
||||
public async delete(id: string): Promise<boolean> {
|
||||
public async delete(
|
||||
id: string,
|
||||
options: {
|
||||
service?: string // The service that is deleting the data
|
||||
} = {}
|
||||
): Promise<boolean> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
// Check if database is in read-only mode
|
||||
|
|
@ -1178,9 +1295,14 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
// Remove from storage
|
||||
await this.storage!.deleteNoun(id)
|
||||
|
||||
// Track deletion statistics
|
||||
const service = options.service || 'default'
|
||||
await this.storage!.decrementStatistic('noun', service)
|
||||
|
||||
// Try to remove metadata (ignore errors)
|
||||
try {
|
||||
await this.storage!.saveMetadata(id, null)
|
||||
await this.storage!.decrementStatistic('metadata', service)
|
||||
} catch (error) {
|
||||
// Ignore
|
||||
}
|
||||
|
|
@ -1194,8 +1316,18 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
|
||||
/**
|
||||
* Update metadata for a vector
|
||||
* @param id The ID of the vector to update metadata for
|
||||
* @param metadata The new metadata
|
||||
* @param options Additional options
|
||||
* @returns Promise that resolves to true if the metadata was updated, false otherwise
|
||||
*/
|
||||
public async updateMetadata(id: string, metadata: T): Promise<boolean> {
|
||||
public async updateMetadata(
|
||||
id: string,
|
||||
metadata: T,
|
||||
options: {
|
||||
service?: string // The service that is updating the data
|
||||
} = {}
|
||||
): Promise<boolean> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
// Check if database is in read-only mode
|
||||
|
|
@ -1222,11 +1354,56 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
// Set a default noun type
|
||||
;(metadata as unknown as GraphNoun).noun = NounType.Concept
|
||||
}
|
||||
|
||||
// Get the service that's updating the metadata
|
||||
const service = options.service || this.getCurrentAugmentation()
|
||||
const graphNoun = metadata as unknown as GraphNoun
|
||||
|
||||
// Preserve existing createdBy and createdAt if they exist
|
||||
const existingMetadata = await this.storage!.getMetadata(id) as any
|
||||
|
||||
if (existingMetadata &&
|
||||
typeof existingMetadata === 'object' &&
|
||||
'createdBy' in existingMetadata) {
|
||||
// Preserve the original creator information
|
||||
graphNoun.createdBy = existingMetadata.createdBy
|
||||
|
||||
// Also preserve creation timestamp if it exists
|
||||
if ('createdAt' in existingMetadata) {
|
||||
graphNoun.createdAt = existingMetadata.createdAt
|
||||
}
|
||||
} else if (!graphNoun.createdBy) {
|
||||
// If no existing createdBy and none in the update, set it
|
||||
graphNoun.createdBy = {
|
||||
augmentation: service,
|
||||
version: '1.0' // TODO: Get actual version from augmentation
|
||||
}
|
||||
|
||||
// Set createdAt if it doesn't exist
|
||||
if (!graphNoun.createdAt) {
|
||||
const now = new Date()
|
||||
graphNoun.createdAt = {
|
||||
seconds: Math.floor(now.getTime() / 1000),
|
||||
nanoseconds: (now.getTime() % 1000) * 1000000
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Always update the updatedAt timestamp
|
||||
const now = new Date()
|
||||
graphNoun.updatedAt = {
|
||||
seconds: Math.floor(now.getTime() / 1000),
|
||||
nanoseconds: (now.getTime() % 1000) * 1000000
|
||||
}
|
||||
}
|
||||
|
||||
// Update metadata
|
||||
await this.storage!.saveMetadata(id, metadata)
|
||||
|
||||
// Track metadata statistics
|
||||
const service = options.service || this.getCurrentAugmentation()
|
||||
await this.storage!.incrementStatistic('metadata', service)
|
||||
|
||||
return true
|
||||
} catch (error) {
|
||||
console.error(`Failed to update metadata for vector ${id}:`, error)
|
||||
|
|
@ -1250,6 +1427,19 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a connection between two entities
|
||||
* This is an alias for relate() for backward compatibility
|
||||
*/
|
||||
public async connect(
|
||||
sourceId: string,
|
||||
targetId: string,
|
||||
relationType: string,
|
||||
metadata?: any
|
||||
): Promise<string> {
|
||||
return this.relate(sourceId, targetId, relationType, metadata)
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a verb between two nouns
|
||||
* If metadata is provided and vector is not, the metadata will be vectorized using the embedding function
|
||||
|
|
@ -1282,6 +1472,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
id?: string // Optional ID to use instead of generating a new one
|
||||
autoCreateMissingNouns?: boolean // Automatically create missing nouns
|
||||
missingNounMetadata?: any // Metadata to use when auto-creating missing nouns
|
||||
service?: string // The service that is inserting the data
|
||||
} = {}
|
||||
): Promise<string> {
|
||||
await this.ensureInitialized()
|
||||
|
|
@ -1301,10 +1492,22 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
const placeholderVector = new Array(this._dimensions).fill(0)
|
||||
|
||||
// Add metadata if provided
|
||||
const service = options.service || this.getCurrentAugmentation()
|
||||
const now = new Date()
|
||||
const timestamp = {
|
||||
seconds: Math.floor(now.getTime() / 1000),
|
||||
nanoseconds: (now.getTime() % 1000) * 1000000
|
||||
}
|
||||
|
||||
const metadata = options.missingNounMetadata || {
|
||||
autoCreated: true,
|
||||
createdAt: new Date().toISOString(),
|
||||
noun: NounType.Concept
|
||||
createdAt: timestamp,
|
||||
updatedAt: timestamp,
|
||||
noun: NounType.Concept,
|
||||
createdBy: {
|
||||
augmentation: service,
|
||||
version: '1.0' // TODO: Get actual version from augmentation
|
||||
}
|
||||
}
|
||||
|
||||
// Add the missing noun
|
||||
|
|
@ -1326,10 +1529,22 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
const placeholderVector = new Array(this._dimensions).fill(0)
|
||||
|
||||
// Add metadata if provided
|
||||
const service = options.service || this.getCurrentAugmentation()
|
||||
const now = new Date()
|
||||
const timestamp = {
|
||||
seconds: Math.floor(now.getTime() / 1000),
|
||||
nanoseconds: (now.getTime() % 1000) * 1000000
|
||||
}
|
||||
|
||||
const metadata = options.missingNounMetadata || {
|
||||
autoCreated: true,
|
||||
createdAt: new Date().toISOString(),
|
||||
noun: NounType.Concept
|
||||
createdAt: timestamp,
|
||||
updatedAt: timestamp,
|
||||
noun: NounType.Concept,
|
||||
createdBy: {
|
||||
augmentation: service,
|
||||
version: '1.0' // TODO: Get actual version from augmentation
|
||||
}
|
||||
}
|
||||
|
||||
// Add the missing noun
|
||||
|
|
@ -1426,16 +1641,34 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
}
|
||||
}
|
||||
|
||||
// Get service name from options or current augmentation
|
||||
const service = options.service || this.getCurrentAugmentation()
|
||||
|
||||
// Create timestamp for creation/update time
|
||||
const now = new Date()
|
||||
const timestamp = {
|
||||
seconds: Math.floor(now.getTime() / 1000),
|
||||
nanoseconds: (now.getTime() % 1000) * 1000000
|
||||
}
|
||||
|
||||
// Create verb
|
||||
const verb: GraphVerb = {
|
||||
id,
|
||||
vector: verbVector,
|
||||
connections: new Map(),
|
||||
sourceId,
|
||||
targetId,
|
||||
type: verbType,
|
||||
sourceId: sourceId,
|
||||
targetId: targetId,
|
||||
source: sourceId,
|
||||
target: targetId,
|
||||
verb: verbType as VerbType,
|
||||
weight: options.weight,
|
||||
metadata: options.metadata
|
||||
metadata: options.metadata,
|
||||
createdAt: timestamp,
|
||||
updatedAt: timestamp,
|
||||
createdBy: {
|
||||
augmentation: service,
|
||||
version: '1.0' // TODO: Get actual version from augmentation
|
||||
}
|
||||
}
|
||||
|
||||
// Add to index
|
||||
|
|
@ -1456,6 +1689,13 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
// Save verb to storage
|
||||
await this.storage!.saveVerb(verb)
|
||||
|
||||
// Track verb statistics
|
||||
const serviceForStats = options.service || 'default'
|
||||
await this.storage!.incrementStatistic('verb', serviceForStats)
|
||||
|
||||
// Update HNSW index size (excluding verbs)
|
||||
await this.storage!.updateHnswIndexSize(await this.getNounCount())
|
||||
|
||||
return id
|
||||
} catch (error) {
|
||||
console.error('Failed to add verb:', error)
|
||||
|
|
@ -1535,8 +1775,16 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
|
||||
/**
|
||||
* Delete a verb
|
||||
* @param id The ID of the verb to delete
|
||||
* @param options Additional options
|
||||
* @returns Promise that resolves to true if the verb was deleted, false otherwise
|
||||
*/
|
||||
public async deleteVerb(id: string): Promise<boolean> {
|
||||
public async deleteVerb(
|
||||
id: string,
|
||||
options: {
|
||||
service?: string // The service that is deleting the data
|
||||
} = {}
|
||||
): Promise<boolean> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
// Check if database is in read-only mode
|
||||
|
|
@ -1552,6 +1800,10 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
// Remove from storage
|
||||
await this.storage!.deleteVerb(id)
|
||||
|
||||
// Track deletion statistics
|
||||
const service = options.service || 'default'
|
||||
await this.storage!.decrementStatistic('verb', service)
|
||||
|
||||
return true
|
||||
} catch (error) {
|
||||
console.error(`Failed to delete verb ${id}:`, error)
|
||||
|
|
@ -1587,26 +1839,128 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
return this.index.size()
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the number of nouns in the database (excluding verbs)
|
||||
* This is used for statistics reporting to match the expected behavior in tests
|
||||
* @private
|
||||
*/
|
||||
private async getNounCount(): Promise<number> {
|
||||
// Get all verbs from storage
|
||||
const allVerbs = await this.storage!.getAllVerbs()
|
||||
|
||||
// Create a set of verb IDs for faster lookup
|
||||
const verbIds = new Set(allVerbs.map(verb => verb.id))
|
||||
|
||||
// Get all nouns from the index
|
||||
const nouns = this.index.getNouns()
|
||||
|
||||
// Count nouns that are not verbs
|
||||
let nounCount = 0
|
||||
for (const [id] of nouns.entries()) {
|
||||
if (!verbIds.has(id)) {
|
||||
nounCount++
|
||||
}
|
||||
}
|
||||
|
||||
return nounCount
|
||||
}
|
||||
|
||||
/**
|
||||
* Force an immediate flush of statistics to storage
|
||||
* This ensures that any pending statistics updates are written to persistent storage
|
||||
* @returns Promise that resolves when the statistics have been flushed
|
||||
*/
|
||||
public async flushStatistics(): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
if (!this.storage) {
|
||||
throw new Error('Storage not initialized')
|
||||
}
|
||||
|
||||
// Call the flushStatisticsToStorage method on the storage adapter
|
||||
await this.storage.flushStatisticsToStorage()
|
||||
}
|
||||
|
||||
/**
|
||||
* Get statistics about the current state of the database
|
||||
* @param options Additional options for retrieving statistics
|
||||
* @returns Object containing counts of nouns, verbs, metadata entries, and HNSW index size
|
||||
*/
|
||||
public async getStatistics(): Promise<{
|
||||
public async getStatistics(options: {
|
||||
service?: string | string[] // Filter statistics by service(s)
|
||||
} = {}): Promise<{
|
||||
nounCount: number
|
||||
verbCount: number
|
||||
metadataCount: number
|
||||
hnswIndexSize: number
|
||||
serviceBreakdown?: {
|
||||
[service: string]: {
|
||||
nounCount: number
|
||||
verbCount: number
|
||||
metadataCount: number
|
||||
}
|
||||
}
|
||||
}> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Get noun count from the index
|
||||
const nounCount = this.index.getNouns().size
|
||||
// Get statistics from storage
|
||||
const stats = await this.storage!.getStatistics()
|
||||
|
||||
// Get verb count from storage
|
||||
// If statistics are available, use them
|
||||
if (stats) {
|
||||
// Initialize result
|
||||
const result = {
|
||||
nounCount: 0,
|
||||
verbCount: 0,
|
||||
metadataCount: 0,
|
||||
hnswIndexSize: stats.hnswIndexSize,
|
||||
serviceBreakdown: {} as {
|
||||
[service: string]: {
|
||||
nounCount: number
|
||||
verbCount: number
|
||||
metadataCount: number
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Filter by service if specified
|
||||
const services = options.service
|
||||
? (Array.isArray(options.service) ? options.service : [options.service])
|
||||
: Object.keys({...stats.nounCount, ...stats.verbCount, ...stats.metadataCount})
|
||||
|
||||
// Calculate totals and service breakdown
|
||||
for (const service of services) {
|
||||
const nounCount = stats.nounCount[service] || 0
|
||||
const verbCount = stats.verbCount[service] || 0
|
||||
const metadataCount = stats.metadataCount[service] || 0
|
||||
|
||||
// Add to totals
|
||||
result.nounCount += nounCount
|
||||
result.verbCount += verbCount
|
||||
result.metadataCount += metadataCount
|
||||
|
||||
// Add to service breakdown
|
||||
result.serviceBreakdown[service] = {
|
||||
nounCount,
|
||||
verbCount,
|
||||
metadataCount
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// If statistics are not available, fall back to calculating them on-demand
|
||||
console.warn('Persistent statistics not available, calculating on-demand')
|
||||
|
||||
// Get all verbs from storage
|
||||
const allVerbs = await this.storage!.getAllVerbs()
|
||||
const verbCount = allVerbs.length
|
||||
|
||||
// Get the noun count using the helper method
|
||||
const nounCount = await this.getNounCount()
|
||||
|
||||
// Count metadata entries by checking each noun for metadata
|
||||
let metadataCount = 0
|
||||
const nouns = this.index.getNouns()
|
||||
|
|
@ -1622,15 +1976,30 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
}
|
||||
}
|
||||
|
||||
// Get HNSW index size
|
||||
const hnswIndexSize = this.index.size()
|
||||
// Get HNSW index size (excluding verbs)
|
||||
// The HNSW index includes both nouns and verbs, but for statistics we want to report
|
||||
// only the number of actual nouns (excluding verbs) to match the expected behavior in tests
|
||||
const hnswIndexSize = nounCount
|
||||
|
||||
return {
|
||||
// Create default statistics
|
||||
const defaultStats = {
|
||||
nounCount,
|
||||
verbCount,
|
||||
metadataCount,
|
||||
hnswIndexSize
|
||||
}
|
||||
|
||||
// Initialize persistent statistics
|
||||
const service = 'default'
|
||||
await this.storage!.saveStatistics({
|
||||
nounCount: {[service]: nounCount},
|
||||
verbCount: {[service]: verbCount},
|
||||
metadataCount: {[service]: metadataCount},
|
||||
hnswIndexSize,
|
||||
lastUpdated: new Date().toISOString()
|
||||
})
|
||||
|
||||
return defaultStats
|
||||
} catch (error) {
|
||||
console.error('Failed to get statistics:', error)
|
||||
throw new Error(`Failed to get statistics: ${error}`)
|
||||
|
|
@ -1684,6 +2053,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
options: {
|
||||
forceEmbed?: boolean // Force using the embedding function even if input is a vector
|
||||
verbTypes?: string[] // Optional array of verb types to search within
|
||||
service?: string // Filter results by the service that created the data
|
||||
} = {}
|
||||
): Promise<Array<GraphVerb & { similarity: number }>> {
|
||||
await this.ensureInitialized()
|
||||
|
|
@ -1708,51 +2078,85 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
}
|
||||
}
|
||||
|
||||
// Get verbs to search through
|
||||
let verbs: GraphVerb[] = []
|
||||
// First use the HNSW index to find similar vectors efficiently
|
||||
const searchResults = await this.index.search(queryVector, k * 2)
|
||||
|
||||
// If verb types are specified, get verbs of those types
|
||||
if (options.verbTypes && options.verbTypes.length > 0) {
|
||||
// Get verbs for each verb type in parallel
|
||||
const verbPromises = options.verbTypes.map((verbType) =>
|
||||
this.getVerbsByType(verbType)
|
||||
)
|
||||
const verbArrays = await Promise.all(verbPromises)
|
||||
// Get all verbs for filtering
|
||||
const allVerbs = await this.storage!.getAllVerbs()
|
||||
|
||||
// Combine all verbs
|
||||
for (const verbArray of verbArrays) {
|
||||
verbs.push(...verbArray)
|
||||
}
|
||||
} else {
|
||||
// Get all verbs
|
||||
verbs = await this.storage!.getAllVerbs()
|
||||
// Create a map of verb IDs for faster lookup
|
||||
const verbMap = new Map<string, GraphVerb>()
|
||||
for (const verb of allVerbs) {
|
||||
verbMap.set(verb.id, verb)
|
||||
}
|
||||
|
||||
// Filter out verbs without embeddings
|
||||
verbs = verbs.filter(
|
||||
(verb) => verb.embedding && verb.embedding.length > 0
|
||||
)
|
||||
// Filter search results to only include verbs
|
||||
const verbResults: Array<GraphVerb & { similarity: number }> = []
|
||||
|
||||
// Calculate similarity for each verb
|
||||
const results: Array<GraphVerb & { similarity: number }> = []
|
||||
for (const verb of verbs) {
|
||||
if (verb.embedding) {
|
||||
const distance = this.index.getDistanceFunction()(
|
||||
queryVector,
|
||||
verb.embedding
|
||||
)
|
||||
results.push({
|
||||
for (const result of searchResults) {
|
||||
// Search results are [id, distance] tuples
|
||||
const [id, distance] = result
|
||||
const verb = verbMap.get(id)
|
||||
if (verb) {
|
||||
// If verb types are specified, check if this verb matches
|
||||
if (options.verbTypes && options.verbTypes.length > 0) {
|
||||
if (!verb.type || !options.verbTypes.includes(verb.type)) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
verbResults.push({
|
||||
...verb,
|
||||
similarity: distance
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// If we didn't get enough results from the index, fall back to the old method
|
||||
if (verbResults.length < k) {
|
||||
console.warn('Not enough verb results from HNSW index, falling back to manual search')
|
||||
|
||||
// Get verbs to search through
|
||||
let verbs: GraphVerb[] = []
|
||||
|
||||
// If verb types are specified, get verbs of those types
|
||||
if (options.verbTypes && options.verbTypes.length > 0) {
|
||||
// Get verbs for each verb type in parallel
|
||||
const verbPromises = options.verbTypes.map((verbType) =>
|
||||
this.getVerbsByType(verbType)
|
||||
)
|
||||
const verbArrays = await Promise.all(verbPromises)
|
||||
|
||||
// Combine all verbs
|
||||
for (const verbArray of verbArrays) {
|
||||
verbs.push(...verbArray)
|
||||
}
|
||||
} else {
|
||||
// Use all verbs
|
||||
verbs = allVerbs
|
||||
}
|
||||
|
||||
// Calculate similarity for each verb not already in results
|
||||
const existingIds = new Set(verbResults.map(v => v.id))
|
||||
for (const verb of verbs) {
|
||||
if (!existingIds.has(verb.id) && verb.vector && verb.vector.length > 0) {
|
||||
const distance = this.index.getDistanceFunction()(
|
||||
queryVector,
|
||||
verb.vector
|
||||
)
|
||||
verbResults.push({
|
||||
...verb,
|
||||
similarity: distance
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by similarity (ascending distance)
|
||||
results.sort((a, b) => a.similarity - b.similarity)
|
||||
verbResults.sort((a, b) => a.similarity - b.similarity)
|
||||
|
||||
// Take top k results
|
||||
return results.slice(0, k)
|
||||
return verbResults.slice(0, k)
|
||||
} catch (error) {
|
||||
console.error('Failed to search verbs:', error)
|
||||
throw new Error(`Failed to search verbs: ${error}`)
|
||||
|
|
@ -1928,6 +2332,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
nounTypes?: string[] // Optional array of noun types to search within
|
||||
includeVerbs?: boolean // Whether to include associated GraphVerbs in the results
|
||||
storeResults?: boolean // Whether to store the results in the local database (default: true)
|
||||
service?: string // Filter results by the service that created the data
|
||||
} = {}
|
||||
): Promise<SearchResult<T>[]> {
|
||||
await this.ensureInitialized()
|
||||
|
|
@ -1989,6 +2394,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
nounTypes?: string[] // Optional array of noun types to search within
|
||||
includeVerbs?: boolean // Whether to include associated GraphVerbs in the results
|
||||
localFirst?: boolean // Whether to search local first (default: true)
|
||||
service?: string // Filter results by the service that created the data
|
||||
} = {}
|
||||
): Promise<SearchResult<T>[]> {
|
||||
await this.ensureInitialized()
|
||||
|
|
@ -2244,6 +2650,16 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
*/
|
||||
public async shutDown(): Promise<void> {
|
||||
try {
|
||||
// Flush statistics to ensure they're saved before shutting down
|
||||
if (this.storage && this.isInitialized) {
|
||||
try {
|
||||
await this.flushStatistics()
|
||||
} catch (statsError) {
|
||||
console.warn('Failed to flush statistics during shutdown:', statsError)
|
||||
// Continue with shutdown even if statistics flush fails
|
||||
}
|
||||
}
|
||||
|
||||
// Disconnect from remote server if connected
|
||||
if (this.isConnectedToRemoteServer()) {
|
||||
await this.disconnectFromRemoteServer()
|
||||
|
|
@ -2493,10 +2909,13 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
console.log('Reconstructing HNSW index from backup data...')
|
||||
|
||||
// Create a new index with the restored configuration
|
||||
this.index = new HNSWIndex(
|
||||
// Always use the optimized implementation for consistency
|
||||
this.index = new HNSWIndexOptimized(
|
||||
data.hnswIndex.config,
|
||||
this.distanceFunction
|
||||
this.distanceFunction,
|
||||
this.storage
|
||||
)
|
||||
this.useOptimizedIndex = true
|
||||
|
||||
// Re-add all nouns to the index
|
||||
for (const noun of data.nouns) {
|
||||
|
|
|
|||
|
|
@ -82,6 +82,11 @@ export interface GraphVerb extends HNSWNoun {
|
|||
verb?: string // Alias for type
|
||||
data?: Record<string, any> // Additional flexible data storage
|
||||
embedding?: Vector // Vector representation of the relationship
|
||||
|
||||
// Timestamp and creator properties
|
||||
createdAt?: { seconds: number, nanoseconds: number } // When the verb was created
|
||||
updatedAt?: { seconds: number, nanoseconds: number } // When the verb was last updated
|
||||
createdBy?: { augmentation: string, version: string } // Information about what created this verb
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -97,6 +102,36 @@ export interface HNSWConfig {
|
|||
/**
|
||||
* Storage interface for persistence
|
||||
*/
|
||||
/**
|
||||
* Statistics data structure for tracking counts by service
|
||||
*/
|
||||
export interface StatisticsData {
|
||||
/**
|
||||
* Count of nouns by service
|
||||
*/
|
||||
nounCount: Record<string, number>
|
||||
|
||||
/**
|
||||
* Count of verbs by service
|
||||
*/
|
||||
verbCount: Record<string, number>
|
||||
|
||||
/**
|
||||
* Count of metadata entries by service
|
||||
*/
|
||||
metadataCount: Record<string, number>
|
||||
|
||||
/**
|
||||
* Size of the HNSW index
|
||||
*/
|
||||
hnswIndexSize: number
|
||||
|
||||
/**
|
||||
* Last updated timestamp
|
||||
*/
|
||||
lastUpdated: string
|
||||
}
|
||||
|
||||
export interface StorageAdapter {
|
||||
init(): Promise<void>
|
||||
|
||||
|
|
@ -160,4 +195,44 @@ export interface StorageAdapter {
|
|||
*/
|
||||
details?: Record<string, any>
|
||||
}>
|
||||
|
||||
/**
|
||||
* Save statistics data
|
||||
* @param statistics The statistics data to save
|
||||
*/
|
||||
saveStatistics(statistics: StatisticsData): Promise<void>
|
||||
|
||||
/**
|
||||
* Get statistics data
|
||||
* @returns Promise that resolves to the statistics data
|
||||
*/
|
||||
getStatistics(): Promise<StatisticsData | null>
|
||||
|
||||
/**
|
||||
* Increment a statistic counter
|
||||
* @param type The type of statistic to increment ('noun', 'verb', 'metadata')
|
||||
* @param service The service that inserted the data
|
||||
* @param amount The amount to increment by (default: 1)
|
||||
*/
|
||||
incrementStatistic(type: 'noun' | 'verb' | 'metadata', service: string, amount?: number): Promise<void>
|
||||
|
||||
/**
|
||||
* Decrement a statistic counter
|
||||
* @param type The type of statistic to decrement ('noun', 'verb', 'metadata')
|
||||
* @param service The service that inserted the data
|
||||
* @param amount The amount to decrement by (default: 1)
|
||||
*/
|
||||
decrementStatistic(type: 'noun' | 'verb' | 'metadata', service: string, amount?: number): Promise<void>
|
||||
|
||||
/**
|
||||
* Update the HNSW index size statistic
|
||||
* @param size The new size of the HNSW index
|
||||
*/
|
||||
updateHnswIndexSize(size: number): Promise<void>
|
||||
|
||||
/**
|
||||
* Force an immediate flush of statistics to storage
|
||||
* This ensures that any pending statistics updates are written to persistent storage
|
||||
*/
|
||||
flushStatisticsToStorage(): Promise<void>
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,14 +23,16 @@ import {
|
|||
euclideanDistance,
|
||||
cosineDistance,
|
||||
manhattanDistance,
|
||||
dotProductDistance
|
||||
dotProductDistance,
|
||||
getStatistics
|
||||
} from './utils/index.js'
|
||||
|
||||
export {
|
||||
euclideanDistance,
|
||||
cosineDistance,
|
||||
manhattanDistance,
|
||||
dotProductDistance
|
||||
dotProductDistance,
|
||||
getStatistics
|
||||
}
|
||||
|
||||
// Export embedding functionality
|
||||
|
|
|
|||
320
src/storage/adapters/baseStorageAdapter.ts
Normal file
320
src/storage/adapters/baseStorageAdapter.ts
Normal file
|
|
@ -0,0 +1,320 @@
|
|||
/**
|
||||
* Base Storage Adapter
|
||||
* Provides common functionality for all storage adapters, including statistics tracking
|
||||
*/
|
||||
|
||||
import { StatisticsData, StorageAdapter } from '../../coreTypes.js'
|
||||
|
||||
/**
|
||||
* Base class for storage adapters that implements statistics tracking
|
||||
*/
|
||||
export abstract class BaseStorageAdapter implements StorageAdapter {
|
||||
// Abstract methods that must be implemented by subclasses
|
||||
abstract init(): Promise<void>
|
||||
abstract saveNoun(noun: any): Promise<void>
|
||||
abstract getNoun(id: string): Promise<any | null>
|
||||
abstract getAllNouns(): Promise<any[]>
|
||||
abstract getNounsByNounType(nounType: string): Promise<any[]>
|
||||
abstract deleteNoun(id: string): Promise<void>
|
||||
abstract saveVerb(verb: any): Promise<void>
|
||||
abstract getVerb(id: string): Promise<any | null>
|
||||
abstract getAllVerbs(): Promise<any[]>
|
||||
abstract getVerbsBySource(sourceId: string): Promise<any[]>
|
||||
abstract getVerbsByTarget(targetId: string): Promise<any[]>
|
||||
abstract getVerbsByType(type: string): Promise<any[]>
|
||||
abstract deleteVerb(id: string): Promise<void>
|
||||
abstract saveMetadata(id: string, metadata: any): Promise<void>
|
||||
abstract getMetadata(id: string): Promise<any | null>
|
||||
abstract clear(): Promise<void>
|
||||
abstract getStorageStatus(): Promise<{
|
||||
type: string
|
||||
used: number
|
||||
quota: number | null
|
||||
details?: Record<string, any>
|
||||
}>
|
||||
|
||||
// Statistics cache
|
||||
protected statisticsCache: StatisticsData | null = null
|
||||
|
||||
// Batch update timer ID
|
||||
protected statisticsBatchUpdateTimerId: NodeJS.Timeout | null = null
|
||||
|
||||
// Flag to indicate if statistics have been modified since last save
|
||||
protected statisticsModified = false
|
||||
|
||||
// Time of last statistics flush to storage
|
||||
protected lastStatisticsFlushTime = 0
|
||||
|
||||
// Minimum time between statistics flushes (5 seconds)
|
||||
protected readonly MIN_FLUSH_INTERVAL_MS = 5000
|
||||
|
||||
// Maximum time to wait before flushing statistics (30 seconds)
|
||||
protected readonly MAX_FLUSH_DELAY_MS = 30000
|
||||
|
||||
// Statistics-specific methods that must be implemented by subclasses
|
||||
protected abstract saveStatisticsData(statistics: StatisticsData): Promise<void>
|
||||
protected abstract getStatisticsData(): Promise<StatisticsData | null>
|
||||
|
||||
/**
|
||||
* Save statistics data
|
||||
* @param statistics The statistics data to save
|
||||
*/
|
||||
async saveStatistics(statistics: StatisticsData): Promise<void> {
|
||||
// Update the cache with a deep copy to avoid reference issues
|
||||
this.statisticsCache = {
|
||||
nounCount: {...statistics.nounCount},
|
||||
verbCount: {...statistics.verbCount},
|
||||
metadataCount: {...statistics.metadataCount},
|
||||
hnswIndexSize: statistics.hnswIndexSize,
|
||||
lastUpdated: statistics.lastUpdated
|
||||
}
|
||||
|
||||
// Schedule a batch update instead of saving immediately
|
||||
this.scheduleBatchUpdate()
|
||||
}
|
||||
|
||||
/**
|
||||
* Get statistics data
|
||||
* @returns Promise that resolves to the statistics data
|
||||
*/
|
||||
async getStatistics(): Promise<StatisticsData | null> {
|
||||
// If we have cached statistics, return a deep copy
|
||||
if (this.statisticsCache) {
|
||||
return {
|
||||
nounCount: {...this.statisticsCache.nounCount},
|
||||
verbCount: {...this.statisticsCache.verbCount},
|
||||
metadataCount: {...this.statisticsCache.metadataCount},
|
||||
hnswIndexSize: this.statisticsCache.hnswIndexSize,
|
||||
lastUpdated: this.statisticsCache.lastUpdated
|
||||
}
|
||||
}
|
||||
|
||||
// Otherwise, get from storage
|
||||
const statistics = await this.getStatisticsData()
|
||||
|
||||
// If we found statistics, update the cache
|
||||
if (statistics) {
|
||||
// Update the cache with a deep copy
|
||||
this.statisticsCache = {
|
||||
nounCount: {...statistics.nounCount},
|
||||
verbCount: {...statistics.verbCount},
|
||||
metadataCount: {...statistics.metadataCount},
|
||||
hnswIndexSize: statistics.hnswIndexSize,
|
||||
lastUpdated: statistics.lastUpdated
|
||||
}
|
||||
}
|
||||
|
||||
return statistics
|
||||
}
|
||||
|
||||
/**
|
||||
* Schedule a batch update of statistics
|
||||
*/
|
||||
protected scheduleBatchUpdate(): void {
|
||||
// Mark statistics as modified
|
||||
this.statisticsModified = true
|
||||
|
||||
// If a timer is already set, don't set another one
|
||||
if (this.statisticsBatchUpdateTimerId !== null) {
|
||||
return
|
||||
}
|
||||
|
||||
// Calculate time since last flush
|
||||
const now = Date.now()
|
||||
const timeSinceLastFlush = now - this.lastStatisticsFlushTime
|
||||
|
||||
// If we've recently flushed, wait longer before the next flush
|
||||
const delayMs = timeSinceLastFlush < this.MIN_FLUSH_INTERVAL_MS
|
||||
? this.MAX_FLUSH_DELAY_MS
|
||||
: this.MIN_FLUSH_INTERVAL_MS
|
||||
|
||||
// Schedule the batch update
|
||||
this.statisticsBatchUpdateTimerId = setTimeout(() => {
|
||||
this.flushStatistics()
|
||||
}, delayMs)
|
||||
}
|
||||
|
||||
/**
|
||||
* Flush statistics to storage
|
||||
*/
|
||||
protected async flushStatistics(): Promise<void> {
|
||||
// Clear the timer
|
||||
if (this.statisticsBatchUpdateTimerId !== null) {
|
||||
clearTimeout(this.statisticsBatchUpdateTimerId)
|
||||
this.statisticsBatchUpdateTimerId = null
|
||||
}
|
||||
|
||||
// If statistics haven't been modified, no need to flush
|
||||
if (!this.statisticsModified || !this.statisticsCache) {
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
// Save the statistics to storage
|
||||
await this.saveStatisticsData(this.statisticsCache)
|
||||
|
||||
// Update the last flush time
|
||||
this.lastStatisticsFlushTime = Date.now()
|
||||
// Reset the modified flag
|
||||
this.statisticsModified = false
|
||||
} catch (error) {
|
||||
console.error('Failed to flush statistics data:', error)
|
||||
// Mark as still modified so we'll try again later
|
||||
this.statisticsModified = true
|
||||
// Don't throw the error to avoid disrupting the application
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Increment a statistic counter
|
||||
* @param type The type of statistic to increment ('noun', 'verb', 'metadata')
|
||||
* @param service The service that inserted the data
|
||||
* @param amount The amount to increment by (default: 1)
|
||||
*/
|
||||
async incrementStatistic(
|
||||
type: 'noun' | 'verb' | 'metadata',
|
||||
service: string,
|
||||
amount: number = 1
|
||||
): Promise<void> {
|
||||
// Get current statistics from cache or storage
|
||||
let statistics = this.statisticsCache
|
||||
if (!statistics) {
|
||||
statistics = await this.getStatisticsData()
|
||||
if (!statistics) {
|
||||
statistics = this.createDefaultStatistics()
|
||||
}
|
||||
|
||||
// Update the cache
|
||||
this.statisticsCache = {
|
||||
nounCount: {...statistics.nounCount},
|
||||
verbCount: {...statistics.verbCount},
|
||||
metadataCount: {...statistics.metadataCount},
|
||||
hnswIndexSize: statistics.hnswIndexSize,
|
||||
lastUpdated: statistics.lastUpdated
|
||||
}
|
||||
}
|
||||
|
||||
// Increment the appropriate counter
|
||||
const counterMap = {
|
||||
noun: this.statisticsCache!.nounCount,
|
||||
verb: this.statisticsCache!.verbCount,
|
||||
metadata: this.statisticsCache!.metadataCount
|
||||
}
|
||||
|
||||
const counter = counterMap[type]
|
||||
counter[service] = (counter[service] || 0) + amount
|
||||
|
||||
// Update timestamp
|
||||
this.statisticsCache!.lastUpdated = new Date().toISOString()
|
||||
|
||||
// Schedule a batch update instead of saving immediately
|
||||
this.scheduleBatchUpdate()
|
||||
}
|
||||
|
||||
/**
|
||||
* Decrement a statistic counter
|
||||
* @param type The type of statistic to decrement ('noun', 'verb', 'metadata')
|
||||
* @param service The service that inserted the data
|
||||
* @param amount The amount to decrement by (default: 1)
|
||||
*/
|
||||
async decrementStatistic(
|
||||
type: 'noun' | 'verb' | 'metadata',
|
||||
service: string,
|
||||
amount: number = 1
|
||||
): Promise<void> {
|
||||
// Get current statistics from cache or storage
|
||||
let statistics = this.statisticsCache
|
||||
if (!statistics) {
|
||||
statistics = await this.getStatisticsData()
|
||||
if (!statistics) {
|
||||
statistics = this.createDefaultStatistics()
|
||||
}
|
||||
|
||||
// Update the cache
|
||||
this.statisticsCache = {
|
||||
nounCount: {...statistics.nounCount},
|
||||
verbCount: {...statistics.verbCount},
|
||||
metadataCount: {...statistics.metadataCount},
|
||||
hnswIndexSize: statistics.hnswIndexSize,
|
||||
lastUpdated: statistics.lastUpdated
|
||||
}
|
||||
}
|
||||
|
||||
// Decrement the appropriate counter
|
||||
const counterMap = {
|
||||
noun: this.statisticsCache!.nounCount,
|
||||
verb: this.statisticsCache!.verbCount,
|
||||
metadata: this.statisticsCache!.metadataCount
|
||||
}
|
||||
|
||||
const counter = counterMap[type]
|
||||
counter[service] = Math.max(0, (counter[service] || 0) - amount)
|
||||
|
||||
// Update timestamp
|
||||
this.statisticsCache!.lastUpdated = new Date().toISOString()
|
||||
|
||||
// Schedule a batch update instead of saving immediately
|
||||
this.scheduleBatchUpdate()
|
||||
}
|
||||
|
||||
/**
|
||||
* Update the HNSW index size statistic
|
||||
* @param size The new size of the HNSW index
|
||||
*/
|
||||
async updateHnswIndexSize(size: number): Promise<void> {
|
||||
// Get current statistics from cache or storage
|
||||
let statistics = this.statisticsCache
|
||||
if (!statistics) {
|
||||
statistics = await this.getStatisticsData()
|
||||
if (!statistics) {
|
||||
statistics = this.createDefaultStatistics()
|
||||
}
|
||||
|
||||
// Update the cache
|
||||
this.statisticsCache = {
|
||||
nounCount: {...statistics.nounCount},
|
||||
verbCount: {...statistics.verbCount},
|
||||
metadataCount: {...statistics.metadataCount},
|
||||
hnswIndexSize: statistics.hnswIndexSize,
|
||||
lastUpdated: statistics.lastUpdated
|
||||
}
|
||||
}
|
||||
|
||||
// Update HNSW index size
|
||||
this.statisticsCache!.hnswIndexSize = size
|
||||
|
||||
// Update timestamp
|
||||
this.statisticsCache!.lastUpdated = new Date().toISOString()
|
||||
|
||||
// Schedule a batch update instead of saving immediately
|
||||
this.scheduleBatchUpdate()
|
||||
}
|
||||
|
||||
/**
|
||||
* Force an immediate flush of statistics to storage
|
||||
* This ensures that any pending statistics updates are written to persistent storage
|
||||
*/
|
||||
async flushStatisticsToStorage(): Promise<void> {
|
||||
// If there are no statistics in cache or they haven't been modified, nothing to flush
|
||||
if (!this.statisticsCache || !this.statisticsModified) {
|
||||
return
|
||||
}
|
||||
|
||||
// Call the protected flushStatistics method to immediately write to storage
|
||||
await this.flushStatistics()
|
||||
}
|
||||
|
||||
/**
|
||||
* Create default statistics data
|
||||
* @returns Default statistics data
|
||||
*/
|
||||
protected createDefaultStatistics(): StatisticsData {
|
||||
return {
|
||||
nounCount: {},
|
||||
verbCount: {},
|
||||
metadataCount: {},
|
||||
hnswIndexSize: 0,
|
||||
lastUpdated: new Date().toISOString()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -3,18 +3,10 @@
|
|||
* In-memory storage adapter for environments where persistent storage is not available or needed
|
||||
*/
|
||||
|
||||
import { GraphVerb, HNSWNoun } from '../../coreTypes.js'
|
||||
import { BaseStorage } from '../baseStorage.js'
|
||||
import { GraphVerb, HNSWNoun, StatisticsData } from '../../coreTypes.js'
|
||||
import { BaseStorage, STATISTICS_KEY } from '../baseStorage.js'
|
||||
|
||||
/**
|
||||
* Type alias for HNSWNoun to make the code more readable
|
||||
*/
|
||||
type HNSWNode = HNSWNoun
|
||||
|
||||
/**
|
||||
* Type alias for GraphVerb to make the code more readable
|
||||
*/
|
||||
type Edge = GraphVerb
|
||||
// No type aliases needed - using the original types directly
|
||||
|
||||
/**
|
||||
* In-memory storage adapter
|
||||
|
|
@ -22,9 +14,10 @@ type Edge = GraphVerb
|
|||
*/
|
||||
export class MemoryStorage extends BaseStorage {
|
||||
// Single map of noun ID to noun
|
||||
private nouns: Map<string, HNSWNode> = new Map()
|
||||
private verbs: Map<string, Edge> = new Map()
|
||||
private nouns: Map<string, HNSWNoun> = new Map()
|
||||
private verbs: Map<string, GraphVerb> = new Map()
|
||||
private metadata: Map<string, any> = new Map()
|
||||
private statistics: StatisticsData | null = null
|
||||
|
||||
constructor() {
|
||||
super()
|
||||
|
|
@ -39,237 +32,270 @@ export class MemoryStorage extends BaseStorage {
|
|||
}
|
||||
|
||||
/**
|
||||
* Save a node to storage
|
||||
* Save a noun to storage
|
||||
*/
|
||||
protected async saveNode(node: HNSWNode): Promise<void> {
|
||||
protected async saveNoun_internal(noun: HNSWNoun): Promise<void> {
|
||||
// Create a deep copy to avoid reference issues
|
||||
const nodeCopy: HNSWNode = {
|
||||
id: node.id,
|
||||
vector: [...node.vector],
|
||||
const nounCopy: HNSWNoun = {
|
||||
id: noun.id,
|
||||
vector: [...noun.vector],
|
||||
connections: new Map()
|
||||
}
|
||||
|
||||
// Copy connections
|
||||
for (const [level, connections] of node.connections.entries()) {
|
||||
nodeCopy.connections.set(level, new Set(connections))
|
||||
for (const [level, connections] of noun.connections.entries()) {
|
||||
nounCopy.connections.set(level, new Set(connections))
|
||||
}
|
||||
|
||||
// Save the node directly in the nouns map
|
||||
this.nouns.set(node.id, nodeCopy)
|
||||
// Save the noun directly in the nouns map
|
||||
this.nouns.set(noun.id, nounCopy)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a node from storage
|
||||
* Get a noun from storage
|
||||
*/
|
||||
protected async getNode(id: string): Promise<HNSWNode | null> {
|
||||
// Get the node directly from the nouns map
|
||||
const node = this.nouns.get(id)
|
||||
protected async getNoun_internal(id: string): Promise<HNSWNoun | null> {
|
||||
// Get the noun directly from the nouns map
|
||||
const noun = this.nouns.get(id)
|
||||
|
||||
// If not found, return null
|
||||
if (!node) {
|
||||
if (!noun) {
|
||||
return null
|
||||
}
|
||||
|
||||
// Return a deep copy to avoid reference issues
|
||||
const nodeCopy: HNSWNode = {
|
||||
id: node.id,
|
||||
vector: [...node.vector],
|
||||
const nounCopy: HNSWNoun = {
|
||||
id: noun.id,
|
||||
vector: [...noun.vector],
|
||||
connections: new Map()
|
||||
}
|
||||
|
||||
// Copy connections
|
||||
for (const [level, connections] of node.connections.entries()) {
|
||||
nodeCopy.connections.set(level, new Set(connections))
|
||||
for (const [level, connections] of noun.connections.entries()) {
|
||||
nounCopy.connections.set(level, new Set(connections))
|
||||
}
|
||||
|
||||
return nodeCopy
|
||||
return nounCopy
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all nodes from storage
|
||||
* Get all nouns from storage
|
||||
*/
|
||||
protected async getAllNodes(): Promise<HNSWNode[]> {
|
||||
const allNodes: HNSWNode[] = []
|
||||
protected async getAllNouns_internal(): Promise<HNSWNoun[]> {
|
||||
const allNouns: HNSWNoun[] = []
|
||||
|
||||
// Iterate through all nodes in the nouns map
|
||||
for (const [nodeId, node] of this.nouns.entries()) {
|
||||
// Iterate through all nouns in the nouns map
|
||||
for (const [nounId, noun] of this.nouns.entries()) {
|
||||
// Return a deep copy to avoid reference issues
|
||||
const nodeCopy: HNSWNode = {
|
||||
id: node.id,
|
||||
vector: [...node.vector],
|
||||
const nounCopy: HNSWNoun = {
|
||||
id: noun.id,
|
||||
vector: [...noun.vector],
|
||||
connections: new Map()
|
||||
}
|
||||
|
||||
// Copy connections
|
||||
for (const [level, connections] of node.connections.entries()) {
|
||||
nodeCopy.connections.set(level, new Set(connections))
|
||||
for (const [level, connections] of noun.connections.entries()) {
|
||||
nounCopy.connections.set(level, new Set(connections))
|
||||
}
|
||||
|
||||
allNodes.push(nodeCopy)
|
||||
allNouns.push(nounCopy)
|
||||
}
|
||||
|
||||
return allNodes
|
||||
return allNouns
|
||||
}
|
||||
|
||||
/**
|
||||
* Get nodes by noun type
|
||||
* Get nouns by noun type
|
||||
* @param nounType The noun type to filter by
|
||||
* @returns Promise that resolves to an array of nodes of the specified noun type
|
||||
* @returns Promise that resolves to an array of nouns of the specified noun type
|
||||
*/
|
||||
protected async getNodesByNounType(nounType: string): Promise<HNSWNode[]> {
|
||||
const nodes: HNSWNode[] = []
|
||||
protected async getNounsByNounType_internal(nounType: string): Promise<HNSWNoun[]> {
|
||||
const nouns: HNSWNoun[] = []
|
||||
|
||||
// Iterate through all nodes and filter by noun type using metadata
|
||||
for (const [nodeId, node] of this.nouns.entries()) {
|
||||
// Iterate through all nouns and filter by noun type using metadata
|
||||
for (const [nounId, noun] of this.nouns.entries()) {
|
||||
// Get the metadata to check the noun type
|
||||
const metadata = await this.getMetadata(nodeId)
|
||||
const metadata = await this.getMetadata(nounId)
|
||||
|
||||
// Include the node if its noun type matches the requested type
|
||||
// Include the noun if its noun type matches the requested type
|
||||
if (metadata && metadata.noun === nounType) {
|
||||
// Return a deep copy to avoid reference issues
|
||||
const nodeCopy: HNSWNode = {
|
||||
id: node.id,
|
||||
vector: [...node.vector],
|
||||
const nounCopy: HNSWNoun = {
|
||||
id: noun.id,
|
||||
vector: [...noun.vector],
|
||||
connections: new Map()
|
||||
}
|
||||
|
||||
// Copy connections
|
||||
for (const [level, connections] of node.connections.entries()) {
|
||||
nodeCopy.connections.set(level, new Set(connections))
|
||||
for (const [level, connections] of noun.connections.entries()) {
|
||||
nounCopy.connections.set(level, new Set(connections))
|
||||
}
|
||||
|
||||
nodes.push(nodeCopy)
|
||||
nouns.push(nounCopy)
|
||||
}
|
||||
}
|
||||
|
||||
return nodes
|
||||
return nouns
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a node from storage
|
||||
* Delete a noun from storage
|
||||
*/
|
||||
protected async deleteNode(id: string): Promise<void> {
|
||||
// Delete the node directly from the nouns map
|
||||
protected async deleteNoun_internal(id: string): Promise<void> {
|
||||
this.nouns.delete(id)
|
||||
}
|
||||
|
||||
/**
|
||||
* Save an edge to storage
|
||||
* Save a verb to storage
|
||||
*/
|
||||
protected async saveEdge(edge: Edge): Promise<void> {
|
||||
protected async saveVerb_internal(verb: GraphVerb): Promise<void> {
|
||||
// Create a deep copy to avoid reference issues
|
||||
const edgeCopy: Edge = {
|
||||
id: edge.id,
|
||||
vector: [...edge.vector],
|
||||
const verbCopy: GraphVerb = {
|
||||
id: verb.id,
|
||||
vector: [...verb.vector],
|
||||
connections: new Map(),
|
||||
sourceId: edge.sourceId,
|
||||
targetId: edge.targetId,
|
||||
type: edge.type,
|
||||
weight: edge.weight,
|
||||
metadata: edge.metadata
|
||||
sourceId: verb.sourceId,
|
||||
targetId: verb.targetId,
|
||||
type: verb.type,
|
||||
weight: verb.weight,
|
||||
metadata: verb.metadata
|
||||
}
|
||||
|
||||
// Copy connections
|
||||
for (const [level, connections] of edge.connections.entries()) {
|
||||
edgeCopy.connections.set(level, new Set(connections))
|
||||
for (const [level, connections] of verb.connections.entries()) {
|
||||
verbCopy.connections.set(level, new Set(connections))
|
||||
}
|
||||
|
||||
// Save the edge directly in the verbs map
|
||||
this.verbs.set(edge.id, edgeCopy)
|
||||
// Save the verb directly in the verbs map
|
||||
this.verbs.set(verb.id, verbCopy)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get an edge from storage
|
||||
* Get a verb from storage
|
||||
*/
|
||||
protected async getEdge(id: string): Promise<Edge | null> {
|
||||
// Get the edge directly from the verbs map
|
||||
const edge = this.verbs.get(id)
|
||||
protected async getVerb_internal(id: string): Promise<GraphVerb | null> {
|
||||
// Get the verb directly from the verbs map
|
||||
const verb = this.verbs.get(id)
|
||||
|
||||
// If not found, return null
|
||||
if (!edge) {
|
||||
if (!verb) {
|
||||
return null
|
||||
}
|
||||
|
||||
// Create default timestamp if not present
|
||||
const defaultTimestamp = {
|
||||
seconds: Math.floor(Date.now() / 1000),
|
||||
nanoseconds: (Date.now() % 1000) * 1000000
|
||||
}
|
||||
|
||||
// Create default createdBy if not present
|
||||
const defaultCreatedBy = {
|
||||
augmentation: 'unknown',
|
||||
version: '1.0'
|
||||
}
|
||||
|
||||
// Return a deep copy to avoid reference issues
|
||||
const edgeCopy: Edge = {
|
||||
id: edge.id,
|
||||
vector: [...edge.vector],
|
||||
const verbCopy: GraphVerb = {
|
||||
id: verb.id,
|
||||
vector: [...verb.vector],
|
||||
connections: new Map(),
|
||||
sourceId: edge.sourceId,
|
||||
targetId: edge.targetId,
|
||||
type: edge.type,
|
||||
weight: edge.weight,
|
||||
metadata: edge.metadata
|
||||
sourceId: (verb.sourceId || verb.source || ""),
|
||||
targetId: (verb.targetId || verb.target || ""),
|
||||
source: (verb.sourceId || verb.source || ""),
|
||||
target: (verb.targetId || verb.target || ""),
|
||||
verb: verb.type || verb.verb,
|
||||
weight: verb.weight,
|
||||
metadata: verb.metadata,
|
||||
createdAt: verb.createdAt || defaultTimestamp,
|
||||
updatedAt: verb.updatedAt || defaultTimestamp,
|
||||
createdBy: verb.createdBy || defaultCreatedBy
|
||||
}
|
||||
|
||||
// Copy connections
|
||||
for (const [level, connections] of edge.connections.entries()) {
|
||||
edgeCopy.connections.set(level, new Set(connections))
|
||||
for (const [level, connections] of verb.connections.entries()) {
|
||||
verbCopy.connections.set(level, new Set(connections))
|
||||
}
|
||||
|
||||
return edgeCopy
|
||||
return verbCopy
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all edges from storage
|
||||
* Get all verbs from storage
|
||||
*/
|
||||
protected async getAllEdges(): Promise<Edge[]> {
|
||||
const allEdges: Edge[] = []
|
||||
protected async getAllVerbs_internal(): Promise<GraphVerb[]> {
|
||||
const allVerbs: GraphVerb[] = []
|
||||
|
||||
// Iterate through all verbs in the verbs map
|
||||
for (const [verbId, verb] of this.verbs.entries()) {
|
||||
// Create default timestamp if not present
|
||||
const defaultTimestamp = {
|
||||
seconds: Math.floor(Date.now() / 1000),
|
||||
nanoseconds: (Date.now() % 1000) * 1000000
|
||||
}
|
||||
|
||||
// Create default createdBy if not present
|
||||
const defaultCreatedBy = {
|
||||
augmentation: 'unknown',
|
||||
version: '1.0'
|
||||
}
|
||||
|
||||
// Iterate through all edges in the verbs map
|
||||
for (const [edgeId, edge] of this.verbs.entries()) {
|
||||
// Return a deep copy to avoid reference issues
|
||||
const edgeCopy: Edge = {
|
||||
id: edge.id,
|
||||
vector: [...edge.vector],
|
||||
const verbCopy: GraphVerb = {
|
||||
id: verb.id,
|
||||
vector: [...verb.vector],
|
||||
connections: new Map(),
|
||||
sourceId: edge.sourceId,
|
||||
targetId: edge.targetId,
|
||||
type: edge.type,
|
||||
weight: edge.weight,
|
||||
metadata: edge.metadata
|
||||
sourceId: (verb.sourceId || verb.source || ""),
|
||||
targetId: (verb.targetId || verb.target || ""),
|
||||
source: (verb.sourceId || verb.source || ""),
|
||||
target: (verb.targetId || verb.target || ""),
|
||||
verb: verb.type || verb.verb,
|
||||
weight: verb.weight,
|
||||
metadata: verb.metadata,
|
||||
createdAt: verb.createdAt || defaultTimestamp,
|
||||
updatedAt: verb.updatedAt || defaultTimestamp,
|
||||
createdBy: verb.createdBy || defaultCreatedBy
|
||||
}
|
||||
|
||||
// Copy connections
|
||||
for (const [level, connections] of edge.connections.entries()) {
|
||||
edgeCopy.connections.set(level, new Set(connections))
|
||||
for (const [level, connections] of verb.connections.entries()) {
|
||||
verbCopy.connections.set(level, new Set(connections))
|
||||
}
|
||||
|
||||
allEdges.push(edgeCopy)
|
||||
allVerbs.push(verbCopy)
|
||||
}
|
||||
|
||||
return allEdges
|
||||
return allVerbs
|
||||
}
|
||||
|
||||
/**
|
||||
* Get edges by source
|
||||
* Get verbs by source
|
||||
*/
|
||||
protected async getEdgesBySource(sourceId: string): Promise<Edge[]> {
|
||||
const edges = await this.getAllEdges()
|
||||
return edges.filter((edge) => edge.sourceId === sourceId)
|
||||
protected async getVerbsBySource_internal(sourceId: string): Promise<GraphVerb[]> {
|
||||
const allVerbs = await this.getAllVerbs_internal()
|
||||
return allVerbs.filter((verb: GraphVerb) => (verb.sourceId || verb.source) === sourceId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get edges by target
|
||||
* Get verbs by target
|
||||
*/
|
||||
protected async getEdgesByTarget(targetId: string): Promise<Edge[]> {
|
||||
const edges = await this.getAllEdges()
|
||||
return edges.filter((edge) => edge.targetId === targetId)
|
||||
protected async getVerbsByTarget_internal(targetId: string): Promise<GraphVerb[]> {
|
||||
const allVerbs = await this.getAllVerbs_internal()
|
||||
return allVerbs.filter((verb: GraphVerb) => (verb.targetId || verb.target) === targetId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get edges by type
|
||||
* Get verbs by type
|
||||
*/
|
||||
protected async getEdgesByType(type: string): Promise<Edge[]> {
|
||||
const edges = await this.getAllEdges()
|
||||
return edges.filter((edge) => edge.type === type)
|
||||
protected async getVerbsByType_internal(type: string): Promise<GraphVerb[]> {
|
||||
const allVerbs = await this.getAllVerbs_internal()
|
||||
return allVerbs.filter((verb: GraphVerb) => (verb.type || verb.verb) === type)
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete an edge from storage
|
||||
* Delete a verb from storage
|
||||
*/
|
||||
protected async deleteEdge(id: string): Promise<void> {
|
||||
// Delete the edge directly from the verbs map
|
||||
protected async deleteVerb_internal(id: string): Promise<void> {
|
||||
// Delete the verb directly from the verbs map
|
||||
this.verbs.delete(id)
|
||||
}
|
||||
|
||||
|
|
@ -321,4 +347,45 @@ export class MemoryStorage extends BaseStorage {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save statistics data to storage
|
||||
* @param statistics The statistics data to save
|
||||
*/
|
||||
protected async saveStatisticsData(statistics: StatisticsData): Promise<void> {
|
||||
// For memory storage, we just need to store the statistics in memory
|
||||
// Create a deep copy to avoid reference issues
|
||||
this.statistics = {
|
||||
nounCount: {...statistics.nounCount},
|
||||
verbCount: {...statistics.verbCount},
|
||||
metadataCount: {...statistics.metadataCount},
|
||||
hnswIndexSize: statistics.hnswIndexSize,
|
||||
lastUpdated: statistics.lastUpdated
|
||||
}
|
||||
|
||||
// Since this is in-memory, there's no need for time-based partitioning
|
||||
// or legacy file handling
|
||||
}
|
||||
|
||||
/**
|
||||
* Get statistics data from storage
|
||||
* @returns Promise that resolves to the statistics data or null if not found
|
||||
*/
|
||||
protected async getStatisticsData(): Promise<StatisticsData | null> {
|
||||
if (!this.statistics) {
|
||||
return null
|
||||
}
|
||||
|
||||
// Return a deep copy to avoid reference issues
|
||||
return {
|
||||
nounCount: {...this.statistics.nounCount},
|
||||
verbCount: {...this.statistics.verbCount},
|
||||
metadataCount: {...this.statistics.metadataCount},
|
||||
hnswIndexSize: this.statistics.hnswIndexSize,
|
||||
lastUpdated: this.statistics.lastUpdated
|
||||
}
|
||||
|
||||
// Since this is in-memory, there's no need for fallback mechanisms
|
||||
// to check multiple storage locations
|
||||
}
|
||||
}
|
||||
|
|
@ -3,10 +3,18 @@
|
|||
* Provides persistent storage for the vector database using the Origin Private File System API
|
||||
*/
|
||||
|
||||
import {GraphVerb, HNSWNoun} from '../../coreTypes.js'
|
||||
import {BaseStorage, NOUNS_DIR, VERBS_DIR, METADATA_DIR, INDEX_DIR} from '../baseStorage.js'
|
||||
import {GraphVerb, HNSWNoun, StatisticsData} from '../../coreTypes.js'
|
||||
import {BaseStorage, NOUNS_DIR, VERBS_DIR, METADATA_DIR, INDEX_DIR, STATISTICS_KEY} from '../baseStorage.js'
|
||||
import '../../types/fileSystemTypes.js'
|
||||
|
||||
// Type alias for HNSWNode
|
||||
type HNSWNode = HNSWNoun
|
||||
|
||||
/**
|
||||
* Type alias for GraphVerb to make the code more readable
|
||||
*/
|
||||
type Edge = GraphVerb
|
||||
|
||||
/**
|
||||
* Helper function to safely get a file from a FileSystemHandle
|
||||
* This is needed because TypeScript doesn't recognize that a FileSystemHandle
|
||||
|
|
@ -18,8 +26,8 @@ async function safeGetFile(handle: FileSystemHandle): Promise<File> {
|
|||
}
|
||||
|
||||
// Type aliases for better readability
|
||||
type HNSWNode = HNSWNoun
|
||||
type Edge = GraphVerb
|
||||
type HNSWNoun_internal = HNSWNoun
|
||||
type Verb = GraphVerb
|
||||
|
||||
// Root directory name for OPFS storage
|
||||
const ROOT_DIR = 'opfs-vector-db'
|
||||
|
|
@ -37,6 +45,7 @@ export class OPFSStorage extends BaseStorage {
|
|||
private isAvailable = false
|
||||
private isPersistentRequested = false
|
||||
private isPersistentGranted = false
|
||||
private statistics: StatisticsData | null = null
|
||||
|
||||
constructor() {
|
||||
super()
|
||||
|
|
@ -148,54 +157,54 @@ export class OPFSStorage extends BaseStorage {
|
|||
}
|
||||
|
||||
/**
|
||||
* Save a node to storage
|
||||
* Save a noun to storage
|
||||
*/
|
||||
protected async saveNode(node: HNSWNode): Promise<void> {
|
||||
protected async saveNoun_internal(noun: HNSWNoun_internal): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Convert connections Map to a serializable format
|
||||
const serializableNode = {
|
||||
...node,
|
||||
connections: this.mapToObject(node.connections, (set) =>
|
||||
const serializableNoun = {
|
||||
...noun,
|
||||
connections: this.mapToObject(noun.connections, (set) =>
|
||||
Array.from(set as Set<string>)
|
||||
)
|
||||
}
|
||||
|
||||
// Create or get the file for this noun
|
||||
const fileHandle = await this.nounsDir!.getFileHandle(node.id, {
|
||||
const fileHandle = await this.nounsDir!.getFileHandle(noun.id, {
|
||||
create: true
|
||||
})
|
||||
|
||||
// Write the noun data to the file
|
||||
const writable = await fileHandle.createWritable()
|
||||
await writable.write(JSON.stringify(serializableNode))
|
||||
await writable.write(JSON.stringify(serializableNoun))
|
||||
await writable.close()
|
||||
} catch (error) {
|
||||
console.error(`Failed to save node ${node.id}:`, error)
|
||||
throw new Error(`Failed to save node ${node.id}: ${error}`)
|
||||
console.error(`Failed to save noun ${noun.id}:`, error)
|
||||
throw new Error(`Failed to save noun ${noun.id}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a node from storage
|
||||
* Get a noun from storage
|
||||
*/
|
||||
protected async getNode(id: string): Promise<HNSWNode | null> {
|
||||
protected async getNoun_internal(id: string): Promise<HNSWNoun_internal | null> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Get the file handle for this node
|
||||
// Get the file handle for this noun
|
||||
const fileHandle = await this.nounsDir!.getFileHandle(id)
|
||||
|
||||
// Read the node data from the file
|
||||
// Read the noun data from the file
|
||||
const file = await fileHandle.getFile()
|
||||
const text = await file.text()
|
||||
const data = JSON.parse(text)
|
||||
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(data.connections)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
for (const [level, nounIds] of Object.entries(data.connections)) {
|
||||
connections.set(Number(level), new Set(nounIds as string[]))
|
||||
}
|
||||
|
||||
return {
|
||||
|
|
@ -204,41 +213,41 @@ export class OPFSStorage extends BaseStorage {
|
|||
connections
|
||||
}
|
||||
} catch (error) {
|
||||
// Node not found or other error
|
||||
// Noun not found or other error
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all nodes from storage
|
||||
* Get all nouns from storage
|
||||
*/
|
||||
protected async getAllNodes(): Promise<HNSWNode[]> {
|
||||
protected async getAllNouns_internal(): Promise<HNSWNoun_internal[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
const allNodes: HNSWNode[] = []
|
||||
const allNouns: HNSWNoun_internal[] = []
|
||||
try {
|
||||
// Iterate through all files in the nouns directory
|
||||
for await (const [name, handle] of this.nounsDir!.entries()) {
|
||||
if (handle.kind === 'file') {
|
||||
try {
|
||||
// Read the node data from the file
|
||||
// Read the noun data from the file
|
||||
const file = await safeGetFile(handle)
|
||||
const text = await file.text()
|
||||
const data = JSON.parse(text)
|
||||
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(data.connections)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
for (const [level, nounIds] of Object.entries(data.connections)) {
|
||||
connections.set(Number(level), new Set(nounIds as string[]))
|
||||
}
|
||||
|
||||
allNodes.push({
|
||||
allNouns.push({
|
||||
id: data.id,
|
||||
vector: data.vector,
|
||||
connections
|
||||
})
|
||||
} catch (error) {
|
||||
console.error(`Error reading node file ${name}:`, error)
|
||||
console.error(`Error reading noun file ${name}:`, error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -246,7 +255,16 @@ export class OPFSStorage extends BaseStorage {
|
|||
console.error('Error reading nouns directory:', error)
|
||||
}
|
||||
|
||||
return allNodes
|
||||
return allNouns
|
||||
}
|
||||
|
||||
/**
|
||||
* Get nouns by noun type (internal implementation)
|
||||
* @param nounType The noun type to filter by
|
||||
* @returns Promise that resolves to an array of nouns of the specified noun type
|
||||
*/
|
||||
protected async getNounsByNounType_internal(nounType: string): Promise<HNSWNoun[]> {
|
||||
return this.getNodesByNounType(nounType)
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -298,6 +316,13 @@ export class OPFSStorage extends BaseStorage {
|
|||
return nodes
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a noun from storage (internal implementation)
|
||||
*/
|
||||
protected async deleteNoun_internal(id: string): Promise<void> {
|
||||
return this.deleteNode(id)
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a node from storage
|
||||
*/
|
||||
|
|
@ -315,6 +340,13 @@ export class OPFSStorage extends BaseStorage {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save a verb to storage (internal implementation)
|
||||
*/
|
||||
protected async saveVerb_internal(verb: GraphVerb): Promise<void> {
|
||||
return this.saveEdge(verb)
|
||||
}
|
||||
|
||||
/**
|
||||
* Save an edge to storage
|
||||
*/
|
||||
|
|
@ -345,6 +377,13 @@ export class OPFSStorage extends BaseStorage {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a verb from storage (internal implementation)
|
||||
*/
|
||||
protected async getVerb_internal(id: string): Promise<GraphVerb | null> {
|
||||
return this.getEdge(id)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get an edge from storage
|
||||
*/
|
||||
|
|
@ -366,15 +405,32 @@ export class OPFSStorage extends BaseStorage {
|
|||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
// Create default timestamp if not present
|
||||
const defaultTimestamp = {
|
||||
seconds: Math.floor(Date.now() / 1000),
|
||||
nanoseconds: (Date.now() % 1000) * 1000000
|
||||
}
|
||||
|
||||
// Create default createdBy if not present
|
||||
const defaultCreatedBy = {
|
||||
augmentation: 'unknown',
|
||||
version: '1.0'
|
||||
}
|
||||
|
||||
return {
|
||||
id: data.id,
|
||||
vector: data.vector,
|
||||
connections,
|
||||
sourceId: data.sourceId,
|
||||
targetId: data.targetId,
|
||||
type: data.type,
|
||||
sourceId: data.sourceId || data.source,
|
||||
targetId: data.targetId || data.target,
|
||||
source: data.sourceId || data.source,
|
||||
target: data.targetId || data.target,
|
||||
verb: data.type || data.verb,
|
||||
weight: data.weight,
|
||||
metadata: data.metadata
|
||||
metadata: data.metadata,
|
||||
createdAt: data.createdAt || defaultTimestamp,
|
||||
updatedAt: data.updatedAt || defaultTimestamp,
|
||||
createdBy: data.createdBy || defaultCreatedBy
|
||||
}
|
||||
} catch (error) {
|
||||
// Edge not found or other error
|
||||
|
|
@ -382,6 +438,13 @@ export class OPFSStorage extends BaseStorage {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all verbs from storage (internal implementation)
|
||||
*/
|
||||
protected async getAllVerbs_internal(): Promise<GraphVerb[]> {
|
||||
return this.getAllEdges()
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all edges from storage
|
||||
*/
|
||||
|
|
@ -405,15 +468,32 @@ export class OPFSStorage extends BaseStorage {
|
|||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
// Create default timestamp if not present
|
||||
const defaultTimestamp = {
|
||||
seconds: Math.floor(Date.now() / 1000),
|
||||
nanoseconds: (Date.now() % 1000) * 1000000
|
||||
}
|
||||
|
||||
// Create default createdBy if not present
|
||||
const defaultCreatedBy = {
|
||||
augmentation: 'unknown',
|
||||
version: '1.0'
|
||||
}
|
||||
|
||||
allEdges.push({
|
||||
id: data.id,
|
||||
vector: data.vector,
|
||||
connections,
|
||||
sourceId: data.sourceId,
|
||||
targetId: data.targetId,
|
||||
type: data.type,
|
||||
sourceId: data.sourceId || data.source,
|
||||
targetId: data.targetId || data.target,
|
||||
source: data.sourceId || data.source,
|
||||
target: data.targetId || data.target,
|
||||
verb: data.type || data.verb,
|
||||
weight: data.weight,
|
||||
metadata: data.metadata
|
||||
metadata: data.metadata,
|
||||
createdAt: data.createdAt || defaultTimestamp,
|
||||
updatedAt: data.updatedAt || defaultTimestamp,
|
||||
createdBy: data.createdBy || defaultCreatedBy
|
||||
})
|
||||
} catch (error) {
|
||||
console.error(`Error reading edge file ${name}:`, error)
|
||||
|
|
@ -427,12 +507,26 @@ export class OPFSStorage extends BaseStorage {
|
|||
return allEdges
|
||||
}
|
||||
|
||||
/**
|
||||
* Get verbs by source (internal implementation)
|
||||
*/
|
||||
protected async getVerbsBySource_internal(sourceId: string): Promise<GraphVerb[]> {
|
||||
return this.getEdgesBySource(sourceId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get edges by source
|
||||
*/
|
||||
protected async getEdgesBySource(sourceId: string): Promise<Edge[]> {
|
||||
const edges = await this.getAllEdges()
|
||||
return edges.filter((edge) => edge.sourceId === sourceId)
|
||||
return edges.filter((edge) => (edge.sourceId || edge.source) === sourceId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get verbs by target (internal implementation)
|
||||
*/
|
||||
protected async getVerbsByTarget_internal(targetId: string): Promise<GraphVerb[]> {
|
||||
return this.getEdgesByTarget(targetId)
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -440,7 +534,14 @@ export class OPFSStorage extends BaseStorage {
|
|||
*/
|
||||
protected async getEdgesByTarget(targetId: string): Promise<Edge[]> {
|
||||
const edges = await this.getAllEdges()
|
||||
return edges.filter((edge) => edge.targetId === targetId)
|
||||
return edges.filter((edge) => (edge.targetId || edge.target) === targetId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get verbs by type (internal implementation)
|
||||
*/
|
||||
protected async getVerbsByType_internal(type: string): Promise<GraphVerb[]> {
|
||||
return this.getEdgesByType(type)
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -448,7 +549,14 @@ export class OPFSStorage extends BaseStorage {
|
|||
*/
|
||||
protected async getEdgesByType(type: string): Promise<Edge[]> {
|
||||
const edges = await this.getAllEdges()
|
||||
return edges.filter((edge) => edge.type === type)
|
||||
return edges.filter((edge) => (edge.type || edge.verb) === type)
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a verb from storage (internal implementation)
|
||||
*/
|
||||
protected async deleteVerb_internal(id: string): Promise<void> {
|
||||
return this.deleteEdge(id)
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -688,4 +796,190 @@ export class OPFSStorage extends BaseStorage {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the statistics key for a specific date
|
||||
* @param date The date to get the key for
|
||||
* @returns The statistics key for the specified date
|
||||
*/
|
||||
private getStatisticsKeyForDate(date: Date): string {
|
||||
const year = date.getUTCFullYear()
|
||||
const month = String(date.getUTCMonth() + 1).padStart(2, '0')
|
||||
const day = String(date.getUTCDate()).padStart(2, '0')
|
||||
return `statistics_${year}${month}${day}.json`
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the current statistics key
|
||||
* @returns The current statistics key
|
||||
*/
|
||||
private getCurrentStatisticsKey(): string {
|
||||
return this.getStatisticsKeyForDate(new Date())
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the legacy statistics key (for backward compatibility)
|
||||
* @returns The legacy statistics key
|
||||
*/
|
||||
private getLegacyStatisticsKey(): string {
|
||||
return 'statistics.json'
|
||||
}
|
||||
|
||||
/**
|
||||
* Save statistics data to storage
|
||||
* @param statistics The statistics data to save
|
||||
*/
|
||||
protected async saveStatisticsData(statistics: StatisticsData): Promise<void> {
|
||||
// Create a deep copy to avoid reference issues
|
||||
this.statistics = {
|
||||
nounCount: {...statistics.nounCount},
|
||||
verbCount: {...statistics.verbCount},
|
||||
metadataCount: {...statistics.metadataCount},
|
||||
hnswIndexSize: statistics.hnswIndexSize,
|
||||
lastUpdated: statistics.lastUpdated
|
||||
}
|
||||
|
||||
try {
|
||||
// Ensure the root directory is initialized
|
||||
await this.ensureInitialized()
|
||||
|
||||
// Get or create the index directory
|
||||
if (!this.indexDir) {
|
||||
throw new Error('Index directory not initialized')
|
||||
}
|
||||
|
||||
// Get the current statistics key
|
||||
const currentKey = this.getCurrentStatisticsKey()
|
||||
|
||||
// Create a file for the statistics data
|
||||
const fileHandle = await this.indexDir.getFileHandle(currentKey, {
|
||||
create: true
|
||||
})
|
||||
|
||||
// Create a writable stream
|
||||
const writable = await fileHandle.createWritable()
|
||||
|
||||
// Write the statistics data to the file
|
||||
await writable.write(JSON.stringify(this.statistics, null, 2))
|
||||
|
||||
// Close the stream
|
||||
await writable.close()
|
||||
|
||||
// Also update the legacy key for backward compatibility, but less frequently
|
||||
if (Math.random() < 0.1) {
|
||||
const legacyKey = this.getLegacyStatisticsKey()
|
||||
const legacyFileHandle = await this.indexDir.getFileHandle(legacyKey, {
|
||||
create: true
|
||||
})
|
||||
const legacyWritable = await legacyFileHandle.createWritable()
|
||||
await legacyWritable.write(JSON.stringify(this.statistics, null, 2))
|
||||
await legacyWritable.close()
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Failed to save statistics data:', error)
|
||||
throw new Error(`Failed to save statistics data: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get statistics data from storage
|
||||
* @returns Promise that resolves to the statistics data or null if not found
|
||||
*/
|
||||
protected async getStatisticsData(): Promise<StatisticsData | null> {
|
||||
// If we have cached statistics, return a deep copy
|
||||
if (this.statistics) {
|
||||
return {
|
||||
nounCount: {...this.statistics.nounCount},
|
||||
verbCount: {...this.statistics.verbCount},
|
||||
metadataCount: {...this.statistics.metadataCount},
|
||||
hnswIndexSize: this.statistics.hnswIndexSize,
|
||||
lastUpdated: this.statistics.lastUpdated
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
// Ensure the root directory is initialized
|
||||
await this.ensureInitialized()
|
||||
|
||||
if (!this.indexDir) {
|
||||
throw new Error('Index directory not initialized')
|
||||
}
|
||||
|
||||
// First try to get statistics from today's file
|
||||
const currentKey = this.getCurrentStatisticsKey()
|
||||
try {
|
||||
const fileHandle = await this.indexDir.getFileHandle(currentKey, {
|
||||
create: false
|
||||
})
|
||||
const file = await fileHandle.getFile()
|
||||
const text = await file.text()
|
||||
this.statistics = JSON.parse(text)
|
||||
|
||||
if (this.statistics) {
|
||||
return {
|
||||
nounCount: {...this.statistics.nounCount},
|
||||
verbCount: {...this.statistics.verbCount},
|
||||
metadataCount: {...this.statistics.metadataCount},
|
||||
hnswIndexSize: this.statistics.hnswIndexSize,
|
||||
lastUpdated: this.statistics.lastUpdated
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
// If today's file doesn't exist, try yesterday's file
|
||||
const yesterday = new Date()
|
||||
yesterday.setDate(yesterday.getDate() - 1)
|
||||
const yesterdayKey = this.getStatisticsKeyForDate(yesterday)
|
||||
|
||||
try {
|
||||
const fileHandle = await this.indexDir.getFileHandle(yesterdayKey, {
|
||||
create: false
|
||||
})
|
||||
const file = await fileHandle.getFile()
|
||||
const text = await file.text()
|
||||
this.statistics = JSON.parse(text)
|
||||
|
||||
if (this.statistics) {
|
||||
return {
|
||||
nounCount: {...this.statistics.nounCount},
|
||||
verbCount: {...this.statistics.verbCount},
|
||||
metadataCount: {...this.statistics.metadataCount},
|
||||
hnswIndexSize: this.statistics.hnswIndexSize,
|
||||
lastUpdated: this.statistics.lastUpdated
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
// If yesterday's file doesn't exist, try the legacy file
|
||||
const legacyKey = this.getLegacyStatisticsKey()
|
||||
|
||||
try {
|
||||
const fileHandle = await this.indexDir.getFileHandle(legacyKey, {
|
||||
create: false
|
||||
})
|
||||
const file = await fileHandle.getFile()
|
||||
const text = await file.text()
|
||||
this.statistics = JSON.parse(text)
|
||||
|
||||
if (this.statistics) {
|
||||
return {
|
||||
nounCount: {...this.statistics.nounCount},
|
||||
verbCount: {...this.statistics.verbCount},
|
||||
metadataCount: {...this.statistics.metadataCount},
|
||||
hnswIndexSize: this.statistics.hnswIndexSize,
|
||||
lastUpdated: this.statistics.lastUpdated
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
// If the legacy file doesn't exist either, return null
|
||||
return null
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If we get here and statistics is null, return default statistics
|
||||
return this.statistics ? this.statistics : null
|
||||
} catch (error) {
|
||||
console.error('Failed to get statistics data:', error)
|
||||
throw new Error(`Failed to get statistics data: ${error}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -3,19 +3,21 @@
|
|||
* Provides common functionality for all storage adapters
|
||||
*/
|
||||
|
||||
import { GraphVerb, HNSWNoun, StorageAdapter } from '../coreTypes.js'
|
||||
import { GraphVerb, HNSWNoun, StatisticsData } from '../coreTypes.js'
|
||||
import { BaseStorageAdapter } from './adapters/baseStorageAdapter.js'
|
||||
|
||||
// Common directory/prefix names
|
||||
export const NOUNS_DIR = 'nouns'
|
||||
export const VERBS_DIR = 'verbs'
|
||||
export const METADATA_DIR = 'metadata'
|
||||
export const INDEX_DIR = 'index'
|
||||
export const STATISTICS_KEY = 'statistics'
|
||||
|
||||
/**
|
||||
* Base storage adapter that implements common functionality
|
||||
* This is an abstract class that should be extended by specific storage adapters
|
||||
*/
|
||||
export abstract class BaseStorage implements StorageAdapter {
|
||||
export abstract class BaseStorage extends BaseStorageAdapter {
|
||||
protected isInitialized = false
|
||||
|
||||
/**
|
||||
|
|
@ -38,7 +40,7 @@ export abstract class BaseStorage implements StorageAdapter {
|
|||
*/
|
||||
public async saveNoun(noun: HNSWNoun): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
return this.saveNode(noun)
|
||||
return this.saveNoun_internal(noun)
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -46,7 +48,7 @@ export abstract class BaseStorage implements StorageAdapter {
|
|||
*/
|
||||
public async getNoun(id: string): Promise<HNSWNoun | null> {
|
||||
await this.ensureInitialized()
|
||||
return this.getNode(id)
|
||||
return this.getNoun_internal(id)
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -54,7 +56,7 @@ export abstract class BaseStorage implements StorageAdapter {
|
|||
*/
|
||||
public async getAllNouns(): Promise<HNSWNoun[]> {
|
||||
await this.ensureInitialized()
|
||||
return this.getAllNodes()
|
||||
return this.getAllNouns_internal()
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -64,7 +66,7 @@ export abstract class BaseStorage implements StorageAdapter {
|
|||
*/
|
||||
public async getNounsByNounType(nounType: string): Promise<HNSWNoun[]> {
|
||||
await this.ensureInitialized()
|
||||
return this.getNodesByNounType(nounType)
|
||||
return this.getNounsByNounType_internal(nounType)
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -72,7 +74,7 @@ export abstract class BaseStorage implements StorageAdapter {
|
|||
*/
|
||||
public async deleteNoun(id: string): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
return this.deleteNode(id)
|
||||
return this.deleteNoun_internal(id)
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -80,7 +82,7 @@ export abstract class BaseStorage implements StorageAdapter {
|
|||
*/
|
||||
public async saveVerb(verb: GraphVerb): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
return this.saveEdge(verb)
|
||||
return this.saveVerb_internal(verb)
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -88,7 +90,7 @@ export abstract class BaseStorage implements StorageAdapter {
|
|||
*/
|
||||
public async getVerb(id: string): Promise<GraphVerb | null> {
|
||||
await this.ensureInitialized()
|
||||
return this.getEdge(id)
|
||||
return this.getVerb_internal(id)
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -96,7 +98,7 @@ export abstract class BaseStorage implements StorageAdapter {
|
|||
*/
|
||||
public async getAllVerbs(): Promise<GraphVerb[]> {
|
||||
await this.ensureInitialized()
|
||||
return this.getAllEdges()
|
||||
return this.getAllVerbs_internal()
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -104,7 +106,7 @@ export abstract class BaseStorage implements StorageAdapter {
|
|||
*/
|
||||
public async getVerbsBySource(sourceId: string): Promise<GraphVerb[]> {
|
||||
await this.ensureInitialized()
|
||||
return this.getEdgesBySource(sourceId)
|
||||
return this.getVerbsBySource_internal(sourceId)
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -112,7 +114,7 @@ export abstract class BaseStorage implements StorageAdapter {
|
|||
*/
|
||||
public async getVerbsByTarget(targetId: string): Promise<GraphVerb[]> {
|
||||
await this.ensureInitialized()
|
||||
return this.getEdgesByTarget(targetId)
|
||||
return this.getVerbsByTarget_internal(targetId)
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -120,7 +122,7 @@ export abstract class BaseStorage implements StorageAdapter {
|
|||
*/
|
||||
public async getVerbsByType(type: string): Promise<GraphVerb[]> {
|
||||
await this.ensureInitialized()
|
||||
return this.getEdgesByType(type)
|
||||
return this.getVerbsByType_internal(type)
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -128,7 +130,7 @@ export abstract class BaseStorage implements StorageAdapter {
|
|||
*/
|
||||
public async deleteVerb(id: string): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
return this.deleteEdge(id)
|
||||
return this.deleteVerb_internal(id)
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -161,76 +163,76 @@ export abstract class BaseStorage implements StorageAdapter {
|
|||
public abstract getMetadata(id: string): Promise<any | null>
|
||||
|
||||
/**
|
||||
* Save a node to storage
|
||||
* Save a noun to storage
|
||||
* This method should be implemented by each specific adapter
|
||||
*/
|
||||
protected abstract saveNode(node: HNSWNoun): Promise<void>
|
||||
protected abstract saveNoun_internal(noun: HNSWNoun): Promise<void>
|
||||
|
||||
/**
|
||||
* Get a node from storage
|
||||
* Get a noun from storage
|
||||
* This method should be implemented by each specific adapter
|
||||
*/
|
||||
protected abstract getNode(id: string): Promise<HNSWNoun | null>
|
||||
protected abstract getNoun_internal(id: string): Promise<HNSWNoun | null>
|
||||
|
||||
/**
|
||||
* Get all nodes from storage
|
||||
* Get all nouns from storage
|
||||
* This method should be implemented by each specific adapter
|
||||
*/
|
||||
protected abstract getAllNodes(): Promise<HNSWNoun[]>
|
||||
protected abstract getAllNouns_internal(): Promise<HNSWNoun[]>
|
||||
|
||||
/**
|
||||
* Get nodes by noun type
|
||||
* Get nouns by noun type
|
||||
* This method should be implemented by each specific adapter
|
||||
*/
|
||||
protected abstract getNodesByNounType(nounType: string): Promise<HNSWNoun[]>
|
||||
protected abstract getNounsByNounType_internal(nounType: string): Promise<HNSWNoun[]>
|
||||
|
||||
/**
|
||||
* Delete a node from storage
|
||||
* Delete a noun from storage
|
||||
* This method should be implemented by each specific adapter
|
||||
*/
|
||||
protected abstract deleteNode(id: string): Promise<void>
|
||||
protected abstract deleteNoun_internal(id: string): Promise<void>
|
||||
|
||||
/**
|
||||
* Save an edge to storage
|
||||
* Save a verb to storage
|
||||
* This method should be implemented by each specific adapter
|
||||
*/
|
||||
protected abstract saveEdge(edge: GraphVerb): Promise<void>
|
||||
protected abstract saveVerb_internal(verb: GraphVerb): Promise<void>
|
||||
|
||||
/**
|
||||
* Get an edge from storage
|
||||
* Get a verb from storage
|
||||
* This method should be implemented by each specific adapter
|
||||
*/
|
||||
protected abstract getEdge(id: string): Promise<GraphVerb | null>
|
||||
protected abstract getVerb_internal(id: string): Promise<GraphVerb | null>
|
||||
|
||||
/**
|
||||
* Get all edges from storage
|
||||
* Get all verbs from storage
|
||||
* This method should be implemented by each specific adapter
|
||||
*/
|
||||
protected abstract getAllEdges(): Promise<GraphVerb[]>
|
||||
protected abstract getAllVerbs_internal(): Promise<GraphVerb[]>
|
||||
|
||||
/**
|
||||
* Get edges by source
|
||||
* Get verbs by source
|
||||
* This method should be implemented by each specific adapter
|
||||
*/
|
||||
protected abstract getEdgesBySource(sourceId: string): Promise<GraphVerb[]>
|
||||
protected abstract getVerbsBySource_internal(sourceId: string): Promise<GraphVerb[]>
|
||||
|
||||
/**
|
||||
* Get edges by target
|
||||
* Get verbs by target
|
||||
* This method should be implemented by each specific adapter
|
||||
*/
|
||||
protected abstract getEdgesByTarget(targetId: string): Promise<GraphVerb[]>
|
||||
protected abstract getVerbsByTarget_internal(targetId: string): Promise<GraphVerb[]>
|
||||
|
||||
/**
|
||||
* Get edges by type
|
||||
* Get verbs by type
|
||||
* This method should be implemented by each specific adapter
|
||||
*/
|
||||
protected abstract getEdgesByType(type: string): Promise<GraphVerb[]>
|
||||
protected abstract getVerbsByType_internal(type: string): Promise<GraphVerb[]>
|
||||
|
||||
/**
|
||||
* Delete an edge from storage
|
||||
* Delete a verb from storage
|
||||
* This method should be implemented by each specific adapter
|
||||
*/
|
||||
protected abstract deleteEdge(id: string): Promise<void>
|
||||
protected abstract deleteVerb_internal(id: string): Promise<void>
|
||||
|
||||
/**
|
||||
* Helper method to convert a Map to a plain object for serialization
|
||||
|
|
@ -245,4 +247,18 @@ export abstract class BaseStorage implements StorageAdapter {
|
|||
}
|
||||
return obj
|
||||
}
|
||||
|
||||
/**
|
||||
* Save statistics data to storage
|
||||
* This method should be implemented by each specific adapter
|
||||
* @param statistics The statistics data to save
|
||||
*/
|
||||
protected abstract saveStatisticsData(statistics: StatisticsData): Promise<void>
|
||||
|
||||
/**
|
||||
* Get statistics data from storage
|
||||
* This method should be implemented by each specific adapter
|
||||
* @returns Promise that resolves to the statistics data or null if not found
|
||||
*/
|
||||
protected abstract getStatisticsData(): Promise<StatisticsData | null>
|
||||
}
|
||||
|
|
@ -1,451 +0,0 @@
|
|||
import { GraphVerb, HNSWNoun, StorageAdapter } from '../coreTypes.js'
|
||||
|
||||
// Dynamically and asynchronously load Node.js modules at the top level.
|
||||
// This ensures they are available as soon as this module is imported,
|
||||
// preventing race conditions with dependencies like TensorFlow.js.
|
||||
let fs: any
|
||||
let path: any
|
||||
|
||||
const nodeModulesPromise = (async () => {
|
||||
// A reliable check for a Node.js environment.
|
||||
const isNode =
|
||||
typeof process !== 'undefined' &&
|
||||
process.versions != null &&
|
||||
process.versions.node != null
|
||||
|
||||
if (!isNode) {
|
||||
return { fs: null, path: null }
|
||||
}
|
||||
|
||||
try {
|
||||
// Use the 'node:' prefix for unambiguous importing of built-in modules.
|
||||
const fsModule = await import('node:fs')
|
||||
const pathModule = await import('node:path')
|
||||
// Return the modules, preferring the default export if it exists.
|
||||
return {
|
||||
fs: fsModule.default || fsModule,
|
||||
path: pathModule.default || pathModule
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(
|
||||
'FileSystemStorage: Failed to load Node.js modules. This adapter is not supported in this environment.',
|
||||
error
|
||||
)
|
||||
return { fs: null, path: null }
|
||||
}
|
||||
})()
|
||||
|
||||
// Immediately assign the modules once the promise resolves.
|
||||
nodeModulesPromise.then((modules) => {
|
||||
fs = modules.fs
|
||||
path = modules.path
|
||||
})
|
||||
|
||||
// --- End of Refactored Code ---
|
||||
|
||||
// Constants for directory and file names
|
||||
const ROOT_DIR = 'brainy-data'
|
||||
const NOUNS_DIR = 'nouns'
|
||||
const VERBS_DIR = 'verbs'
|
||||
const METADATA_DIR = 'metadata'
|
||||
|
||||
// All nouns now use the same directory - no separate directories per noun type
|
||||
|
||||
/**
|
||||
* File system storage adapter for Node.js environments
|
||||
*/
|
||||
export class FileSystemStorage implements StorageAdapter {
|
||||
private rootDir: string
|
||||
private nounsDir: string
|
||||
private verbsDir: string
|
||||
private metadataDir: string
|
||||
private isInitialized = false
|
||||
|
||||
constructor(rootDirectory?: string) {
|
||||
// We'll set the paths in the init method after dynamically importing the modules
|
||||
this.rootDir = rootDirectory || ''
|
||||
this.nounsDir = ''
|
||||
this.verbsDir = ''
|
||||
this.metadataDir = ''
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the storage adapter
|
||||
*/
|
||||
public async init(): Promise<void> {
|
||||
if (this.isInitialized) {
|
||||
return
|
||||
}
|
||||
|
||||
// Wait for the top-level module loading to complete.
|
||||
await nodeModulesPromise
|
||||
|
||||
// Check if the modules were loaded successfully.
|
||||
if (!fs || !path) {
|
||||
throw new Error(
|
||||
'FileSystemStorage requires a Node.js environment, but `fs` and `path` modules could not be loaded.'
|
||||
)
|
||||
}
|
||||
|
||||
try {
|
||||
// Now set up the directory paths
|
||||
const rootDir = this.rootDir || process.cwd()
|
||||
|
||||
// Check if rootDir already ends with ROOT_DIR to prevent duplication
|
||||
if (rootDir.endsWith(ROOT_DIR)) {
|
||||
this.rootDir = rootDir
|
||||
} else {
|
||||
this.rootDir = path.resolve(rootDir, ROOT_DIR)
|
||||
}
|
||||
|
||||
this.nounsDir = path.join(this.rootDir, NOUNS_DIR)
|
||||
this.verbsDir = path.join(this.rootDir, VERBS_DIR)
|
||||
this.metadataDir = path.join(this.rootDir, METADATA_DIR)
|
||||
|
||||
// Create directories if they don't exist
|
||||
await this.ensureDirectoryExists(this.rootDir)
|
||||
await this.ensureDirectoryExists(this.nounsDir)
|
||||
await this.ensureDirectoryExists(this.verbsDir)
|
||||
await this.ensureDirectoryExists(this.metadataDir)
|
||||
|
||||
this.isInitialized = true
|
||||
} catch (error: any) {
|
||||
console.error('Error initializing FileSystemStorage:', error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
private async ensureDirectoryExists(dirPath: string): Promise<void> {
|
||||
try {
|
||||
await fs.promises.mkdir(dirPath, { recursive: true })
|
||||
} catch (error: any) {
|
||||
// Ignore EEXIST error, which means the directory already exists
|
||||
if (error.code !== 'EEXIST') {
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private getNounPath(id: string, nounType?: string): string {
|
||||
// All nouns now use the same directory regardless of type
|
||||
return path.join(this.nounsDir, `${id}.json`)
|
||||
}
|
||||
|
||||
public async saveNoun(
|
||||
noun: HNSWNoun & { metadata?: { noun?: string } }
|
||||
): Promise<void> {
|
||||
if (!this.isInitialized) await this.init()
|
||||
const nounType = (noun as any).metadata?.noun
|
||||
const filePath = this.getNounPath(noun.id, nounType)
|
||||
await this.ensureDirectoryExists(path.dirname(filePath))
|
||||
await fs.promises.writeFile(filePath, JSON.stringify(noun, null, 2))
|
||||
}
|
||||
|
||||
public async getNoun(id: string): Promise<HNSWNoun | null> {
|
||||
if (!this.isInitialized) await this.init()
|
||||
const filePath = path.join(this.nounsDir, `${id}.json`)
|
||||
try {
|
||||
const data = await fs.promises.readFile(filePath, 'utf-8')
|
||||
return JSON.parse(data)
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error reading noun ${id}:`, error)
|
||||
}
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
public async deleteNoun(id: string): Promise<void> {
|
||||
if (!this.isInitialized) await this.init()
|
||||
const noun = await this.getNoun(id)
|
||||
if (noun) {
|
||||
const nounType = (noun as any).metadata?.noun
|
||||
const filePath = this.getNounPath(id, nounType)
|
||||
try {
|
||||
await fs.promises.unlink(filePath)
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error deleting noun file ${filePath}:`, error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public async getAllNouns(): Promise<HNSWNoun[]> {
|
||||
if (!this.isInitialized) await this.init()
|
||||
const allNouns: HNSWNoun[] = []
|
||||
try {
|
||||
const files = await fs.promises.readdir(this.nounsDir)
|
||||
for (const file of files) {
|
||||
if (file.endsWith('.json')) {
|
||||
const filePath = path.join(this.nounsDir, file)
|
||||
const data = await fs.promises.readFile(filePath, 'utf-8')
|
||||
allNouns.push(JSON.parse(data))
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error reading directory ${this.nounsDir}:`, error)
|
||||
}
|
||||
}
|
||||
return allNouns
|
||||
}
|
||||
|
||||
/**
|
||||
* Get nouns by noun type
|
||||
* @param nounType The noun type to filter by
|
||||
* @returns Promise that resolves to an array of nouns of the specified noun type
|
||||
*/
|
||||
public async getNounsByNounType(nounType: string): Promise<HNSWNoun[]> {
|
||||
if (!this.isInitialized) await this.init()
|
||||
|
||||
const nouns: HNSWNoun[] = []
|
||||
try {
|
||||
const files = await fs.promises.readdir(this.nounsDir)
|
||||
for (const file of files) {
|
||||
if (file.endsWith('.json')) {
|
||||
const filePath = path.join(this.nounsDir, file)
|
||||
const data = await fs.promises.readFile(filePath, 'utf-8')
|
||||
const noun = JSON.parse(data)
|
||||
|
||||
// Filter by noun type using metadata
|
||||
const metadata = await this.getMetadata(noun.id)
|
||||
if (metadata && metadata.noun === nounType) {
|
||||
nouns.push(noun)
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error reading directory ${this.nounsDir}:`, error)
|
||||
}
|
||||
}
|
||||
|
||||
return nouns
|
||||
}
|
||||
|
||||
public async saveVerb(verb: GraphVerb): Promise<void> {
|
||||
if (!this.isInitialized) await this.init()
|
||||
const filePath = path.join(this.verbsDir, `${verb.id}.json`)
|
||||
await fs.promises.writeFile(filePath, JSON.stringify(verb, null, 2))
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a verb by its ID
|
||||
* @param id The ID of the verb to retrieve
|
||||
* @returns Promise that resolves to the verb or null if not found
|
||||
*/
|
||||
public async getVerb(id: string): Promise<GraphVerb | null> {
|
||||
if (!this.isInitialized) await this.init()
|
||||
const filePath = path.join(this.verbsDir, `${id}.json`)
|
||||
try {
|
||||
const data = await fs.promises.readFile(filePath, 'utf-8')
|
||||
return JSON.parse(data)
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error reading verb ${id}:`, error)
|
||||
}
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
public async getVerbsBySource(sourceId: string): Promise<GraphVerb[]> {
|
||||
if (!this.isInitialized) await this.init()
|
||||
const allVerbs = await this.getAllVerbs()
|
||||
return allVerbs.filter((verb) => verb.sourceId === sourceId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get verbs by target ID
|
||||
* @param targetId The target ID to filter by
|
||||
* @returns Promise that resolves to an array of verbs with the specified target ID
|
||||
*/
|
||||
public async getVerbsByTarget(targetId: string): Promise<GraphVerb[]> {
|
||||
if (!this.isInitialized) await this.init()
|
||||
const allVerbs = await this.getAllVerbs()
|
||||
return allVerbs.filter((verb) => verb.targetId === targetId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get verbs by type
|
||||
* @param type The verb type to filter by
|
||||
* @returns Promise that resolves to an array of verbs of the specified type
|
||||
*/
|
||||
public async getVerbsByType(type: string): Promise<GraphVerb[]> {
|
||||
if (!this.isInitialized) await this.init()
|
||||
const allVerbs = await this.getAllVerbs()
|
||||
return allVerbs.filter((verb) => verb.type === type)
|
||||
}
|
||||
|
||||
public async getAllVerbs(): Promise<GraphVerb[]> {
|
||||
if (!this.isInitialized) await this.init()
|
||||
const allVerbs: GraphVerb[] = []
|
||||
try {
|
||||
const files = await fs.promises.readdir(this.verbsDir)
|
||||
for (const file of files) {
|
||||
if (file.endsWith('.json')) {
|
||||
const filePath = path.join(this.verbsDir, file)
|
||||
const data = await fs.promises.readFile(filePath, 'utf-8')
|
||||
allVerbs.push(JSON.parse(data))
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error reading verbs directory ${this.verbsDir}:`, error)
|
||||
}
|
||||
}
|
||||
return allVerbs
|
||||
}
|
||||
|
||||
public async deleteVerb(id: string): Promise<void> {
|
||||
if (!this.isInitialized) await this.init()
|
||||
const filePath = path.join(this.verbsDir, `${id}.json`)
|
||||
try {
|
||||
await fs.promises.unlink(filePath)
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error deleting verb file ${filePath}:`, error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save metadata for an entity
|
||||
* @param id The ID of the entity
|
||||
* @param metadata The metadata to save
|
||||
*/
|
||||
public async saveMetadata(id: string, metadata: any): Promise<void> {
|
||||
if (!this.isInitialized) await this.init()
|
||||
const filePath = path.join(this.metadataDir, `${id}.json`)
|
||||
await this.ensureDirectoryExists(path.dirname(filePath))
|
||||
await fs.promises.writeFile(filePath, JSON.stringify(metadata, null, 2))
|
||||
}
|
||||
|
||||
/**
|
||||
* Get metadata for an entity
|
||||
* @param id The ID of the entity
|
||||
* @returns Promise that resolves to the metadata or null if not found
|
||||
*/
|
||||
public async getMetadata(id: string): Promise<any | null> {
|
||||
if (!this.isInitialized) await this.init()
|
||||
const filePath = path.join(this.metadataDir, `${id}.json`)
|
||||
try {
|
||||
const data = await fs.promises.readFile(filePath, 'utf-8')
|
||||
return JSON.parse(data)
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error reading metadata for ${id}:`, error)
|
||||
}
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
public async clear(): Promise<void> {
|
||||
if (!this.isInitialized) await this.init()
|
||||
|
||||
// Helper function to recursively remove directory contents
|
||||
const removeDirectoryContents = async (dirPath: string): Promise<void> => {
|
||||
try {
|
||||
const files = await fs.promises.readdir(dirPath, { withFileTypes: true })
|
||||
|
||||
for (const file of files) {
|
||||
const fullPath = path.join(dirPath, file.name)
|
||||
|
||||
if (file.isDirectory()) {
|
||||
await removeDirectoryContents(fullPath)
|
||||
// Use fs.promises.rm with recursive option instead of rmdir
|
||||
try {
|
||||
await fs.promises.rm(fullPath, { recursive: true, force: true })
|
||||
} catch (rmError: any) {
|
||||
// Fallback to rmdir if rm fails
|
||||
await fs.promises.rmdir(fullPath)
|
||||
}
|
||||
} else {
|
||||
await fs.promises.unlink(fullPath)
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Error removing directory contents ${dirPath}:`, error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
// First try the modern approach
|
||||
await fs.promises.rm(this.rootDir, { recursive: true, force: true })
|
||||
} catch (error: any) {
|
||||
console.warn('Modern rm failed, falling back to manual cleanup:', error)
|
||||
|
||||
// Fallback: manually remove contents then directory
|
||||
try {
|
||||
await removeDirectoryContents(this.rootDir)
|
||||
// Use fs.promises.rm with recursive option instead of rmdir
|
||||
try {
|
||||
await fs.promises.rm(this.rootDir, { recursive: true, force: true })
|
||||
} catch (rmError: any) {
|
||||
// Final fallback to rmdir if rm fails
|
||||
await fs.promises.rmdir(this.rootDir)
|
||||
}
|
||||
} catch (fallbackError: any) {
|
||||
if (fallbackError.code !== 'ENOENT') {
|
||||
console.error('Manual cleanup also failed:', fallbackError)
|
||||
throw fallbackError
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.isInitialized = false // Reset state
|
||||
await this.init() // Re-create directories
|
||||
}
|
||||
|
||||
public async getStorageStatus(): Promise<{
|
||||
type: string
|
||||
used: number
|
||||
quota: number | null
|
||||
details?: Record<string, any>
|
||||
}> {
|
||||
if (!this.isInitialized) await this.init()
|
||||
|
||||
const calculateSize = async (dirPath: string): Promise<number> => {
|
||||
let size = 0
|
||||
try {
|
||||
const files = await fs.promises.readdir(dirPath, {
|
||||
withFileTypes: true
|
||||
})
|
||||
for (const file of files) {
|
||||
const fullPath = path.join(dirPath, file.name)
|
||||
if (file.isDirectory()) {
|
||||
size += await calculateSize(fullPath)
|
||||
} else {
|
||||
const stats = await fs.promises.stat(fullPath)
|
||||
size += stats.size
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`Could not calculate size for ${dirPath}:`, error)
|
||||
}
|
||||
}
|
||||
return size
|
||||
}
|
||||
|
||||
const totalSize = await calculateSize(this.rootDir)
|
||||
const nouns = await this.getAllNouns()
|
||||
const verbs = await this.getAllVerbs()
|
||||
|
||||
return {
|
||||
type: 'FileSystem',
|
||||
used: totalSize,
|
||||
quota: null, // File system quota is not easily available from Node.js
|
||||
details: {
|
||||
rootDir: this.rootDir,
|
||||
nounCount: nouns.length,
|
||||
verbCount: verbs.length
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -1,952 +0,0 @@
|
|||
import { GraphVerb, HNSWNoun, StorageAdapter } from '../coreTypes.js'
|
||||
|
||||
// Type aliases for compatibility
|
||||
type HNSWNode = HNSWNoun
|
||||
type Edge = GraphVerb
|
||||
|
||||
// Export R2Storage as an alias for S3CompatibleStorage
|
||||
export { S3CompatibleStorage as R2Storage }
|
||||
|
||||
// Constants for S3 bucket prefixes
|
||||
const NOUNS_PREFIX = 'nouns/'
|
||||
const VERBS_PREFIX = 'verbs/'
|
||||
const EDGES_PREFIX = 'verbs/' // Alias for VERBS_PREFIX for edge operations
|
||||
const METADATA_PREFIX = 'metadata/'
|
||||
|
||||
// All nouns now use the same prefix - no separate directories per noun type
|
||||
const NOUN_PREFIX = 'nouns/' // Single directory for all noun types
|
||||
|
||||
/**
|
||||
* S3-compatible storage adapter for server environments
|
||||
* Uses the AWS S3 client to interact with S3-compatible storage services
|
||||
* including Amazon S3, Cloudflare R2, and Google Cloud Storage
|
||||
*
|
||||
* To use this adapter with Cloudflare R2, you need to provide:
|
||||
* - bucketName: Your bucket name
|
||||
* - accountId: Your Cloudflare account ID
|
||||
* - accessKeyId: R2 access key ID
|
||||
* - secretAccessKey: R2 secret access key
|
||||
* - serviceType: 'r2'
|
||||
*
|
||||
* To use this adapter with Amazon S3, you need to provide:
|
||||
* - bucketName: Your S3 bucket name
|
||||
* - accessKeyId: AWS access key ID
|
||||
* - secretAccessKey: AWS secret access key
|
||||
* - region: AWS region (e.g., 'us-east-1')
|
||||
* - serviceType: 's3'
|
||||
*
|
||||
* To use this adapter with Google Cloud Storage, you need to provide:
|
||||
* - bucketName: Your GCS bucket name
|
||||
* - accessKeyId: HMAC access key
|
||||
* - secretAccessKey: HMAC secret
|
||||
* - endpoint: GCS endpoint (e.g., 'https://storage.googleapis.com')
|
||||
* - serviceType: 'gcs'
|
||||
*
|
||||
* For other S3-compatible services, provide:
|
||||
* - bucketName: Your bucket name
|
||||
* - accessKeyId: Access key ID
|
||||
* - secretAccessKey: Secret access key
|
||||
* - endpoint: Service endpoint URL
|
||||
* - region: Region (if required)
|
||||
* - serviceType: 'custom'
|
||||
*/
|
||||
export class S3CompatibleStorage implements StorageAdapter {
|
||||
private bucketName: string
|
||||
private accessKeyId: string
|
||||
private secretAccessKey: string
|
||||
private endpoint?: string
|
||||
private region?: string
|
||||
private accountId?: string
|
||||
private serviceType: 'r2' | 's3' | 'gcs' | 'custom'
|
||||
private s3Client: any // Will be initialized in init()
|
||||
private isInitialized = false
|
||||
|
||||
// Alias methods to match StorageAdapter interface
|
||||
public async saveNoun(noun: HNSWNoun): Promise<void> {
|
||||
return this.saveNode(noun)
|
||||
}
|
||||
|
||||
public async getNoun(id: string): Promise<HNSWNoun | null> {
|
||||
return this.getNode(id)
|
||||
}
|
||||
|
||||
public async getAllNouns(): Promise<HNSWNoun[]> {
|
||||
return this.getAllNodes()
|
||||
}
|
||||
|
||||
public async getNounsByNounType(nounType: string): Promise<HNSWNoun[]> {
|
||||
return this.getNodesByNounType(nounType)
|
||||
}
|
||||
|
||||
public async deleteNoun(id: string): Promise<void> {
|
||||
return this.deleteNode(id)
|
||||
}
|
||||
|
||||
public async saveVerb(verb: GraphVerb): Promise<void> {
|
||||
return this.saveEdge(verb)
|
||||
}
|
||||
|
||||
public async getVerb(id: string): Promise<GraphVerb | null> {
|
||||
return this.getEdge(id)
|
||||
}
|
||||
|
||||
public async getAllVerbs(): Promise<GraphVerb[]> {
|
||||
return this.getAllEdges()
|
||||
}
|
||||
|
||||
public async getVerbsBySource(sourceId: string): Promise<GraphVerb[]> {
|
||||
return this.getEdgesBySource(sourceId)
|
||||
}
|
||||
|
||||
public async getVerbsByTarget(targetId: string): Promise<GraphVerb[]> {
|
||||
return this.getEdgesByTarget(targetId)
|
||||
}
|
||||
|
||||
public async getVerbsByType(type: string): Promise<GraphVerb[]> {
|
||||
return this.getEdgesByType(type)
|
||||
}
|
||||
|
||||
public async deleteVerb(id: string): Promise<void> {
|
||||
return this.deleteEdge(id)
|
||||
}
|
||||
|
||||
constructor(options: {
|
||||
bucketName: string
|
||||
accessKeyId: string
|
||||
secretAccessKey: string
|
||||
serviceType: 'r2' | 's3' | 'gcs' | 'custom'
|
||||
accountId?: string
|
||||
region?: string
|
||||
endpoint?: string
|
||||
}) {
|
||||
this.bucketName = options.bucketName
|
||||
this.accessKeyId = options.accessKeyId
|
||||
this.secretAccessKey = options.secretAccessKey
|
||||
this.serviceType = options.serviceType
|
||||
this.accountId = options.accountId
|
||||
this.region = options.region
|
||||
this.endpoint = options.endpoint
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the storage adapter
|
||||
*/
|
||||
public async init(): Promise<void> {
|
||||
if (this.isInitialized) {
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
// Dynamically import the AWS SDK to avoid bundling it in browser environments
|
||||
try {
|
||||
const { S3Client } = await import('@aws-sdk/client-s3')
|
||||
|
||||
// Configure the S3 client based on the service type
|
||||
const clientConfig: any = {
|
||||
credentials: {
|
||||
accessKeyId: this.accessKeyId,
|
||||
secretAccessKey: this.secretAccessKey
|
||||
}
|
||||
}
|
||||
|
||||
switch (this.serviceType) {
|
||||
case 'r2':
|
||||
if (!this.accountId) {
|
||||
throw new Error('accountId is required for Cloudflare R2')
|
||||
}
|
||||
clientConfig.region = 'auto' // R2 uses 'auto' as the region
|
||||
clientConfig.endpoint = `https://${this.accountId}.r2.cloudflarestorage.com`
|
||||
break
|
||||
|
||||
case 's3':
|
||||
if (!this.region) {
|
||||
throw new Error('region is required for Amazon S3')
|
||||
}
|
||||
clientConfig.region = this.region
|
||||
// No endpoint needed for standard S3
|
||||
break
|
||||
|
||||
case 'gcs':
|
||||
if (!this.endpoint) {
|
||||
// Default GCS endpoint if not provided
|
||||
this.endpoint = 'https://storage.googleapis.com'
|
||||
}
|
||||
clientConfig.endpoint = this.endpoint
|
||||
clientConfig.region = this.region || 'auto'
|
||||
break
|
||||
|
||||
case 'custom':
|
||||
if (!this.endpoint) {
|
||||
throw new Error(
|
||||
'endpoint is required for custom S3-compatible services'
|
||||
)
|
||||
}
|
||||
clientConfig.endpoint = this.endpoint
|
||||
if (this.region) {
|
||||
clientConfig.region = this.region
|
||||
}
|
||||
break
|
||||
|
||||
default:
|
||||
throw new Error(`Unsupported service type: ${this.serviceType}`)
|
||||
}
|
||||
|
||||
// Initialize the S3 client with the configured options
|
||||
this.s3Client = new S3Client(clientConfig)
|
||||
|
||||
this.isInitialized = true
|
||||
} catch (importError) {
|
||||
throw new Error(
|
||||
`Failed to import AWS SDK: ${importError}. Make sure @aws-sdk/client-s3 is installed.`
|
||||
)
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Failed to initialize ${this.serviceType} storage:`, error)
|
||||
throw new Error(
|
||||
`Failed to initialize ${this.serviceType} storage: ${error}`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save a node to storage
|
||||
*/
|
||||
public async saveNode(node: HNSWNode): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Convert connections Map to a serializable format
|
||||
const serializableNode = {
|
||||
...node,
|
||||
connections: this.mapToObject(node.connections, (set) =>
|
||||
Array.from(set as Set<string>)
|
||||
)
|
||||
}
|
||||
|
||||
// Get the appropriate prefix based on the node's metadata
|
||||
const nodePrefix = await this.getNodePrefix(node.id)
|
||||
|
||||
// Import the PutObjectCommand only when needed
|
||||
const { PutObjectCommand } = await import('@aws-sdk/client-s3')
|
||||
|
||||
// Save the node to S3-compatible storage
|
||||
await this.s3Client.send(
|
||||
new PutObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: `${nodePrefix}${node.id}.json`,
|
||||
Body: JSON.stringify(serializableNode, null, 2),
|
||||
ContentType: 'application/json'
|
||||
})
|
||||
)
|
||||
} catch (error) {
|
||||
console.error(`Failed to save node ${node.id}:`, error)
|
||||
throw new Error(`Failed to save node ${node.id}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a node from storage
|
||||
*/
|
||||
public async getNode(id: string): Promise<HNSWNode | null> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Import the GetObjectCommand only when needed
|
||||
const { GetObjectCommand } = await import('@aws-sdk/client-s3')
|
||||
|
||||
// Try to get the node from the consolidated nouns directory
|
||||
const response = await this.s3Client.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: `${NOUN_PREFIX}${id}.json`
|
||||
})
|
||||
)
|
||||
|
||||
// Convert the response body to a string
|
||||
const bodyContents = await response.Body.transformToString()
|
||||
const parsedNode = JSON.parse(bodyContents)
|
||||
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(parsedNode.connections)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
return {
|
||||
id: parsedNode.id,
|
||||
vector: parsedNode.vector,
|
||||
connections
|
||||
}
|
||||
} catch (error) {
|
||||
// Node not found or other error
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get nodes by noun type
|
||||
* @param nounType The noun type to filter by
|
||||
* @returns Promise that resolves to an array of nodes of the specified noun type
|
||||
*/
|
||||
public async getNodesByNounType(nounType: string): Promise<HNSWNode[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Import the ListObjectsV2Command and GetObjectCommand only when needed
|
||||
const { ListObjectsV2Command, GetObjectCommand } = await import(
|
||||
'@aws-sdk/client-s3'
|
||||
)
|
||||
|
||||
// List all objects in the consolidated nouns directory
|
||||
const listResponse = await this.s3Client.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: this.bucketName,
|
||||
Prefix: NOUN_PREFIX
|
||||
})
|
||||
)
|
||||
|
||||
const nodes: HNSWNode[] = []
|
||||
|
||||
// If there are no objects, return an empty array
|
||||
if (!listResponse.Contents || listResponse.Contents.length === 0) {
|
||||
return nodes
|
||||
}
|
||||
|
||||
// Get each node and filter by noun type
|
||||
const nodePromises = listResponse.Contents.map(
|
||||
async (object: { Key: string }) => {
|
||||
try {
|
||||
// Extract node ID from the key (remove prefix and .json extension)
|
||||
const nodeId = object.Key.replace(NOUN_PREFIX, '').replace('.json', '')
|
||||
|
||||
// Get the metadata to check the noun type
|
||||
const metadata = await this.getMetadata(nodeId)
|
||||
|
||||
// Skip if metadata doesn't exist or noun type doesn't match
|
||||
if (!metadata || metadata.noun !== nounType) {
|
||||
return null
|
||||
}
|
||||
|
||||
const response = await this.s3Client.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: object.Key
|
||||
})
|
||||
)
|
||||
|
||||
const bodyContents = await response.Body.transformToString()
|
||||
const parsedNode = JSON.parse(bodyContents)
|
||||
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(
|
||||
parsedNode.connections
|
||||
)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
return {
|
||||
id: parsedNode.id,
|
||||
vector: parsedNode.vector,
|
||||
connections
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Failed to get node from ${object.Key}:`, error)
|
||||
return null
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
const nodeResults = await Promise.all(nodePromises)
|
||||
return nodeResults.filter((node): node is HNSWNode => node !== null)
|
||||
} catch (error) {
|
||||
console.error(`Failed to get nodes for noun type ${nounType}:`, error)
|
||||
throw new Error(`Failed to get nodes for noun type ${nounType}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all nodes from storage
|
||||
*/
|
||||
public async getAllNodes(): Promise<HNSWNode[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Import the ListObjectsV2Command and GetObjectCommand only when needed
|
||||
const { ListObjectsV2Command, GetObjectCommand } = await import(
|
||||
'@aws-sdk/client-s3'
|
||||
)
|
||||
|
||||
// List all objects in the consolidated nouns directory
|
||||
const listResponse = await this.s3Client.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: this.bucketName,
|
||||
Prefix: NOUN_PREFIX
|
||||
})
|
||||
)
|
||||
|
||||
const nodes: HNSWNode[] = []
|
||||
|
||||
// If there are no objects, return an empty array
|
||||
if (!listResponse.Contents || listResponse.Contents.length === 0) {
|
||||
return nodes
|
||||
}
|
||||
|
||||
// Get each node
|
||||
const nodePromises = listResponse.Contents.map(
|
||||
async (object: { Key: string }) => {
|
||||
try {
|
||||
const response = await this.s3Client.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: object.Key
|
||||
})
|
||||
)
|
||||
|
||||
const bodyContents = await response.Body.transformToString()
|
||||
const parsedNode = JSON.parse(bodyContents)
|
||||
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(
|
||||
parsedNode.connections
|
||||
)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
return {
|
||||
id: parsedNode.id,
|
||||
vector: parsedNode.vector,
|
||||
connections
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Failed to get node from ${object.Key}:`, error)
|
||||
return null
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
const nodeResults = await Promise.all(nodePromises)
|
||||
return nodeResults.filter((node): node is HNSWNode => node !== null)
|
||||
} catch (error) {
|
||||
console.error('Failed to get all nodes:', error)
|
||||
throw new Error(`Failed to get all nodes: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a node from storage
|
||||
*/
|
||||
public async deleteNode(id: string): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Import the DeleteObjectCommand only when needed
|
||||
const { DeleteObjectCommand } = await import('@aws-sdk/client-s3')
|
||||
|
||||
// Delete the node from the consolidated nouns directory
|
||||
await this.s3Client.send(
|
||||
new DeleteObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: `${NOUN_PREFIX}${id}.json`
|
||||
})
|
||||
)
|
||||
} catch (error) {
|
||||
console.error(`Failed to delete node ${id}:`, error)
|
||||
throw new Error(`Failed to delete node ${id}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save an edge to storage
|
||||
*/
|
||||
public async saveEdge(edge: Edge): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Convert connections Map to a serializable format
|
||||
const serializableEdge = {
|
||||
...edge,
|
||||
connections: this.mapToObject(edge.connections, (set) =>
|
||||
Array.from(set as Set<string>)
|
||||
)
|
||||
}
|
||||
|
||||
// Import the PutObjectCommand only when needed
|
||||
const { PutObjectCommand } = await import('@aws-sdk/client-s3')
|
||||
|
||||
// Save the edge to S3-compatible storage
|
||||
await this.s3Client.send(
|
||||
new PutObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: `${VERBS_PREFIX}${edge.id}.json`,
|
||||
Body: JSON.stringify(serializableEdge, null, 2),
|
||||
ContentType: 'application/json'
|
||||
})
|
||||
)
|
||||
} catch (error) {
|
||||
console.error(`Failed to save edge ${edge.id}:`, error)
|
||||
throw new Error(`Failed to save edge ${edge.id}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get an edge from storage
|
||||
*/
|
||||
public async getEdge(id: string): Promise<Edge | null> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Import the GetObjectCommand only when needed
|
||||
const { GetObjectCommand } = await import('@aws-sdk/client-s3')
|
||||
|
||||
try {
|
||||
// Try to get the edge from S3-compatible storage
|
||||
const response = await this.s3Client.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: `${VERBS_PREFIX}${id}.json`
|
||||
})
|
||||
)
|
||||
|
||||
// Convert the response body to a string
|
||||
const bodyContents = await response.Body.transformToString()
|
||||
const parsedEdge = JSON.parse(bodyContents)
|
||||
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(parsedEdge.connections)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
return {
|
||||
id: parsedEdge.id,
|
||||
vector: parsedEdge.vector,
|
||||
connections,
|
||||
sourceId: parsedEdge.sourceId,
|
||||
targetId: parsedEdge.targetId,
|
||||
type: parsedEdge.type,
|
||||
weight: parsedEdge.weight,
|
||||
metadata: parsedEdge.metadata
|
||||
}
|
||||
} catch {
|
||||
return null // Edge not found
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Failed to get edge ${id}:`, error)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all edges from storage
|
||||
*/
|
||||
public async getAllEdges(): Promise<Edge[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Import the ListObjectsV2Command and GetObjectCommand only when needed
|
||||
const { ListObjectsV2Command, GetObjectCommand } = await import(
|
||||
'@aws-sdk/client-s3'
|
||||
)
|
||||
|
||||
// List all objects with the edges prefix
|
||||
const listResponse = await this.s3Client.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: this.bucketName,
|
||||
Prefix: VERBS_PREFIX
|
||||
})
|
||||
)
|
||||
|
||||
const edges: Edge[] = []
|
||||
|
||||
// If there are no objects, return an empty array
|
||||
if (!listResponse.Contents || listResponse.Contents.length === 0) {
|
||||
return edges
|
||||
}
|
||||
|
||||
// Get each edge
|
||||
const edgePromises = listResponse.Contents.map(
|
||||
async (object: { Key: string }) => {
|
||||
try {
|
||||
const response = await this.s3Client.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: object.Key
|
||||
})
|
||||
)
|
||||
|
||||
const bodyContents = await response.Body.transformToString()
|
||||
const parsedEdge = JSON.parse(bodyContents)
|
||||
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nodeIds] of Object.entries(
|
||||
parsedEdge.connections
|
||||
)) {
|
||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||
}
|
||||
|
||||
return {
|
||||
id: parsedEdge.id,
|
||||
vector: parsedEdge.vector,
|
||||
connections,
|
||||
sourceId: parsedEdge.sourceId,
|
||||
targetId: parsedEdge.targetId,
|
||||
type: parsedEdge.type,
|
||||
weight: parsedEdge.weight,
|
||||
metadata: parsedEdge.metadata
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Failed to get edge from ${object.Key}:`, error)
|
||||
return null
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
const edgeResults = await Promise.all(edgePromises)
|
||||
return edgeResults.filter((edge): edge is Edge => edge !== null)
|
||||
} catch (error) {
|
||||
console.error('Failed to get all edges:', error)
|
||||
throw new Error(`Failed to get all edges: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get edges by source node ID
|
||||
*/
|
||||
public async getEdgesBySource(sourceId: string): Promise<Edge[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
const allEdges = await this.getAllEdges()
|
||||
return allEdges.filter((edge) => edge.sourceId === sourceId)
|
||||
} catch (error) {
|
||||
console.error(`Failed to get edges by source ${sourceId}:`, error)
|
||||
throw new Error(`Failed to get edges by source ${sourceId}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get edges by target node ID
|
||||
*/
|
||||
public async getEdgesByTarget(targetId: string): Promise<Edge[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
const allEdges = await this.getAllEdges()
|
||||
return allEdges.filter((edge) => edge.targetId === targetId)
|
||||
} catch (error) {
|
||||
console.error(`Failed to get edges by target ${targetId}:`, error)
|
||||
throw new Error(`Failed to get edges by target ${targetId}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get edges by type
|
||||
*/
|
||||
public async getEdgesByType(type: string): Promise<Edge[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
const allEdges = await this.getAllEdges()
|
||||
return allEdges.filter((edge) => edge.type === type)
|
||||
} catch (error) {
|
||||
console.error(`Failed to get edges by type ${type}:`, error)
|
||||
throw new Error(`Failed to get edges by type ${type}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete an edge from storage
|
||||
*/
|
||||
public async deleteEdge(id: string): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Import the DeleteObjectCommand and GetObjectCommand only when needed
|
||||
const { DeleteObjectCommand, GetObjectCommand } = await import(
|
||||
'@aws-sdk/client-s3'
|
||||
)
|
||||
|
||||
try {
|
||||
// Check if the edge exists before deleting
|
||||
await this.s3Client.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: `${EDGES_PREFIX}${id}.json`
|
||||
})
|
||||
)
|
||||
|
||||
// Delete the edge
|
||||
await this.s3Client.send(
|
||||
new DeleteObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: `${EDGES_PREFIX}${id}.json`
|
||||
})
|
||||
)
|
||||
} catch {
|
||||
// Edge not found, nothing to delete
|
||||
return
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Failed to delete edge ${id}:`, error)
|
||||
throw new Error(`Failed to delete edge ${id}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save metadata to storage
|
||||
*/
|
||||
public async saveMetadata(id: string, metadata: any): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Import the PutObjectCommand only when needed
|
||||
const { PutObjectCommand } = await import('@aws-sdk/client-s3')
|
||||
|
||||
// Save the metadata to S3-compatible storage
|
||||
await this.s3Client.send(
|
||||
new PutObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: `${METADATA_PREFIX}${id}.json`,
|
||||
Body: JSON.stringify(metadata, null, 2),
|
||||
ContentType: 'application/json'
|
||||
})
|
||||
)
|
||||
} catch (error) {
|
||||
console.error(`Failed to save metadata for ${id}:`, error)
|
||||
throw new Error(`Failed to save metadata for ${id}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get metadata from storage
|
||||
*/
|
||||
public async getMetadata(id: string): Promise<any | null> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Import the GetObjectCommand only when needed
|
||||
const { GetObjectCommand } = await import('@aws-sdk/client-s3')
|
||||
|
||||
try {
|
||||
// Try to get the metadata from S3-compatible storage
|
||||
const response = await this.s3Client.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: `${METADATA_PREFIX}${id}.json`
|
||||
})
|
||||
)
|
||||
|
||||
// Convert the response body to a string
|
||||
const bodyContents = await response.Body.transformToString()
|
||||
return JSON.parse(bodyContents)
|
||||
} catch {
|
||||
return null // Metadata not found
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Failed to get metadata for ${id}:`, error)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear all data from storage
|
||||
*/
|
||||
public async clear(): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Import the ListObjectsV2Command and DeleteObjectCommand only when needed
|
||||
const { ListObjectsV2Command, DeleteObjectCommand } = await import(
|
||||
'@aws-sdk/client-s3'
|
||||
)
|
||||
|
||||
// List all objects in the bucket
|
||||
const listResponse = await this.s3Client.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: this.bucketName
|
||||
})
|
||||
)
|
||||
|
||||
// If there are no objects, return
|
||||
if (!listResponse.Contents || listResponse.Contents.length === 0) {
|
||||
return
|
||||
}
|
||||
|
||||
// Delete each object
|
||||
const deletePromises = listResponse.Contents.map(
|
||||
async (object: { Key: string }) => {
|
||||
try {
|
||||
await this.s3Client.send(
|
||||
new DeleteObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: object.Key
|
||||
})
|
||||
)
|
||||
} catch (error) {
|
||||
console.error(`Failed to delete object ${object.Key}:`, error)
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
await Promise.all(deletePromises)
|
||||
} catch (error) {
|
||||
console.error('Failed to clear storage:', error)
|
||||
throw new Error(`Failed to clear storage: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get information about storage usage and capacity
|
||||
*/
|
||||
public async getStorageStatus(): Promise<{
|
||||
type: string
|
||||
used: number
|
||||
quota: number | null
|
||||
details?: Record<string, any>
|
||||
}> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
// Import the ListObjectsV2Command only when needed
|
||||
const { ListObjectsV2Command } = await import('@aws-sdk/client-s3')
|
||||
|
||||
// List all objects in the bucket to calculate total size
|
||||
const listResponse = await this.s3Client.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: this.bucketName
|
||||
})
|
||||
)
|
||||
|
||||
let totalSize = 0
|
||||
let nodeCount = 0
|
||||
let edgeCount = 0
|
||||
let metadataCount = 0
|
||||
|
||||
// Calculate total size and counts
|
||||
if (listResponse.Contents) {
|
||||
for (const object of listResponse.Contents) {
|
||||
totalSize += object.Size || 0
|
||||
|
||||
const key = object.Key || ''
|
||||
if (key.startsWith(NOUNS_PREFIX)) {
|
||||
nodeCount++
|
||||
} else if (key.startsWith(VERBS_PREFIX)) {
|
||||
edgeCount++
|
||||
} else if (key.startsWith(METADATA_PREFIX)) {
|
||||
metadataCount++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Count nodes by noun type by examining metadata
|
||||
const nounTypeCounts: Record<string, number> = {
|
||||
person: 0,
|
||||
place: 0,
|
||||
thing: 0,
|
||||
event: 0,
|
||||
concept: 0,
|
||||
content: 0,
|
||||
group: 0,
|
||||
list: 0,
|
||||
category: 0,
|
||||
default: 0
|
||||
}
|
||||
|
||||
// List all noun objects and count by type using metadata
|
||||
const nounsListResponse = await this.s3Client.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: this.bucketName,
|
||||
Prefix: NOUN_PREFIX
|
||||
})
|
||||
)
|
||||
|
||||
if (nounsListResponse.Contents) {
|
||||
for (const object of nounsListResponse.Contents) {
|
||||
try {
|
||||
// Extract node ID from the key
|
||||
const nodeId = object.Key?.replace(NOUN_PREFIX, '').replace('.json', '')
|
||||
if (nodeId) {
|
||||
// Get metadata to determine noun type
|
||||
const metadata = await this.getMetadata(nodeId)
|
||||
const nounType = metadata?.noun || 'default'
|
||||
|
||||
if (nounType in nounTypeCounts) {
|
||||
nounTypeCounts[nounType]++
|
||||
} else {
|
||||
nounTypeCounts.default++
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
// If we can't get metadata, count as default
|
||||
nounTypeCounts.default++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
type: this.serviceType,
|
||||
used: totalSize,
|
||||
quota: null, // S3-compatible services typically don't provide quota information through the API
|
||||
details: {
|
||||
nodeCount,
|
||||
edgeCount,
|
||||
metadataCount,
|
||||
nounTypes: {
|
||||
person: { count: nounTypeCounts.person },
|
||||
place: { count: nounTypeCounts.place },
|
||||
thing: { count: nounTypeCounts.thing },
|
||||
event: { count: nounTypeCounts.event },
|
||||
concept: { count: nounTypeCounts.concept },
|
||||
content: { count: nounTypeCounts.content },
|
||||
group: { count: nounTypeCounts.group },
|
||||
list: { count: nounTypeCounts.list },
|
||||
category: { count: nounTypeCounts.category },
|
||||
default: { count: nounTypeCounts.default }
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Failed to get storage status:', error)
|
||||
return {
|
||||
type: this.serviceType,
|
||||
used: 0,
|
||||
quota: null,
|
||||
details: { error: String(error) }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure the storage adapter is initialized
|
||||
*/
|
||||
private async ensureInitialized(): Promise<void> {
|
||||
if (!this.isInitialized) {
|
||||
await this.init()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the appropriate prefix for a node - now all nouns use the same prefix
|
||||
*/
|
||||
private async getNodePrefix(id: string): Promise<string> {
|
||||
// All nouns now use the same prefix regardless of type
|
||||
return NOUN_PREFIX
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a Map to a plain object for serialization
|
||||
*/
|
||||
private mapToObject<K extends string | number, V>(
|
||||
map: Map<K, V>,
|
||||
valueTransformer: (value: V) => any = (v) => v
|
||||
): Record<string, any> {
|
||||
const obj: Record<string, any> = {}
|
||||
for (const [key, value] of map.entries()) {
|
||||
obj[key.toString()] = valueTransformer(value)
|
||||
}
|
||||
return obj
|
||||
}
|
||||
}
|
||||
|
|
@ -34,7 +34,7 @@ export interface GraphNoun {
|
|||
}
|
||||
|
||||
/**
|
||||
* Base interface for edges (verbs) in the graph
|
||||
* Base interface for verbs in the graph
|
||||
* Represents relationships between nouns
|
||||
*/
|
||||
export interface GraphVerb {
|
||||
|
|
@ -45,6 +45,7 @@ export interface GraphVerb {
|
|||
verb: VerbType // Type of relationship
|
||||
createdAt: Timestamp // When the verb was created
|
||||
updatedAt: Timestamp // When the verb was last updated
|
||||
createdBy: CreatorMetadata // Information about what created this verb
|
||||
data?: Record<string, any> // Additional flexible data storage
|
||||
embedding?: number[] // Vector representation of the relationship
|
||||
confidence?: number // Confidence score (0-1)
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
export * from './distance.js'
|
||||
export * from './embedding.js'
|
||||
export * from './workerUtils.js'
|
||||
export * from './statistics.js'
|
||||
|
|
|
|||
44
src/utils/statistics.ts
Normal file
44
src/utils/statistics.ts
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
/**
|
||||
* Utility functions for retrieving statistics from Brainy
|
||||
*/
|
||||
|
||||
import { BrainyData } from '../brainyData.js'
|
||||
|
||||
/**
|
||||
* Get statistics about the current state of a BrainyData instance
|
||||
* This function provides access to statistics at the root level of the library
|
||||
*
|
||||
* @param instance A BrainyData instance to get statistics from
|
||||
* @param options Additional options for retrieving statistics
|
||||
* @returns Object containing counts of nouns, verbs, metadata entries, and HNSW index size
|
||||
* @throws Error if the instance is not provided or if statistics retrieval fails
|
||||
*/
|
||||
export async function getStatistics(
|
||||
instance: BrainyData,
|
||||
options: {
|
||||
service?: string | string[] // Filter statistics by service(s)
|
||||
} = {}
|
||||
): Promise<{
|
||||
nounCount: number
|
||||
verbCount: number
|
||||
metadataCount: number
|
||||
hnswIndexSize: number
|
||||
serviceBreakdown?: {
|
||||
[service: string]: {
|
||||
nounCount: number
|
||||
verbCount: number
|
||||
metadataCount: number
|
||||
}
|
||||
}
|
||||
}> {
|
||||
if (!instance) {
|
||||
throw new Error('BrainyData instance must be provided to getStatistics')
|
||||
}
|
||||
|
||||
try {
|
||||
return await instance.getStatistics(options)
|
||||
} catch (error) {
|
||||
console.error('Failed to get statistics:', error)
|
||||
throw new Error(`Failed to get statistics: ${error}`)
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue