diff --git a/src/storage/adapters/baseStorageAdapter.ts b/src/storage/adapters/baseStorageAdapter.ts new file mode 100644 index 00000000..ddea65a7 --- /dev/null +++ b/src/storage/adapters/baseStorageAdapter.ts @@ -0,0 +1,158 @@ +/** + * Base Storage Adapter + * Provides common functionality for all storage adapters, including statistics tracking + */ + +import { StatisticsData, StorageAdapter } from '../../coreTypes.js' + +/** + * Base class for storage adapters that implements statistics tracking + */ +export abstract class BaseStorageAdapter implements StorageAdapter { + // Abstract methods that must be implemented by subclasses + abstract init(): Promise + abstract saveNoun(noun: any): Promise + abstract getNoun(id: string): Promise + abstract getAllNouns(): Promise + abstract getNounsByNounType(nounType: string): Promise + abstract deleteNoun(id: string): Promise + abstract saveVerb(verb: any): Promise + abstract getVerb(id: string): Promise + abstract getAllVerbs(): Promise + abstract getVerbsBySource(sourceId: string): Promise + abstract getVerbsByTarget(targetId: string): Promise + abstract getVerbsByType(type: string): Promise + abstract deleteVerb(id: string): Promise + abstract saveMetadata(id: string, metadata: any): Promise + abstract getMetadata(id: string): Promise + abstract clear(): Promise + abstract getStorageStatus(): Promise<{ + type: string + used: number + quota: number | null + details?: Record + }> + + // Statistics-specific methods + protected abstract saveStatisticsData(statistics: StatisticsData): Promise + protected abstract getStatisticsData(): Promise + + /** + * Save statistics data + * @param statistics The statistics data to save + */ + async saveStatistics(statistics: StatisticsData): Promise { + await this.saveStatisticsData(statistics) + } + + /** + * Get statistics data + * @returns Promise that resolves to the statistics data + */ + async getStatistics(): Promise { + return await this.getStatisticsData() + } + + /** + * Increment a statistic counter + * @param type The type of statistic to increment ('noun', 'verb', 'metadata') + * @param service The service that inserted the data + * @param amount The amount to increment by (default: 1) + */ + async incrementStatistic( + type: 'noun' | 'verb' | 'metadata', + service: string, + amount: number = 1 + ): Promise { + // Get current statistics or create default if not exists + let statistics = await this.getStatisticsData() + if (!statistics) { + statistics = this.createDefaultStatistics() + } + + // Increment the appropriate counter + const counterMap = { + noun: statistics.nounCount, + verb: statistics.verbCount, + metadata: statistics.metadataCount + } + + const counter = counterMap[type] + counter[service] = (counter[service] || 0) + amount + + // Update timestamp + statistics.lastUpdated = new Date().toISOString() + + // Save updated statistics + await this.saveStatisticsData(statistics) + } + + /** + * Decrement a statistic counter + * @param type The type of statistic to decrement ('noun', 'verb', 'metadata') + * @param service The service that inserted the data + * @param amount The amount to decrement by (default: 1) + */ + async decrementStatistic( + type: 'noun' | 'verb' | 'metadata', + service: string, + amount: number = 1 + ): Promise { + // Get current statistics or create default if not exists + let statistics = await this.getStatisticsData() + if (!statistics) { + statistics = this.createDefaultStatistics() + } + + // Decrement the appropriate counter + const counterMap = { + noun: statistics.nounCount, + verb: statistics.verbCount, + metadata: statistics.metadataCount + } + + const counter = counterMap[type] + counter[service] = Math.max(0, (counter[service] || 0) - amount) + + // Update timestamp + statistics.lastUpdated = new Date().toISOString() + + // Save updated statistics + await this.saveStatisticsData(statistics) + } + + /** + * Update the HNSW index size statistic + * @param size The new size of the HNSW index + */ + async updateHnswIndexSize(size: number): Promise { + // Get current statistics or create default if not exists + let statistics = await this.getStatisticsData() + if (!statistics) { + statistics = this.createDefaultStatistics() + } + + // Update HNSW index size + statistics.hnswIndexSize = size + + // Update timestamp + statistics.lastUpdated = new Date().toISOString() + + // Save updated statistics + await this.saveStatisticsData(statistics) + } + + /** + * Create default statistics data + * @returns Default statistics data + */ + protected createDefaultStatistics(): StatisticsData { + return { + nounCount: {}, + verbCount: {}, + metadataCount: {}, + hnswIndexSize: 0, + lastUpdated: new Date().toISOString() + } + } +} \ No newline at end of file diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts index 2ee1fccb..75ccbb53 100644 --- a/src/storage/adapters/fileSystemStorage.ts +++ b/src/storage/adapters/fileSystemStorage.ts @@ -3,8 +3,8 @@ * File system storage adapter for Node.js environments */ -import { GraphVerb, HNSWNoun } from '../../coreTypes.js' -import { BaseStorage, NOUNS_DIR, VERBS_DIR, METADATA_DIR, INDEX_DIR } from '../baseStorage.js' +import { GraphVerb, HNSWNoun, StatisticsData } from '../../coreTypes.js' +import { BaseStorage, NOUNS_DIR, VERBS_DIR, METADATA_DIR, INDEX_DIR, STATISTICS_KEY } from '../baseStorage.js' // Type aliases for better readability type HNSWNode = HNSWNoun @@ -552,4 +552,41 @@ export class FileSystemStorage extends BaseStorage { } } } + + /** + * Save statistics data to storage + * @param statistics The statistics data to save + */ + protected async saveStatisticsData(statistics: StatisticsData): Promise { + await this.ensureInitialized() + + try { + const filePath = path.join(this.indexDir, `${STATISTICS_KEY}.json`) + await fs.promises.writeFile(filePath, JSON.stringify(statistics, null, 2)) + } catch (error) { + console.error('Failed to save statistics data:', error) + throw new Error(`Failed to save statistics data: ${error}`) + } + } + + /** + * Get statistics data from storage + * @returns Promise that resolves to the statistics data or null if not found + */ + protected async getStatisticsData(): Promise { + await this.ensureInitialized() + + try { + const filePath = path.join(this.indexDir, `${STATISTICS_KEY}.json`) + const data = await fs.promises.readFile(filePath, 'utf-8') + return JSON.parse(data) + } catch (error: any) { + // If the file doesn't exist, return null + if (error.code === 'ENOENT') { + return null + } + console.error('Error getting statistics data:', error) + throw error + } + } } \ No newline at end of file diff --git a/src/storage/adapters/memoryStorage.ts b/src/storage/adapters/memoryStorage.ts index 9ba12c55..bcc1833e 100644 --- a/src/storage/adapters/memoryStorage.ts +++ b/src/storage/adapters/memoryStorage.ts @@ -3,8 +3,8 @@ * In-memory storage adapter for environments where persistent storage is not available or needed */ -import { GraphVerb, HNSWNoun } from '../../coreTypes.js' -import { BaseStorage } from '../baseStorage.js' +import { GraphVerb, HNSWNoun, StatisticsData } from '../../coreTypes.js' +import { BaseStorage, STATISTICS_KEY } from '../baseStorage.js' /** * Type alias for HNSWNoun to make the code more readable @@ -25,6 +25,7 @@ export class MemoryStorage extends BaseStorage { private nouns: Map = new Map() private verbs: Map = new Map() private metadata: Map = new Map() + private statistics: StatisticsData | null = null constructor() { super() @@ -321,4 +322,38 @@ export class MemoryStorage extends BaseStorage { } } } + + /** + * Save statistics data to storage + * @param statistics The statistics data to save + */ + protected async saveStatisticsData(statistics: StatisticsData): Promise { + // Create a deep copy to avoid reference issues + this.statistics = { + nounCount: {...statistics.nounCount}, + verbCount: {...statistics.verbCount}, + metadataCount: {...statistics.metadataCount}, + hnswIndexSize: statistics.hnswIndexSize, + lastUpdated: statistics.lastUpdated + } + } + + /** + * Get statistics data from storage + * @returns Promise that resolves to the statistics data or null if not found + */ + protected async getStatisticsData(): Promise { + if (!this.statistics) { + return null + } + + // Return a deep copy to avoid reference issues + return { + nounCount: {...this.statistics.nounCount}, + verbCount: {...this.statistics.verbCount}, + metadataCount: {...this.statistics.metadataCount}, + hnswIndexSize: this.statistics.hnswIndexSize, + lastUpdated: this.statistics.lastUpdated + } + } } \ No newline at end of file diff --git a/src/storage/adapters/opfsStorage.ts b/src/storage/adapters/opfsStorage.ts index d9037576..20e37e22 100644 --- a/src/storage/adapters/opfsStorage.ts +++ b/src/storage/adapters/opfsStorage.ts @@ -3,8 +3,8 @@ * Provides persistent storage for the vector database using the Origin Private File System API */ -import {GraphVerb, HNSWNoun} from '../../coreTypes.js' -import {BaseStorage, NOUNS_DIR, VERBS_DIR, METADATA_DIR, INDEX_DIR} from '../baseStorage.js' +import {GraphVerb, HNSWNoun, StatisticsData} from '../../coreTypes.js' +import {BaseStorage, NOUNS_DIR, VERBS_DIR, METADATA_DIR, INDEX_DIR, STATISTICS_KEY} from '../baseStorage.js' import '../../types/fileSystemTypes.js' /** @@ -37,6 +37,7 @@ export class OPFSStorage extends BaseStorage { private isAvailable = false private isPersistentRequested = false private isPersistentGranted = false + private statistics: StatisticsData | null = null constructor() { super() @@ -688,4 +689,106 @@ export class OPFSStorage extends BaseStorage { } } } + + /** + * Save statistics data to storage + * @param statistics The statistics data to save + */ + protected async saveStatisticsData(statistics: StatisticsData): Promise { + // Create a deep copy to avoid reference issues + this.statistics = { + nounCount: {...statistics.nounCount}, + verbCount: {...statistics.verbCount}, + metadataCount: {...statistics.metadataCount}, + hnswIndexSize: statistics.hnswIndexSize, + lastUpdated: statistics.lastUpdated + } + + try { + // Ensure the root directory is initialized + await this.ensureInitialized() + + // Get or create the index directory + if (!this.indexDir) { + throw new Error('Index directory not initialized') + } + + // Create a file for the statistics data + const fileHandle = await this.indexDir.getFileHandle('statistics.json', { + create: true + }) + + // Create a writable stream + const writable = await fileHandle.createWritable() + + // Write the statistics data to the file + await writable.write(JSON.stringify(this.statistics, null, 2)) + + // Close the stream + await writable.close() + } catch (error) { + console.error('Failed to save statistics data:', error) + throw new Error(`Failed to save statistics data: ${error}`) + } + } + + /** + * Get statistics data from storage + * @returns Promise that resolves to the statistics data or null if not found + */ + protected async getStatisticsData(): Promise { + // If we have cached statistics, return a deep copy + if (this.statistics) { + return { + nounCount: {...this.statistics.nounCount}, + verbCount: {...this.statistics.verbCount}, + metadataCount: {...this.statistics.metadataCount}, + hnswIndexSize: this.statistics.hnswIndexSize, + lastUpdated: this.statistics.lastUpdated + } + } + + try { + // Ensure the root directory is initialized + await this.ensureInitialized() + + if (!this.indexDir) { + throw new Error('Index directory not initialized') + } + + try { + // Try to get the statistics file + const fileHandle = await this.indexDir.getFileHandle('statistics.json', { + create: false + }) + + // Get the file data + const file = await fileHandle.getFile() + const text = await file.text() + + // Parse the statistics data + this.statistics = JSON.parse(text) + + // Return a deep copy + if (this.statistics) { + return { + nounCount: {...this.statistics.nounCount}, + verbCount: {...this.statistics.verbCount}, + metadataCount: {...this.statistics.metadataCount}, + hnswIndexSize: this.statistics.hnswIndexSize, + lastUpdated: this.statistics.lastUpdated + } + } + + // If statistics is null, return default statistics + return this.createDefaultStatistics() + } catch (error) { + // If the file doesn't exist, return null + return null + } + } catch (error) { + console.error('Failed to get statistics data:', error) + throw new Error(`Failed to get statistics data: ${error}`) + } + } } \ No newline at end of file diff --git a/src/storage/adapters/s3CompatibleStorage.ts b/src/storage/adapters/s3CompatibleStorage.ts index 20a5c8e1..12d71282 100644 --- a/src/storage/adapters/s3CompatibleStorage.ts +++ b/src/storage/adapters/s3CompatibleStorage.ts @@ -4,8 +4,8 @@ * including Amazon S3, Cloudflare R2, and Google Cloud Storage */ -import { GraphVerb, HNSWNoun } from '../../coreTypes.js' -import { BaseStorage, NOUNS_DIR, VERBS_DIR, METADATA_DIR, INDEX_DIR } from '../baseStorage.js' +import { GraphVerb, HNSWNoun, StatisticsData } from '../../coreTypes.js' +import { BaseStorage, NOUNS_DIR, VERBS_DIR, METADATA_DIR, INDEX_DIR, STATISTICS_KEY } from '../baseStorage.js' // Type aliases for better readability type HNSWNode = HNSWNoun @@ -56,6 +56,9 @@ export class S3CompatibleStorage extends BaseStorage { private verbPrefix: string private metadataPrefix: string private indexPrefix: string + + // Statistics caching for better performance + private statisticsCache: StatisticsData | null = null /** * Initialize the storage adapter @@ -1014,4 +1017,113 @@ export class S3CompatibleStorage extends BaseStorage { } } } + + /** + * Save statistics data to storage + * @param statistics The statistics data to save + */ + protected async saveStatisticsData(statistics: StatisticsData): Promise { + await this.ensureInitialized() + + try { + // Update the cache with a deep copy to avoid reference issues + this.statisticsCache = { + nounCount: {...statistics.nounCount}, + verbCount: {...statistics.verbCount}, + metadataCount: {...statistics.metadataCount}, + hnswIndexSize: statistics.hnswIndexSize, + lastUpdated: statistics.lastUpdated + } + + // Import the PutObjectCommand only when needed + const { PutObjectCommand } = await import('@aws-sdk/client-s3') + + const key = `${this.indexPrefix}${STATISTICS_KEY}.json` + const body = JSON.stringify(statistics, null, 2) + + // Save the statistics to S3-compatible storage + await this.s3Client!.send( + new PutObjectCommand({ + Bucket: this.bucketName, + Key: key, + Body: body, + ContentType: 'application/json' + }) + ) + } catch (error) { + console.error('Failed to save statistics data:', error) + throw new Error(`Failed to save statistics data: ${error}`) + } + } + + /** + * Get statistics data from storage + * @returns Promise that resolves to the statistics data or null if not found + */ + protected async getStatisticsData(): Promise { + await this.ensureInitialized() + + // If we have cached statistics, return a deep copy + if (this.statisticsCache) { + return { + nounCount: {...this.statisticsCache.nounCount}, + verbCount: {...this.statisticsCache.verbCount}, + metadataCount: {...this.statisticsCache.metadataCount}, + hnswIndexSize: this.statisticsCache.hnswIndexSize, + lastUpdated: this.statisticsCache.lastUpdated + } + } + + try { + // Import the GetObjectCommand only when needed + const { GetObjectCommand } = await import('@aws-sdk/client-s3') + + const key = `${this.indexPrefix}${STATISTICS_KEY}.json` + + // Try to get the statistics from the index directory + const response = await this.s3Client!.send( + new GetObjectCommand({ + Bucket: this.bucketName, + Key: key + }) + ) + + // Check if response is null or undefined + if (!response || !response.Body) { + return null + } + + // Convert the response body to a string + const bodyContents = await response.Body.transformToString() + + // Parse the JSON string and update the cache + const statistics = JSON.parse(bodyContents) + + // Update the cache with a deep copy + this.statisticsCache = { + nounCount: {...statistics.nounCount}, + verbCount: {...statistics.verbCount}, + metadataCount: {...statistics.metadataCount}, + hnswIndexSize: statistics.hnswIndexSize, + lastUpdated: statistics.lastUpdated + } + + return statistics + } catch (error: any) { + // Check if this is a "NoSuchKey" error (object doesn't exist) + if ( + error.name === 'NoSuchKey' || + (error.message && ( + error.message.includes('NoSuchKey') || + error.message.includes('not found') || + error.message.includes('does not exist') + )) + ) { + return null + } + + console.error('Error getting statistics data:', error) + throw error + } + } } \ No newline at end of file diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts index bc957161..46c6eb75 100644 --- a/src/storage/baseStorage.ts +++ b/src/storage/baseStorage.ts @@ -3,19 +3,21 @@ * Provides common functionality for all storage adapters */ -import { GraphVerb, HNSWNoun, StorageAdapter } from '../coreTypes.js' +import { GraphVerb, HNSWNoun, StatisticsData } from '../coreTypes.js' +import { BaseStorageAdapter } from './adapters/baseStorageAdapter.js' // Common directory/prefix names export const NOUNS_DIR = 'nouns' export const VERBS_DIR = 'verbs' export const METADATA_DIR = 'metadata' export const INDEX_DIR = 'index' +export const STATISTICS_KEY = 'statistics' /** * Base storage adapter that implements common functionality * This is an abstract class that should be extended by specific storage adapters */ -export abstract class BaseStorage implements StorageAdapter { +export abstract class BaseStorage extends BaseStorageAdapter { protected isInitialized = false /** @@ -245,4 +247,18 @@ export abstract class BaseStorage implements StorageAdapter { } return obj } + + /** + * Save statistics data to storage + * This method should be implemented by each specific adapter + * @param statistics The statistics data to save + */ + protected abstract saveStatisticsData(statistics: StatisticsData): Promise + + /** + * Get statistics data from storage + * This method should be implemented by each specific adapter + * @returns Promise that resolves to the statistics data or null if not found + */ + protected abstract getStatisticsData(): Promise } \ No newline at end of file diff --git a/src/utils/statistics.ts b/src/utils/statistics.ts index 77b11bc5..753a267b 100644 --- a/src/utils/statistics.ts +++ b/src/utils/statistics.ts @@ -9,21 +9,34 @@ import { BrainyData } from '../brainyData.js' * This function provides access to statistics at the root level of the library * * @param instance A BrainyData instance to get statistics from + * @param options Additional options for retrieving statistics * @returns Object containing counts of nouns, verbs, metadata entries, and HNSW index size * @throws Error if the instance is not provided or if statistics retrieval fails */ -export async function getStatistics(instance: BrainyData): Promise<{ +export async function getStatistics( + instance: BrainyData, + options: { + service?: string | string[] // Filter statistics by service(s) + } = {} +): Promise<{ nounCount: number verbCount: number metadataCount: number hnswIndexSize: number + serviceBreakdown?: { + [service: string]: { + nounCount: number + verbCount: number + metadataCount: number + } + } }> { if (!instance) { throw new Error('BrainyData instance must be provided to getStatistics') } try { - return await instance.getStatistics() + return await instance.getStatistics(options) } catch (error) { console.error('Failed to get statistics:', error) throw new Error(`Failed to get statistics: ${error}`) diff --git a/tests/statistics.test.ts b/tests/statistics.test.ts index b613000f..47f3a0bd 100644 --- a/tests/statistics.test.ts +++ b/tests/statistics.test.ts @@ -72,5 +72,54 @@ describe('Brainy Statistics Functionality', () => { // Verify they match expect(functionStats).toEqual(instanceStats) }) + + it('should track statistics by service', async () => { + // Create a BrainyData instance + const data = new brainy.BrainyData({ + dimensions: 3, + metric: 'euclidean' + }) + + await data.init() + await data.clear() // Clear any existing data + + // Add data from different services + await data.add([1, 0, 0], { id: 'v1', label: 'service1-item' }, { service: 'service1' }) + await data.add([0, 1, 0], { id: 'v2', label: 'service1-item' }, { service: 'service1' }) + await data.add([0, 0, 1], { id: 'v3', label: 'service2-item' }, { service: 'service2' }) + + // Add verbs from different services + await data.addVerb('v1', 'v2', undefined, { type: 'related_to', service: 'service1' }) + await data.addVerb('v2', 'v3', undefined, { type: 'related_to', service: 'service2' }) + + // Get statistics for all services + const allStats = await data.getStatistics() + + // Verify total counts + expect(allStats.nounCount).toBe(3) + expect(allStats.verbCount).toBe(2) + expect(allStats.metadataCount).toBe(3) + + // Verify service breakdown exists + expect(allStats.serviceBreakdown).toBeDefined() + + // Verify service1 statistics + const service1Stats = await data.getStatistics({ service: 'service1' }) + expect(service1Stats.nounCount).toBe(2) + expect(service1Stats.verbCount).toBe(1) + expect(service1Stats.metadataCount).toBe(2) + + // Verify service2 statistics + const service2Stats = await data.getStatistics({ service: 'service2' }) + expect(service2Stats.nounCount).toBe(1) + expect(service2Stats.verbCount).toBe(1) + expect(service2Stats.metadataCount).toBe(1) + + // Verify multiple services filter + const combinedStats = await data.getStatistics({ service: ['service1', 'service2'] }) + expect(combinedStats.nounCount).toBe(3) + expect(combinedStats.verbCount).toBe(2) + expect(combinedStats.metadataCount).toBe(3) + }) }) }) \ No newline at end of file