**feat(core, storage, tests): add service-level statistics tracking and storage adapter enhancements**

- **Core**: Enhanced `getStatistics` function to support `service` and `service[]` filters, enabling statistics breakdown by service. Modified return structure to include `serviceBreakdown` for detailed insights.
- **Storage**: Implemented a new `BaseStorageAdapter` abstract class to centralize statistics-related functionality, such as incrementing/decrementing counters and updating HNSW index size. Refactored all storage adapters (`FileSystemStorage`, `S3CompatibleStorage`, `MemoryStorage`, `OPFSStorage`) to extend `BaseStorageAdapter`, ensuring consistent statistics tracking.
- **Tests**: Added new test cases in `statistics.test.ts` to validate service-level statistics tracking, breakdown accuracy, and multi-service filtering.

**Purpose**: Improve insight into data trends by tracking service-specific usage in statistics. Enhance maintainability and consistency through storage adapter centralization and robust testing.

src/storage/adapters/baseStorageAdapter.ts

@@ -0,0 +1,158 @@
+/**
+ * Base Storage Adapter
+ * Provides common functionality for all storage adapters, including statistics tracking
+ */
+
+import { StatisticsData, StorageAdapter } from '../../coreTypes.js'
+
+/**
+ * Base class for storage adapters that implements statistics tracking
+ */
+export abstract class BaseStorageAdapter implements StorageAdapter {
+  // Abstract methods that must be implemented by subclasses
+  abstract init(): Promise<void>
+  abstract saveNoun(noun: any): Promise<void>
+  abstract getNoun(id: string): Promise<any | null>
+  abstract getAllNouns(): Promise<any[]>
+  abstract getNounsByNounType(nounType: string): Promise<any[]>
+  abstract deleteNoun(id: string): Promise<void>
+  abstract saveVerb(verb: any): Promise<void>
+  abstract getVerb(id: string): Promise<any | null>
+  abstract getAllVerbs(): Promise<any[]>
+  abstract getVerbsBySource(sourceId: string): Promise<any[]>
+  abstract getVerbsByTarget(targetId: string): Promise<any[]>
+  abstract getVerbsByType(type: string): Promise<any[]>
+  abstract deleteVerb(id: string): Promise<void>
+  abstract saveMetadata(id: string, metadata: any): Promise<void>
+  abstract getMetadata(id: string): Promise<any | null>
+  abstract clear(): Promise<void>
+  abstract getStorageStatus(): Promise<{
+    type: string
+    used: number
+    quota: number | null
+    details?: Record<string, any>
+  }>
+
+  // Statistics-specific methods
+  protected abstract saveStatisticsData(statistics: StatisticsData): Promise<void>
+  protected abstract getStatisticsData(): Promise<StatisticsData | null>
+
+  /**
+   * Save statistics data
+   * @param statistics The statistics data to save
+   */
+  async saveStatistics(statistics: StatisticsData): Promise<void> {
+    await this.saveStatisticsData(statistics)
+  }
+
+  /**
+   * Get statistics data
+   * @returns Promise that resolves to the statistics data
+   */
+  async getStatistics(): Promise<StatisticsData | null> {
+    return await this.getStatisticsData()
+  }
+
+  /**
+   * Increment a statistic counter
+   * @param type The type of statistic to increment ('noun', 'verb', 'metadata')
+   * @param service The service that inserted the data
+   * @param amount The amount to increment by (default: 1)
+   */
+  async incrementStatistic(
+    type: 'noun' | 'verb' | 'metadata',
+    service: string,
+    amount: number = 1
+  ): Promise<void> {
+    // Get current statistics or create default if not exists
+    let statistics = await this.getStatisticsData()
+    if (!statistics) {
+      statistics = this.createDefaultStatistics()
+    }
+
+    // Increment the appropriate counter
+    const counterMap = {
+      noun: statistics.nounCount,
+      verb: statistics.verbCount,
+      metadata: statistics.metadataCount
+    }
+
+    const counter = counterMap[type]
+    counter[service] = (counter[service] || 0) + amount
+
+    // Update timestamp
+    statistics.lastUpdated = new Date().toISOString()
+
+    // Save updated statistics
+    await this.saveStatisticsData(statistics)
+  }
+
+  /**
+   * Decrement a statistic counter
+   * @param type The type of statistic to decrement ('noun', 'verb', 'metadata')
+   * @param service The service that inserted the data
+   * @param amount The amount to decrement by (default: 1)
+   */
+  async decrementStatistic(
+    type: 'noun' | 'verb' | 'metadata',
+    service: string,
+    amount: number = 1
+  ): Promise<void> {
+    // Get current statistics or create default if not exists
+    let statistics = await this.getStatisticsData()
+    if (!statistics) {
+      statistics = this.createDefaultStatistics()
+    }
+
+    // Decrement the appropriate counter
+    const counterMap = {
+      noun: statistics.nounCount,
+      verb: statistics.verbCount,
+      metadata: statistics.metadataCount
+    }
+
+    const counter = counterMap[type]
+    counter[service] = Math.max(0, (counter[service] || 0) - amount)
+
+    // Update timestamp
+    statistics.lastUpdated = new Date().toISOString()
+
+    // Save updated statistics
+    await this.saveStatisticsData(statistics)
+  }
+
+  /**
+   * Update the HNSW index size statistic
+   * @param size The new size of the HNSW index
+   */
+  async updateHnswIndexSize(size: number): Promise<void> {
+    // Get current statistics or create default if not exists
+    let statistics = await this.getStatisticsData()
+    if (!statistics) {
+      statistics = this.createDefaultStatistics()
+    }
+
+    // Update HNSW index size
+    statistics.hnswIndexSize = size
+
+    // Update timestamp
+    statistics.lastUpdated = new Date().toISOString()
+
+    // Save updated statistics
+    await this.saveStatisticsData(statistics)
+  }
+
+  /**
+   * Create default statistics data
+   * @returns Default statistics data
+   */
+  protected createDefaultStatistics(): StatisticsData {
+    return {
+      nounCount: {},
+      verbCount: {},
+      metadataCount: {},
+      hnswIndexSize: 0,
+      lastUpdated: new Date().toISOString()
+    }
+  }
+}

src/storage/adapters/fileSystemStorage.ts

@@ -3,8 +3,8 @@
  * File system storage adapter for Node.js environments
  */
 
-import { GraphVerb, HNSWNoun } from '../../coreTypes.js'
+import { GraphVerb, HNSWNoun, StatisticsData } from '../../coreTypes.js'
-import { BaseStorage, NOUNS_DIR, VERBS_DIR, METADATA_DIR, INDEX_DIR } from '../baseStorage.js'
+import { BaseStorage, NOUNS_DIR, VERBS_DIR, METADATA_DIR, INDEX_DIR, STATISTICS_KEY } from '../baseStorage.js'
 
 // Type aliases for better readability
 type HNSWNode = HNSWNoun
@@ -552,4 +552,41 @@ export class FileSystemStorage extends BaseStorage {
       }
     }
   }
+
+  /**
+   * Save statistics data to storage
+   * @param statistics The statistics data to save
+   */
+  protected async saveStatisticsData(statistics: StatisticsData): Promise<void> {
+    await this.ensureInitialized()
+
+    try {
+      const filePath = path.join(this.indexDir, `${STATISTICS_KEY}.json`)
+      await fs.promises.writeFile(filePath, JSON.stringify(statistics, null, 2))
+    } catch (error) {
+      console.error('Failed to save statistics data:', error)
+      throw new Error(`Failed to save statistics data: ${error}`)
+    }
+  }
+
+  /**
+   * Get statistics data from storage
+   * @returns Promise that resolves to the statistics data or null if not found
+   */
+  protected async getStatisticsData(): Promise<StatisticsData | null> {
+    await this.ensureInitialized()
+
+    try {
+      const filePath = path.join(this.indexDir, `${STATISTICS_KEY}.json`)
+      const data = await fs.promises.readFile(filePath, 'utf-8')
+      return JSON.parse(data)
+    } catch (error: any) {
+      // If the file doesn't exist, return null
+      if (error.code === 'ENOENT') {
+        return null
+      }
+      console.error('Error getting statistics data:', error)
+      throw error
+    }
+  }
 }

src/storage/adapters/memoryStorage.ts

@@ -3,8 +3,8 @@
  * In-memory storage adapter for environments where persistent storage is not available or needed
  */
 
-import { GraphVerb, HNSWNoun } from '../../coreTypes.js'
+import { GraphVerb, HNSWNoun, StatisticsData } from '../../coreTypes.js'
-import { BaseStorage } from '../baseStorage.js'
+import { BaseStorage, STATISTICS_KEY } from '../baseStorage.js'
 
 /**
  * Type alias for HNSWNoun to make the code more readable
@@ -25,6 +25,7 @@ export class MemoryStorage extends BaseStorage {
   private nouns: Map<string, HNSWNode> = new Map()
   private verbs: Map<string, Edge> = new Map()
   private metadata: Map<string, any> = new Map()
+  private statistics: StatisticsData | null = null
 
   constructor() {
     super()
@@ -321,4 +322,38 @@ export class MemoryStorage extends BaseStorage {
       }
     }
   }
+
+  /**
+   * Save statistics data to storage
+   * @param statistics The statistics data to save
+   */
+  protected async saveStatisticsData(statistics: StatisticsData): Promise<void> {
+    // Create a deep copy to avoid reference issues
+    this.statistics = {
+      nounCount: {...statistics.nounCount},
+      verbCount: {...statistics.verbCount},
+      metadataCount: {...statistics.metadataCount},
+      hnswIndexSize: statistics.hnswIndexSize,
+      lastUpdated: statistics.lastUpdated
+    }
+  }
+
+  /**
+   * Get statistics data from storage
+   * @returns Promise that resolves to the statistics data or null if not found
+   */
+  protected async getStatisticsData(): Promise<StatisticsData | null> {
+    if (!this.statistics) {
+      return null
+    }
+
+    // Return a deep copy to avoid reference issues
+    return {
+      nounCount: {...this.statistics.nounCount},
+      verbCount: {...this.statistics.verbCount},
+      metadataCount: {...this.statistics.metadataCount},
+      hnswIndexSize: this.statistics.hnswIndexSize,
+      lastUpdated: this.statistics.lastUpdated
+    }
+  }
 }

src/storage/adapters/opfsStorage.ts

@@ -3,8 +3,8 @@
  * Provides persistent storage for the vector database using the Origin Private File System API
  */
 
-import {GraphVerb, HNSWNoun} from '../../coreTypes.js'
+import {GraphVerb, HNSWNoun, StatisticsData} from '../../coreTypes.js'
-import {BaseStorage, NOUNS_DIR, VERBS_DIR, METADATA_DIR, INDEX_DIR} from '../baseStorage.js'
+import {BaseStorage, NOUNS_DIR, VERBS_DIR, METADATA_DIR, INDEX_DIR, STATISTICS_KEY} from '../baseStorage.js'
 import '../../types/fileSystemTypes.js'
 
 /**
@@ -37,6 +37,7 @@ export class OPFSStorage extends BaseStorage {
     private isAvailable = false
     private isPersistentRequested = false
     private isPersistentGranted = false
+    private statistics: StatisticsData | null = null
 
     constructor() {
         super()
@@ -688,4 +689,106 @@ export class OPFSStorage extends BaseStorage {
             }
         }
     }
+
+    /**
+     * Save statistics data to storage
+     * @param statistics The statistics data to save
+     */
+    protected async saveStatisticsData(statistics: StatisticsData): Promise<void> {
+        // Create a deep copy to avoid reference issues
+        this.statistics = {
+            nounCount: {...statistics.nounCount},
+            verbCount: {...statistics.verbCount},
+            metadataCount: {...statistics.metadataCount},
+            hnswIndexSize: statistics.hnswIndexSize,
+            lastUpdated: statistics.lastUpdated
+        }
+
+        try {
+            // Ensure the root directory is initialized
+            await this.ensureInitialized()
+
+            // Get or create the index directory
+            if (!this.indexDir) {
+                throw new Error('Index directory not initialized')
+            }
+
+            // Create a file for the statistics data
+            const fileHandle = await this.indexDir.getFileHandle('statistics.json', {
+                create: true
+            })
+
+            // Create a writable stream
+            const writable = await fileHandle.createWritable()
+
+            // Write the statistics data to the file
+            await writable.write(JSON.stringify(this.statistics, null, 2))
+
+            // Close the stream
+            await writable.close()
+        } catch (error) {
+            console.error('Failed to save statistics data:', error)
+            throw new Error(`Failed to save statistics data: ${error}`)
+        }
+    }
+
+    /**
+     * Get statistics data from storage
+     * @returns Promise that resolves to the statistics data or null if not found
+     */
+    protected async getStatisticsData(): Promise<StatisticsData | null> {
+        // If we have cached statistics, return a deep copy
+        if (this.statistics) {
+            return {
+                nounCount: {...this.statistics.nounCount},
+                verbCount: {...this.statistics.verbCount},
+                metadataCount: {...this.statistics.metadataCount},
+                hnswIndexSize: this.statistics.hnswIndexSize,
+                lastUpdated: this.statistics.lastUpdated
+            }
+        }
+
+        try {
+            // Ensure the root directory is initialized
+            await this.ensureInitialized()
+
+            if (!this.indexDir) {
+                throw new Error('Index directory not initialized')
+            }
+
+            try {
+                // Try to get the statistics file
+                const fileHandle = await this.indexDir.getFileHandle('statistics.json', {
+                    create: false
+                })
+
+                // Get the file data
+                const file = await fileHandle.getFile()
+                const text = await file.text()
+
+                // Parse the statistics data
+                this.statistics = JSON.parse(text)
+
+                // Return a deep copy
+                if (this.statistics) {
+                    return {
+                        nounCount: {...this.statistics.nounCount},
+                        verbCount: {...this.statistics.verbCount},
+                        metadataCount: {...this.statistics.metadataCount},
+                        hnswIndexSize: this.statistics.hnswIndexSize,
+                        lastUpdated: this.statistics.lastUpdated
+                    }
+                }
+                
+                // If statistics is null, return default statistics
+                return this.createDefaultStatistics()
+            } catch (error) {
+                // If the file doesn't exist, return null
+                return null
+            }
+        } catch (error) {
+            console.error('Failed to get statistics data:', error)
+            throw new Error(`Failed to get statistics data: ${error}`)
+        }
+    }
 }

src/storage/adapters/s3CompatibleStorage.ts

@@ -4,8 +4,8 @@
  * including Amazon S3, Cloudflare R2, and Google Cloud Storage
  */
 
-import { GraphVerb, HNSWNoun } from '../../coreTypes.js'
+import { GraphVerb, HNSWNoun, StatisticsData } from '../../coreTypes.js'
-import { BaseStorage, NOUNS_DIR, VERBS_DIR, METADATA_DIR, INDEX_DIR } from '../baseStorage.js'
+import { BaseStorage, NOUNS_DIR, VERBS_DIR, METADATA_DIR, INDEX_DIR, STATISTICS_KEY } from '../baseStorage.js'
 
 // Type aliases for better readability
 type HNSWNode = HNSWNoun
@@ -56,6 +56,9 @@ export class S3CompatibleStorage extends BaseStorage {
   private verbPrefix: string
   private metadataPrefix: string
   private indexPrefix: string
+  
+  // Statistics caching for better performance
+  private statisticsCache: StatisticsData | null = null
 
   /**
    * Initialize the storage adapter
@@ -1014,4 +1017,113 @@ export class S3CompatibleStorage extends BaseStorage {
       }
     }
   }
+  
+  /**
+   * Save statistics data to storage
+   * @param statistics The statistics data to save
+   */
+  protected async saveStatisticsData(statistics: StatisticsData): Promise<void> {
+    await this.ensureInitialized()
+
+    try {
+      // Update the cache with a deep copy to avoid reference issues
+      this.statisticsCache = {
+        nounCount: {...statistics.nounCount},
+        verbCount: {...statistics.verbCount},
+        metadataCount: {...statistics.metadataCount},
+        hnswIndexSize: statistics.hnswIndexSize,
+        lastUpdated: statistics.lastUpdated
+      }
+      
+      // Import the PutObjectCommand only when needed
+      const { PutObjectCommand } = await import('@aws-sdk/client-s3')
+
+      const key = `${this.indexPrefix}${STATISTICS_KEY}.json`
+      const body = JSON.stringify(statistics, null, 2)
+      
+      // Save the statistics to S3-compatible storage
+      await this.s3Client!.send(
+        new PutObjectCommand({
+          Bucket: this.bucketName,
+          Key: key,
+          Body: body,
+          ContentType: 'application/json'
+        })
+      )
+    } catch (error) {
+      console.error('Failed to save statistics data:', error)
+      throw new Error(`Failed to save statistics data: ${error}`)
+    }
+  }
+
+  /**
+   * Get statistics data from storage
+   * @returns Promise that resolves to the statistics data or null if not found
+   */
+  protected async getStatisticsData(): Promise<StatisticsData | null> {
+    await this.ensureInitialized()
+
+    // If we have cached statistics, return a deep copy
+    if (this.statisticsCache) {
+      return {
+        nounCount: {...this.statisticsCache.nounCount},
+        verbCount: {...this.statisticsCache.verbCount},
+        metadataCount: {...this.statisticsCache.metadataCount},
+        hnswIndexSize: this.statisticsCache.hnswIndexSize,
+        lastUpdated: this.statisticsCache.lastUpdated
+      }
+    }
+
+    try {
+      // Import the GetObjectCommand only when needed
+      const { GetObjectCommand } = await import('@aws-sdk/client-s3')
+
+      const key = `${this.indexPrefix}${STATISTICS_KEY}.json`
+      
+      // Try to get the statistics from the index directory
+      const response = await this.s3Client!.send(
+        new GetObjectCommand({
+          Bucket: this.bucketName,
+          Key: key
+        })
+      )
+
+      // Check if response is null or undefined
+      if (!response || !response.Body) {
+        return null
+      }
+
+      // Convert the response body to a string
+      const bodyContents = await response.Body.transformToString()
+      
+      // Parse the JSON string and update the cache
+      const statistics = JSON.parse(bodyContents)
+      
+      // Update the cache with a deep copy
+      this.statisticsCache = {
+        nounCount: {...statistics.nounCount},
+        verbCount: {...statistics.verbCount},
+        metadataCount: {...statistics.metadataCount},
+        hnswIndexSize: statistics.hnswIndexSize,
+        lastUpdated: statistics.lastUpdated
+      }
+      
+      return statistics
+    } catch (error: any) {
+      // Check if this is a "NoSuchKey" error (object doesn't exist)
+      if (
+        error.name === 'NoSuchKey' || 
+        (error.message && (
+          error.message.includes('NoSuchKey') || 
+          error.message.includes('not found') || 
+          error.message.includes('does not exist')
+        ))
+      ) {
+        return null
+      }
+      
+      console.error('Error getting statistics data:', error)
+      throw error
+    }
+  }
 }

src/storage/baseStorage.ts

@@ -3,19 +3,21 @@
  * Provides common functionality for all storage adapters
  */
 
-import { GraphVerb, HNSWNoun, StorageAdapter } from '../coreTypes.js'
+import { GraphVerb, HNSWNoun, StatisticsData } from '../coreTypes.js'
+import { BaseStorageAdapter } from './adapters/baseStorageAdapter.js'
 
 // Common directory/prefix names
 export const NOUNS_DIR = 'nouns'
 export const VERBS_DIR = 'verbs'
 export const METADATA_DIR = 'metadata'
 export const INDEX_DIR = 'index'
+export const STATISTICS_KEY = 'statistics'
 
 /**
  * Base storage adapter that implements common functionality
  * This is an abstract class that should be extended by specific storage adapters
  */
-export abstract class BaseStorage implements StorageAdapter {
+export abstract class BaseStorage extends BaseStorageAdapter {
   protected isInitialized = false
 
   /**
@@ -245,4 +247,18 @@ export abstract class BaseStorage implements StorageAdapter {
     }
     return obj
   }
+
+  /**
+   * Save statistics data to storage
+   * This method should be implemented by each specific adapter
+   * @param statistics The statistics data to save
+   */
+  protected abstract saveStatisticsData(statistics: StatisticsData): Promise<void>
+
+  /**
+   * Get statistics data from storage
+   * This method should be implemented by each specific adapter
+   * @returns Promise that resolves to the statistics data or null if not found
+   */
+  protected abstract getStatisticsData(): Promise<StatisticsData | null>
 }

src/utils/statistics.ts

@@ -9,21 +9,34 @@ import { BrainyData } from '../brainyData.js'
  * This function provides access to statistics at the root level of the library
  * 
  * @param instance A BrainyData instance to get statistics from
+ * @param options Additional options for retrieving statistics
  * @returns Object containing counts of nouns, verbs, metadata entries, and HNSW index size
  * @throws Error if the instance is not provided or if statistics retrieval fails
  */
-export async function getStatistics(instance: BrainyData): Promise<{
+export async function getStatistics(
+    instance: BrainyData,
+    options: {
+        service?: string | string[] // Filter statistics by service(s)
+    } = {}
+): Promise<{
     nounCount: number
     verbCount: number
     metadataCount: number
     hnswIndexSize: number
+    serviceBreakdown?: {
+        [service: string]: {
+            nounCount: number
+            verbCount: number
+            metadataCount: number
+        }
+    }
 }> {
     if (!instance) {
         throw new Error('BrainyData instance must be provided to getStatistics')
     }
     
     try {
-        return await instance.getStatistics()
+        return await instance.getStatistics(options)
     } catch (error) {
         console.error('Failed to get statistics:', error)
         throw new Error(`Failed to get statistics: ${error}`)

tests/statistics.test.ts

@@ -72,5 +72,54 @@ describe('Brainy Statistics Functionality', () => {
       // Verify they match
       expect(functionStats).toEqual(instanceStats)
     })
+
+    it('should track statistics by service', async () => {
+      // Create a BrainyData instance
+      const data = new brainy.BrainyData({
+        dimensions: 3,
+        metric: 'euclidean'
+      })
+
+      await data.init()
+      await data.clear() // Clear any existing data
+
+      // Add data from different services
+      await data.add([1, 0, 0], { id: 'v1', label: 'service1-item' }, { service: 'service1' })
+      await data.add([0, 1, 0], { id: 'v2', label: 'service1-item' }, { service: 'service1' })
+      await data.add([0, 0, 1], { id: 'v3', label: 'service2-item' }, { service: 'service2' })
+      
+      // Add verbs from different services
+      await data.addVerb('v1', 'v2', undefined, { type: 'related_to', service: 'service1' })
+      await data.addVerb('v2', 'v3', undefined, { type: 'related_to', service: 'service2' })
+
+      // Get statistics for all services
+      const allStats = await data.getStatistics()
+      
+      // Verify total counts
+      expect(allStats.nounCount).toBe(3)
+      expect(allStats.verbCount).toBe(2)
+      expect(allStats.metadataCount).toBe(3)
+      
+      // Verify service breakdown exists
+      expect(allStats.serviceBreakdown).toBeDefined()
+      
+      // Verify service1 statistics
+      const service1Stats = await data.getStatistics({ service: 'service1' })
+      expect(service1Stats.nounCount).toBe(2)
+      expect(service1Stats.verbCount).toBe(1)
+      expect(service1Stats.metadataCount).toBe(2)
+      
+      // Verify service2 statistics
+      const service2Stats = await data.getStatistics({ service: 'service2' })
+      expect(service2Stats.nounCount).toBe(1)
+      expect(service2Stats.verbCount).toBe(1)
+      expect(service2Stats.metadataCount).toBe(1)
+      
+      // Verify multiple services filter
+      const combinedStats = await data.getStatistics({ service: ['service1', 'service2'] })
+      expect(combinedStats.nounCount).toBe(3)
+      expect(combinedStats.verbCount).toBe(2)
+      expect(combinedStats.metadataCount).toBe(3)
+    })
   })
 })