soulcraft/brainy
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

**feat(core, storage, tests): add service-level statistics tracking and storage adapter enhancements**

Browse source 
- **Core**: Enhanced `getStatistics` function to support `service` and `service[]` filters, enabling statistics breakdown by service. Modified return structure to include `serviceBreakdown` for detailed insights.
- **Storage**: Implemented a new `BaseStorageAdapter` abstract class to centralize statistics-related functionality, such as incrementing/decrementing counters and updating HNSW index size. Refactored all storage adapters (`FileSystemStorage`, `S3CompatibleStorage`, `MemoryStorage`, `OPFSStorage`) to extend `BaseStorageAdapter`, ensuring consistent statistics tracking.
- **Tests**: Added new test cases in `statistics.test.ts` to validate service-level statistics tracking, breakdown accuracy, and multi-service filtering.

**Purpose**: Improve insight into data trends by tracking service-specific usage in statistics. Enhance maintainability and consistency through storage adapter centralization and robust testing.
This commit is contained in:
David Snelling 2025-07-24 11:35:52 -07:00
parent 3c95ee011d
commit 85e5a6bfb9
8 changed files with 535 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

158
src/storage/adapters/baseStorageAdapter.ts Normal file
View file

@ -0,0 +1,158 @@
/**
* Base Storage Adapter
* Provides common functionality for all storage adapters, including statistics tracking
*/
import { StatisticsData, StorageAdapter } from '../../coreTypes.js'
/**
* Base class for storage adapters that implements statistics tracking
*/
export abstract class BaseStorageAdapter implements StorageAdapter {
// Abstract methods that must be implemented by subclasses
abstract init(): Promise<void>
abstract saveNoun(noun: any): Promise<void>
abstract getNoun(id: string): Promise<any | null>
abstract getAllNouns(): Promise<any[]>
abstract getNounsByNounType(nounType: string): Promise<any[]>
abstract deleteNoun(id: string): Promise<void>
abstract saveVerb(verb: any): Promise<void>
abstract getVerb(id: string): Promise<any | null>
abstract getAllVerbs(): Promise<any[]>
abstract getVerbsBySource(sourceId: string): Promise<any[]>
abstract getVerbsByTarget(targetId: string): Promise<any[]>
abstract getVerbsByType(type: string): Promise<any[]>
abstract deleteVerb(id: string): Promise<void>
abstract saveMetadata(id: string, metadata: any): Promise<void>
abstract getMetadata(id: string): Promise<any | null>
abstract clear(): Promise<void>
abstract getStorageStatus(): Promise<{
type: string
used: number
quota: number | null
details?: Record<string, any>
}>
// Statistics-specific methods
protected abstract saveStatisticsData(statistics: StatisticsData): Promise<void>
protected abstract getStatisticsData(): Promise<StatisticsData | null>
/**
* Save statistics data
* @param statistics The statistics data to save
*/
async saveStatistics(statistics: StatisticsData): Promise<void> {
await this.saveStatisticsData(statistics)
}
/**
* Get statistics data
* @returns Promise that resolves to the statistics data
*/
async getStatistics(): Promise<StatisticsData | null> {
return await this.getStatisticsData()
}
/**
* Increment a statistic counter
* @param type The type of statistic to increment ('noun', 'verb', 'metadata')
* @param service The service that inserted the data
* @param amount The amount to increment by (default: 1)
*/
async incrementStatistic(
type: 'noun' | 'verb' | 'metadata',
service: string,
amount: number = 1
): Promise<void> {
// Get current statistics or create default if not exists
let statistics = await this.getStatisticsData()
if (!statistics) {
statistics = this.createDefaultStatistics()
}
// Increment the appropriate counter
const counterMap = {
noun: statistics.nounCount,
verb: statistics.verbCount,
metadata: statistics.metadataCount
}
const counter = counterMap[type]
counter[service] = (counter[service] || 0) + amount
// Update timestamp
statistics.lastUpdated = new Date().toISOString()
// Save updated statistics
await this.saveStatisticsData(statistics)
}
/**
* Decrement a statistic counter
* @param type The type of statistic to decrement ('noun', 'verb', 'metadata')
* @param service The service that inserted the data
* @param amount The amount to decrement by (default: 1)
*/
async decrementStatistic(
type: 'noun' | 'verb' | 'metadata',
service: string,
amount: number = 1
): Promise<void> {
// Get current statistics or create default if not exists
let statistics = await this.getStatisticsData()
if (!statistics) {
statistics = this.createDefaultStatistics()
}
// Decrement the appropriate counter
const counterMap = {
noun: statistics.nounCount,
verb: statistics.verbCount,
metadata: statistics.metadataCount
}
const counter = counterMap[type]
counter[service] = Math.max(0, (counter[service] || 0) - amount)
// Update timestamp
statistics.lastUpdated = new Date().toISOString()
// Save updated statistics
await this.saveStatisticsData(statistics)
}
/**
* Update the HNSW index size statistic
* @param size The new size of the HNSW index
*/
async updateHnswIndexSize(size: number): Promise<void> {
// Get current statistics or create default if not exists
let statistics = await this.getStatisticsData()
if (!statistics) {
statistics = this.createDefaultStatistics()
}
// Update HNSW index size
statistics.hnswIndexSize = size
// Update timestamp
statistics.lastUpdated = new Date().toISOString()
// Save updated statistics
await this.saveStatisticsData(statistics)
}
/**
* Create default statistics data
* @returns Default statistics data
*/
protected createDefaultStatistics(): StatisticsData {
return {
nounCount: {},
verbCount: {},
metadataCount: {},
hnswIndexSize: 0,
lastUpdated: new Date().toISOString()
}
}
}

41
src/storage/adapters/fileSystemStorage.ts
View file

@ -3,8 +3,8 @@
* File system storage adapter for Node.js environments
*/
import { GraphVerb, HNSWNoun } from '../../coreTypes.js'
import { BaseStorage, NOUNS_DIR, VERBS_DIR, METADATA_DIR, INDEX_DIR } from '../baseStorage.js'
import { GraphVerb, HNSWNoun, StatisticsData } from '../../coreTypes.js'
import { BaseStorage, NOUNS_DIR, VERBS_DIR, METADATA_DIR, INDEX_DIR, STATISTICS_KEY } from '../baseStorage.js'
// Type aliases for better readability
type HNSWNode = HNSWNoun
@ -552,4 +552,41 @@ export class FileSystemStorage extends BaseStorage {
}
}
}
/**
* Save statistics data to storage
* @param statistics The statistics data to save
*/
protected async saveStatisticsData(statistics: StatisticsData): Promise<void> {
await this.ensureInitialized()
try {
const filePath = path.join(this.indexDir, `${STATISTICS_KEY}.json`)
await fs.promises.writeFile(filePath, JSON.stringify(statistics, null, 2))
} catch (error) {
console.error('Failed to save statistics data:', error)
throw new Error(`Failed to save statistics data: ${error}`)
}
}
/**
* Get statistics data from storage
* @returns Promise that resolves to the statistics data or null if not found
*/
protected async getStatisticsData(): Promise<StatisticsData | null> {
await this.ensureInitialized()
try {
const filePath = path.join(this.indexDir, `${STATISTICS_KEY}.json`)
const data = await fs.promises.readFile(filePath, 'utf-8')
return JSON.parse(data)
} catch (error: any) {
// If the file doesn't exist, return null
if (error.code === 'ENOENT') {
return null
}
console.error('Error getting statistics data:', error)
throw error
}
}
}

39
src/storage/adapters/memoryStorage.ts
View file

@ -3,8 +3,8 @@
* In-memory storage adapter for environments where persistent storage is not available or needed
*/
import { GraphVerb, HNSWNoun } from '../../coreTypes.js'
import { BaseStorage } from '../baseStorage.js'
import { GraphVerb, HNSWNoun, StatisticsData } from '../../coreTypes.js'
import { BaseStorage, STATISTICS_KEY } from '../baseStorage.js'
/**
* Type alias for HNSWNoun to make the code more readable
@ -25,6 +25,7 @@ export class MemoryStorage extends BaseStorage {
private nouns: Map<string, HNSWNode> = new Map()
private verbs: Map<string, Edge> = new Map()
private metadata: Map<string, any> = new Map()
private statistics: StatisticsData | null = null
constructor() {
super()
@ -321,4 +322,38 @@ export class MemoryStorage extends BaseStorage {
}
}
}
/**
* Save statistics data to storage
* @param statistics The statistics data to save
*/
protected async saveStatisticsData(statistics: StatisticsData): Promise<void> {
// Create a deep copy to avoid reference issues
this.statistics = {
nounCount: {...statistics.nounCount},
verbCount: {...statistics.verbCount},
metadataCount: {...statistics.metadataCount},
hnswIndexSize: statistics.hnswIndexSize,
lastUpdated: statistics.lastUpdated
}
}
/**
* Get statistics data from storage
* @returns Promise that resolves to the statistics data or null if not found
*/
protected async getStatisticsData(): Promise<StatisticsData | null> {
if (!this.statistics) {
return null
}
// Return a deep copy to avoid reference issues
return {
nounCount: {...this.statistics.nounCount},
verbCount: {...this.statistics.verbCount},
metadataCount: {...this.statistics.metadataCount},
hnswIndexSize: this.statistics.hnswIndexSize,
lastUpdated: this.statistics.lastUpdated
}
}
}

107
src/storage/adapters/opfsStorage.ts
View file

@ -3,8 +3,8 @@
* Provides persistent storage for the vector database using the Origin Private File System API
*/
import {GraphVerb, HNSWNoun} from '../../coreTypes.js'
import {BaseStorage, NOUNS_DIR, VERBS_DIR, METADATA_DIR, INDEX_DIR} from '../baseStorage.js'
import {GraphVerb, HNSWNoun, StatisticsData} from '../../coreTypes.js'
import {BaseStorage, NOUNS_DIR, VERBS_DIR, METADATA_DIR, INDEX_DIR, STATISTICS_KEY} from '../baseStorage.js'
import '../../types/fileSystemTypes.js'
/**
@ -37,6 +37,7 @@ export class OPFSStorage extends BaseStorage {
private isAvailable = false
private isPersistentRequested = false
private isPersistentGranted = false
private statistics: StatisticsData | null = null
constructor() {
super()
@ -688,4 +689,106 @@ export class OPFSStorage extends BaseStorage {
}
}
}
/**
* Save statistics data to storage
* @param statistics The statistics data to save
*/
protected async saveStatisticsData(statistics: StatisticsData): Promise<void> {
// Create a deep copy to avoid reference issues
this.statistics = {
nounCount: {...statistics.nounCount},
verbCount: {...statistics.verbCount},
metadataCount: {...statistics.metadataCount},
hnswIndexSize: statistics.hnswIndexSize,
lastUpdated: statistics.lastUpdated
}
try {
// Ensure the root directory is initialized
await this.ensureInitialized()
// Get or create the index directory
if (!this.indexDir) {
throw new Error('Index directory not initialized')
}
// Create a file for the statistics data
const fileHandle = await this.indexDir.getFileHandle('statistics.json', {
create: true
})
// Create a writable stream
const writable = await fileHandle.createWritable()
// Write the statistics data to the file
await writable.write(JSON.stringify(this.statistics, null, 2))
// Close the stream
await writable.close()
} catch (error) {
console.error('Failed to save statistics data:', error)
throw new Error(`Failed to save statistics data: ${error}`)
}
}
/**
* Get statistics data from storage
* @returns Promise that resolves to the statistics data or null if not found
*/
protected async getStatisticsData(): Promise<StatisticsData | null> {
// If we have cached statistics, return a deep copy
if (this.statistics) {
return {
nounCount: {...this.statistics.nounCount},
verbCount: {...this.statistics.verbCount},
metadataCount: {...this.statistics.metadataCount},
hnswIndexSize: this.statistics.hnswIndexSize,
lastUpdated: this.statistics.lastUpdated
}
}
try {
// Ensure the root directory is initialized
await this.ensureInitialized()
if (!this.indexDir) {
throw new Error('Index directory not initialized')
}
try {
// Try to get the statistics file
const fileHandle = await this.indexDir.getFileHandle('statistics.json', {
create: false
})
// Get the file data
const file = await fileHandle.getFile()
const text = await file.text()
// Parse the statistics data
this.statistics = JSON.parse(text)
// Return a deep copy
if (this.statistics) {
return {
nounCount: {...this.statistics.nounCount},
verbCount: {...this.statistics.verbCount},
metadataCount: {...this.statistics.metadataCount},
hnswIndexSize: this.statistics.hnswIndexSize,
lastUpdated: this.statistics.lastUpdated
}
}
// If statistics is null, return default statistics
return this.createDefaultStatistics()
} catch (error) {
// If the file doesn't exist, return null
return null
}
} catch (error) {
console.error('Failed to get statistics data:', error)
throw new Error(`Failed to get statistics data: ${error}`)
}
}
}

116
src/storage/adapters/s3CompatibleStorage.ts
View file

@ -4,8 +4,8 @@
* including Amazon S3, Cloudflare R2, and Google Cloud Storage
*/
import { GraphVerb, HNSWNoun } from '../../coreTypes.js'
import { BaseStorage, NOUNS_DIR, VERBS_DIR, METADATA_DIR, INDEX_DIR } from '../baseStorage.js'
import { GraphVerb, HNSWNoun, StatisticsData } from '../../coreTypes.js'
import { BaseStorage, NOUNS_DIR, VERBS_DIR, METADATA_DIR, INDEX_DIR, STATISTICS_KEY } from '../baseStorage.js'
// Type aliases for better readability
type HNSWNode = HNSWNoun
@ -56,6 +56,9 @@ export class S3CompatibleStorage extends BaseStorage {
private verbPrefix: string
private metadataPrefix: string
private indexPrefix: string
// Statistics caching for better performance
private statisticsCache: StatisticsData | null = null
/**
* Initialize the storage adapter
@ -1014,4 +1017,113 @@ export class S3CompatibleStorage extends BaseStorage {
}
}
}
/**
* Save statistics data to storage
* @param statistics The statistics data to save
*/
protected async saveStatisticsData(statistics: StatisticsData): Promise<void> {
await this.ensureInitialized()
try {
// Update the cache with a deep copy to avoid reference issues
this.statisticsCache = {
nounCount: {...statistics.nounCount},
verbCount: {...statistics.verbCount},
metadataCount: {...statistics.metadataCount},
hnswIndexSize: statistics.hnswIndexSize,
lastUpdated: statistics.lastUpdated
}
// Import the PutObjectCommand only when needed
const { PutObjectCommand } = await import('@aws-sdk/client-s3')
const key = `${this.indexPrefix}${STATISTICS_KEY}.json`
const body = JSON.stringify(statistics, null, 2)
// Save the statistics to S3-compatible storage
await this.s3Client!.send(
new PutObjectCommand({
Bucket: this.bucketName,
Key: key,
Body: body,
ContentType: 'application/json'
})
)
} catch (error) {
console.error('Failed to save statistics data:', error)
throw new Error(`Failed to save statistics data: ${error}`)
}
}
/**
* Get statistics data from storage
* @returns Promise that resolves to the statistics data or null if not found
*/
protected async getStatisticsData(): Promise<StatisticsData | null> {
await this.ensureInitialized()
// If we have cached statistics, return a deep copy
if (this.statisticsCache) {
return {
nounCount: {...this.statisticsCache.nounCount},
verbCount: {...this.statisticsCache.verbCount},
metadataCount: {...this.statisticsCache.metadataCount},
hnswIndexSize: this.statisticsCache.hnswIndexSize,
lastUpdated: this.statisticsCache.lastUpdated
}
}
try {
// Import the GetObjectCommand only when needed
const { GetObjectCommand } = await import('@aws-sdk/client-s3')
const key = `${this.indexPrefix}${STATISTICS_KEY}.json`
// Try to get the statistics from the index directory
const response = await this.s3Client!.send(
new GetObjectCommand({
Bucket: this.bucketName,
Key: key
})
)
// Check if response is null or undefined
if (!response || !response.Body) {
return null
}
// Convert the response body to a string
const bodyContents = await response.Body.transformToString()
// Parse the JSON string and update the cache
const statistics = JSON.parse(bodyContents)
// Update the cache with a deep copy
this.statisticsCache = {
nounCount: {...statistics.nounCount},
verbCount: {...statistics.verbCount},
metadataCount: {...statistics.metadataCount},
hnswIndexSize: statistics.hnswIndexSize,
lastUpdated: statistics.lastUpdated
}
return statistics
} catch (error: any) {
// Check if this is a "NoSuchKey" error (object doesn't exist)
if (
error.name === 'NoSuchKey' ||
(error.message && (
error.message.includes('NoSuchKey') ||
error.message.includes('not found') ||
error.message.includes('does not exist')
))
) {
return null
}
console.error('Error getting statistics data:', error)
throw error
}
}
}

20
src/storage/baseStorage.ts
View file

@ -3,19 +3,21 @@
* Provides common functionality for all storage adapters
*/
import { GraphVerb, HNSWNoun, StorageAdapter } from '../coreTypes.js'
import { GraphVerb, HNSWNoun, StatisticsData } from '../coreTypes.js'
import { BaseStorageAdapter } from './adapters/baseStorageAdapter.js'
// Common directory/prefix names
export const NOUNS_DIR = 'nouns'
export const VERBS_DIR = 'verbs'
export const METADATA_DIR = 'metadata'
export const INDEX_DIR = 'index'
export const STATISTICS_KEY = 'statistics'
/**
* Base storage adapter that implements common functionality
* This is an abstract class that should be extended by specific storage adapters
*/
export abstract class BaseStorage implements StorageAdapter {
export abstract class BaseStorage extends BaseStorageAdapter {
protected isInitialized = false
/**
@ -245,4 +247,18 @@ export abstract class BaseStorage implements StorageAdapter {
}
return obj
}
/**
* Save statistics data to storage
* This method should be implemented by each specific adapter
* @param statistics The statistics data to save
*/
protected abstract saveStatisticsData(statistics: StatisticsData): Promise<void>
/**
* Get statistics data from storage
* This method should be implemented by each specific adapter
* @returns Promise that resolves to the statistics data or null if not found
*/
protected abstract getStatisticsData(): Promise<StatisticsData | null>
}

17
src/utils/statistics.ts
View file

@ -9,21 +9,34 @@ import { BrainyData } from '../brainyData.js'
* This function provides access to statistics at the root level of the library
*
* @param instance A BrainyData instance to get statistics from
* @param options Additional options for retrieving statistics
* @returns Object containing counts of nouns, verbs, metadata entries, and HNSW index size
* @throws Error if the instance is not provided or if statistics retrieval fails
*/
export async function getStatistics(instance: BrainyData): Promise<{
export async function getStatistics(
instance: BrainyData,
options: {
service?: string | string[] // Filter statistics by service(s)
} = {}
): Promise<{
nounCount: number
verbCount: number
metadataCount: number
hnswIndexSize: number
serviceBreakdown?: {
[service: string]: {
nounCount: number
verbCount: number
metadataCount: number
}
}
}> {
if (!instance) {
throw new Error('BrainyData instance must be provided to getStatistics')
}
try {
return await instance.getStatistics()
return await instance.getStatistics(options)
} catch (error) {
console.error('Failed to get statistics:', error)
throw new Error(`Failed to get statistics: ${error}`)

49
tests/statistics.test.ts
View file

@ -72,5 +72,54 @@ describe('Brainy Statistics Functionality', () => {
// Verify they match
expect(functionStats).toEqual(instanceStats)
})
it('should track statistics by service', async () => {
// Create a BrainyData instance
const data = new brainy.BrainyData({
dimensions: 3,
metric: 'euclidean'
})
await data.init()
await data.clear() // Clear any existing data
// Add data from different services
await data.add([1, 0, 0], { id: 'v1', label: 'service1-item' }, { service: 'service1' })
await data.add([0, 1, 0], { id: 'v2', label: 'service1-item' }, { service: 'service1' })
await data.add([0, 0, 1], { id: 'v3', label: 'service2-item' }, { service: 'service2' })
// Add verbs from different services
await data.addVerb('v1', 'v2', undefined, { type: 'related_to', service: 'service1' })
await data.addVerb('v2', 'v3', undefined, { type: 'related_to', service: 'service2' })
// Get statistics for all services
const allStats = await data.getStatistics()
// Verify total counts
expect(allStats.nounCount).toBe(3)
expect(allStats.verbCount).toBe(2)
expect(allStats.metadataCount).toBe(3)
// Verify service breakdown exists
expect(allStats.serviceBreakdown).toBeDefined()
// Verify service1 statistics
const service1Stats = await data.getStatistics({ service: 'service1' })
expect(service1Stats.nounCount).toBe(2)
expect(service1Stats.verbCount).toBe(1)
expect(service1Stats.metadataCount).toBe(2)
// Verify service2 statistics
const service2Stats = await data.getStatistics({ service: 'service2' })
expect(service2Stats.nounCount).toBe(1)
expect(service2Stats.verbCount).toBe(1)
expect(service2Stats.metadataCount).toBe(1)
// Verify multiple services filter
const combinedStats = await data.getStatistics({ service: ['service1', 'service2'] })
expect(combinedStats.nounCount).toBe(3)
expect(combinedStats.verbCount).toBe(2)
expect(combinedStats.metadataCount).toBe(3)
})
})
})