feat: add comprehensive per-service statistics tracking

Add full support for tracking and analyzing data by service in multi-tenant deployments.

## Features Added

- **Service Statistics Tracking**: Track nouns, verbs, and metadata counts per service
- **Service Activity Monitoring**: Track first/last activity timestamps and operation counts
- **New API Methods**:
  - `listServices()`: List all services with their statistics and status
  - `getServiceStatistics(service)`: Get detailed stats for a specific service
  - Enhanced `getStatistics()` with service filtering and breakdown

- **Service Filtering**: Filter search results and queries by service
- **Storage Enhancements**: BaseStorageAdapter tracks service activity with timestamps
- **Type Definitions**: Added ServiceStatistics interface and extended StatisticsData

## Implementation Details

- Services automatically tracked via defaultService config or per-operation override
- Service status detection (active/inactive/read-only) based on activity
- Memory-efficient tracking at statistics level, not per noun/verb
- Backward compatible - existing data tracked under 'default' service

## Documentation

- Comprehensive guide in docs/guides/per-service-statistics.md
- Examples for multi-tenant apps, health monitoring, and auditing
- API reference and migration guide included

## Testing

- Full test suite in tests/service-statistics.test.ts
- Coverage of all new methods and filtering capabilities

This enables better observability, debugging, and management of multi-service Brainy deployments, addressing the need to track individual service performance when multiple services share storage.
This commit is contained in:
David Snelling 2025-08-06 10:17:28 -07:00
parent e838327a22
commit d2ddb9199e
7 changed files with 1232 additions and 4 deletions

View file

@ -138,7 +138,17 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
verbCount: { ...statistics.verbCount },
metadataCount: { ...statistics.metadataCount },
hnswIndexSize: statistics.hnswIndexSize,
lastUpdated: statistics.lastUpdated
lastUpdated: statistics.lastUpdated,
// Include serviceActivity if present
...(statistics.serviceActivity && {
serviceActivity: Object.fromEntries(
Object.entries(statistics.serviceActivity).map(([k, v]) => [k, {...v}])
)
}),
// Include services if present
...(statistics.services && {
services: statistics.services.map(s => ({...s}))
})
}
// Schedule a batch update instead of saving immediately
@ -263,7 +273,17 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
verbCount: { ...statistics.verbCount },
metadataCount: { ...statistics.metadataCount },
hnswIndexSize: statistics.hnswIndexSize,
lastUpdated: statistics.lastUpdated
lastUpdated: statistics.lastUpdated,
// Include serviceActivity if present
...(statistics.serviceActivity && {
serviceActivity: Object.fromEntries(
Object.entries(statistics.serviceActivity).map(([k, v]) => [k, {...v}])
)
}),
// Include services if present
...(statistics.services && {
services: statistics.services.map(s => ({...s}))
})
}
}
@ -277,6 +297,9 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
const counter = counterMap[type]
counter[service] = (counter[service] || 0) + amount
// Track service activity
this.trackServiceActivity(service, 'add')
// Update timestamp
this.statisticsCache!.lastUpdated = new Date().toISOString()
@ -284,6 +307,41 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
this.scheduleBatchUpdate()
}
/**
* Track service activity (first/last activity, operation counts)
* @param service The service name
* @param operation The operation type
*/
protected trackServiceActivity(
service: string,
operation: 'add' | 'update' | 'delete'
): void {
if (!this.statisticsCache) {
return
}
// Initialize serviceActivity if it doesn't exist
if (!this.statisticsCache.serviceActivity) {
this.statisticsCache.serviceActivity = {}
}
const now = new Date().toISOString()
const activity = this.statisticsCache.serviceActivity[service]
if (!activity) {
// First activity for this service
this.statisticsCache.serviceActivity[service] = {
firstActivity: now,
lastActivity: now,
totalOperations: 1
}
} else {
// Update existing activity
activity.lastActivity = now
activity.totalOperations++
}
}
/**
* Decrement a statistic counter
* @param type The type of statistic to decrement ('noun', 'verb', 'metadata')
@ -309,7 +367,17 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
verbCount: { ...statistics.verbCount },
metadataCount: { ...statistics.metadataCount },
hnswIndexSize: statistics.hnswIndexSize,
lastUpdated: statistics.lastUpdated
lastUpdated: statistics.lastUpdated,
// Include serviceActivity if present
...(statistics.serviceActivity && {
serviceActivity: Object.fromEntries(
Object.entries(statistics.serviceActivity).map(([k, v]) => [k, {...v}])
)
}),
// Include services if present
...(statistics.services && {
services: statistics.services.map(s => ({...s}))
})
}
}
@ -323,6 +391,9 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
const counter = counterMap[type]
counter[service] = Math.max(0, (counter[service] || 0) - amount)
// Track service activity
this.trackServiceActivity(service, 'delete')
// Update timestamp
this.statisticsCache!.lastUpdated = new Date().toISOString()
@ -349,7 +420,17 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
verbCount: { ...statistics.verbCount },
metadataCount: { ...statistics.metadataCount },
hnswIndexSize: statistics.hnswIndexSize,
lastUpdated: statistics.lastUpdated
lastUpdated: statistics.lastUpdated,
// Include serviceActivity if present
...(statistics.serviceActivity && {
serviceActivity: Object.fromEntries(
Object.entries(statistics.serviceActivity).map(([k, v]) => [k, {...v}])
)
}),
// Include services if present
...(statistics.services && {
services: statistics.services.map(s => ({...s}))
})
}
}

View file

@ -622,6 +622,16 @@ export class MemoryStorage extends BaseStorage {
metadataCount: {...statistics.metadataCount},
hnswIndexSize: statistics.hnswIndexSize,
lastUpdated: statistics.lastUpdated,
// Include serviceActivity if present
...(statistics.serviceActivity && {
serviceActivity: Object.fromEntries(
Object.entries(statistics.serviceActivity).map(([k, v]) => [k, {...v}])
)
}),
// Include services if present
...(statistics.services && {
services: statistics.services.map(s => ({...s}))
}),
// Include distributedConfig if present
...(statistics.distributedConfig && {
distributedConfig: JSON.parse(JSON.stringify(statistics.distributedConfig))
@ -648,6 +658,16 @@ export class MemoryStorage extends BaseStorage {
metadataCount: {...this.statistics.metadataCount},
hnswIndexSize: this.statistics.hnswIndexSize,
lastUpdated: this.statistics.lastUpdated,
// Include serviceActivity if present
...(this.statistics.serviceActivity && {
serviceActivity: Object.fromEntries(
Object.entries(this.statistics.serviceActivity).map(([k, v]) => [k, {...v}])
)
}),
// Include services if present
...(this.statistics.services && {
services: this.statistics.services.map(s => ({...s}))
}),
// Include distributedConfig if present
...(this.statistics.distributedConfig && {
distributedConfig: JSON.parse(JSON.stringify(this.statistics.distributedConfig))