feat: add comprehensive per-service statistics tracking
Add full support for tracking and analyzing data by service in multi-tenant deployments. ## Features Added - **Service Statistics Tracking**: Track nouns, verbs, and metadata counts per service - **Service Activity Monitoring**: Track first/last activity timestamps and operation counts - **New API Methods**: - `listServices()`: List all services with their statistics and status - `getServiceStatistics(service)`: Get detailed stats for a specific service - Enhanced `getStatistics()` with service filtering and breakdown - **Service Filtering**: Filter search results and queries by service - **Storage Enhancements**: BaseStorageAdapter tracks service activity with timestamps - **Type Definitions**: Added ServiceStatistics interface and extended StatisticsData ## Implementation Details - Services automatically tracked via defaultService config or per-operation override - Service status detection (active/inactive/read-only) based on activity - Memory-efficient tracking at statistics level, not per noun/verb - Backward compatible - existing data tracked under 'default' service ## Documentation - Comprehensive guide in docs/guides/per-service-statistics.md - Examples for multi-tenant apps, health monitoring, and auditing - API reference and migration guide included ## Testing - Full test suite in tests/service-statistics.test.ts - Coverage of all new methods and filtering capabilities This enables better observability, debugging, and management of multi-service Brainy deployments, addressing the need to track individual service performance when multiple services share storage.
This commit is contained in:
parent
e838327a22
commit
d2ddb9199e
7 changed files with 1232 additions and 4 deletions
|
|
@ -4177,6 +4177,154 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* List all services that have written data to the database
|
||||
* @returns Array of service statistics
|
||||
*/
|
||||
public async listServices(): Promise<import('./coreTypes.js').ServiceStatistics[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
const stats = await this.storage!.getStatistics()
|
||||
if (!stats) {
|
||||
return []
|
||||
}
|
||||
|
||||
// Get unique service names from all counters
|
||||
const services = new Set<string>()
|
||||
Object.keys(stats.nounCount).forEach(s => services.add(s))
|
||||
Object.keys(stats.verbCount).forEach(s => services.add(s))
|
||||
Object.keys(stats.metadataCount).forEach(s => services.add(s))
|
||||
|
||||
// Build service statistics for each service
|
||||
const result: import('./coreTypes.js').ServiceStatistics[] = []
|
||||
|
||||
for (const service of services) {
|
||||
const serviceStats: import('./coreTypes.js').ServiceStatistics = {
|
||||
name: service,
|
||||
totalNouns: stats.nounCount[service] || 0,
|
||||
totalVerbs: stats.verbCount[service] || 0,
|
||||
totalMetadata: stats.metadataCount[service] || 0
|
||||
}
|
||||
|
||||
// Add activity timestamps if available
|
||||
if (stats.serviceActivity && stats.serviceActivity[service]) {
|
||||
const activity = stats.serviceActivity[service]
|
||||
serviceStats.firstActivity = activity.firstActivity
|
||||
serviceStats.lastActivity = activity.lastActivity
|
||||
serviceStats.operations = {
|
||||
adds: activity.totalOperations,
|
||||
updates: 0,
|
||||
deletes: 0
|
||||
}
|
||||
}
|
||||
|
||||
// Determine status based on recent activity
|
||||
if (serviceStats.lastActivity) {
|
||||
const lastActivityTime = new Date(serviceStats.lastActivity).getTime()
|
||||
const now = Date.now()
|
||||
const hourAgo = now - 3600000
|
||||
|
||||
if (lastActivityTime > hourAgo) {
|
||||
serviceStats.status = 'active'
|
||||
} else {
|
||||
serviceStats.status = 'inactive'
|
||||
}
|
||||
} else {
|
||||
serviceStats.status = 'inactive'
|
||||
}
|
||||
|
||||
// Check if service is read-only (has no write operations)
|
||||
if (serviceStats.totalNouns === 0 && serviceStats.totalVerbs === 0) {
|
||||
serviceStats.status = 'read-only'
|
||||
}
|
||||
|
||||
result.push(serviceStats)
|
||||
}
|
||||
|
||||
// Sort by last activity (most recent first)
|
||||
result.sort((a, b) => {
|
||||
if (!a.lastActivity && !b.lastActivity) return 0
|
||||
if (!a.lastActivity) return 1
|
||||
if (!b.lastActivity) return -1
|
||||
return new Date(b.lastActivity).getTime() - new Date(a.lastActivity).getTime()
|
||||
})
|
||||
|
||||
return result
|
||||
} catch (error) {
|
||||
console.error('Failed to list services:', error)
|
||||
throw new Error(`Failed to list services: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get statistics for a specific service
|
||||
* @param service The service name to get statistics for
|
||||
* @returns Service statistics or null if service not found
|
||||
*/
|
||||
public async getServiceStatistics(
|
||||
service: string
|
||||
): Promise<import('./coreTypes.js').ServiceStatistics | null> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
const stats = await this.storage!.getStatistics()
|
||||
if (!stats) {
|
||||
return null
|
||||
}
|
||||
|
||||
// Check if service exists in any counter
|
||||
const hasData =
|
||||
(stats.nounCount[service] || 0) > 0 ||
|
||||
(stats.verbCount[service] || 0) > 0 ||
|
||||
(stats.metadataCount[service] || 0) > 0
|
||||
|
||||
if (!hasData && !stats.serviceActivity?.[service]) {
|
||||
return null
|
||||
}
|
||||
|
||||
const serviceStats: import('./coreTypes.js').ServiceStatistics = {
|
||||
name: service,
|
||||
totalNouns: stats.nounCount[service] || 0,
|
||||
totalVerbs: stats.verbCount[service] || 0,
|
||||
totalMetadata: stats.metadataCount[service] || 0
|
||||
}
|
||||
|
||||
// Add activity timestamps if available
|
||||
if (stats.serviceActivity && stats.serviceActivity[service]) {
|
||||
const activity = stats.serviceActivity[service]
|
||||
serviceStats.firstActivity = activity.firstActivity
|
||||
serviceStats.lastActivity = activity.lastActivity
|
||||
serviceStats.operations = {
|
||||
adds: activity.totalOperations,
|
||||
updates: 0,
|
||||
deletes: 0
|
||||
}
|
||||
}
|
||||
|
||||
// Determine status
|
||||
if (serviceStats.lastActivity) {
|
||||
const lastActivityTime = new Date(serviceStats.lastActivity).getTime()
|
||||
const now = Date.now()
|
||||
const hourAgo = now - 3600000
|
||||
|
||||
serviceStats.status = lastActivityTime > hourAgo ? 'active' : 'inactive'
|
||||
} else {
|
||||
serviceStats.status = 'inactive'
|
||||
}
|
||||
|
||||
// Check if service is read-only
|
||||
if (serviceStats.totalNouns === 0 && serviceStats.totalVerbs === 0) {
|
||||
serviceStats.status = 'read-only'
|
||||
}
|
||||
|
||||
return serviceStats
|
||||
} catch (error) {
|
||||
console.error(`Failed to get statistics for service ${service}:`, error)
|
||||
throw new Error(`Failed to get statistics for service ${service}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the database is in read-only mode
|
||||
* @returns True if the database is in read-only mode, false otherwise
|
||||
|
|
|
|||
|
|
@ -140,6 +140,60 @@ export interface HNSWConfig {
|
|||
/**
|
||||
* Statistics data structure for tracking counts by service
|
||||
*/
|
||||
/**
|
||||
* Per-service statistics tracking
|
||||
*/
|
||||
export interface ServiceStatistics {
|
||||
/**
|
||||
* Service name
|
||||
*/
|
||||
name: string
|
||||
|
||||
/**
|
||||
* Total number of nouns created by this service
|
||||
*/
|
||||
totalNouns: number
|
||||
|
||||
/**
|
||||
* Total number of verbs created by this service
|
||||
*/
|
||||
totalVerbs: number
|
||||
|
||||
/**
|
||||
* Total number of metadata entries created by this service
|
||||
*/
|
||||
totalMetadata: number
|
||||
|
||||
/**
|
||||
* First activity timestamp for this service
|
||||
*/
|
||||
firstActivity?: string
|
||||
|
||||
/**
|
||||
* Last activity timestamp for this service
|
||||
*/
|
||||
lastActivity?: string
|
||||
|
||||
/**
|
||||
* Error count for this service
|
||||
*/
|
||||
errorCount?: number
|
||||
|
||||
/**
|
||||
* Operation breakdown for this service
|
||||
*/
|
||||
operations?: {
|
||||
adds: number
|
||||
updates: number
|
||||
deletes: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Status of the service (active, inactive, read-only)
|
||||
*/
|
||||
status?: 'active' | 'inactive' | 'read-only'
|
||||
}
|
||||
|
||||
export interface StatisticsData {
|
||||
/**
|
||||
* Count of nouns by service
|
||||
|
|
@ -252,6 +306,20 @@ export interface StatisticsData {
|
|||
averageConnectionsPerVerb: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Service-level activity timestamps
|
||||
*/
|
||||
serviceActivity?: Record<string, {
|
||||
firstActivity: string
|
||||
lastActivity: string
|
||||
totalOperations: number
|
||||
}>
|
||||
|
||||
/**
|
||||
* List of all services that have written data
|
||||
*/
|
||||
services?: ServiceStatistics[]
|
||||
|
||||
/**
|
||||
* Last updated timestamp
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -138,7 +138,17 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
|||
verbCount: { ...statistics.verbCount },
|
||||
metadataCount: { ...statistics.metadataCount },
|
||||
hnswIndexSize: statistics.hnswIndexSize,
|
||||
lastUpdated: statistics.lastUpdated
|
||||
lastUpdated: statistics.lastUpdated,
|
||||
// Include serviceActivity if present
|
||||
...(statistics.serviceActivity && {
|
||||
serviceActivity: Object.fromEntries(
|
||||
Object.entries(statistics.serviceActivity).map(([k, v]) => [k, {...v}])
|
||||
)
|
||||
}),
|
||||
// Include services if present
|
||||
...(statistics.services && {
|
||||
services: statistics.services.map(s => ({...s}))
|
||||
})
|
||||
}
|
||||
|
||||
// Schedule a batch update instead of saving immediately
|
||||
|
|
@ -263,7 +273,17 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
|||
verbCount: { ...statistics.verbCount },
|
||||
metadataCount: { ...statistics.metadataCount },
|
||||
hnswIndexSize: statistics.hnswIndexSize,
|
||||
lastUpdated: statistics.lastUpdated
|
||||
lastUpdated: statistics.lastUpdated,
|
||||
// Include serviceActivity if present
|
||||
...(statistics.serviceActivity && {
|
||||
serviceActivity: Object.fromEntries(
|
||||
Object.entries(statistics.serviceActivity).map(([k, v]) => [k, {...v}])
|
||||
)
|
||||
}),
|
||||
// Include services if present
|
||||
...(statistics.services && {
|
||||
services: statistics.services.map(s => ({...s}))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -277,6 +297,9 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
|||
const counter = counterMap[type]
|
||||
counter[service] = (counter[service] || 0) + amount
|
||||
|
||||
// Track service activity
|
||||
this.trackServiceActivity(service, 'add')
|
||||
|
||||
// Update timestamp
|
||||
this.statisticsCache!.lastUpdated = new Date().toISOString()
|
||||
|
||||
|
|
@ -284,6 +307,41 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
|||
this.scheduleBatchUpdate()
|
||||
}
|
||||
|
||||
/**
|
||||
* Track service activity (first/last activity, operation counts)
|
||||
* @param service The service name
|
||||
* @param operation The operation type
|
||||
*/
|
||||
protected trackServiceActivity(
|
||||
service: string,
|
||||
operation: 'add' | 'update' | 'delete'
|
||||
): void {
|
||||
if (!this.statisticsCache) {
|
||||
return
|
||||
}
|
||||
|
||||
// Initialize serviceActivity if it doesn't exist
|
||||
if (!this.statisticsCache.serviceActivity) {
|
||||
this.statisticsCache.serviceActivity = {}
|
||||
}
|
||||
|
||||
const now = new Date().toISOString()
|
||||
const activity = this.statisticsCache.serviceActivity[service]
|
||||
|
||||
if (!activity) {
|
||||
// First activity for this service
|
||||
this.statisticsCache.serviceActivity[service] = {
|
||||
firstActivity: now,
|
||||
lastActivity: now,
|
||||
totalOperations: 1
|
||||
}
|
||||
} else {
|
||||
// Update existing activity
|
||||
activity.lastActivity = now
|
||||
activity.totalOperations++
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Decrement a statistic counter
|
||||
* @param type The type of statistic to decrement ('noun', 'verb', 'metadata')
|
||||
|
|
@ -309,7 +367,17 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
|||
verbCount: { ...statistics.verbCount },
|
||||
metadataCount: { ...statistics.metadataCount },
|
||||
hnswIndexSize: statistics.hnswIndexSize,
|
||||
lastUpdated: statistics.lastUpdated
|
||||
lastUpdated: statistics.lastUpdated,
|
||||
// Include serviceActivity if present
|
||||
...(statistics.serviceActivity && {
|
||||
serviceActivity: Object.fromEntries(
|
||||
Object.entries(statistics.serviceActivity).map(([k, v]) => [k, {...v}])
|
||||
)
|
||||
}),
|
||||
// Include services if present
|
||||
...(statistics.services && {
|
||||
services: statistics.services.map(s => ({...s}))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -323,6 +391,9 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
|||
const counter = counterMap[type]
|
||||
counter[service] = Math.max(0, (counter[service] || 0) - amount)
|
||||
|
||||
// Track service activity
|
||||
this.trackServiceActivity(service, 'delete')
|
||||
|
||||
// Update timestamp
|
||||
this.statisticsCache!.lastUpdated = new Date().toISOString()
|
||||
|
||||
|
|
@ -349,7 +420,17 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
|||
verbCount: { ...statistics.verbCount },
|
||||
metadataCount: { ...statistics.metadataCount },
|
||||
hnswIndexSize: statistics.hnswIndexSize,
|
||||
lastUpdated: statistics.lastUpdated
|
||||
lastUpdated: statistics.lastUpdated,
|
||||
// Include serviceActivity if present
|
||||
...(statistics.serviceActivity && {
|
||||
serviceActivity: Object.fromEntries(
|
||||
Object.entries(statistics.serviceActivity).map(([k, v]) => [k, {...v}])
|
||||
)
|
||||
}),
|
||||
// Include services if present
|
||||
...(statistics.services && {
|
||||
services: statistics.services.map(s => ({...s}))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -622,6 +622,16 @@ export class MemoryStorage extends BaseStorage {
|
|||
metadataCount: {...statistics.metadataCount},
|
||||
hnswIndexSize: statistics.hnswIndexSize,
|
||||
lastUpdated: statistics.lastUpdated,
|
||||
// Include serviceActivity if present
|
||||
...(statistics.serviceActivity && {
|
||||
serviceActivity: Object.fromEntries(
|
||||
Object.entries(statistics.serviceActivity).map(([k, v]) => [k, {...v}])
|
||||
)
|
||||
}),
|
||||
// Include services if present
|
||||
...(statistics.services && {
|
||||
services: statistics.services.map(s => ({...s}))
|
||||
}),
|
||||
// Include distributedConfig if present
|
||||
...(statistics.distributedConfig && {
|
||||
distributedConfig: JSON.parse(JSON.stringify(statistics.distributedConfig))
|
||||
|
|
@ -648,6 +658,16 @@ export class MemoryStorage extends BaseStorage {
|
|||
metadataCount: {...this.statistics.metadataCount},
|
||||
hnswIndexSize: this.statistics.hnswIndexSize,
|
||||
lastUpdated: this.statistics.lastUpdated,
|
||||
// Include serviceActivity if present
|
||||
...(this.statistics.serviceActivity && {
|
||||
serviceActivity: Object.fromEntries(
|
||||
Object.entries(this.statistics.serviceActivity).map(([k, v]) => [k, {...v}])
|
||||
)
|
||||
}),
|
||||
// Include services if present
|
||||
...(this.statistics.services && {
|
||||
services: this.statistics.services.map(s => ({...s}))
|
||||
}),
|
||||
// Include distributedConfig if present
|
||||
...(this.statistics.distributedConfig && {
|
||||
distributedConfig: JSON.parse(JSON.stringify(this.statistics.distributedConfig))
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue