**feat(core, storage, tests): add service-level statistics tracking and storage adapter enhancements**
- **Core**: Enhanced `getStatistics` function to support `service` and `service[]` filters, enabling statistics breakdown by service. Modified return structure to include `serviceBreakdown` for detailed insights. - **Storage**: Implemented a new `BaseStorageAdapter` abstract class to centralize statistics-related functionality, such as incrementing/decrementing counters and updating HNSW index size. Refactored all storage adapters (`FileSystemStorage`, `S3CompatibleStorage`, `MemoryStorage`, `OPFSStorage`) to extend `BaseStorageAdapter`, ensuring consistent statistics tracking. - **Tests**: Added new test cases in `statistics.test.ts` to validate service-level statistics tracking, breakdown accuracy, and multi-service filtering. **Purpose**: Improve insight into data trends by tracking service-specific usage in statistics. Enhance maintainability and consistency through storage adapter centralization and robust testing.
This commit is contained in:
parent
3c95ee011d
commit
85e5a6bfb9
8 changed files with 535 additions and 12 deletions
|
|
@ -3,8 +3,8 @@
|
|||
* Provides persistent storage for the vector database using the Origin Private File System API
|
||||
*/
|
||||
|
||||
import {GraphVerb, HNSWNoun} from '../../coreTypes.js'
|
||||
import {BaseStorage, NOUNS_DIR, VERBS_DIR, METADATA_DIR, INDEX_DIR} from '../baseStorage.js'
|
||||
import {GraphVerb, HNSWNoun, StatisticsData} from '../../coreTypes.js'
|
||||
import {BaseStorage, NOUNS_DIR, VERBS_DIR, METADATA_DIR, INDEX_DIR, STATISTICS_KEY} from '../baseStorage.js'
|
||||
import '../../types/fileSystemTypes.js'
|
||||
|
||||
/**
|
||||
|
|
@ -37,6 +37,7 @@ export class OPFSStorage extends BaseStorage {
|
|||
private isAvailable = false
|
||||
private isPersistentRequested = false
|
||||
private isPersistentGranted = false
|
||||
private statistics: StatisticsData | null = null
|
||||
|
||||
constructor() {
|
||||
super()
|
||||
|
|
@ -688,4 +689,106 @@ export class OPFSStorage extends BaseStorage {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save statistics data to storage
|
||||
* @param statistics The statistics data to save
|
||||
*/
|
||||
protected async saveStatisticsData(statistics: StatisticsData): Promise<void> {
|
||||
// Create a deep copy to avoid reference issues
|
||||
this.statistics = {
|
||||
nounCount: {...statistics.nounCount},
|
||||
verbCount: {...statistics.verbCount},
|
||||
metadataCount: {...statistics.metadataCount},
|
||||
hnswIndexSize: statistics.hnswIndexSize,
|
||||
lastUpdated: statistics.lastUpdated
|
||||
}
|
||||
|
||||
try {
|
||||
// Ensure the root directory is initialized
|
||||
await this.ensureInitialized()
|
||||
|
||||
// Get or create the index directory
|
||||
if (!this.indexDir) {
|
||||
throw new Error('Index directory not initialized')
|
||||
}
|
||||
|
||||
// Create a file for the statistics data
|
||||
const fileHandle = await this.indexDir.getFileHandle('statistics.json', {
|
||||
create: true
|
||||
})
|
||||
|
||||
// Create a writable stream
|
||||
const writable = await fileHandle.createWritable()
|
||||
|
||||
// Write the statistics data to the file
|
||||
await writable.write(JSON.stringify(this.statistics, null, 2))
|
||||
|
||||
// Close the stream
|
||||
await writable.close()
|
||||
} catch (error) {
|
||||
console.error('Failed to save statistics data:', error)
|
||||
throw new Error(`Failed to save statistics data: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get statistics data from storage
|
||||
* @returns Promise that resolves to the statistics data or null if not found
|
||||
*/
|
||||
protected async getStatisticsData(): Promise<StatisticsData | null> {
|
||||
// If we have cached statistics, return a deep copy
|
||||
if (this.statistics) {
|
||||
return {
|
||||
nounCount: {...this.statistics.nounCount},
|
||||
verbCount: {...this.statistics.verbCount},
|
||||
metadataCount: {...this.statistics.metadataCount},
|
||||
hnswIndexSize: this.statistics.hnswIndexSize,
|
||||
lastUpdated: this.statistics.lastUpdated
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
// Ensure the root directory is initialized
|
||||
await this.ensureInitialized()
|
||||
|
||||
if (!this.indexDir) {
|
||||
throw new Error('Index directory not initialized')
|
||||
}
|
||||
|
||||
try {
|
||||
// Try to get the statistics file
|
||||
const fileHandle = await this.indexDir.getFileHandle('statistics.json', {
|
||||
create: false
|
||||
})
|
||||
|
||||
// Get the file data
|
||||
const file = await fileHandle.getFile()
|
||||
const text = await file.text()
|
||||
|
||||
// Parse the statistics data
|
||||
this.statistics = JSON.parse(text)
|
||||
|
||||
// Return a deep copy
|
||||
if (this.statistics) {
|
||||
return {
|
||||
nounCount: {...this.statistics.nounCount},
|
||||
verbCount: {...this.statistics.verbCount},
|
||||
metadataCount: {...this.statistics.metadataCount},
|
||||
hnswIndexSize: this.statistics.hnswIndexSize,
|
||||
lastUpdated: this.statistics.lastUpdated
|
||||
}
|
||||
}
|
||||
|
||||
// If statistics is null, return default statistics
|
||||
return this.createDefaultStatistics()
|
||||
} catch (error) {
|
||||
// If the file doesn't exist, return null
|
||||
return null
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Failed to get statistics data:', error)
|
||||
throw new Error(`Failed to get statistics data: ${error}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue