**feat(tests, docs, storage): add statistics storage tests and enhance documentation**
- **Tests**: Added new `statistics-storage.test.ts` to validate statistics storage functionality across scenarios including saving, retrieving, time-based partitioning, and backward compatibility. Ensured tests dynamically handle missing environment variables by skipping S3-related tests when credentials are unavailable. - **Docs**: Enhanced `statistics.md` with detailed explanations of scalability improvements, including adaptive flush timing, batched updates, and time-based partitioning. Improved readability and structure. - **Storage**: Updated all storage adapters to integrate time-based partitioning and maintain backward compatibility with legacy statistics storage formats. - **Dependencies**: Added `dotenv` to support environmental variable management for storage adapter tests. **Purpose**: Strengthen system reliability by adding comprehensive test coverage for statistics storage, improve scalability documentation, and ensure consistency across storage adapters with robust implementations.
This commit is contained in:
parent
290a6f927e
commit
38d1ee3d7e
12 changed files with 1955 additions and 1301 deletions
|
|
@ -328,6 +328,7 @@ export class MemoryStorage extends BaseStorage {
|
|||
* @param statistics The statistics data to save
|
||||
*/
|
||||
protected async saveStatisticsData(statistics: StatisticsData): Promise<void> {
|
||||
// For memory storage, we just need to store the statistics in memory
|
||||
// Create a deep copy to avoid reference issues
|
||||
this.statistics = {
|
||||
nounCount: {...statistics.nounCount},
|
||||
|
|
@ -336,6 +337,9 @@ export class MemoryStorage extends BaseStorage {
|
|||
hnswIndexSize: statistics.hnswIndexSize,
|
||||
lastUpdated: statistics.lastUpdated
|
||||
}
|
||||
|
||||
// Since this is in-memory, there's no need for time-based partitioning
|
||||
// or legacy file handling
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -355,5 +359,8 @@ export class MemoryStorage extends BaseStorage {
|
|||
hnswIndexSize: this.statistics.hnswIndexSize,
|
||||
lastUpdated: this.statistics.lastUpdated
|
||||
}
|
||||
|
||||
// Since this is in-memory, there's no need for fallback mechanisms
|
||||
// to check multiple storage locations
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue