soulcraft/brainy
2
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

**feat(tests, docs, storage): add statistics storage tests and enhance documentation**

Browse source 
- **Tests**: Added new `statistics-storage.test.ts` to validate statistics storage functionality across scenarios including saving, retrieving, time-based partitioning, and backward compatibility. Ensured tests dynamically handle missing environment variables by skipping S3-related tests when credentials are unavailable.
- **Docs**: Enhanced `statistics.md` with detailed explanations of scalability improvements, including adaptive flush timing, batched updates, and time-based partitioning. Improved readability and structure.
- **Storage**: Updated all storage adapters to integrate time-based partitioning and maintain backward compatibility with legacy statistics storage formats.
- **Dependencies**: Added `dotenv` to support environmental variable management for storage adapter tests.

**Purpose**: Strengthen system reliability by adding comprehensive test coverage for statistics storage, improve scalability documentation, and ensure consistency across storage adapters with robust implementations.
This commit is contained in:
David Snelling 2025-07-24 16:24:02 -07:00
parent 5839e0b556
commit 23c34d5e55
12 changed files with 1955 additions and 1301 deletions
Download patch file Download diff file Expand all files Collapse all files

204
src/storage/adapters/baseStorageAdapter.ts
View file

@ -33,7 +33,25 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
details?: Record<string, any>
}>
// Statistics-specific methods
// Statistics cache
protected statisticsCache: StatisticsData | null = null
// Batch update timer ID
protected statisticsBatchUpdateTimerId: NodeJS.Timeout | null = null
// Flag to indicate if statistics have been modified since last save
protected statisticsModified = false
// Time of last statistics flush to storage
protected lastStatisticsFlushTime = 0
// Minimum time between statistics flushes (5 seconds)
protected readonly MIN_FLUSH_INTERVAL_MS = 5000
// Maximum time to wait before flushing statistics (30 seconds)
protected readonly MAX_FLUSH_DELAY_MS = 30000
// Statistics-specific methods that must be implemented by subclasses
protected abstract saveStatisticsData(statistics: StatisticsData): Promise<void>
protected abstract getStatisticsData(): Promise<StatisticsData | null>
@ -42,7 +60,17 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
* @param statistics The statistics data to save
*/
async saveStatistics(statistics: StatisticsData): Promise<void> {
await this.saveStatisticsData(statistics)
// Update the cache with a deep copy to avoid reference issues
this.statisticsCache = {
nounCount: {...statistics.nounCount},
verbCount: {...statistics.verbCount},
metadataCount: {...statistics.metadataCount},
hnswIndexSize: statistics.hnswIndexSize,
lastUpdated: statistics.lastUpdated
}
// Schedule a batch update instead of saving immediately
this.scheduleBatchUpdate()
}
/**
@ -50,7 +78,91 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
* @returns Promise that resolves to the statistics data
*/
async getStatistics(): Promise<StatisticsData | null> {
return await this.getStatisticsData()
// If we have cached statistics, return a deep copy
if (this.statisticsCache) {
return {
nounCount: {...this.statisticsCache.nounCount},
verbCount: {...this.statisticsCache.verbCount},
metadataCount: {...this.statisticsCache.metadataCount},
hnswIndexSize: this.statisticsCache.hnswIndexSize,
lastUpdated: this.statisticsCache.lastUpdated
}
}
// Otherwise, get from storage
const statistics = await this.getStatisticsData()
// If we found statistics, update the cache
if (statistics) {
// Update the cache with a deep copy
this.statisticsCache = {
nounCount: {...statistics.nounCount},
verbCount: {...statistics.verbCount},
metadataCount: {...statistics.metadataCount},
hnswIndexSize: statistics.hnswIndexSize,
lastUpdated: statistics.lastUpdated
}
}
return statistics
}
/**
* Schedule a batch update of statistics
*/
protected scheduleBatchUpdate(): void {
// Mark statistics as modified
this.statisticsModified = true
// If a timer is already set, don't set another one
if (this.statisticsBatchUpdateTimerId !== null) {
return
}
// Calculate time since last flush
const now = Date.now()
const timeSinceLastFlush = now - this.lastStatisticsFlushTime
// If we've recently flushed, wait longer before the next flush
const delayMs = timeSinceLastFlush < this.MIN_FLUSH_INTERVAL_MS
? this.MAX_FLUSH_DELAY_MS
: this.MIN_FLUSH_INTERVAL_MS
// Schedule the batch update
this.statisticsBatchUpdateTimerId = setTimeout(() => {
this.flushStatistics()
}, delayMs)
}
/**
* Flush statistics to storage
*/
protected async flushStatistics(): Promise<void> {
// Clear the timer
if (this.statisticsBatchUpdateTimerId !== null) {
clearTimeout(this.statisticsBatchUpdateTimerId)
this.statisticsBatchUpdateTimerId = null
}
// If statistics haven't been modified, no need to flush
if (!this.statisticsModified || !this.statisticsCache) {
return
}
try {
// Save the statistics to storage
await this.saveStatisticsData(this.statisticsCache)
// Update the last flush time
this.lastStatisticsFlushTime = Date.now()
// Reset the modified flag
this.statisticsModified = false
} catch (error) {
console.error('Failed to flush statistics data:', error)
// Mark as still modified so we'll try again later
this.statisticsModified = true
// Don't throw the error to avoid disrupting the application
}
}
/**
@ -64,27 +176,39 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
service: string,
amount: number = 1
): Promise<void> {
// Get current statistics or create default if not exists
let statistics = await this.getStatisticsData()
// Get current statistics from cache or storage
let statistics = this.statisticsCache
if (!statistics) {
statistics = this.createDefaultStatistics()
statistics = await this.getStatisticsData()
if (!statistics) {
statistics = this.createDefaultStatistics()
}
// Update the cache
this.statisticsCache = {
nounCount: {...statistics.nounCount},
verbCount: {...statistics.verbCount},
metadataCount: {...statistics.metadataCount},
hnswIndexSize: statistics.hnswIndexSize,
lastUpdated: statistics.lastUpdated
}
}
// Increment the appropriate counter
const counterMap = {
noun: statistics.nounCount,
verb: statistics.verbCount,
metadata: statistics.metadataCount
noun: this.statisticsCache!.nounCount,
verb: this.statisticsCache!.verbCount,
metadata: this.statisticsCache!.metadataCount
}
const counter = counterMap[type]
counter[service] = (counter[service] || 0) + amount
// Update timestamp
statistics.lastUpdated = new Date().toISOString()
this.statisticsCache!.lastUpdated = new Date().toISOString()
// Save updated statistics
await this.saveStatisticsData(statistics)
// Schedule a batch update instead of saving immediately
this.scheduleBatchUpdate()
}
/**
@ -98,27 +222,39 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
service: string,
amount: number = 1
): Promise<void> {
// Get current statistics or create default if not exists
let statistics = await this.getStatisticsData()
// Get current statistics from cache or storage
let statistics = this.statisticsCache
if (!statistics) {
statistics = this.createDefaultStatistics()
statistics = await this.getStatisticsData()
if (!statistics) {
statistics = this.createDefaultStatistics()
}
// Update the cache
this.statisticsCache = {
nounCount: {...statistics.nounCount},
verbCount: {...statistics.verbCount},
metadataCount: {...statistics.metadataCount},
hnswIndexSize: statistics.hnswIndexSize,
lastUpdated: statistics.lastUpdated
}
}
// Decrement the appropriate counter
const counterMap = {
noun: statistics.nounCount,
verb: statistics.verbCount,
metadata: statistics.metadataCount
noun: this.statisticsCache!.nounCount,
verb: this.statisticsCache!.verbCount,
metadata: this.statisticsCache!.metadataCount
}
const counter = counterMap[type]
counter[service] = Math.max(0, (counter[service] || 0) - amount)
// Update timestamp
statistics.lastUpdated = new Date().toISOString()
this.statisticsCache!.lastUpdated = new Date().toISOString()
// Save updated statistics
await this.saveStatisticsData(statistics)
// Schedule a batch update instead of saving immediately
this.scheduleBatchUpdate()
}
/**
@ -126,20 +262,32 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
* @param size The new size of the HNSW index
*/
async updateHnswIndexSize(size: number): Promise<void> {
// Get current statistics or create default if not exists
let statistics = await this.getStatisticsData()
// Get current statistics from cache or storage
let statistics = this.statisticsCache
if (!statistics) {
statistics = this.createDefaultStatistics()
statistics = await this.getStatisticsData()
if (!statistics) {
statistics = this.createDefaultStatistics()
}
// Update the cache
this.statisticsCache = {
nounCount: {...statistics.nounCount},
verbCount: {...statistics.verbCount},
metadataCount: {...statistics.metadataCount},
hnswIndexSize: statistics.hnswIndexSize,
lastUpdated: statistics.lastUpdated
}
}
// Update HNSW index size
statistics.hnswIndexSize = size
this.statisticsCache!.hnswIndexSize = size
// Update timestamp
statistics.lastUpdated = new Date().toISOString()
this.statisticsCache!.lastUpdated = new Date().toISOString()
// Save updated statistics
await this.saveStatisticsData(statistics)
// Schedule a batch update instead of saving immediately
this.scheduleBatchUpdate()
}
/**

73
src/storage/adapters/fileSystemStorage.ts
View file

@ -561,8 +561,22 @@ export class FileSystemStorage extends BaseStorage {
await this.ensureInitialized()
try {
const filePath = path.join(this.indexDir, `${STATISTICS_KEY}.json`)
await fs.promises.writeFile(filePath, JSON.stringify(statistics, null, 2))
// Get the current date for time-based partitioning
const now = new Date()
const year = now.getUTCFullYear()
const month = String(now.getUTCMonth() + 1).padStart(2, '0')
const day = String(now.getUTCDate()).padStart(2, '0')
const dateStr = `${year}${month}${day}`
// Save to the current day's file
const currentFilePath = path.join(this.indexDir, `${STATISTICS_KEY}_${dateStr}.json`)
await fs.promises.writeFile(currentFilePath, JSON.stringify(statistics, null, 2))
// Also update the legacy file for backward compatibility (less frequently)
if (Math.random() < 0.1) {
const legacyFilePath = path.join(this.indexDir, `${STATISTICS_KEY}.json`)
await fs.promises.writeFile(legacyFilePath, JSON.stringify(statistics, null, 2))
}
} catch (error) {
console.error('Failed to save statistics data:', error)
throw new Error(`Failed to save statistics data: ${error}`)
@ -577,14 +591,55 @@ export class FileSystemStorage extends BaseStorage {
await this.ensureInitialized()
try {
const filePath = path.join(this.indexDir, `${STATISTICS_KEY}.json`)
const data = await fs.promises.readFile(filePath, 'utf-8')
return JSON.parse(data)
} catch (error: any) {
// If the file doesn't exist, return null
if (error.code === 'ENOENT') {
return null
// Try to get statistics from today's file first
const now = new Date()
const year = now.getUTCFullYear()
const month = String(now.getUTCMonth() + 1).padStart(2, '0')
const day = String(now.getUTCDate()).padStart(2, '0')
const dateStr = `${year}${month}${day}`
const currentFilePath = path.join(this.indexDir, `${STATISTICS_KEY}_${dateStr}.json`)
try {
const data = await fs.promises.readFile(currentFilePath, 'utf-8')
return JSON.parse(data)
} catch (currentError: any) {
// If today's file doesn't exist, try yesterday's file
if (currentError.code === 'ENOENT') {
const yesterday = new Date()
yesterday.setDate(yesterday.getDate() - 1)
const yesterdayYear = yesterday.getUTCFullYear()
const yesterdayMonth = String(yesterday.getUTCMonth() + 1).padStart(2, '0')
const yesterdayDay = String(yesterday.getUTCDate()).padStart(2, '0')
const yesterdayDateStr = `${yesterdayYear}${yesterdayMonth}${yesterdayDay}`
const yesterdayFilePath = path.join(this.indexDir, `${STATISTICS_KEY}_${yesterdayDateStr}.json`)
try {
const yesterdayData = await fs.promises.readFile(yesterdayFilePath, 'utf-8')
return JSON.parse(yesterdayData)
} catch (yesterdayError: any) {
// If yesterday's file doesn't exist, try the legacy file
if (yesterdayError.code === 'ENOENT') {
const legacyFilePath = path.join(this.indexDir, `${STATISTICS_KEY}.json`)
try {
const legacyData = await fs.promises.readFile(legacyFilePath, 'utf-8')
return JSON.parse(legacyData)
} catch (legacyError: any) {
// If the legacy file doesn't exist either, return null
if (legacyError.code === 'ENOENT') {
return null
}
throw legacyError
}
}
throw yesterdayError
}
}
throw currentError
}
} catch (error) {
console.error('Error getting statistics data:', error)
throw error
}

7
src/storage/adapters/memoryStorage.ts
View file

@ -328,6 +328,7 @@ export class MemoryStorage extends BaseStorage {
* @param statistics The statistics data to save
*/
protected async saveStatisticsData(statistics: StatisticsData): Promise<void> {
// For memory storage, we just need to store the statistics in memory
// Create a deep copy to avoid reference issues
this.statistics = {
nounCount: {...statistics.nounCount},
@ -336,6 +337,9 @@ export class MemoryStorage extends BaseStorage {
hnswIndexSize: statistics.hnswIndexSize,
lastUpdated: statistics.lastUpdated
}
// Since this is in-memory, there's no need for time-based partitioning
// or legacy file handling
}
/**
@ -355,5 +359,8 @@ export class MemoryStorage extends BaseStorage {
hnswIndexSize: this.statistics.hnswIndexSize,
lastUpdated: this.statistics.lastUpdated
}
// Since this is in-memory, there's no need for fallback mechanisms
// to check multiple storage locations
}
}

112
src/storage/adapters/opfsStorage.ts
View file

@ -690,6 +690,34 @@ export class OPFSStorage extends BaseStorage {
}
}
/**
* Get the statistics key for a specific date
* @param date The date to get the key for
* @returns The statistics key for the specified date
*/
private getStatisticsKeyForDate(date: Date): string {
const year = date.getUTCFullYear()
const month = String(date.getUTCMonth() + 1).padStart(2, '0')
const day = String(date.getUTCDate()).padStart(2, '0')
return `statistics_${year}${month}${day}.json`
}
/**
* Get the current statistics key
* @returns The current statistics key
*/
private getCurrentStatisticsKey(): string {
return this.getStatisticsKeyForDate(new Date())
}
/**
* Get the legacy statistics key (for backward compatibility)
* @returns The legacy statistics key
*/
private getLegacyStatisticsKey(): string {
return 'statistics.json'
}
/**
* Save statistics data to storage
* @param statistics The statistics data to save
@ -713,8 +741,11 @@ export class OPFSStorage extends BaseStorage {
throw new Error('Index directory not initialized')
}
// Get the current statistics key
const currentKey = this.getCurrentStatisticsKey()
// Create a file for the statistics data
const fileHandle = await this.indexDir.getFileHandle('statistics.json', {
const fileHandle = await this.indexDir.getFileHandle(currentKey, {
create: true
})
@ -726,6 +757,17 @@ export class OPFSStorage extends BaseStorage {
// Close the stream
await writable.close()
// Also update the legacy key for backward compatibility, but less frequently
if (Math.random() < 0.1) {
const legacyKey = this.getLegacyStatisticsKey()
const legacyFileHandle = await this.indexDir.getFileHandle(legacyKey, {
create: true
})
const legacyWritable = await legacyFileHandle.createWritable()
await legacyWritable.write(JSON.stringify(this.statistics, null, 2))
await legacyWritable.close()
}
} catch (error) {
console.error('Failed to save statistics data:', error)
throw new Error(`Failed to save statistics data: ${error}`)
@ -756,20 +798,16 @@ export class OPFSStorage extends BaseStorage {
throw new Error('Index directory not initialized')
}
// First try to get statistics from today's file
const currentKey = this.getCurrentStatisticsKey()
try {
// Try to get the statistics file
const fileHandle = await this.indexDir.getFileHandle('statistics.json', {
const fileHandle = await this.indexDir.getFileHandle(currentKey, {
create: false
})
// Get the file data
const file = await fileHandle.getFile()
const text = await file.text()
// Parse the statistics data
this.statistics = JSON.parse(text)
// Return a deep copy
if (this.statistics) {
return {
nounCount: {...this.statistics.nounCount},
@ -779,13 +817,59 @@ export class OPFSStorage extends BaseStorage {
lastUpdated: this.statistics.lastUpdated
}
}
// If statistics is null, return default statistics
return this.createDefaultStatistics()
} catch (error) {
// If the file doesn't exist, return null
return null
// If today's file doesn't exist, try yesterday's file
const yesterday = new Date()
yesterday.setDate(yesterday.getDate() - 1)
const yesterdayKey = this.getStatisticsKeyForDate(yesterday)
try {
const fileHandle = await this.indexDir.getFileHandle(yesterdayKey, {
create: false
})
const file = await fileHandle.getFile()
const text = await file.text()
this.statistics = JSON.parse(text)
if (this.statistics) {
return {
nounCount: {...this.statistics.nounCount},
verbCount: {...this.statistics.verbCount},
metadataCount: {...this.statistics.metadataCount},
hnswIndexSize: this.statistics.hnswIndexSize,
lastUpdated: this.statistics.lastUpdated
}
}
} catch (error) {
// If yesterday's file doesn't exist, try the legacy file
const legacyKey = this.getLegacyStatisticsKey()
try {
const fileHandle = await this.indexDir.getFileHandle(legacyKey, {
create: false
})
const file = await fileHandle.getFile()
const text = await file.text()
this.statistics = JSON.parse(text)
if (this.statistics) {
return {
nounCount: {...this.statistics.nounCount},
verbCount: {...this.statistics.verbCount},
metadataCount: {...this.statistics.metadataCount},
hnswIndexSize: this.statistics.hnswIndexSize,
lastUpdated: this.statistics.lastUpdated
}
}
} catch (error) {
// If the legacy file doesn't exist either, return null
return null
}
}
}
// If we get here and statistics is null, return default statistics
return this.statistics ? this.statistics : null
} catch (error) {
console.error('Failed to get statistics data:', error)
throw new Error(`Failed to get statistics data: ${error}`)

2188
src/storage/adapters/s3CompatibleStorage.ts
View file

File diff suppressed because it is too large Load diff