Initial commit: Brainy - Multi-Dimensional AI Database
Open source vector database with HNSW indexing, graph relationships, and metadata facets. Features CLI with professional augmentation registry integration for discovering extensions and capabilities.
This commit is contained in:
commit
f8c45f2d8d
448 changed files with 103294 additions and 0 deletions
366
dist/utils/statisticsCollector.js
vendored
Normal file
366
dist/utils/statisticsCollector.js
vendored
Normal file
|
|
@ -0,0 +1,366 @@
|
|||
/**
|
||||
* Lightweight statistics collector for Brainy
|
||||
* Designed to have minimal performance impact even with millions of entries
|
||||
*/
|
||||
export class StatisticsCollector {
|
||||
constructor() {
|
||||
// Content type tracking (lightweight counters)
|
||||
this.contentTypes = new Map();
|
||||
// Data freshness tracking (only track timestamps, not full data)
|
||||
this.oldestTimestamp = Date.now();
|
||||
this.newestTimestamp = Date.now();
|
||||
this.updateTimestamps = [];
|
||||
// Search performance tracking (rolling window)
|
||||
this.searchMetrics = {
|
||||
totalSearches: 0,
|
||||
totalSearchTimeMs: 0,
|
||||
searchTimestamps: [],
|
||||
topSearchTerms: new Map()
|
||||
};
|
||||
// Verb type tracking
|
||||
this.verbTypes = new Map();
|
||||
// Storage size estimates (updated periodically, not on every operation)
|
||||
this.storageSizeCache = {
|
||||
lastUpdated: 0,
|
||||
sizes: {
|
||||
nouns: 0,
|
||||
verbs: 0,
|
||||
metadata: 0,
|
||||
index: 0
|
||||
}
|
||||
};
|
||||
// Throttling metrics
|
||||
this.throttlingMetrics = {
|
||||
currentlyThrottled: false,
|
||||
lastThrottleTime: 0,
|
||||
consecutiveThrottleEvents: 0,
|
||||
currentBackoffMs: 1000,
|
||||
totalThrottleEvents: 0,
|
||||
throttleEventsByHour: new Array(24).fill(0),
|
||||
throttleReasons: new Map(),
|
||||
delayedOperations: 0,
|
||||
retriedOperations: 0,
|
||||
failedDueToThrottling: 0,
|
||||
totalDelayMs: 0,
|
||||
serviceThrottling: new Map()
|
||||
};
|
||||
this.MAX_TIMESTAMPS = 1000; // Keep last 1000 timestamps
|
||||
this.MAX_SEARCH_TERMS = 100; // Track top 100 search terms
|
||||
this.SIZE_UPDATE_INTERVAL = 60000; // Update sizes every minute
|
||||
}
|
||||
/**
|
||||
* Track content type (very lightweight)
|
||||
*/
|
||||
trackContentType(type) {
|
||||
this.contentTypes.set(type, (this.contentTypes.get(type) || 0) + 1);
|
||||
}
|
||||
/**
|
||||
* Track data update timestamp (lightweight)
|
||||
*/
|
||||
trackUpdate(timestamp) {
|
||||
const ts = timestamp || Date.now();
|
||||
// Update oldest/newest
|
||||
if (ts < this.oldestTimestamp)
|
||||
this.oldestTimestamp = ts;
|
||||
if (ts > this.newestTimestamp)
|
||||
this.newestTimestamp = ts;
|
||||
// Add to rolling window
|
||||
this.updateTimestamps.push({ timestamp: ts, count: 1 });
|
||||
// Keep window size limited
|
||||
if (this.updateTimestamps.length > this.MAX_TIMESTAMPS) {
|
||||
this.updateTimestamps.shift();
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Track search performance (lightweight)
|
||||
*/
|
||||
trackSearch(searchTerm, durationMs) {
|
||||
this.searchMetrics.totalSearches++;
|
||||
this.searchMetrics.totalSearchTimeMs += durationMs;
|
||||
// Add to rolling window
|
||||
this.searchMetrics.searchTimestamps.push({
|
||||
timestamp: Date.now(),
|
||||
count: 1
|
||||
});
|
||||
// Keep window size limited
|
||||
if (this.searchMetrics.searchTimestamps.length > this.MAX_TIMESTAMPS) {
|
||||
this.searchMetrics.searchTimestamps.shift();
|
||||
}
|
||||
// Track search term (limit to top N)
|
||||
const termCount = (this.searchMetrics.topSearchTerms.get(searchTerm) || 0) + 1;
|
||||
this.searchMetrics.topSearchTerms.set(searchTerm, termCount);
|
||||
// Prune if too many terms
|
||||
if (this.searchMetrics.topSearchTerms.size > this.MAX_SEARCH_TERMS * 2) {
|
||||
this.pruneSearchTerms();
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Track verb type (lightweight)
|
||||
*/
|
||||
trackVerbType(type) {
|
||||
this.verbTypes.set(type, (this.verbTypes.get(type) || 0) + 1);
|
||||
}
|
||||
/**
|
||||
* Update storage size estimates (called periodically, not on every operation)
|
||||
*/
|
||||
updateStorageSizes(sizes) {
|
||||
this.storageSizeCache = {
|
||||
lastUpdated: Date.now(),
|
||||
sizes
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Track a throttling event
|
||||
*/
|
||||
trackThrottlingEvent(reason, service) {
|
||||
this.throttlingMetrics.currentlyThrottled = true;
|
||||
this.throttlingMetrics.consecutiveThrottleEvents++;
|
||||
this.throttlingMetrics.lastThrottleTime = Date.now();
|
||||
this.throttlingMetrics.totalThrottleEvents++;
|
||||
// Track by hour
|
||||
const hourIndex = new Date().getHours();
|
||||
this.throttlingMetrics.throttleEventsByHour[hourIndex]++;
|
||||
// Track reason
|
||||
const reasonCount = this.throttlingMetrics.throttleReasons.get(reason) || 0;
|
||||
this.throttlingMetrics.throttleReasons.set(reason, reasonCount + 1);
|
||||
// Track service-level throttling
|
||||
if (service) {
|
||||
const serviceInfo = this.throttlingMetrics.serviceThrottling.get(service) || {
|
||||
throttleCount: 0,
|
||||
lastThrottle: 0,
|
||||
status: 'normal'
|
||||
};
|
||||
serviceInfo.throttleCount++;
|
||||
serviceInfo.lastThrottle = Date.now();
|
||||
serviceInfo.status = 'throttled';
|
||||
this.throttlingMetrics.serviceThrottling.set(service, serviceInfo);
|
||||
}
|
||||
// Exponential backoff
|
||||
this.throttlingMetrics.currentBackoffMs = Math.min(this.throttlingMetrics.currentBackoffMs * 2, 30000 // Max 30 seconds
|
||||
);
|
||||
}
|
||||
/**
|
||||
* Clear throttling state after successful operations
|
||||
*/
|
||||
clearThrottlingState() {
|
||||
if (this.throttlingMetrics.consecutiveThrottleEvents > 0) {
|
||||
this.throttlingMetrics.consecutiveThrottleEvents = 0;
|
||||
this.throttlingMetrics.currentBackoffMs = 1000; // Reset to initial backoff
|
||||
this.throttlingMetrics.currentlyThrottled = false;
|
||||
// Update service statuses
|
||||
for (const [, info] of this.throttlingMetrics.serviceThrottling) {
|
||||
if (info.status === 'throttled') {
|
||||
info.status = 'recovering';
|
||||
}
|
||||
else if (info.status === 'recovering') {
|
||||
const timeSinceThrottle = Date.now() - info.lastThrottle;
|
||||
if (timeSinceThrottle > 60000) { // 1 minute recovery period
|
||||
info.status = 'normal';
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Track delayed operation
|
||||
*/
|
||||
trackDelayedOperation(delayMs) {
|
||||
this.throttlingMetrics.delayedOperations++;
|
||||
this.throttlingMetrics.totalDelayMs += delayMs;
|
||||
}
|
||||
/**
|
||||
* Track retried operation
|
||||
*/
|
||||
trackRetriedOperation() {
|
||||
this.throttlingMetrics.retriedOperations++;
|
||||
}
|
||||
/**
|
||||
* Track operation failed due to throttling
|
||||
*/
|
||||
trackFailedDueToThrottling() {
|
||||
this.throttlingMetrics.failedDueToThrottling++;
|
||||
}
|
||||
/**
|
||||
* Update throttling metrics from storage adapter
|
||||
*/
|
||||
updateThrottlingMetrics(metrics) {
|
||||
this.throttlingMetrics.currentlyThrottled = metrics.currentlyThrottled;
|
||||
this.throttlingMetrics.lastThrottleTime = metrics.lastThrottleTime;
|
||||
this.throttlingMetrics.consecutiveThrottleEvents = metrics.consecutiveThrottleEvents;
|
||||
this.throttlingMetrics.currentBackoffMs = metrics.currentBackoffMs;
|
||||
this.throttlingMetrics.totalThrottleEvents = metrics.totalThrottleEvents;
|
||||
this.throttlingMetrics.throttleEventsByHour = [...metrics.throttleEventsByHour];
|
||||
// Update throttle reasons map
|
||||
this.throttlingMetrics.throttleReasons.clear();
|
||||
for (const [reason, count] of Object.entries(metrics.throttleReasons)) {
|
||||
this.throttlingMetrics.throttleReasons.set(reason, count);
|
||||
}
|
||||
this.throttlingMetrics.delayedOperations = metrics.delayedOperations;
|
||||
this.throttlingMetrics.retriedOperations = metrics.retriedOperations;
|
||||
this.throttlingMetrics.failedDueToThrottling = metrics.failedDueToThrottling;
|
||||
this.throttlingMetrics.totalDelayMs = metrics.totalDelayMs;
|
||||
}
|
||||
/**
|
||||
* Get comprehensive statistics
|
||||
*/
|
||||
getStatistics() {
|
||||
const now = Date.now();
|
||||
const hourAgo = now - 3600000;
|
||||
const dayAgo = now - 86400000;
|
||||
const weekAgo = now - 604800000;
|
||||
const monthAgo = now - 2592000000;
|
||||
// Calculate data freshness
|
||||
const updatesLastHour = this.updateTimestamps.filter(t => t.timestamp > hourAgo).length;
|
||||
const updatesLastDay = this.updateTimestamps.filter(t => t.timestamp > dayAgo).length;
|
||||
// Calculate age distribution
|
||||
const ageDistribution = {
|
||||
last24h: 0,
|
||||
last7d: 0,
|
||||
last30d: 0,
|
||||
older: 0
|
||||
};
|
||||
// Estimate based on update patterns (not scanning all data)
|
||||
const totalUpdates = this.updateTimestamps.length;
|
||||
if (totalUpdates > 0) {
|
||||
const recentUpdates = this.updateTimestamps.filter(t => t.timestamp > dayAgo).length;
|
||||
const weekUpdates = this.updateTimestamps.filter(t => t.timestamp > weekAgo).length;
|
||||
const monthUpdates = this.updateTimestamps.filter(t => t.timestamp > monthAgo).length;
|
||||
ageDistribution.last24h = Math.round((recentUpdates / totalUpdates) * 100);
|
||||
ageDistribution.last7d = Math.round(((weekUpdates - recentUpdates) / totalUpdates) * 100);
|
||||
ageDistribution.last30d = Math.round(((monthUpdates - weekUpdates) / totalUpdates) * 100);
|
||||
ageDistribution.older = 100 - ageDistribution.last24h - ageDistribution.last7d - ageDistribution.last30d;
|
||||
}
|
||||
// Calculate search metrics
|
||||
const searchesLastHour = this.searchMetrics.searchTimestamps.filter(t => t.timestamp > hourAgo).length;
|
||||
const searchesLastDay = this.searchMetrics.searchTimestamps.filter(t => t.timestamp > dayAgo).length;
|
||||
const avgSearchTime = this.searchMetrics.totalSearches > 0
|
||||
? this.searchMetrics.totalSearchTimeMs / this.searchMetrics.totalSearches
|
||||
: 0;
|
||||
// Get top search terms
|
||||
const topSearchTerms = Array.from(this.searchMetrics.topSearchTerms.entries())
|
||||
.sort((a, b) => b[1] - a[1])
|
||||
.slice(0, 10)
|
||||
.map(([term]) => term);
|
||||
// Calculate storage metrics
|
||||
const totalSize = Object.values(this.storageSizeCache.sizes).reduce((a, b) => a + b, 0);
|
||||
// Calculate average delay for throttling
|
||||
const averageDelayMs = this.throttlingMetrics.delayedOperations > 0
|
||||
? this.throttlingMetrics.totalDelayMs / this.throttlingMetrics.delayedOperations
|
||||
: 0;
|
||||
// Convert service throttling map to record
|
||||
const serviceThrottlingRecord = {};
|
||||
for (const [service, info] of this.throttlingMetrics.serviceThrottling) {
|
||||
serviceThrottlingRecord[service] = {
|
||||
throttleCount: info.throttleCount,
|
||||
lastThrottle: new Date(info.lastThrottle).toISOString(),
|
||||
status: info.status
|
||||
};
|
||||
}
|
||||
return {
|
||||
contentTypes: Object.fromEntries(this.contentTypes),
|
||||
dataFreshness: {
|
||||
oldestEntry: new Date(this.oldestTimestamp).toISOString(),
|
||||
newestEntry: new Date(this.newestTimestamp).toISOString(),
|
||||
updatesLastHour,
|
||||
updatesLastDay,
|
||||
ageDistribution
|
||||
},
|
||||
storageMetrics: {
|
||||
totalSizeBytes: totalSize,
|
||||
nounsSizeBytes: this.storageSizeCache.sizes.nouns,
|
||||
verbsSizeBytes: this.storageSizeCache.sizes.verbs,
|
||||
metadataSizeBytes: this.storageSizeCache.sizes.metadata,
|
||||
indexSizeBytes: this.storageSizeCache.sizes.index
|
||||
},
|
||||
searchMetrics: {
|
||||
totalSearches: this.searchMetrics.totalSearches,
|
||||
averageSearchTimeMs: avgSearchTime,
|
||||
searchesLastHour,
|
||||
searchesLastDay,
|
||||
topSearchTerms
|
||||
},
|
||||
verbStatistics: {
|
||||
totalVerbs: Array.from(this.verbTypes.values()).reduce((a, b) => a + b, 0),
|
||||
verbTypes: Object.fromEntries(this.verbTypes),
|
||||
averageConnectionsPerVerb: 2 // Verbs connect 2 nouns
|
||||
},
|
||||
throttlingMetrics: {
|
||||
storage: {
|
||||
currentlyThrottled: this.throttlingMetrics.currentlyThrottled,
|
||||
lastThrottleTime: this.throttlingMetrics.lastThrottleTime > 0
|
||||
? new Date(this.throttlingMetrics.lastThrottleTime).toISOString()
|
||||
: undefined,
|
||||
consecutiveThrottleEvents: this.throttlingMetrics.consecutiveThrottleEvents,
|
||||
currentBackoffMs: this.throttlingMetrics.currentBackoffMs,
|
||||
totalThrottleEvents: this.throttlingMetrics.totalThrottleEvents,
|
||||
throttleEventsByHour: [...this.throttlingMetrics.throttleEventsByHour],
|
||||
throttleReasons: Object.fromEntries(this.throttlingMetrics.throttleReasons)
|
||||
},
|
||||
operationImpact: {
|
||||
delayedOperations: this.throttlingMetrics.delayedOperations,
|
||||
retriedOperations: this.throttlingMetrics.retriedOperations,
|
||||
failedDueToThrottling: this.throttlingMetrics.failedDueToThrottling,
|
||||
averageDelayMs,
|
||||
totalDelayMs: this.throttlingMetrics.totalDelayMs
|
||||
},
|
||||
serviceThrottling: Object.keys(serviceThrottlingRecord).length > 0
|
||||
? serviceThrottlingRecord
|
||||
: undefined
|
||||
}
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Merge statistics from storage (for distributed systems)
|
||||
*/
|
||||
mergeFromStorage(stored) {
|
||||
// Merge content types
|
||||
if (stored.contentTypes) {
|
||||
for (const [type, count] of Object.entries(stored.contentTypes)) {
|
||||
this.contentTypes.set(type, count);
|
||||
}
|
||||
}
|
||||
// Merge verb types
|
||||
if (stored.verbStatistics?.verbTypes) {
|
||||
for (const [type, count] of Object.entries(stored.verbStatistics.verbTypes)) {
|
||||
this.verbTypes.set(type, count);
|
||||
}
|
||||
}
|
||||
// Merge search metrics
|
||||
if (stored.searchMetrics) {
|
||||
this.searchMetrics.totalSearches = stored.searchMetrics.totalSearches || 0;
|
||||
this.searchMetrics.totalSearchTimeMs = (stored.searchMetrics.averageSearchTimeMs || 0) * this.searchMetrics.totalSearches;
|
||||
}
|
||||
// Merge data freshness
|
||||
if (stored.dataFreshness) {
|
||||
this.oldestTimestamp = new Date(stored.dataFreshness.oldestEntry).getTime();
|
||||
this.newestTimestamp = new Date(stored.dataFreshness.newestEntry).getTime();
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Reset statistics (for testing)
|
||||
*/
|
||||
reset() {
|
||||
this.contentTypes.clear();
|
||||
this.verbTypes.clear();
|
||||
this.updateTimestamps = [];
|
||||
this.searchMetrics = {
|
||||
totalSearches: 0,
|
||||
totalSearchTimeMs: 0,
|
||||
searchTimestamps: [],
|
||||
topSearchTerms: new Map()
|
||||
};
|
||||
this.oldestTimestamp = Date.now();
|
||||
this.newestTimestamp = Date.now();
|
||||
}
|
||||
pruneSearchTerms() {
|
||||
// Keep only top N search terms
|
||||
const sorted = Array.from(this.searchMetrics.topSearchTerms.entries())
|
||||
.sort((a, b) => b[1] - a[1])
|
||||
.slice(0, this.MAX_SEARCH_TERMS);
|
||||
this.searchMetrics.topSearchTerms.clear();
|
||||
for (const [term, count] of sorted) {
|
||||
this.searchMetrics.topSearchTerms.set(term, count);
|
||||
}
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=statisticsCollector.js.map
|
||||
Loading…
Add table
Add a link
Reference in a new issue