diff --git a/src/brainyData.ts b/src/brainyData.ts index c21c2a73..97fd4a08 100644 --- a/src/brainyData.ts +++ b/src/brainyData.ts @@ -59,6 +59,7 @@ import { } from './distributed/index.js' import { SearchCache, SearchCacheConfig } from './utils/searchCache.js' import { CacheAutoConfigurator } from './utils/cacheAutoConfig.js' +import { StatisticsCollector } from './utils/statisticsCollector.js' export interface BrainyDataConfig { /** @@ -283,7 +284,7 @@ export interface BrainyDataConfig { * Enables coordination across multiple Brainy instances */ distributed?: DistributedConfig | boolean - + /** * Cache configuration for optimizing search performance * Controls how the system caches data for faster access @@ -404,7 +405,7 @@ export class BrainyData implements BrainyDataInterface { private remoteServerConfig: BrainyDataConfig['remoteServer'] | null = null private serverSearchConduit: ServerSearchConduitAugmentation | null = null private serverConnection: WebSocketConnection | null = null - + // Distributed mode properties private distributedConfig: DistributedConfig | null = null private configManager: DistributedConfigManager | null = null @@ -413,6 +414,9 @@ export class BrainyData implements BrainyDataInterface { private domainDetector: DomainDetector | null = null private healthMonitor: HealthMonitor | null = null + // Statistics collector + private statisticsCollector: StatisticsCollector = new StatisticsCollector() + /** * Get the vector dimensions */ @@ -547,7 +551,7 @@ export class BrainyData implements BrainyDataInterface { ...config.cache } } - + // Store distributed configuration if (config.distributed) { if (typeof config.distributed === 'boolean') { @@ -560,10 +564,10 @@ export class BrainyData implements BrainyDataInterface { this.distributedConfig = config.distributed } } - + // Initialize cache auto-configurator first this.cacheAutoConfigurator = new CacheAutoConfigurator() - + // Auto-detect optimal cache configuration if not explicitly provided let finalSearchCacheConfig = config.searchCache if (!config.searchCache || Object.keys(config.searchCache).length === 0) { @@ -571,7 +575,7 @@ export class BrainyData implements BrainyDataInterface { config.storage ) finalSearchCacheConfig = autoConfig.cacheConfig - + // Apply auto-detected real-time update configuration if not explicitly set if (!config.realtimeUpdates && autoConfig.realtimeConfig.enabled) { this.realtimeUpdateConfig = { @@ -579,12 +583,12 @@ export class BrainyData implements BrainyDataInterface { ...autoConfig.realtimeConfig } } - + if (this.loggingConfig?.verbose) { console.log(this.cacheAutoConfigurator.getConfigExplanation(autoConfig)) } } - + // Initialize search cache with final configuration this.searchCache = new SearchCache(finalSearchCacheConfig) } @@ -768,7 +772,10 @@ export class BrainyData implements BrainyDataInterface { // Adapt cache configuration based on performance (every few updates) // Only adapt every 5th update to avoid over-optimization - const updateCount = Math.floor((Date.now() - (this.lastUpdateTime || 0)) / this.realtimeUpdateConfig.interval) + const updateCount = Math.floor( + (Date.now() - (this.lastUpdateTime || 0)) / + this.realtimeUpdateConfig.interval + ) if (updateCount % 5 === 0) { this.adaptCacheConfiguration() } @@ -1108,7 +1115,7 @@ export class BrainyData implements BrainyDataInterface { // Initialize storage await this.storage!.init() - + // Initialize distributed mode if configured if (this.distributedConfig) { await this.initializeDistributedMode() @@ -1172,6 +1179,16 @@ export class BrainyData implements BrainyDataInterface { } } + // Initialize statistics collector with existing data + try { + const existingStats = await this.storage!.getStatistics() + if (existingStats) { + this.statisticsCollector.mergeFromStorage(existingStats) + } + } catch (e) { + // Ignore errors loading existing statistics + } + this.isInitialized = true this.isInitializing = false @@ -1192,17 +1209,17 @@ export class BrainyData implements BrainyDataInterface { if (!this.storage) { throw new Error('Storage must be initialized before distributed mode') } - + // Create configuration manager with mode hints this.configManager = new DistributedConfigManager( this.storage, this.distributedConfig || undefined, { readOnly: this.readOnly, writeOnly: this.writeOnly } ) - + // Initialize configuration const sharedConfig = await this.configManager.initialize() - + // Create partitioner based on strategy if (sharedConfig.settings.partitionStrategy === 'hash') { this.partitioner = new HashPartitioner(sharedConfig) @@ -1210,27 +1227,33 @@ export class BrainyData implements BrainyDataInterface { // Default to hash partitioner for now this.partitioner = new HashPartitioner(sharedConfig) } - + // Create operational mode based on role const role = this.configManager.getRole() this.operationalMode = OperationalModeFactory.createMode(role) - + // Validate that role matches the configured mode // Don't override explicitly set readOnly/writeOnly if (role === 'reader' && !this.readOnly) { - console.warn('Distributed role is "reader" but readOnly is not set. Setting readOnly=true for consistency.') + console.warn( + 'Distributed role is "reader" but readOnly is not set. Setting readOnly=true for consistency.' + ) this.readOnly = true this.writeOnly = false } else if (role === 'writer' && !this.writeOnly) { - console.warn('Distributed role is "writer" but writeOnly is not set. Setting writeOnly=true for consistency.') + console.warn( + 'Distributed role is "writer" but writeOnly is not set. Setting writeOnly=true for consistency.' + ) this.readOnly = false this.writeOnly = true } else if (role === 'hybrid' && (this.readOnly || this.writeOnly)) { - console.warn('Distributed role is "hybrid" but readOnly or writeOnly is set. Clearing both for hybrid mode.') + console.warn( + 'Distributed role is "hybrid" but readOnly or writeOnly is set. Clearing both for hybrid mode.' + ) this.readOnly = false this.writeOnly = false } - + // Apply cache configuration from operational mode const modeCache = this.operationalMode.cacheStrategy if (modeCache) { @@ -1241,30 +1264,32 @@ export class BrainyData implements BrainyDataInterface { warmCacheTTL: modeCache.ttl, batchSize: modeCache.writeBufferSize || 100 } - + // Update storage cache config if it supports it if (this.storage && 'updateCacheConfig' in this.storage) { - (this.storage as any).updateCacheConfig(this.cacheConfig) + ;(this.storage as any).updateCacheConfig(this.cacheConfig) } } - + // Initialize domain detector this.domainDetector = new DomainDetector() - + // Initialize health monitor this.healthMonitor = new HealthMonitor(this.configManager) this.healthMonitor.start() - + // Set up config update listener this.configManager.setOnConfigUpdate((config) => { this.handleDistributedConfigUpdate(config) }) - + if (this.loggingConfig?.verbose) { - console.log(`Distributed mode initialized as ${role} with ${sharedConfig.settings.partitionStrategy} partitioning`) + console.log( + `Distributed mode initialized as ${role} with ${sharedConfig.settings.partitionStrategy} partitioning` + ) } } - + /** * Handle distributed configuration updates */ @@ -1273,13 +1298,13 @@ export class BrainyData implements BrainyDataInterface { if (this.partitioner && config.settings) { this.partitioner = new HashPartitioner(config) } - + // Log configuration update if (this.loggingConfig?.verbose) { console.log('Distributed configuration updated:', config.version) } } - + /** * Get distributed health status * @returns Health status if distributed mode is enabled @@ -1290,7 +1315,7 @@ export class BrainyData implements BrainyDataInterface { } return null } - + /** * Connect to a remote Brainy server for search operations * @param serverUrl WebSocket URL of the remote Brainy server @@ -1430,26 +1455,31 @@ export class BrainyData implements BrainyDataInterface { try { if (this.writeOnly) { // In write-only mode, check storage directly - existingNoun = await this.storage!.getNoun(options.id) ?? undefined + existingNoun = + (await this.storage!.getNoun(options.id)) ?? undefined } else { // In normal mode, check index first, then storage existingNoun = this.index.getNouns().get(options.id) if (!existingNoun) { - existingNoun = await this.storage!.getNoun(options.id) ?? undefined + existingNoun = + (await this.storage!.getNoun(options.id)) ?? undefined } } if (existingNoun) { // Check if existing noun is a placeholder const existingMetadata = await this.storage!.getMetadata(options.id) - const isPlaceholder = existingMetadata && - typeof existingMetadata === 'object' && + const isPlaceholder = + existingMetadata && + typeof existingMetadata === 'object' && (existingMetadata as any).isPlaceholder if (isPlaceholder) { // Replace placeholder with real data if (this.loggingConfig?.verbose) { - console.log(`Replacing placeholder noun ${options.id} with real data`) + console.log( + `Replacing placeholder noun ${options.id} with real data` + ) } } else { // Real noun already exists, update it @@ -1472,6 +1502,7 @@ export class BrainyData implements BrainyDataInterface { id, vector, connections: new Map(), + level: 0, // Default level for new nodes metadata: undefined // Will be set separately } } else { @@ -1553,29 +1584,35 @@ export class BrainyData implements BrainyDataInterface { if (metadata && typeof metadata === 'object') { // Always make a copy without adding the ID metadataToSave = { ...metadata } - + // Add domain metadata if distributed mode is enabled if (this.domainDetector) { // First check if domain is already in metadata if ((metadataToSave as any).domain) { // Domain already specified, keep it - const domainInfo = this.domainDetector.detectDomain(metadataToSave) + const domainInfo = + this.domainDetector.detectDomain(metadataToSave) if (domainInfo.domainMetadata) { - (metadataToSave as any).domainMetadata = domainInfo.domainMetadata + ;(metadataToSave as any).domainMetadata = + domainInfo.domainMetadata } } else { // Try to detect domain from the data - const dataToAnalyze = Array.isArray(vectorOrData) ? metadata : vectorOrData - const domainInfo = this.domainDetector.detectDomain(dataToAnalyze) + const dataToAnalyze = Array.isArray(vectorOrData) + ? metadata + : vectorOrData + const domainInfo = + this.domainDetector.detectDomain(dataToAnalyze) if (domainInfo.domain) { - (metadataToSave as any).domain = domainInfo.domain + ;(metadataToSave as any).domain = domainInfo.domain if (domainInfo.domainMetadata) { - (metadataToSave as any).domainMetadata = domainInfo.domainMetadata + ;(metadataToSave as any).domainMetadata = + domainInfo.domainMetadata } } } } - + // Add partition information if distributed mode is enabled if (this.partitioner) { const partition = this.partitioner.getPartition(id) @@ -1588,12 +1625,27 @@ export class BrainyData implements BrainyDataInterface { // Track metadata statistics const metadataService = this.getServiceName(options) await this.storage!.incrementStatistic('metadata', metadataService) + + // Track content type if it's a GraphNoun + if ( + metadataToSave && + typeof metadataToSave === 'object' && + 'noun' in metadataToSave + ) { + this.statisticsCollector.trackContentType( + (metadataToSave as any).noun + ) + } + + // Track update timestamp + this.statisticsCollector.trackUpdate() } } - // Update HNSW index size (excluding verbs) - await this.storage!.updateHnswIndexSize(await this.getNounCount()) - + // Update HNSW index size with actual index size + const indexSize = this.index.size() + await this.storage!.updateHnswIndexSize(indexSize) + // Update health metrics if in distributed mode if (this.healthMonitor) { const vectorCount = await this.getNounCount() @@ -1617,12 +1669,12 @@ export class BrainyData implements BrainyDataInterface { return id } catch (error) { console.error('Failed to add vector:', error) - + // Track error in health monitor if (this.healthMonitor) { this.healthMonitor.recordRequest(0, true) } - + throw new Error(`Failed to add vector: ${error}`) } } @@ -2005,7 +2057,7 @@ export class BrainyData implements BrainyDataInterface { // When using offset, we need to fetch more results and then slice const offset = options.offset || 0 const totalNeeded = k + offset - + // Search in the index for totalNeeded results const results = await this.index.search(queryVector, totalNeeded) @@ -2198,9 +2250,13 @@ export class BrainyData implements BrainyDataInterface { // Default behavior (backward compatible): search locally try { // Check cache first (transparent to user) - const cacheKey = this.searchCache.getCacheKey(queryVectorOrData, k, options) + const cacheKey = this.searchCache.getCacheKey( + queryVectorOrData, + k, + options + ) const cachedResults = this.searchCache.get(cacheKey) - + if (cachedResults) { // Track cache hit in health monitor if (this.healthMonitor) { @@ -2210,22 +2266,22 @@ export class BrainyData implements BrainyDataInterface { } return cachedResults } - + // Cache miss - perform actual search const results = await this.searchLocal(queryVectorOrData, k, options) - + // Cache results for future queries (unless explicitly disabled) if (!options.skipCache) { this.searchCache.set(cacheKey, results) } - + // Track successful search in health monitor if (this.healthMonitor) { const latency = Date.now() - startTime this.healthMonitor.recordRequest(latency, false) this.healthMonitor.recordCacheAccess(false) } - + return results } catch (error) { // Track error in health monitor @@ -2260,23 +2316,24 @@ export class BrainyData implements BrainyDataInterface { ): Promise> { // For cursor-based search, we need to fetch more results and filter const searchK = options.cursor ? k + 20 : k // Get extra results for filtering - + // Perform regular search const allResults = await this.search(queryVectorOrData, searchK, { ...options, skipCache: options.skipCache }) - + let results = allResults let startIndex = 0 - + // If cursor provided, find starting position if (options.cursor) { - startIndex = allResults.findIndex(r => - r.id === options.cursor!.lastId && - Math.abs(r.score - options.cursor!.lastScore) < 0.0001 + startIndex = allResults.findIndex( + (r) => + r.id === options.cursor!.lastId && + Math.abs(r.score - options.cursor!.lastScore) < 0.0001 ) - + if (startIndex >= 0) { startIndex += 1 // Start after the cursor position results = allResults.slice(startIndex, startIndex + k) @@ -2288,11 +2345,13 @@ export class BrainyData implements BrainyDataInterface { } else { results = allResults.slice(0, k) } - + // Create cursor for next page let nextCursor: SearchCursor | undefined - const hasMoreResults = (startIndex + results.length) < allResults.length || allResults.length >= searchK - + const hasMoreResults = + startIndex + results.length < allResults.length || + allResults.length >= searchK + if (results.length > 0 && hasMoreResults) { const lastResult = results[results.length - 1] nextCursor = { @@ -2301,7 +2360,7 @@ export class BrainyData implements BrainyDataInterface { position: startIndex + results.length } } - + return { results, cursor: nextCursor, @@ -2407,14 +2466,14 @@ export class BrainyData implements BrainyDataInterface { } // Filter out placeholder nouns from search results - searchResults = searchResults.filter(result => { + searchResults = searchResults.filter((result) => { if (result.metadata && typeof result.metadata === 'object') { const metadata = result.metadata as Record // Exclude placeholder nouns from search results if (metadata.isPlaceholder) { return false } - + // Apply domain filter if specified if (options.filter?.domain) { if (metadata.domain !== options.filter.domain) { @@ -2544,7 +2603,7 @@ export class BrainyData implements BrainyDataInterface { // In write-only mode, query storage directly since index is not loaded if (this.writeOnly) { try { - noun = await this.storage!.getNoun(id) ?? undefined + noun = (await this.storage!.getNoun(id)) ?? undefined } catch (storageError) { // If storage lookup fails, return null (noun doesn't exist) return null @@ -2552,11 +2611,11 @@ export class BrainyData implements BrainyDataInterface { } else { // Normal mode: Get noun from index first noun = this.index.getNouns().get(id) - + // If not found in index, fallback to storage (for race conditions) if (!noun && this.storage) { try { - noun = await this.storage.getNoun(id) ?? undefined + noun = (await this.storage.getNoun(id)) ?? undefined } catch (storageError) { // Storage lookup failed, noun doesn't exist return null @@ -3061,6 +3120,7 @@ export class BrainyData implements BrainyDataInterface { id: sourceId, vector: sourcePlaceholderVector, connections: new Map(), + level: 0, metadata: sourceMetadata } @@ -3083,6 +3143,7 @@ export class BrainyData implements BrainyDataInterface { id: targetId, vector: targetPlaceholderVector, connections: new Map(), + level: 0, metadata: targetMetadata } @@ -3377,8 +3438,12 @@ export class BrainyData implements BrainyDataInterface { const serviceForStats = this.getServiceName(options) await this.storage!.incrementStatistic('verb', serviceForStats) - // Update HNSW index size (excluding verbs) - await this.storage!.updateHnswIndexSize(await this.getNounCount()) + // Track verb type + this.statisticsCollector.trackVerbType(verbMetadata.verb) + + // Update HNSW index size with actual index size + const indexSize = this.index.size() + await this.storage!.updateHnswIndexSize(indexSize) // Invalidate search cache since verb data has changed this.searchCache.invalidateOnDataChange('add') @@ -3406,7 +3471,9 @@ export class BrainyData implements BrainyDataInterface { // Get the verb metadata const metadata = await this.storage!.getVerbMetadata(id) if (!metadata) { - console.warn(`Verb ${id} found but no metadata - creating minimal GraphVerb`) + console.warn( + `Verb ${id} found but no metadata - creating minimal GraphVerb` + ) // Return minimal GraphVerb if metadata is missing return { id: hnswVerb.id, @@ -3451,7 +3518,7 @@ export class BrainyData implements BrainyDataInterface { try { // Get all lightweight verbs from storage const hnswVerbs = await this.storage!.getAllVerbs() - + // Convert each HNSWVerb to GraphVerb by loading metadata const graphVerbs: GraphVerb[] = [] for (const hnswVerb of hnswVerbs) { @@ -3478,7 +3545,7 @@ export class BrainyData implements BrainyDataInterface { console.warn(`Verb ${hnswVerb.id} found but no metadata - skipping`) } } - + return graphVerbs } catch (error) { console.error('Failed to get all verbs:', error) @@ -3649,7 +3716,10 @@ export class BrainyData implements BrainyDataInterface { // Clear storage await this.storage!.clear() - + + // Reset statistics collector + this.statisticsCollector = new StatisticsCollector() + // Clear search cache since all data has been removed this.searchCache.invalidateOnDataChange('delete') } catch (error) { @@ -3682,7 +3752,7 @@ export class BrainyData implements BrainyDataInterface { public clearCache(): void { this.searchCache.clear() } - + /** * Adapt cache configuration based on current performance metrics * This method analyzes usage patterns and automatically optimizes cache settings @@ -3692,7 +3762,7 @@ export class BrainyData implements BrainyDataInterface { const stats = this.searchCache.getStats() const memoryUsage = this.searchCache.getMemoryUsage() const currentConfig = this.searchCache.getConfig() - + // Prepare performance metrics for adaptation const performanceMetrics = { hitRate: stats.hitRate, @@ -3701,27 +3771,29 @@ export class BrainyData implements BrainyDataInterface { externalChangesDetected: 0, // Would be tracked from real-time updates timeSinceLastChange: Date.now() - this.lastUpdateTime } - + // Try to adapt configuration const newConfig = this.cacheAutoConfigurator.adaptConfiguration( currentConfig, performanceMetrics ) - + if (newConfig) { // Apply new cache configuration this.searchCache.updateConfig(newConfig.cacheConfig) - + // Apply new real-time update configuration if needed - if (newConfig.realtimeConfig.enabled !== this.realtimeUpdateConfig.enabled || - newConfig.realtimeConfig.interval !== this.realtimeUpdateConfig.interval) { - + if ( + newConfig.realtimeConfig.enabled !== + this.realtimeUpdateConfig.enabled || + newConfig.realtimeConfig.interval !== this.realtimeUpdateConfig.interval + ) { const wasEnabled = this.realtimeUpdateConfig.enabled this.realtimeUpdateConfig = { ...this.realtimeUpdateConfig, ...newConfig.realtimeConfig } - + // Restart real-time updates with new configuration if (wasEnabled) { this.stopRealtimeUpdates() @@ -3730,7 +3802,7 @@ export class BrainyData implements BrainyDataInterface { this.startRealtimeUpdates() } } - + if (this.loggingConfig?.verbose) { console.log('🔧 Auto-adapted cache configuration:') console.log(this.cacheAutoConfigurator.getConfigExplanation(newConfig)) @@ -3823,6 +3895,55 @@ export class BrainyData implements BrainyDataInterface { await this.storage.flushStatisticsToStorage() } + /** + * Update storage sizes if needed (called periodically for performance) + */ + private async updateStorageSizesIfNeeded(): Promise { + // Only update every minute to avoid performance impact + const now = Date.now() + const lastUpdate = (this as any).lastStorageSizeUpdate || 0 + + if (now - lastUpdate < 60000) { + return // Skip if updated recently + } + + ;(this as any).lastStorageSizeUpdate = now + + try { + // Estimate sizes based on counts and average sizes + const stats = await this.storage!.getStatistics() + if (stats) { + const avgNounSize = 2048 // ~2KB per noun (vector + metadata) + const avgVerbSize = 512 // ~0.5KB per verb + const avgMetadataSize = 256 // ~0.25KB per metadata entry + const avgIndexEntrySize = 128 // ~128 bytes per index entry + + // Calculate total counts + const totalNouns = Object.values(stats.nounCount).reduce( + (a, b) => a + b, + 0 + ) + const totalVerbs = Object.values(stats.verbCount).reduce( + (a, b) => a + b, + 0 + ) + const totalMetadata = Object.values(stats.metadataCount).reduce( + (a, b) => a + b, + 0 + ) + + this.statisticsCollector.updateStorageSizes({ + nouns: totalNouns * avgNounSize, + verbs: totalVerbs * avgVerbSize, + metadata: totalMetadata * avgMetadataSize, + index: stats.hnswIndexSize * avgIndexEntrySize + }) + } + } catch (error) { + // Ignore errors in size calculation + } + } + /** * Get statistics about the current state of the database * @param options Additional options for retrieving statistics @@ -3941,6 +4062,42 @@ export class BrainyData implements BrainyDataInterface { total: result.nounCount + result.verbCount + result.metadataCount } + // Add extended statistics if requested + if (true) { + // Always include for now + // Add index health metrics + try { + const indexHealth = this.index.getIndexHealth() + ;(result as any).indexHealth = indexHealth + } catch (e) { + // Index health not available + } + + // Add cache metrics + try { + const cacheStats = this.searchCache.getStats() + ;(result as any).cacheMetrics = cacheStats + } catch (e) { + // Cache stats not available + } + + // Add memory usage + if (typeof process !== 'undefined' && process.memoryUsage) { + ;(result as any).memoryUsage = process.memoryUsage().heapUsed + } + + // Add last updated timestamp + ;(result as any).lastUpdated = + stats.lastUpdated || new Date().toISOString() + + // Add enhanced statistics from collector + const collectorStats = this.statisticsCollector.getStatistics() + Object.assign(result as any, collectorStats) + + // Update storage sizes if needed (only periodically for performance) + await this.updateStorageSizesIfNeeded() + } + return result } @@ -4398,16 +4555,24 @@ export class BrainyData implements BrainyDataInterface { // Check if database is in write-only mode this.checkWriteOnly() + const searchStartTime = Date.now() + try { // Embed the query text const queryVector = await this.embed(query) // Search using the embedded vector - return await this.search(queryVector, k, { + const results = await this.search(queryVector, k, { nounTypes: options.nounTypes, includeVerbs: options.includeVerbs, searchMode: options.searchMode }) + + // Track search performance + const duration = Date.now() - searchStartTime + this.statisticsCollector.trackSearch(query, duration) + + return results } catch (error) { console.error('Failed to search with text query:', error) throw new Error(`Failed to search with text query: ${error}`) @@ -4466,7 +4631,7 @@ export class BrainyData implements BrainyDataInterface { // When using offset, fetch more results and slice const offset = options.offset || 0 const totalNeeded = k + offset - + // Search the remote server for totalNeeded results const searchResult = await this.serverSearchConduit.searchServer( this.serverConnection.connectionId, @@ -5348,7 +5513,7 @@ export class BrainyData implements BrainyDataInterface { // Sort by score and limit to k results return allResults.sort((a, b) => b.score - a.score).slice(0, k) } - + /** * Cleanup distributed resources * Should be called when shutting down the instance @@ -5359,16 +5524,16 @@ export class BrainyData implements BrainyDataInterface { clearInterval(this.updateTimerId) this.updateTimerId = null } - + // Clean up distributed mode resources if (this.healthMonitor) { this.healthMonitor.stop() } - + if (this.configManager) { await this.configManager.cleanup() } - + // Clean up worker pools await cleanupWorkerPools() } diff --git a/src/coreTypes.ts b/src/coreTypes.ts index f389076c..021851dc 100644 --- a/src/coreTypes.ts +++ b/src/coreTypes.ts @@ -82,6 +82,7 @@ export interface HNSWNoun { id: string vector: Vector connections: Map> // level -> set of connected noun ids + level: number // The highest layer this noun appears in metadata?: any // Optional metadata for the noun } @@ -199,6 +200,58 @@ export interface StatisticsData { */ standardFieldMappings?: Record> + /** + * Content type breakdown (e.g., Person, Repository, Issue, etc.) + */ + contentTypes?: Record + + /** + * Data freshness metrics + */ + dataFreshness?: { + oldestEntry: string + newestEntry: string + updatesLastHour: number + updatesLastDay: number + ageDistribution: { + last24h: number + last7d: number + last30d: number + older: number + } + } + + /** + * Storage utilization metrics + */ + storageMetrics?: { + totalSizeBytes: number + nounsSizeBytes: number + verbsSizeBytes: number + metadataSizeBytes: number + indexSizeBytes: number + } + + /** + * Search performance metrics + */ + searchMetrics?: { + totalSearches: number + averageSearchTimeMs: number + searchesLastHour: number + searchesLastDay: number + topSearchTerms?: string[] + } + + /** + * Verb statistics similar to nouns + */ + verbStatistics?: { + totalVerbs: number + verbTypes: Record + averageConnectionsPerVerb: number + } + /** * Last updated timestamp */ diff --git a/src/hnsw/hnswIndex.ts b/src/hnsw/hnswIndex.ts index e0f655ed..43f12a18 100644 --- a/src/hnsw/hnswIndex.ts +++ b/src/hnsw/hnswIndex.ts @@ -140,7 +140,8 @@ export class HNSWIndex { const noun: HNSWNoun = { id, vector, - connections: new Map() + connections: new Map(), + level: nounLevel } // Initialize empty connection sets for each level @@ -558,6 +559,38 @@ export class HNSWIndex { return { ...this.config } } + /** + * Get index health metrics + */ + public getIndexHealth(): { + averageConnections: number + layerDistribution: number[] + maxLayer: number + totalNodes: number + } { + let totalConnections = 0 + const layerCounts = new Array(this.maxLevel + 1).fill(0) + + // Count connections and layer distribution + this.nouns.forEach(noun => { + // Count connections at each layer + for (let level = 0; level <= noun.level; level++) { + totalConnections += noun.connections.get(level)?.size || 0 + layerCounts[level]++ + } + }) + + const totalNodes = this.nouns.size + const averageConnections = totalNodes > 0 ? totalConnections / totalNodes : 0 + + return { + averageConnections, + layerDistribution: layerCounts, + maxLayer: this.maxLevel, + totalNodes + } + } + /** * Search within a specific layer * Returns a map of noun IDs to distances, sorted by distance diff --git a/src/hnsw/hnswIndexOptimized.ts b/src/hnsw/hnswIndexOptimized.ts index 08031531..f6eec182 100644 --- a/src/hnsw/hnswIndexOptimized.ts +++ b/src/hnsw/hnswIndexOptimized.ts @@ -411,7 +411,8 @@ export class HNSWIndexOptimized extends HNSWIndex { const noun: HNSWNoun = { id, vector, - connections: new Map() + connections: new Map(), + level: 0 } // Store the noun diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts index 1bb1f2c8..b828d68a 100644 --- a/src/storage/adapters/fileSystemStorage.ts +++ b/src/storage/adapters/fileSystemStorage.ts @@ -193,7 +193,8 @@ export class FileSystemStorage extends BaseStorage { return { id: parsedNode.id, vector: parsedNode.vector, - connections + connections, + level: parsedNode.level || 0 } } catch (error: any) { if (error.code !== 'ENOENT') { @@ -229,7 +230,8 @@ export class FileSystemStorage extends BaseStorage { allNodes.push({ id: parsedNode.id, vector: parsedNode.vector, - connections + connections, + level: parsedNode.level || 0 }) } } @@ -273,7 +275,8 @@ export class FileSystemStorage extends BaseStorage { nouns.push({ id: parsedNode.id, vector: parsedNode.vector, - connections + connections, + level: parsedNode.level || 0 }) } } @@ -575,6 +578,10 @@ export class FileSystemStorage extends BaseStorage { // Remove all files in the index directory await removeDirectoryContents(this.indexDir) + + // Clear the statistics cache + this.statisticsCache = null + this.statisticsModified = false } /** diff --git a/src/storage/adapters/memoryStorage.ts b/src/storage/adapters/memoryStorage.ts index 9f43e803..eb34891b 100644 --- a/src/storage/adapters/memoryStorage.ts +++ b/src/storage/adapters/memoryStorage.ts @@ -42,7 +42,8 @@ export class MemoryStorage extends BaseStorage { const nounCopy: HNSWNoun = { id: noun.id, vector: [...noun.vector], - connections: new Map() + connections: new Map(), + level: noun.level || 0 } // Copy connections @@ -70,7 +71,8 @@ export class MemoryStorage extends BaseStorage { const nounCopy: HNSWNoun = { id: noun.id, vector: [...noun.vector], - connections: new Map() + connections: new Map(), + level: noun.level || 0 } // Copy connections @@ -93,7 +95,8 @@ export class MemoryStorage extends BaseStorage { const nounCopy: HNSWNoun = { id: noun.id, vector: [...noun.vector], - connections: new Map() + connections: new Map(), + level: noun.level || 0 } // Copy connections @@ -193,7 +196,8 @@ export class MemoryStorage extends BaseStorage { const nounCopy: HNSWNoun = { id: noun.id, vector: [...noun.vector], - connections: new Map() + connections: new Map(), + level: noun.level || 0 } // Copy connections @@ -578,6 +582,10 @@ export class MemoryStorage extends BaseStorage { this.nounMetadata.clear() this.verbMetadata.clear() this.statistics = null + + // Clear the statistics cache + this.statisticsCache = null + this.statisticsModified = false } /** diff --git a/src/storage/adapters/opfsStorage.ts b/src/storage/adapters/opfsStorage.ts index 5ff0f5e1..448b27ee 100644 --- a/src/storage/adapters/opfsStorage.ts +++ b/src/storage/adapters/opfsStorage.ts @@ -224,7 +224,8 @@ export class OPFSStorage extends BaseStorage { return { id: data.id, vector: data.vector, - connections + connections, + level: data.level || 0 } } catch (error) { // Noun not found or other error @@ -258,7 +259,8 @@ export class OPFSStorage extends BaseStorage { allNouns.push({ id: data.id, vector: data.vector, - connections + connections, + level: data.level || 0 }) } catch (error) { console.error(`Error reading noun file ${name}:`, error) @@ -315,7 +317,8 @@ export class OPFSStorage extends BaseStorage { nodes.push({ id: data.id, vector: data.vector, - connections + connections, + level: data.level || 0 }) } } catch (error) { @@ -732,6 +735,10 @@ export class OPFSStorage extends BaseStorage { // Remove all files in the index directory await removeDirectoryContents(this.indexDir!) + + // Clear the statistics cache + this.statisticsCache = null + this.statisticsModified = false } catch (error) { console.error('Error clearing storage:', error) throw error diff --git a/src/storage/adapters/s3CompatibleStorage.ts b/src/storage/adapters/s3CompatibleStorage.ts index d45e4d3a..92e00148 100644 --- a/src/storage/adapters/s3CompatibleStorage.ts +++ b/src/storage/adapters/s3CompatibleStorage.ts @@ -440,7 +440,8 @@ export class S3CompatibleStorage extends BaseStorage { const node = { id: parsedNode.id, vector: parsedNode.vector, - connections + connections, + level: parsedNode.level || 0 } console.log(`Successfully retrieved node ${id}:`, node) @@ -1585,6 +1586,10 @@ export class S3CompatibleStorage extends BaseStorage { // Delete all objects in the index directory await deleteObjectsWithPrefix(this.indexPrefix) + + // Clear the statistics cache + this.statisticsCache = null + this.statisticsModified = false } catch (error) { console.error('Failed to clear storage:', error) throw new Error(`Failed to clear storage: ${error}`) diff --git a/src/storage/cacheManager.ts b/src/storage/cacheManager.ts index 015185b3..e29b1a1d 100644 --- a/src/storage/cacheManager.ts +++ b/src/storage/cacheManager.ts @@ -7,7 +7,7 @@ * - Level 3: Cold storage (all nodes) - OPFS, Filesystem or S3 depending on environment */ -import { HNSWNoun, GraphVerb } from '../coreTypes.js' +import { HNSWNoun, GraphVerb, HNSWVerb } from '../coreTypes.js' import { BrainyError } from '../errors/brainyError.js' // Extend Navigator interface to include deviceMemory property @@ -71,7 +71,7 @@ enum StorageType { /** * Multi-level cache manager for efficient data access */ -export class CacheManager { +export class CacheManager { // Hot cache (RAM) private hotCache = new Map>() diff --git a/src/utils/statisticsCollector.ts b/src/utils/statisticsCollector.ts new file mode 100644 index 00000000..ccb9dbf3 --- /dev/null +++ b/src/utils/statisticsCollector.ts @@ -0,0 +1,269 @@ +/** + * Lightweight statistics collector for Brainy + * Designed to have minimal performance impact even with millions of entries + */ + +import { StatisticsData } from '../coreTypes.js' + +interface TimeSeriesData { + timestamp: number + count: number +} + +export class StatisticsCollector { + // Content type tracking (lightweight counters) + private contentTypes: Map = new Map() + + // Data freshness tracking (only track timestamps, not full data) + private oldestTimestamp: number = Date.now() + private newestTimestamp: number = Date.now() + private updateTimestamps: TimeSeriesData[] = [] + + // Search performance tracking (rolling window) + private searchMetrics = { + totalSearches: 0, + totalSearchTimeMs: 0, + searchTimestamps: [] as TimeSeriesData[], + topSearchTerms: new Map() + } + + // Verb type tracking + private verbTypes: Map = new Map() + + // Storage size estimates (updated periodically, not on every operation) + private storageSizeCache = { + lastUpdated: 0, + sizes: { + nouns: 0, + verbs: 0, + metadata: 0, + index: 0 + } + } + + private readonly MAX_TIMESTAMPS = 1000 // Keep last 1000 timestamps + private readonly MAX_SEARCH_TERMS = 100 // Track top 100 search terms + private readonly SIZE_UPDATE_INTERVAL = 60000 // Update sizes every minute + + /** + * Track content type (very lightweight) + */ + trackContentType(type: string): void { + this.contentTypes.set(type, (this.contentTypes.get(type) || 0) + 1) + } + + /** + * Track data update timestamp (lightweight) + */ + trackUpdate(timestamp?: number): void { + const ts = timestamp || Date.now() + + // Update oldest/newest + if (ts < this.oldestTimestamp) this.oldestTimestamp = ts + if (ts > this.newestTimestamp) this.newestTimestamp = ts + + // Add to rolling window + this.updateTimestamps.push({ timestamp: ts, count: 1 }) + + // Keep window size limited + if (this.updateTimestamps.length > this.MAX_TIMESTAMPS) { + this.updateTimestamps.shift() + } + } + + /** + * Track search performance (lightweight) + */ + trackSearch(searchTerm: string, durationMs: number): void { + this.searchMetrics.totalSearches++ + this.searchMetrics.totalSearchTimeMs += durationMs + + // Add to rolling window + this.searchMetrics.searchTimestamps.push({ + timestamp: Date.now(), + count: 1 + }) + + // Keep window size limited + if (this.searchMetrics.searchTimestamps.length > this.MAX_TIMESTAMPS) { + this.searchMetrics.searchTimestamps.shift() + } + + // Track search term (limit to top N) + const termCount = (this.searchMetrics.topSearchTerms.get(searchTerm) || 0) + 1 + this.searchMetrics.topSearchTerms.set(searchTerm, termCount) + + // Prune if too many terms + if (this.searchMetrics.topSearchTerms.size > this.MAX_SEARCH_TERMS * 2) { + this.pruneSearchTerms() + } + } + + /** + * Track verb type (lightweight) + */ + trackVerbType(type: string): void { + this.verbTypes.set(type, (this.verbTypes.get(type) || 0) + 1) + } + + /** + * Update storage size estimates (called periodically, not on every operation) + */ + updateStorageSizes(sizes: { + nouns: number + verbs: number + metadata: number + index: number + }): void { + this.storageSizeCache = { + lastUpdated: Date.now(), + sizes + } + } + + /** + * Get comprehensive statistics + */ + getStatistics(): Partial { + const now = Date.now() + const hourAgo = now - 3600000 + const dayAgo = now - 86400000 + const weekAgo = now - 604800000 + const monthAgo = now - 2592000000 + + // Calculate data freshness + const updatesLastHour = this.updateTimestamps.filter(t => t.timestamp > hourAgo).length + const updatesLastDay = this.updateTimestamps.filter(t => t.timestamp > dayAgo).length + + // Calculate age distribution + const ageDistribution = { + last24h: 0, + last7d: 0, + last30d: 0, + older: 0 + } + + // Estimate based on update patterns (not scanning all data) + const totalUpdates = this.updateTimestamps.length + if (totalUpdates > 0) { + const recentUpdates = this.updateTimestamps.filter(t => t.timestamp > dayAgo).length + const weekUpdates = this.updateTimestamps.filter(t => t.timestamp > weekAgo).length + const monthUpdates = this.updateTimestamps.filter(t => t.timestamp > monthAgo).length + + ageDistribution.last24h = Math.round((recentUpdates / totalUpdates) * 100) + ageDistribution.last7d = Math.round(((weekUpdates - recentUpdates) / totalUpdates) * 100) + ageDistribution.last30d = Math.round(((monthUpdates - weekUpdates) / totalUpdates) * 100) + ageDistribution.older = 100 - ageDistribution.last24h - ageDistribution.last7d - ageDistribution.last30d + } + + // Calculate search metrics + const searchesLastHour = this.searchMetrics.searchTimestamps.filter(t => t.timestamp > hourAgo).length + const searchesLastDay = this.searchMetrics.searchTimestamps.filter(t => t.timestamp > dayAgo).length + const avgSearchTime = this.searchMetrics.totalSearches > 0 + ? this.searchMetrics.totalSearchTimeMs / this.searchMetrics.totalSearches + : 0 + + // Get top search terms + const topSearchTerms = Array.from(this.searchMetrics.topSearchTerms.entries()) + .sort((a, b) => b[1] - a[1]) + .slice(0, 10) + .map(([term]) => term) + + // Calculate storage metrics + const totalSize = Object.values(this.storageSizeCache.sizes).reduce((a, b) => a + b, 0) + + return { + contentTypes: Object.fromEntries(this.contentTypes), + + dataFreshness: { + oldestEntry: new Date(this.oldestTimestamp).toISOString(), + newestEntry: new Date(this.newestTimestamp).toISOString(), + updatesLastHour, + updatesLastDay, + ageDistribution + }, + + storageMetrics: { + totalSizeBytes: totalSize, + nounsSizeBytes: this.storageSizeCache.sizes.nouns, + verbsSizeBytes: this.storageSizeCache.sizes.verbs, + metadataSizeBytes: this.storageSizeCache.sizes.metadata, + indexSizeBytes: this.storageSizeCache.sizes.index + }, + + searchMetrics: { + totalSearches: this.searchMetrics.totalSearches, + averageSearchTimeMs: avgSearchTime, + searchesLastHour, + searchesLastDay, + topSearchTerms + }, + + verbStatistics: { + totalVerbs: Array.from(this.verbTypes.values()).reduce((a, b) => a + b, 0), + verbTypes: Object.fromEntries(this.verbTypes), + averageConnectionsPerVerb: 2 // Verbs connect 2 nouns + } + } + } + + /** + * Merge statistics from storage (for distributed systems) + */ + mergeFromStorage(stored: Partial): void { + // Merge content types + if (stored.contentTypes) { + for (const [type, count] of Object.entries(stored.contentTypes)) { + this.contentTypes.set(type, count) + } + } + + // Merge verb types + if (stored.verbStatistics?.verbTypes) { + for (const [type, count] of Object.entries(stored.verbStatistics.verbTypes)) { + this.verbTypes.set(type, count) + } + } + + // Merge search metrics + if (stored.searchMetrics) { + this.searchMetrics.totalSearches = stored.searchMetrics.totalSearches || 0 + this.searchMetrics.totalSearchTimeMs = (stored.searchMetrics.averageSearchTimeMs || 0) * this.searchMetrics.totalSearches + } + + // Merge data freshness + if (stored.dataFreshness) { + this.oldestTimestamp = new Date(stored.dataFreshness.oldestEntry).getTime() + this.newestTimestamp = new Date(stored.dataFreshness.newestEntry).getTime() + } + } + + /** + * Reset statistics (for testing) + */ + reset(): void { + this.contentTypes.clear() + this.verbTypes.clear() + this.updateTimestamps = [] + this.searchMetrics = { + totalSearches: 0, + totalSearchTimeMs: 0, + searchTimestamps: [], + topSearchTerms: new Map() + } + this.oldestTimestamp = Date.now() + this.newestTimestamp = Date.now() + } + + private pruneSearchTerms(): void { + // Keep only top N search terms + const sorted = Array.from(this.searchMetrics.topSearchTerms.entries()) + .sort((a, b) => b[1] - a[1]) + .slice(0, this.MAX_SEARCH_TERMS) + + this.searchMetrics.topSearchTerms.clear() + for (const [term, count] of sorted) { + this.searchMetrics.topSearchTerms.set(term, count) + } + } +} \ No newline at end of file diff --git a/tests/core.test.ts b/tests/core.test.ts index ce8ea32f..2738b806 100644 --- a/tests/core.test.ts +++ b/tests/core.test.ts @@ -389,7 +389,7 @@ describe('Brainy Core Functionality', () => { // Verify counts expect(stats.nounCount).toBe(3) expect(stats.verbCount).toBe(2) - expect(stats.hnswIndexSize).toBe(1) + expect(stats.hnswIndexSize).toBe(5) }) }) }) diff --git a/tests/statistics.test.ts b/tests/statistics.test.ts index c6222263..bd7daa10 100644 --- a/tests/statistics.test.ts +++ b/tests/statistics.test.ts @@ -57,7 +57,7 @@ describe('Brainy Statistics Functionality', () => { expect(stats.nounCount).toBe(3) expect(stats.verbCount).toBe(1) expect(stats.metadataCount).toBe(3) // Each noun has metadata - expect(stats.hnswIndexSize).toBe(2) + expect(stats.hnswIndexSize).toBe(4) // 3 nouns + 1 verb (verbs are also added to HNSW index) }) it('should throw an error when no instance is provided', async () => { @@ -77,8 +77,16 @@ describe('Brainy Statistics Functionality', () => { const instanceStats = await data.getStatistics() const functionStats = await brainy.getStatistics(data) - // Verify they match - expect(functionStats).toEqual(instanceStats) + // Verify core statistics match (ignoring volatile fields like memoryUsage and timestamps) + expect(functionStats.nounCount).toBe(instanceStats.nounCount) + expect(functionStats.verbCount).toBe(instanceStats.verbCount) + expect(functionStats.metadataCount).toBe(instanceStats.metadataCount) + expect(functionStats.hnswIndexSize).toBe(instanceStats.hnswIndexSize) + + // If serviceBreakdown exists, verify it matches + if (instanceStats.serviceBreakdown) { + expect(functionStats.serviceBreakdown).toEqual(instanceStats.serviceBreakdown) + } }) it('should track statistics by service', async () => {