soulcraft/brainy
2
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

fix(core): resolve TypeScript compilation errors and test failures

Browse source 
- Add missing 'level' property to HNSWNoun objects in storage adapters
- Fix HNSWVerb type compatibility in CacheManager imports
- Clear statistics cache when clearing storage to prevent stale data
- Update test expectations to match actual HNSW index behavior (includes both nouns and verbs)
- Add StatisticsCollector utility for enhanced metrics tracking
- Improve statistics comparison in tests to handle volatile fields
This commit is contained in:
David Snelling 2025-08-04 20:00:38 -07:00
parent 8c1c9a08d3
commit ba890170bf
12 changed files with 668 additions and 112 deletions
Download patch file Download diff file Expand all files Collapse all files

231
src/brainyData.ts
View file

@ -59,6 +59,7 @@ import {
} from './distributed/index.js'
import { SearchCache, SearchCacheConfig } from './utils/searchCache.js'
import { CacheAutoConfigurator } from './utils/cacheAutoConfig.js'
import { StatisticsCollector } from './utils/statisticsCollector.js'
export interface BrainyDataConfig {
/**
@ -413,6 +414,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
private domainDetector: DomainDetector | null = null
private healthMonitor: HealthMonitor | null = null
// Statistics collector
private statisticsCollector: StatisticsCollector = new StatisticsCollector()
/**
* Get the vector dimensions
*/
@ -768,7 +772,10 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
// Adapt cache configuration based on performance (every few updates)
// Only adapt every 5th update to avoid over-optimization
const updateCount = Math.floor((Date.now() - (this.lastUpdateTime || 0)) / this.realtimeUpdateConfig.interval)
const updateCount = Math.floor(
(Date.now() - (this.lastUpdateTime || 0)) /
this.realtimeUpdateConfig.interval
)
if (updateCount % 5 === 0) {
this.adaptCacheConfiguration()
}
@ -1172,6 +1179,16 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
}
}
// Initialize statistics collector with existing data
try {
const existingStats = await this.storage!.getStatistics()
if (existingStats) {
this.statisticsCollector.mergeFromStorage(existingStats)
}
} catch (e) {
// Ignore errors loading existing statistics
}
this.isInitialized = true
this.isInitializing = false
@ -1218,15 +1235,21 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
// Validate that role matches the configured mode
// Don't override explicitly set readOnly/writeOnly
if (role === 'reader' && !this.readOnly) {
console.warn('Distributed role is "reader" but readOnly is not set. Setting readOnly=true for consistency.')
console.warn(
'Distributed role is "reader" but readOnly is not set. Setting readOnly=true for consistency.'
)
this.readOnly = true
this.writeOnly = false
} else if (role === 'writer' && !this.writeOnly) {
console.warn('Distributed role is "writer" but writeOnly is not set. Setting writeOnly=true for consistency.')
console.warn(
'Distributed role is "writer" but writeOnly is not set. Setting writeOnly=true for consistency.'
)
this.readOnly = false
this.writeOnly = true
} else if (role === 'hybrid' && (this.readOnly || this.writeOnly)) {
console.warn('Distributed role is "hybrid" but readOnly or writeOnly is set. Clearing both for hybrid mode.')
console.warn(
'Distributed role is "hybrid" but readOnly or writeOnly is set. Clearing both for hybrid mode.'
)
this.readOnly = false
this.writeOnly = false
}
@ -1244,7 +1267,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
// Update storage cache config if it supports it
if (this.storage && 'updateCacheConfig' in this.storage) {
(this.storage as any).updateCacheConfig(this.cacheConfig)
;(this.storage as any).updateCacheConfig(this.cacheConfig)
}
}
@ -1261,7 +1284,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
})
if (this.loggingConfig?.verbose) {
console.log(`Distributed mode initialized as ${role} with ${sharedConfig.settings.partitionStrategy} partitioning`)
console.log(
`Distributed mode initialized as ${role} with ${sharedConfig.settings.partitionStrategy} partitioning`
)
}
}
@ -1430,26 +1455,31 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
try {
if (this.writeOnly) {
// In write-only mode, check storage directly
existingNoun = await this.storage!.getNoun(options.id) ?? undefined
existingNoun =
(await this.storage!.getNoun(options.id)) ?? undefined
} else {
// In normal mode, check index first, then storage
existingNoun = this.index.getNouns().get(options.id)
if (!existingNoun) {
existingNoun = await this.storage!.getNoun(options.id) ?? undefined
existingNoun =
(await this.storage!.getNoun(options.id)) ?? undefined
}
}
if (existingNoun) {
// Check if existing noun is a placeholder
const existingMetadata = await this.storage!.getMetadata(options.id)
const isPlaceholder = existingMetadata &&
const isPlaceholder =
existingMetadata &&
typeof existingMetadata === 'object' &&
(existingMetadata as any).isPlaceholder
if (isPlaceholder) {
// Replace placeholder with real data
if (this.loggingConfig?.verbose) {
console.log(`Replacing placeholder noun ${options.id} with real data`)
console.log(
`Replacing placeholder noun ${options.id} with real data`
)
}
} else {
// Real noun already exists, update it
@ -1472,6 +1502,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
id,
vector,
connections: new Map(),
level: 0, // Default level for new nodes
metadata: undefined // Will be set separately
}
} else {
@ -1559,18 +1590,24 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
// First check if domain is already in metadata
if ((metadataToSave as any).domain) {
// Domain already specified, keep it
const domainInfo = this.domainDetector.detectDomain(metadataToSave)
const domainInfo =
this.domainDetector.detectDomain(metadataToSave)
if (domainInfo.domainMetadata) {
(metadataToSave as any).domainMetadata = domainInfo.domainMetadata
;(metadataToSave as any).domainMetadata =
domainInfo.domainMetadata
}
} else {
// Try to detect domain from the data
const dataToAnalyze = Array.isArray(vectorOrData) ? metadata : vectorOrData
const domainInfo = this.domainDetector.detectDomain(dataToAnalyze)
const dataToAnalyze = Array.isArray(vectorOrData)
? metadata
: vectorOrData
const domainInfo =
this.domainDetector.detectDomain(dataToAnalyze)
if (domainInfo.domain) {
(metadataToSave as any).domain = domainInfo.domain
;(metadataToSave as any).domain = domainInfo.domain
if (domainInfo.domainMetadata) {
(metadataToSave as any).domainMetadata = domainInfo.domainMetadata
;(metadataToSave as any).domainMetadata =
domainInfo.domainMetadata
}
}
}
@ -1588,11 +1625,26 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
// Track metadata statistics
const metadataService = this.getServiceName(options)
await this.storage!.incrementStatistic('metadata', metadataService)
// Track content type if it's a GraphNoun
if (
metadataToSave &&
typeof metadataToSave === 'object' &&
'noun' in metadataToSave
) {
this.statisticsCollector.trackContentType(
(metadataToSave as any).noun
)
}
// Track update timestamp
this.statisticsCollector.trackUpdate()
}
}
// Update HNSW index size (excluding verbs)
await this.storage!.updateHnswIndexSize(await this.getNounCount())
// Update HNSW index size with actual index size
const indexSize = this.index.size()
await this.storage!.updateHnswIndexSize(indexSize)
// Update health metrics if in distributed mode
if (this.healthMonitor) {
@ -2198,7 +2250,11 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
// Default behavior (backward compatible): search locally
try {
// Check cache first (transparent to user)
const cacheKey = this.searchCache.getCacheKey(queryVectorOrData, k, options)
const cacheKey = this.searchCache.getCacheKey(
queryVectorOrData,
k,
options
)
const cachedResults = this.searchCache.get(cacheKey)
if (cachedResults) {
@ -2272,9 +2328,10 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
// If cursor provided, find starting position
if (options.cursor) {
startIndex = allResults.findIndex(r =>
r.id === options.cursor!.lastId &&
Math.abs(r.score - options.cursor!.lastScore) < 0.0001
startIndex = allResults.findIndex(
(r) =>
r.id === options.cursor!.lastId &&
Math.abs(r.score - options.cursor!.lastScore) < 0.0001
)
if (startIndex >= 0) {
@ -2291,7 +2348,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
// Create cursor for next page
let nextCursor: SearchCursor | undefined
const hasMoreResults = (startIndex + results.length) < allResults.length || allResults.length >= searchK
const hasMoreResults =
startIndex + results.length < allResults.length ||
allResults.length >= searchK
if (results.length > 0 && hasMoreResults) {
const lastResult = results[results.length - 1]
@ -2407,7 +2466,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
}
// Filter out placeholder nouns from search results
searchResults = searchResults.filter(result => {
searchResults = searchResults.filter((result) => {
if (result.metadata && typeof result.metadata === 'object') {
const metadata = result.metadata as Record<string, any>
// Exclude placeholder nouns from search results
@ -2544,7 +2603,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
// In write-only mode, query storage directly since index is not loaded
if (this.writeOnly) {
try {
noun = await this.storage!.getNoun(id) ?? undefined
noun = (await this.storage!.getNoun(id)) ?? undefined
} catch (storageError) {
// If storage lookup fails, return null (noun doesn't exist)
return null
@ -2556,7 +2615,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
// If not found in index, fallback to storage (for race conditions)
if (!noun && this.storage) {
try {
noun = await this.storage.getNoun(id) ?? undefined
noun = (await this.storage.getNoun(id)) ?? undefined
} catch (storageError) {
// Storage lookup failed, noun doesn't exist
return null
@ -3061,6 +3120,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
id: sourceId,
vector: sourcePlaceholderVector,
connections: new Map(),
level: 0,
metadata: sourceMetadata
}
@ -3083,6 +3143,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
id: targetId,
vector: targetPlaceholderVector,
connections: new Map(),
level: 0,
metadata: targetMetadata
}
@ -3377,8 +3438,12 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
const serviceForStats = this.getServiceName(options)
await this.storage!.incrementStatistic('verb', serviceForStats)
// Update HNSW index size (excluding verbs)
await this.storage!.updateHnswIndexSize(await this.getNounCount())
// Track verb type
this.statisticsCollector.trackVerbType(verbMetadata.verb)
// Update HNSW index size with actual index size
const indexSize = this.index.size()
await this.storage!.updateHnswIndexSize(indexSize)
// Invalidate search cache since verb data has changed
this.searchCache.invalidateOnDataChange('add')
@ -3406,7 +3471,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
// Get the verb metadata
const metadata = await this.storage!.getVerbMetadata(id)
if (!metadata) {
console.warn(`Verb ${id} found but no metadata - creating minimal GraphVerb`)
console.warn(
`Verb ${id} found but no metadata - creating minimal GraphVerb`
)
// Return minimal GraphVerb if metadata is missing
return {
id: hnswVerb.id,
@ -3650,6 +3717,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
// Clear storage
await this.storage!.clear()
// Reset statistics collector
this.statisticsCollector = new StatisticsCollector()
// Clear search cache since all data has been removed
this.searchCache.invalidateOnDataChange('delete')
} catch (error) {
@ -3713,9 +3783,11 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
this.searchCache.updateConfig(newConfig.cacheConfig)
// Apply new real-time update configuration if needed
if (newConfig.realtimeConfig.enabled !== this.realtimeUpdateConfig.enabled ||
newConfig.realtimeConfig.interval !== this.realtimeUpdateConfig.interval) {
if (
newConfig.realtimeConfig.enabled !==
this.realtimeUpdateConfig.enabled ||
newConfig.realtimeConfig.interval !== this.realtimeUpdateConfig.interval
) {
const wasEnabled = this.realtimeUpdateConfig.enabled
this.realtimeUpdateConfig = {
...this.realtimeUpdateConfig,
@ -3823,6 +3895,55 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
await this.storage.flushStatisticsToStorage()
}
/**
* Update storage sizes if needed (called periodically for performance)
*/
private async updateStorageSizesIfNeeded(): Promise<void> {
// Only update every minute to avoid performance impact
const now = Date.now()
const lastUpdate = (this as any).lastStorageSizeUpdate || 0
if (now - lastUpdate < 60000) {
return // Skip if updated recently
}
;(this as any).lastStorageSizeUpdate = now
try {
// Estimate sizes based on counts and average sizes
const stats = await this.storage!.getStatistics()
if (stats) {
const avgNounSize = 2048 // ~2KB per noun (vector + metadata)
const avgVerbSize = 512 // ~0.5KB per verb
const avgMetadataSize = 256 // ~0.25KB per metadata entry
const avgIndexEntrySize = 128 // ~128 bytes per index entry
// Calculate total counts
const totalNouns = Object.values(stats.nounCount).reduce(
(a, b) => a + b,
0
)
const totalVerbs = Object.values(stats.verbCount).reduce(
(a, b) => a + b,
0
)
const totalMetadata = Object.values(stats.metadataCount).reduce(
(a, b) => a + b,
0
)
this.statisticsCollector.updateStorageSizes({
nouns: totalNouns * avgNounSize,
verbs: totalVerbs * avgVerbSize,
metadata: totalMetadata * avgMetadataSize,
index: stats.hnswIndexSize * avgIndexEntrySize
})
}
} catch (error) {
// Ignore errors in size calculation
}
}
/**
* Get statistics about the current state of the database
* @param options Additional options for retrieving statistics
@ -3941,6 +4062,42 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
total: result.nounCount + result.verbCount + result.metadataCount
}
// Add extended statistics if requested
if (true) {
// Always include for now
// Add index health metrics
try {
const indexHealth = this.index.getIndexHealth()
;(result as any).indexHealth = indexHealth
} catch (e) {
// Index health not available
}
// Add cache metrics
try {
const cacheStats = this.searchCache.getStats()
;(result as any).cacheMetrics = cacheStats
} catch (e) {
// Cache stats not available
}
// Add memory usage
if (typeof process !== 'undefined' && process.memoryUsage) {
;(result as any).memoryUsage = process.memoryUsage().heapUsed
}
// Add last updated timestamp
;(result as any).lastUpdated =
stats.lastUpdated || new Date().toISOString()
// Add enhanced statistics from collector
const collectorStats = this.statisticsCollector.getStatistics()
Object.assign(result as any, collectorStats)
// Update storage sizes if needed (only periodically for performance)
await this.updateStorageSizesIfNeeded()
}
return result
}
@ -4398,16 +4555,24 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
// Check if database is in write-only mode
this.checkWriteOnly()
const searchStartTime = Date.now()
try {
// Embed the query text
const queryVector = await this.embed(query)
// Search using the embedded vector
return await this.search(queryVector, k, {
const results = await this.search(queryVector, k, {
nounTypes: options.nounTypes,
includeVerbs: options.includeVerbs,
searchMode: options.searchMode
})
// Track search performance
const duration = Date.now() - searchStartTime
this.statisticsCollector.trackSearch(query, duration)
return results
} catch (error) {
console.error('Failed to search with text query:', error)
throw new Error(`Failed to search with text query: ${error}`)

53
src/coreTypes.ts
View file

@ -82,6 +82,7 @@ export interface HNSWNoun {
id: string
vector: Vector
connections: Map<number, Set<string>> // level -> set of connected noun ids
level: number // The highest layer this noun appears in
metadata?: any // Optional metadata for the noun
}
@ -199,6 +200,58 @@ export interface StatisticsData {
*/
standardFieldMappings?: Record<string, Record<string, string[]>>
/**
* Content type breakdown (e.g., Person, Repository, Issue, etc.)
*/
contentTypes?: Record<string, number>
/**
* Data freshness metrics
*/
dataFreshness?: {
oldestEntry: string
newestEntry: string
updatesLastHour: number
updatesLastDay: number
ageDistribution: {
last24h: number
last7d: number
last30d: number
older: number
}
}
/**
* Storage utilization metrics
*/
storageMetrics?: {
totalSizeBytes: number
nounsSizeBytes: number
verbsSizeBytes: number
metadataSizeBytes: number
indexSizeBytes: number
}
/**
* Search performance metrics
*/
searchMetrics?: {
totalSearches: number
averageSearchTimeMs: number
searchesLastHour: number
searchesLastDay: number
topSearchTerms?: string[]
}
/**
* Verb statistics similar to nouns
*/
verbStatistics?: {
totalVerbs: number
verbTypes: Record<string, number>
averageConnectionsPerVerb: number
}
/**
* Last updated timestamp
*/

35
src/hnsw/hnswIndex.ts
View file

@ -140,7 +140,8 @@ export class HNSWIndex {
const noun: HNSWNoun = {
id,
vector,
connections: new Map()
connections: new Map(),
level: nounLevel
}
// Initialize empty connection sets for each level
@ -558,6 +559,38 @@ export class HNSWIndex {
return { ...this.config }
}
/**
* Get index health metrics
*/
public getIndexHealth(): {
averageConnections: number
layerDistribution: number[]
maxLayer: number
totalNodes: number
} {
let totalConnections = 0
const layerCounts = new Array(this.maxLevel + 1).fill(0)
// Count connections and layer distribution
this.nouns.forEach(noun => {
// Count connections at each layer
for (let level = 0; level <= noun.level; level++) {
totalConnections += noun.connections.get(level)?.size || 0
layerCounts[level]++
}
})
const totalNodes = this.nouns.size
const averageConnections = totalNodes > 0 ? totalConnections / totalNodes : 0
return {
averageConnections,
layerDistribution: layerCounts,
maxLayer: this.maxLevel,
totalNodes
}
}
/**
* Search within a specific layer
* Returns a map of noun IDs to distances, sorted by distance

3
src/hnsw/hnswIndexOptimized.ts
View file

@ -411,7 +411,8 @@ export class HNSWIndexOptimized extends HNSWIndex {
const noun: HNSWNoun = {
id,
vector,
connections: new Map()
connections: new Map(),
level: 0
}
// Store the noun

13
src/storage/adapters/fileSystemStorage.ts
View file

@ -193,7 +193,8 @@ export class FileSystemStorage extends BaseStorage {
return {
id: parsedNode.id,
vector: parsedNode.vector,
connections
connections,
level: parsedNode.level || 0
}
} catch (error: any) {
if (error.code !== 'ENOENT') {
@ -229,7 +230,8 @@ export class FileSystemStorage extends BaseStorage {
allNodes.push({
id: parsedNode.id,
vector: parsedNode.vector,
connections
connections,
level: parsedNode.level || 0
})
}
}
@ -273,7 +275,8 @@ export class FileSystemStorage extends BaseStorage {
nouns.push({
id: parsedNode.id,
vector: parsedNode.vector,
connections
connections,
level: parsedNode.level || 0
})
}
}
@ -575,6 +578,10 @@ export class FileSystemStorage extends BaseStorage {
// Remove all files in the index directory
await removeDirectoryContents(this.indexDir)
// Clear the statistics cache
this.statisticsCache = null
this.statisticsModified = false
}
/**

16
src/storage/adapters/memoryStorage.ts
View file

@ -42,7 +42,8 @@ export class MemoryStorage extends BaseStorage {
const nounCopy: HNSWNoun = {
id: noun.id,
vector: [...noun.vector],
connections: new Map()
connections: new Map(),
level: noun.level || 0
}
// Copy connections
@ -70,7 +71,8 @@ export class MemoryStorage extends BaseStorage {
const nounCopy: HNSWNoun = {
id: noun.id,
vector: [...noun.vector],
connections: new Map()
connections: new Map(),
level: noun.level || 0
}
// Copy connections
@ -93,7 +95,8 @@ export class MemoryStorage extends BaseStorage {
const nounCopy: HNSWNoun = {
id: noun.id,
vector: [...noun.vector],
connections: new Map()
connections: new Map(),
level: noun.level || 0
}
// Copy connections
@ -193,7 +196,8 @@ export class MemoryStorage extends BaseStorage {
const nounCopy: HNSWNoun = {
id: noun.id,
vector: [...noun.vector],
connections: new Map()
connections: new Map(),
level: noun.level || 0
}
// Copy connections
@ -578,6 +582,10 @@ export class MemoryStorage extends BaseStorage {
this.nounMetadata.clear()
this.verbMetadata.clear()
this.statistics = null
// Clear the statistics cache
this.statisticsCache = null
this.statisticsModified = false
}
/**

13
src/storage/adapters/opfsStorage.ts
View file

@ -224,7 +224,8 @@ export class OPFSStorage extends BaseStorage {
return {
id: data.id,
vector: data.vector,
connections
connections,
level: data.level || 0
}
} catch (error) {
// Noun not found or other error
@ -258,7 +259,8 @@ export class OPFSStorage extends BaseStorage {
allNouns.push({
id: data.id,
vector: data.vector,
connections
connections,
level: data.level || 0
})
} catch (error) {
console.error(`Error reading noun file ${name}:`, error)
@ -315,7 +317,8 @@ export class OPFSStorage extends BaseStorage {
nodes.push({
id: data.id,
vector: data.vector,
connections
connections,
level: data.level || 0
})
}
} catch (error) {
@ -732,6 +735,10 @@ export class OPFSStorage extends BaseStorage {
// Remove all files in the index directory
await removeDirectoryContents(this.indexDir!)
// Clear the statistics cache
this.statisticsCache = null
this.statisticsModified = false
} catch (error) {
console.error('Error clearing storage:', error)
throw error

7
src/storage/adapters/s3CompatibleStorage.ts
View file

@ -440,7 +440,8 @@ export class S3CompatibleStorage extends BaseStorage {
const node = {
id: parsedNode.id,
vector: parsedNode.vector,
connections
connections,
level: parsedNode.level || 0
}
console.log(`Successfully retrieved node ${id}:`, node)
@ -1585,6 +1586,10 @@ export class S3CompatibleStorage extends BaseStorage {
// Delete all objects in the index directory
await deleteObjectsWithPrefix(this.indexPrefix)
// Clear the statistics cache
this.statisticsCache = null
this.statisticsModified = false
} catch (error) {
console.error('Failed to clear storage:', error)
throw new Error(`Failed to clear storage: ${error}`)

4
src/storage/cacheManager.ts
View file

@ -7,7 +7,7 @@
* - Level 3: Cold storage (all nodes) - OPFS, Filesystem or S3 depending on environment
*/
import { HNSWNoun, GraphVerb } from '../coreTypes.js'
import { HNSWNoun, GraphVerb, HNSWVerb } from '../coreTypes.js'
import { BrainyError } from '../errors/brainyError.js'
// Extend Navigator interface to include deviceMemory property
@ -71,7 +71,7 @@ enum StorageType {
/**
* Multi-level cache manager for efficient data access
*/
export class CacheManager<T extends HNSWNode | Edge> {
export class CacheManager<T extends HNSWNode | Edge | HNSWVerb> {
// Hot cache (RAM)
private hotCache = new Map<string, CacheEntry<T>>()

269
src/utils/statisticsCollector.ts Normal file
View file

@ -0,0 +1,269 @@
/**
* Lightweight statistics collector for Brainy
* Designed to have minimal performance impact even with millions of entries
*/
import { StatisticsData } from '../coreTypes.js'
interface TimeSeriesData {
timestamp: number
count: number
}
export class StatisticsCollector {
// Content type tracking (lightweight counters)
private contentTypes: Map<string, number> = new Map()
// Data freshness tracking (only track timestamps, not full data)
private oldestTimestamp: number = Date.now()
private newestTimestamp: number = Date.now()
private updateTimestamps: TimeSeriesData[] = []
// Search performance tracking (rolling window)
private searchMetrics = {
totalSearches: 0,
totalSearchTimeMs: 0,
searchTimestamps: [] as TimeSeriesData[],
topSearchTerms: new Map<string, number>()
}
// Verb type tracking
private verbTypes: Map<string, number> = new Map()
// Storage size estimates (updated periodically, not on every operation)
private storageSizeCache = {
lastUpdated: 0,
sizes: {
nouns: 0,
verbs: 0,
metadata: 0,
index: 0
}
}
private readonly MAX_TIMESTAMPS = 1000 // Keep last 1000 timestamps
private readonly MAX_SEARCH_TERMS = 100 // Track top 100 search terms
private readonly SIZE_UPDATE_INTERVAL = 60000 // Update sizes every minute
/**
* Track content type (very lightweight)
*/
trackContentType(type: string): void {
this.contentTypes.set(type, (this.contentTypes.get(type) || 0) + 1)
}
/**
* Track data update timestamp (lightweight)
*/
trackUpdate(timestamp?: number): void {
const ts = timestamp || Date.now()
// Update oldest/newest
if (ts < this.oldestTimestamp) this.oldestTimestamp = ts
if (ts > this.newestTimestamp) this.newestTimestamp = ts
// Add to rolling window
this.updateTimestamps.push({ timestamp: ts, count: 1 })
// Keep window size limited
if (this.updateTimestamps.length > this.MAX_TIMESTAMPS) {
this.updateTimestamps.shift()
}
}
/**
* Track search performance (lightweight)
*/
trackSearch(searchTerm: string, durationMs: number): void {
this.searchMetrics.totalSearches++
this.searchMetrics.totalSearchTimeMs += durationMs
// Add to rolling window
this.searchMetrics.searchTimestamps.push({
timestamp: Date.now(),
count: 1
})
// Keep window size limited
if (this.searchMetrics.searchTimestamps.length > this.MAX_TIMESTAMPS) {
this.searchMetrics.searchTimestamps.shift()
}
// Track search term (limit to top N)
const termCount = (this.searchMetrics.topSearchTerms.get(searchTerm) || 0) + 1
this.searchMetrics.topSearchTerms.set(searchTerm, termCount)
// Prune if too many terms
if (this.searchMetrics.topSearchTerms.size > this.MAX_SEARCH_TERMS * 2) {
this.pruneSearchTerms()
}
}
/**
* Track verb type (lightweight)
*/
trackVerbType(type: string): void {
this.verbTypes.set(type, (this.verbTypes.get(type) || 0) + 1)
}
/**
* Update storage size estimates (called periodically, not on every operation)
*/
updateStorageSizes(sizes: {
nouns: number
verbs: number
metadata: number
index: number
}): void {
this.storageSizeCache = {
lastUpdated: Date.now(),
sizes
}
}
/**
* Get comprehensive statistics
*/
getStatistics(): Partial<StatisticsData> {
const now = Date.now()
const hourAgo = now - 3600000
const dayAgo = now - 86400000
const weekAgo = now - 604800000
const monthAgo = now - 2592000000
// Calculate data freshness
const updatesLastHour = this.updateTimestamps.filter(t => t.timestamp > hourAgo).length
const updatesLastDay = this.updateTimestamps.filter(t => t.timestamp > dayAgo).length
// Calculate age distribution
const ageDistribution = {
last24h: 0,
last7d: 0,
last30d: 0,
older: 0
}
// Estimate based on update patterns (not scanning all data)
const totalUpdates = this.updateTimestamps.length
if (totalUpdates > 0) {
const recentUpdates = this.updateTimestamps.filter(t => t.timestamp > dayAgo).length
const weekUpdates = this.updateTimestamps.filter(t => t.timestamp > weekAgo).length
const monthUpdates = this.updateTimestamps.filter(t => t.timestamp > monthAgo).length
ageDistribution.last24h = Math.round((recentUpdates / totalUpdates) * 100)
ageDistribution.last7d = Math.round(((weekUpdates - recentUpdates) / totalUpdates) * 100)
ageDistribution.last30d = Math.round(((monthUpdates - weekUpdates) / totalUpdates) * 100)
ageDistribution.older = 100 - ageDistribution.last24h - ageDistribution.last7d - ageDistribution.last30d
}
// Calculate search metrics
const searchesLastHour = this.searchMetrics.searchTimestamps.filter(t => t.timestamp > hourAgo).length
const searchesLastDay = this.searchMetrics.searchTimestamps.filter(t => t.timestamp > dayAgo).length
const avgSearchTime = this.searchMetrics.totalSearches > 0
? this.searchMetrics.totalSearchTimeMs / this.searchMetrics.totalSearches
: 0
// Get top search terms
const topSearchTerms = Array.from(this.searchMetrics.topSearchTerms.entries())
.sort((a, b) => b[1] - a[1])
.slice(0, 10)
.map(([term]) => term)
// Calculate storage metrics
const totalSize = Object.values(this.storageSizeCache.sizes).reduce((a, b) => a + b, 0)
return {
contentTypes: Object.fromEntries(this.contentTypes),
dataFreshness: {
oldestEntry: new Date(this.oldestTimestamp).toISOString(),
newestEntry: new Date(this.newestTimestamp).toISOString(),
updatesLastHour,
updatesLastDay,
ageDistribution
},
storageMetrics: {
totalSizeBytes: totalSize,
nounsSizeBytes: this.storageSizeCache.sizes.nouns,
verbsSizeBytes: this.storageSizeCache.sizes.verbs,
metadataSizeBytes: this.storageSizeCache.sizes.metadata,
indexSizeBytes: this.storageSizeCache.sizes.index
},
searchMetrics: {
totalSearches: this.searchMetrics.totalSearches,
averageSearchTimeMs: avgSearchTime,
searchesLastHour,
searchesLastDay,
topSearchTerms
},
verbStatistics: {
totalVerbs: Array.from(this.verbTypes.values()).reduce((a, b) => a + b, 0),
verbTypes: Object.fromEntries(this.verbTypes),
averageConnectionsPerVerb: 2 // Verbs connect 2 nouns
}
}
}
/**
* Merge statistics from storage (for distributed systems)
*/
mergeFromStorage(stored: Partial<StatisticsData>): void {
// Merge content types
if (stored.contentTypes) {
for (const [type, count] of Object.entries(stored.contentTypes)) {
this.contentTypes.set(type, count)
}
}
// Merge verb types
if (stored.verbStatistics?.verbTypes) {
for (const [type, count] of Object.entries(stored.verbStatistics.verbTypes)) {
this.verbTypes.set(type, count)
}
}
// Merge search metrics
if (stored.searchMetrics) {
this.searchMetrics.totalSearches = stored.searchMetrics.totalSearches || 0
this.searchMetrics.totalSearchTimeMs = (stored.searchMetrics.averageSearchTimeMs || 0) * this.searchMetrics.totalSearches
}
// Merge data freshness
if (stored.dataFreshness) {
this.oldestTimestamp = new Date(stored.dataFreshness.oldestEntry).getTime()
this.newestTimestamp = new Date(stored.dataFreshness.newestEntry).getTime()
}
}
/**
* Reset statistics (for testing)
*/
reset(): void {
this.contentTypes.clear()
this.verbTypes.clear()
this.updateTimestamps = []
this.searchMetrics = {
totalSearches: 0,
totalSearchTimeMs: 0,
searchTimestamps: [],
topSearchTerms: new Map()
}
this.oldestTimestamp = Date.now()
this.newestTimestamp = Date.now()
}
private pruneSearchTerms(): void {
// Keep only top N search terms
const sorted = Array.from(this.searchMetrics.topSearchTerms.entries())
.sort((a, b) => b[1] - a[1])
.slice(0, this.MAX_SEARCH_TERMS)
this.searchMetrics.topSearchTerms.clear()
for (const [term, count] of sorted) {
this.searchMetrics.topSearchTerms.set(term, count)
}
}
}

2
tests/core.test.ts
View file

@ -389,7 +389,7 @@ describe('Brainy Core Functionality', () => {
// Verify counts
expect(stats.nounCount).toBe(3)
expect(stats.verbCount).toBe(2)
expect(stats.hnswIndexSize).toBe(1)
expect(stats.hnswIndexSize).toBe(5)
})
})
})

14
tests/statistics.test.ts
View file

@ -57,7 +57,7 @@ describe('Brainy Statistics Functionality', () => {
expect(stats.nounCount).toBe(3)
expect(stats.verbCount).toBe(1)
expect(stats.metadataCount).toBe(3) // Each noun has metadata
expect(stats.hnswIndexSize).toBe(2)
expect(stats.hnswIndexSize).toBe(4) // 3 nouns + 1 verb (verbs are also added to HNSW index)
})
it('should throw an error when no instance is provided', async () => {
@ -77,8 +77,16 @@ describe('Brainy Statistics Functionality', () => {
const instanceStats = await data.getStatistics()
const functionStats = await brainy.getStatistics(data)
// Verify they match
expect(functionStats).toEqual(instanceStats)
// Verify core statistics match (ignoring volatile fields like memoryUsage and timestamps)
expect(functionStats.nounCount).toBe(instanceStats.nounCount)
expect(functionStats.verbCount).toBe(instanceStats.verbCount)
expect(functionStats.metadataCount).toBe(instanceStats.metadataCount)
expect(functionStats.hnswIndexSize).toBe(instanceStats.hnswIndexSize)
// If serviceBreakdown exists, verify it matches
if (instanceStats.serviceBreakdown) {
expect(functionStats.serviceBreakdown).toEqual(instanceStats.serviceBreakdown)
}
})
it('should track statistics by service', async () => {