fix(core): resolve TypeScript compilation errors and test failures
- Add missing 'level' property to HNSWNoun objects in storage adapters - Fix HNSWVerb type compatibility in CacheManager imports - Clear statistics cache when clearing storage to prevent stale data - Update test expectations to match actual HNSW index behavior (includes both nouns and verbs) - Add StatisticsCollector utility for enhanced metrics tracking - Improve statistics comparison in tests to handle volatile fields
This commit is contained in:
parent
649e452ff9
commit
cfaf2f8b83
12 changed files with 668 additions and 112 deletions
|
|
@ -59,6 +59,7 @@ import {
|
||||||
} from './distributed/index.js'
|
} from './distributed/index.js'
|
||||||
import { SearchCache, SearchCacheConfig } from './utils/searchCache.js'
|
import { SearchCache, SearchCacheConfig } from './utils/searchCache.js'
|
||||||
import { CacheAutoConfigurator } from './utils/cacheAutoConfig.js'
|
import { CacheAutoConfigurator } from './utils/cacheAutoConfig.js'
|
||||||
|
import { StatisticsCollector } from './utils/statisticsCollector.js'
|
||||||
|
|
||||||
export interface BrainyDataConfig {
|
export interface BrainyDataConfig {
|
||||||
/**
|
/**
|
||||||
|
|
@ -283,7 +284,7 @@ export interface BrainyDataConfig {
|
||||||
* Enables coordination across multiple Brainy instances
|
* Enables coordination across multiple Brainy instances
|
||||||
*/
|
*/
|
||||||
distributed?: DistributedConfig | boolean
|
distributed?: DistributedConfig | boolean
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Cache configuration for optimizing search performance
|
* Cache configuration for optimizing search performance
|
||||||
* Controls how the system caches data for faster access
|
* Controls how the system caches data for faster access
|
||||||
|
|
@ -404,7 +405,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
private remoteServerConfig: BrainyDataConfig['remoteServer'] | null = null
|
private remoteServerConfig: BrainyDataConfig['remoteServer'] | null = null
|
||||||
private serverSearchConduit: ServerSearchConduitAugmentation | null = null
|
private serverSearchConduit: ServerSearchConduitAugmentation | null = null
|
||||||
private serverConnection: WebSocketConnection | null = null
|
private serverConnection: WebSocketConnection | null = null
|
||||||
|
|
||||||
// Distributed mode properties
|
// Distributed mode properties
|
||||||
private distributedConfig: DistributedConfig | null = null
|
private distributedConfig: DistributedConfig | null = null
|
||||||
private configManager: DistributedConfigManager | null = null
|
private configManager: DistributedConfigManager | null = null
|
||||||
|
|
@ -413,6 +414,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
private domainDetector: DomainDetector | null = null
|
private domainDetector: DomainDetector | null = null
|
||||||
private healthMonitor: HealthMonitor | null = null
|
private healthMonitor: HealthMonitor | null = null
|
||||||
|
|
||||||
|
// Statistics collector
|
||||||
|
private statisticsCollector: StatisticsCollector = new StatisticsCollector()
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get the vector dimensions
|
* Get the vector dimensions
|
||||||
*/
|
*/
|
||||||
|
|
@ -547,7 +551,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
...config.cache
|
...config.cache
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Store distributed configuration
|
// Store distributed configuration
|
||||||
if (config.distributed) {
|
if (config.distributed) {
|
||||||
if (typeof config.distributed === 'boolean') {
|
if (typeof config.distributed === 'boolean') {
|
||||||
|
|
@ -560,10 +564,10 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
this.distributedConfig = config.distributed
|
this.distributedConfig = config.distributed
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Initialize cache auto-configurator first
|
// Initialize cache auto-configurator first
|
||||||
this.cacheAutoConfigurator = new CacheAutoConfigurator()
|
this.cacheAutoConfigurator = new CacheAutoConfigurator()
|
||||||
|
|
||||||
// Auto-detect optimal cache configuration if not explicitly provided
|
// Auto-detect optimal cache configuration if not explicitly provided
|
||||||
let finalSearchCacheConfig = config.searchCache
|
let finalSearchCacheConfig = config.searchCache
|
||||||
if (!config.searchCache || Object.keys(config.searchCache).length === 0) {
|
if (!config.searchCache || Object.keys(config.searchCache).length === 0) {
|
||||||
|
|
@ -571,7 +575,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
config.storage
|
config.storage
|
||||||
)
|
)
|
||||||
finalSearchCacheConfig = autoConfig.cacheConfig
|
finalSearchCacheConfig = autoConfig.cacheConfig
|
||||||
|
|
||||||
// Apply auto-detected real-time update configuration if not explicitly set
|
// Apply auto-detected real-time update configuration if not explicitly set
|
||||||
if (!config.realtimeUpdates && autoConfig.realtimeConfig.enabled) {
|
if (!config.realtimeUpdates && autoConfig.realtimeConfig.enabled) {
|
||||||
this.realtimeUpdateConfig = {
|
this.realtimeUpdateConfig = {
|
||||||
|
|
@ -579,12 +583,12 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
...autoConfig.realtimeConfig
|
...autoConfig.realtimeConfig
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (this.loggingConfig?.verbose) {
|
if (this.loggingConfig?.verbose) {
|
||||||
console.log(this.cacheAutoConfigurator.getConfigExplanation(autoConfig))
|
console.log(this.cacheAutoConfigurator.getConfigExplanation(autoConfig))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Initialize search cache with final configuration
|
// Initialize search cache with final configuration
|
||||||
this.searchCache = new SearchCache<T>(finalSearchCacheConfig)
|
this.searchCache = new SearchCache<T>(finalSearchCacheConfig)
|
||||||
}
|
}
|
||||||
|
|
@ -768,7 +772,10 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
|
|
||||||
// Adapt cache configuration based on performance (every few updates)
|
// Adapt cache configuration based on performance (every few updates)
|
||||||
// Only adapt every 5th update to avoid over-optimization
|
// Only adapt every 5th update to avoid over-optimization
|
||||||
const updateCount = Math.floor((Date.now() - (this.lastUpdateTime || 0)) / this.realtimeUpdateConfig.interval)
|
const updateCount = Math.floor(
|
||||||
|
(Date.now() - (this.lastUpdateTime || 0)) /
|
||||||
|
this.realtimeUpdateConfig.interval
|
||||||
|
)
|
||||||
if (updateCount % 5 === 0) {
|
if (updateCount % 5 === 0) {
|
||||||
this.adaptCacheConfiguration()
|
this.adaptCacheConfiguration()
|
||||||
}
|
}
|
||||||
|
|
@ -1108,7 +1115,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
|
|
||||||
// Initialize storage
|
// Initialize storage
|
||||||
await this.storage!.init()
|
await this.storage!.init()
|
||||||
|
|
||||||
// Initialize distributed mode if configured
|
// Initialize distributed mode if configured
|
||||||
if (this.distributedConfig) {
|
if (this.distributedConfig) {
|
||||||
await this.initializeDistributedMode()
|
await this.initializeDistributedMode()
|
||||||
|
|
@ -1172,6 +1179,16 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Initialize statistics collector with existing data
|
||||||
|
try {
|
||||||
|
const existingStats = await this.storage!.getStatistics()
|
||||||
|
if (existingStats) {
|
||||||
|
this.statisticsCollector.mergeFromStorage(existingStats)
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
// Ignore errors loading existing statistics
|
||||||
|
}
|
||||||
|
|
||||||
this.isInitialized = true
|
this.isInitialized = true
|
||||||
this.isInitializing = false
|
this.isInitializing = false
|
||||||
|
|
||||||
|
|
@ -1192,17 +1209,17 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
if (!this.storage) {
|
if (!this.storage) {
|
||||||
throw new Error('Storage must be initialized before distributed mode')
|
throw new Error('Storage must be initialized before distributed mode')
|
||||||
}
|
}
|
||||||
|
|
||||||
// Create configuration manager with mode hints
|
// Create configuration manager with mode hints
|
||||||
this.configManager = new DistributedConfigManager(
|
this.configManager = new DistributedConfigManager(
|
||||||
this.storage,
|
this.storage,
|
||||||
this.distributedConfig || undefined,
|
this.distributedConfig || undefined,
|
||||||
{ readOnly: this.readOnly, writeOnly: this.writeOnly }
|
{ readOnly: this.readOnly, writeOnly: this.writeOnly }
|
||||||
)
|
)
|
||||||
|
|
||||||
// Initialize configuration
|
// Initialize configuration
|
||||||
const sharedConfig = await this.configManager.initialize()
|
const sharedConfig = await this.configManager.initialize()
|
||||||
|
|
||||||
// Create partitioner based on strategy
|
// Create partitioner based on strategy
|
||||||
if (sharedConfig.settings.partitionStrategy === 'hash') {
|
if (sharedConfig.settings.partitionStrategy === 'hash') {
|
||||||
this.partitioner = new HashPartitioner(sharedConfig)
|
this.partitioner = new HashPartitioner(sharedConfig)
|
||||||
|
|
@ -1210,27 +1227,33 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
// Default to hash partitioner for now
|
// Default to hash partitioner for now
|
||||||
this.partitioner = new HashPartitioner(sharedConfig)
|
this.partitioner = new HashPartitioner(sharedConfig)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Create operational mode based on role
|
// Create operational mode based on role
|
||||||
const role = this.configManager.getRole()
|
const role = this.configManager.getRole()
|
||||||
this.operationalMode = OperationalModeFactory.createMode(role)
|
this.operationalMode = OperationalModeFactory.createMode(role)
|
||||||
|
|
||||||
// Validate that role matches the configured mode
|
// Validate that role matches the configured mode
|
||||||
// Don't override explicitly set readOnly/writeOnly
|
// Don't override explicitly set readOnly/writeOnly
|
||||||
if (role === 'reader' && !this.readOnly) {
|
if (role === 'reader' && !this.readOnly) {
|
||||||
console.warn('Distributed role is "reader" but readOnly is not set. Setting readOnly=true for consistency.')
|
console.warn(
|
||||||
|
'Distributed role is "reader" but readOnly is not set. Setting readOnly=true for consistency.'
|
||||||
|
)
|
||||||
this.readOnly = true
|
this.readOnly = true
|
||||||
this.writeOnly = false
|
this.writeOnly = false
|
||||||
} else if (role === 'writer' && !this.writeOnly) {
|
} else if (role === 'writer' && !this.writeOnly) {
|
||||||
console.warn('Distributed role is "writer" but writeOnly is not set. Setting writeOnly=true for consistency.')
|
console.warn(
|
||||||
|
'Distributed role is "writer" but writeOnly is not set. Setting writeOnly=true for consistency.'
|
||||||
|
)
|
||||||
this.readOnly = false
|
this.readOnly = false
|
||||||
this.writeOnly = true
|
this.writeOnly = true
|
||||||
} else if (role === 'hybrid' && (this.readOnly || this.writeOnly)) {
|
} else if (role === 'hybrid' && (this.readOnly || this.writeOnly)) {
|
||||||
console.warn('Distributed role is "hybrid" but readOnly or writeOnly is set. Clearing both for hybrid mode.')
|
console.warn(
|
||||||
|
'Distributed role is "hybrid" but readOnly or writeOnly is set. Clearing both for hybrid mode.'
|
||||||
|
)
|
||||||
this.readOnly = false
|
this.readOnly = false
|
||||||
this.writeOnly = false
|
this.writeOnly = false
|
||||||
}
|
}
|
||||||
|
|
||||||
// Apply cache configuration from operational mode
|
// Apply cache configuration from operational mode
|
||||||
const modeCache = this.operationalMode.cacheStrategy
|
const modeCache = this.operationalMode.cacheStrategy
|
||||||
if (modeCache) {
|
if (modeCache) {
|
||||||
|
|
@ -1241,30 +1264,32 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
warmCacheTTL: modeCache.ttl,
|
warmCacheTTL: modeCache.ttl,
|
||||||
batchSize: modeCache.writeBufferSize || 100
|
batchSize: modeCache.writeBufferSize || 100
|
||||||
}
|
}
|
||||||
|
|
||||||
// Update storage cache config if it supports it
|
// Update storage cache config if it supports it
|
||||||
if (this.storage && 'updateCacheConfig' in this.storage) {
|
if (this.storage && 'updateCacheConfig' in this.storage) {
|
||||||
(this.storage as any).updateCacheConfig(this.cacheConfig)
|
;(this.storage as any).updateCacheConfig(this.cacheConfig)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Initialize domain detector
|
// Initialize domain detector
|
||||||
this.domainDetector = new DomainDetector()
|
this.domainDetector = new DomainDetector()
|
||||||
|
|
||||||
// Initialize health monitor
|
// Initialize health monitor
|
||||||
this.healthMonitor = new HealthMonitor(this.configManager)
|
this.healthMonitor = new HealthMonitor(this.configManager)
|
||||||
this.healthMonitor.start()
|
this.healthMonitor.start()
|
||||||
|
|
||||||
// Set up config update listener
|
// Set up config update listener
|
||||||
this.configManager.setOnConfigUpdate((config) => {
|
this.configManager.setOnConfigUpdate((config) => {
|
||||||
this.handleDistributedConfigUpdate(config)
|
this.handleDistributedConfigUpdate(config)
|
||||||
})
|
})
|
||||||
|
|
||||||
if (this.loggingConfig?.verbose) {
|
if (this.loggingConfig?.verbose) {
|
||||||
console.log(`Distributed mode initialized as ${role} with ${sharedConfig.settings.partitionStrategy} partitioning`)
|
console.log(
|
||||||
|
`Distributed mode initialized as ${role} with ${sharedConfig.settings.partitionStrategy} partitioning`
|
||||||
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Handle distributed configuration updates
|
* Handle distributed configuration updates
|
||||||
*/
|
*/
|
||||||
|
|
@ -1273,13 +1298,13 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
if (this.partitioner && config.settings) {
|
if (this.partitioner && config.settings) {
|
||||||
this.partitioner = new HashPartitioner(config)
|
this.partitioner = new HashPartitioner(config)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Log configuration update
|
// Log configuration update
|
||||||
if (this.loggingConfig?.verbose) {
|
if (this.loggingConfig?.verbose) {
|
||||||
console.log('Distributed configuration updated:', config.version)
|
console.log('Distributed configuration updated:', config.version)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get distributed health status
|
* Get distributed health status
|
||||||
* @returns Health status if distributed mode is enabled
|
* @returns Health status if distributed mode is enabled
|
||||||
|
|
@ -1290,7 +1315,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
}
|
}
|
||||||
return null
|
return null
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Connect to a remote Brainy server for search operations
|
* Connect to a remote Brainy server for search operations
|
||||||
* @param serverUrl WebSocket URL of the remote Brainy server
|
* @param serverUrl WebSocket URL of the remote Brainy server
|
||||||
|
|
@ -1430,26 +1455,31 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
try {
|
try {
|
||||||
if (this.writeOnly) {
|
if (this.writeOnly) {
|
||||||
// In write-only mode, check storage directly
|
// In write-only mode, check storage directly
|
||||||
existingNoun = await this.storage!.getNoun(options.id) ?? undefined
|
existingNoun =
|
||||||
|
(await this.storage!.getNoun(options.id)) ?? undefined
|
||||||
} else {
|
} else {
|
||||||
// In normal mode, check index first, then storage
|
// In normal mode, check index first, then storage
|
||||||
existingNoun = this.index.getNouns().get(options.id)
|
existingNoun = this.index.getNouns().get(options.id)
|
||||||
if (!existingNoun) {
|
if (!existingNoun) {
|
||||||
existingNoun = await this.storage!.getNoun(options.id) ?? undefined
|
existingNoun =
|
||||||
|
(await this.storage!.getNoun(options.id)) ?? undefined
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (existingNoun) {
|
if (existingNoun) {
|
||||||
// Check if existing noun is a placeholder
|
// Check if existing noun is a placeholder
|
||||||
const existingMetadata = await this.storage!.getMetadata(options.id)
|
const existingMetadata = await this.storage!.getMetadata(options.id)
|
||||||
const isPlaceholder = existingMetadata &&
|
const isPlaceholder =
|
||||||
typeof existingMetadata === 'object' &&
|
existingMetadata &&
|
||||||
|
typeof existingMetadata === 'object' &&
|
||||||
(existingMetadata as any).isPlaceholder
|
(existingMetadata as any).isPlaceholder
|
||||||
|
|
||||||
if (isPlaceholder) {
|
if (isPlaceholder) {
|
||||||
// Replace placeholder with real data
|
// Replace placeholder with real data
|
||||||
if (this.loggingConfig?.verbose) {
|
if (this.loggingConfig?.verbose) {
|
||||||
console.log(`Replacing placeholder noun ${options.id} with real data`)
|
console.log(
|
||||||
|
`Replacing placeholder noun ${options.id} with real data`
|
||||||
|
)
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// Real noun already exists, update it
|
// Real noun already exists, update it
|
||||||
|
|
@ -1472,6 +1502,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
id,
|
id,
|
||||||
vector,
|
vector,
|
||||||
connections: new Map(),
|
connections: new Map(),
|
||||||
|
level: 0, // Default level for new nodes
|
||||||
metadata: undefined // Will be set separately
|
metadata: undefined // Will be set separately
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
|
@ -1553,29 +1584,35 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
if (metadata && typeof metadata === 'object') {
|
if (metadata && typeof metadata === 'object') {
|
||||||
// Always make a copy without adding the ID
|
// Always make a copy without adding the ID
|
||||||
metadataToSave = { ...metadata }
|
metadataToSave = { ...metadata }
|
||||||
|
|
||||||
// Add domain metadata if distributed mode is enabled
|
// Add domain metadata if distributed mode is enabled
|
||||||
if (this.domainDetector) {
|
if (this.domainDetector) {
|
||||||
// First check if domain is already in metadata
|
// First check if domain is already in metadata
|
||||||
if ((metadataToSave as any).domain) {
|
if ((metadataToSave as any).domain) {
|
||||||
// Domain already specified, keep it
|
// Domain already specified, keep it
|
||||||
const domainInfo = this.domainDetector.detectDomain(metadataToSave)
|
const domainInfo =
|
||||||
|
this.domainDetector.detectDomain(metadataToSave)
|
||||||
if (domainInfo.domainMetadata) {
|
if (domainInfo.domainMetadata) {
|
||||||
(metadataToSave as any).domainMetadata = domainInfo.domainMetadata
|
;(metadataToSave as any).domainMetadata =
|
||||||
|
domainInfo.domainMetadata
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// Try to detect domain from the data
|
// Try to detect domain from the data
|
||||||
const dataToAnalyze = Array.isArray(vectorOrData) ? metadata : vectorOrData
|
const dataToAnalyze = Array.isArray(vectorOrData)
|
||||||
const domainInfo = this.domainDetector.detectDomain(dataToAnalyze)
|
? metadata
|
||||||
|
: vectorOrData
|
||||||
|
const domainInfo =
|
||||||
|
this.domainDetector.detectDomain(dataToAnalyze)
|
||||||
if (domainInfo.domain) {
|
if (domainInfo.domain) {
|
||||||
(metadataToSave as any).domain = domainInfo.domain
|
;(metadataToSave as any).domain = domainInfo.domain
|
||||||
if (domainInfo.domainMetadata) {
|
if (domainInfo.domainMetadata) {
|
||||||
(metadataToSave as any).domainMetadata = domainInfo.domainMetadata
|
;(metadataToSave as any).domainMetadata =
|
||||||
|
domainInfo.domainMetadata
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Add partition information if distributed mode is enabled
|
// Add partition information if distributed mode is enabled
|
||||||
if (this.partitioner) {
|
if (this.partitioner) {
|
||||||
const partition = this.partitioner.getPartition(id)
|
const partition = this.partitioner.getPartition(id)
|
||||||
|
|
@ -1588,12 +1625,27 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
// Track metadata statistics
|
// Track metadata statistics
|
||||||
const metadataService = this.getServiceName(options)
|
const metadataService = this.getServiceName(options)
|
||||||
await this.storage!.incrementStatistic('metadata', metadataService)
|
await this.storage!.incrementStatistic('metadata', metadataService)
|
||||||
|
|
||||||
|
// Track content type if it's a GraphNoun
|
||||||
|
if (
|
||||||
|
metadataToSave &&
|
||||||
|
typeof metadataToSave === 'object' &&
|
||||||
|
'noun' in metadataToSave
|
||||||
|
) {
|
||||||
|
this.statisticsCollector.trackContentType(
|
||||||
|
(metadataToSave as any).noun
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Track update timestamp
|
||||||
|
this.statisticsCollector.trackUpdate()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Update HNSW index size (excluding verbs)
|
// Update HNSW index size with actual index size
|
||||||
await this.storage!.updateHnswIndexSize(await this.getNounCount())
|
const indexSize = this.index.size()
|
||||||
|
await this.storage!.updateHnswIndexSize(indexSize)
|
||||||
|
|
||||||
// Update health metrics if in distributed mode
|
// Update health metrics if in distributed mode
|
||||||
if (this.healthMonitor) {
|
if (this.healthMonitor) {
|
||||||
const vectorCount = await this.getNounCount()
|
const vectorCount = await this.getNounCount()
|
||||||
|
|
@ -1617,12 +1669,12 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
return id
|
return id
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('Failed to add vector:', error)
|
console.error('Failed to add vector:', error)
|
||||||
|
|
||||||
// Track error in health monitor
|
// Track error in health monitor
|
||||||
if (this.healthMonitor) {
|
if (this.healthMonitor) {
|
||||||
this.healthMonitor.recordRequest(0, true)
|
this.healthMonitor.recordRequest(0, true)
|
||||||
}
|
}
|
||||||
|
|
||||||
throw new Error(`Failed to add vector: ${error}`)
|
throw new Error(`Failed to add vector: ${error}`)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -2005,7 +2057,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
// When using offset, we need to fetch more results and then slice
|
// When using offset, we need to fetch more results and then slice
|
||||||
const offset = options.offset || 0
|
const offset = options.offset || 0
|
||||||
const totalNeeded = k + offset
|
const totalNeeded = k + offset
|
||||||
|
|
||||||
// Search in the index for totalNeeded results
|
// Search in the index for totalNeeded results
|
||||||
const results = await this.index.search(queryVector, totalNeeded)
|
const results = await this.index.search(queryVector, totalNeeded)
|
||||||
|
|
||||||
|
|
@ -2198,9 +2250,13 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
// Default behavior (backward compatible): search locally
|
// Default behavior (backward compatible): search locally
|
||||||
try {
|
try {
|
||||||
// Check cache first (transparent to user)
|
// Check cache first (transparent to user)
|
||||||
const cacheKey = this.searchCache.getCacheKey(queryVectorOrData, k, options)
|
const cacheKey = this.searchCache.getCacheKey(
|
||||||
|
queryVectorOrData,
|
||||||
|
k,
|
||||||
|
options
|
||||||
|
)
|
||||||
const cachedResults = this.searchCache.get(cacheKey)
|
const cachedResults = this.searchCache.get(cacheKey)
|
||||||
|
|
||||||
if (cachedResults) {
|
if (cachedResults) {
|
||||||
// Track cache hit in health monitor
|
// Track cache hit in health monitor
|
||||||
if (this.healthMonitor) {
|
if (this.healthMonitor) {
|
||||||
|
|
@ -2210,22 +2266,22 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
}
|
}
|
||||||
return cachedResults
|
return cachedResults
|
||||||
}
|
}
|
||||||
|
|
||||||
// Cache miss - perform actual search
|
// Cache miss - perform actual search
|
||||||
const results = await this.searchLocal(queryVectorOrData, k, options)
|
const results = await this.searchLocal(queryVectorOrData, k, options)
|
||||||
|
|
||||||
// Cache results for future queries (unless explicitly disabled)
|
// Cache results for future queries (unless explicitly disabled)
|
||||||
if (!options.skipCache) {
|
if (!options.skipCache) {
|
||||||
this.searchCache.set(cacheKey, results)
|
this.searchCache.set(cacheKey, results)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Track successful search in health monitor
|
// Track successful search in health monitor
|
||||||
if (this.healthMonitor) {
|
if (this.healthMonitor) {
|
||||||
const latency = Date.now() - startTime
|
const latency = Date.now() - startTime
|
||||||
this.healthMonitor.recordRequest(latency, false)
|
this.healthMonitor.recordRequest(latency, false)
|
||||||
this.healthMonitor.recordCacheAccess(false)
|
this.healthMonitor.recordCacheAccess(false)
|
||||||
}
|
}
|
||||||
|
|
||||||
return results
|
return results
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
// Track error in health monitor
|
// Track error in health monitor
|
||||||
|
|
@ -2260,23 +2316,24 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
): Promise<PaginatedSearchResult<T>> {
|
): Promise<PaginatedSearchResult<T>> {
|
||||||
// For cursor-based search, we need to fetch more results and filter
|
// For cursor-based search, we need to fetch more results and filter
|
||||||
const searchK = options.cursor ? k + 20 : k // Get extra results for filtering
|
const searchK = options.cursor ? k + 20 : k // Get extra results for filtering
|
||||||
|
|
||||||
// Perform regular search
|
// Perform regular search
|
||||||
const allResults = await this.search(queryVectorOrData, searchK, {
|
const allResults = await this.search(queryVectorOrData, searchK, {
|
||||||
...options,
|
...options,
|
||||||
skipCache: options.skipCache
|
skipCache: options.skipCache
|
||||||
})
|
})
|
||||||
|
|
||||||
let results = allResults
|
let results = allResults
|
||||||
let startIndex = 0
|
let startIndex = 0
|
||||||
|
|
||||||
// If cursor provided, find starting position
|
// If cursor provided, find starting position
|
||||||
if (options.cursor) {
|
if (options.cursor) {
|
||||||
startIndex = allResults.findIndex(r =>
|
startIndex = allResults.findIndex(
|
||||||
r.id === options.cursor!.lastId &&
|
(r) =>
|
||||||
Math.abs(r.score - options.cursor!.lastScore) < 0.0001
|
r.id === options.cursor!.lastId &&
|
||||||
|
Math.abs(r.score - options.cursor!.lastScore) < 0.0001
|
||||||
)
|
)
|
||||||
|
|
||||||
if (startIndex >= 0) {
|
if (startIndex >= 0) {
|
||||||
startIndex += 1 // Start after the cursor position
|
startIndex += 1 // Start after the cursor position
|
||||||
results = allResults.slice(startIndex, startIndex + k)
|
results = allResults.slice(startIndex, startIndex + k)
|
||||||
|
|
@ -2288,11 +2345,13 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
} else {
|
} else {
|
||||||
results = allResults.slice(0, k)
|
results = allResults.slice(0, k)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Create cursor for next page
|
// Create cursor for next page
|
||||||
let nextCursor: SearchCursor | undefined
|
let nextCursor: SearchCursor | undefined
|
||||||
const hasMoreResults = (startIndex + results.length) < allResults.length || allResults.length >= searchK
|
const hasMoreResults =
|
||||||
|
startIndex + results.length < allResults.length ||
|
||||||
|
allResults.length >= searchK
|
||||||
|
|
||||||
if (results.length > 0 && hasMoreResults) {
|
if (results.length > 0 && hasMoreResults) {
|
||||||
const lastResult = results[results.length - 1]
|
const lastResult = results[results.length - 1]
|
||||||
nextCursor = {
|
nextCursor = {
|
||||||
|
|
@ -2301,7 +2360,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
position: startIndex + results.length
|
position: startIndex + results.length
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
results,
|
results,
|
||||||
cursor: nextCursor,
|
cursor: nextCursor,
|
||||||
|
|
@ -2407,14 +2466,14 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Filter out placeholder nouns from search results
|
// Filter out placeholder nouns from search results
|
||||||
searchResults = searchResults.filter(result => {
|
searchResults = searchResults.filter((result) => {
|
||||||
if (result.metadata && typeof result.metadata === 'object') {
|
if (result.metadata && typeof result.metadata === 'object') {
|
||||||
const metadata = result.metadata as Record<string, any>
|
const metadata = result.metadata as Record<string, any>
|
||||||
// Exclude placeholder nouns from search results
|
// Exclude placeholder nouns from search results
|
||||||
if (metadata.isPlaceholder) {
|
if (metadata.isPlaceholder) {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
// Apply domain filter if specified
|
// Apply domain filter if specified
|
||||||
if (options.filter?.domain) {
|
if (options.filter?.domain) {
|
||||||
if (metadata.domain !== options.filter.domain) {
|
if (metadata.domain !== options.filter.domain) {
|
||||||
|
|
@ -2544,7 +2603,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
// In write-only mode, query storage directly since index is not loaded
|
// In write-only mode, query storage directly since index is not loaded
|
||||||
if (this.writeOnly) {
|
if (this.writeOnly) {
|
||||||
try {
|
try {
|
||||||
noun = await this.storage!.getNoun(id) ?? undefined
|
noun = (await this.storage!.getNoun(id)) ?? undefined
|
||||||
} catch (storageError) {
|
} catch (storageError) {
|
||||||
// If storage lookup fails, return null (noun doesn't exist)
|
// If storage lookup fails, return null (noun doesn't exist)
|
||||||
return null
|
return null
|
||||||
|
|
@ -2552,11 +2611,11 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
} else {
|
} else {
|
||||||
// Normal mode: Get noun from index first
|
// Normal mode: Get noun from index first
|
||||||
noun = this.index.getNouns().get(id)
|
noun = this.index.getNouns().get(id)
|
||||||
|
|
||||||
// If not found in index, fallback to storage (for race conditions)
|
// If not found in index, fallback to storage (for race conditions)
|
||||||
if (!noun && this.storage) {
|
if (!noun && this.storage) {
|
||||||
try {
|
try {
|
||||||
noun = await this.storage.getNoun(id) ?? undefined
|
noun = (await this.storage.getNoun(id)) ?? undefined
|
||||||
} catch (storageError) {
|
} catch (storageError) {
|
||||||
// Storage lookup failed, noun doesn't exist
|
// Storage lookup failed, noun doesn't exist
|
||||||
return null
|
return null
|
||||||
|
|
@ -3061,6 +3120,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
id: sourceId,
|
id: sourceId,
|
||||||
vector: sourcePlaceholderVector,
|
vector: sourcePlaceholderVector,
|
||||||
connections: new Map(),
|
connections: new Map(),
|
||||||
|
level: 0,
|
||||||
metadata: sourceMetadata
|
metadata: sourceMetadata
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -3083,6 +3143,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
id: targetId,
|
id: targetId,
|
||||||
vector: targetPlaceholderVector,
|
vector: targetPlaceholderVector,
|
||||||
connections: new Map(),
|
connections: new Map(),
|
||||||
|
level: 0,
|
||||||
metadata: targetMetadata
|
metadata: targetMetadata
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -3377,8 +3438,12 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
const serviceForStats = this.getServiceName(options)
|
const serviceForStats = this.getServiceName(options)
|
||||||
await this.storage!.incrementStatistic('verb', serviceForStats)
|
await this.storage!.incrementStatistic('verb', serviceForStats)
|
||||||
|
|
||||||
// Update HNSW index size (excluding verbs)
|
// Track verb type
|
||||||
await this.storage!.updateHnswIndexSize(await this.getNounCount())
|
this.statisticsCollector.trackVerbType(verbMetadata.verb)
|
||||||
|
|
||||||
|
// Update HNSW index size with actual index size
|
||||||
|
const indexSize = this.index.size()
|
||||||
|
await this.storage!.updateHnswIndexSize(indexSize)
|
||||||
|
|
||||||
// Invalidate search cache since verb data has changed
|
// Invalidate search cache since verb data has changed
|
||||||
this.searchCache.invalidateOnDataChange('add')
|
this.searchCache.invalidateOnDataChange('add')
|
||||||
|
|
@ -3406,7 +3471,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
// Get the verb metadata
|
// Get the verb metadata
|
||||||
const metadata = await this.storage!.getVerbMetadata(id)
|
const metadata = await this.storage!.getVerbMetadata(id)
|
||||||
if (!metadata) {
|
if (!metadata) {
|
||||||
console.warn(`Verb ${id} found but no metadata - creating minimal GraphVerb`)
|
console.warn(
|
||||||
|
`Verb ${id} found but no metadata - creating minimal GraphVerb`
|
||||||
|
)
|
||||||
// Return minimal GraphVerb if metadata is missing
|
// Return minimal GraphVerb if metadata is missing
|
||||||
return {
|
return {
|
||||||
id: hnswVerb.id,
|
id: hnswVerb.id,
|
||||||
|
|
@ -3451,7 +3518,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
try {
|
try {
|
||||||
// Get all lightweight verbs from storage
|
// Get all lightweight verbs from storage
|
||||||
const hnswVerbs = await this.storage!.getAllVerbs()
|
const hnswVerbs = await this.storage!.getAllVerbs()
|
||||||
|
|
||||||
// Convert each HNSWVerb to GraphVerb by loading metadata
|
// Convert each HNSWVerb to GraphVerb by loading metadata
|
||||||
const graphVerbs: GraphVerb[] = []
|
const graphVerbs: GraphVerb[] = []
|
||||||
for (const hnswVerb of hnswVerbs) {
|
for (const hnswVerb of hnswVerbs) {
|
||||||
|
|
@ -3478,7 +3545,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
console.warn(`Verb ${hnswVerb.id} found but no metadata - skipping`)
|
console.warn(`Verb ${hnswVerb.id} found but no metadata - skipping`)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return graphVerbs
|
return graphVerbs
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('Failed to get all verbs:', error)
|
console.error('Failed to get all verbs:', error)
|
||||||
|
|
@ -3649,7 +3716,10 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
|
|
||||||
// Clear storage
|
// Clear storage
|
||||||
await this.storage!.clear()
|
await this.storage!.clear()
|
||||||
|
|
||||||
|
// Reset statistics collector
|
||||||
|
this.statisticsCollector = new StatisticsCollector()
|
||||||
|
|
||||||
// Clear search cache since all data has been removed
|
// Clear search cache since all data has been removed
|
||||||
this.searchCache.invalidateOnDataChange('delete')
|
this.searchCache.invalidateOnDataChange('delete')
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
|
@ -3682,7 +3752,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
public clearCache(): void {
|
public clearCache(): void {
|
||||||
this.searchCache.clear()
|
this.searchCache.clear()
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Adapt cache configuration based on current performance metrics
|
* Adapt cache configuration based on current performance metrics
|
||||||
* This method analyzes usage patterns and automatically optimizes cache settings
|
* This method analyzes usage patterns and automatically optimizes cache settings
|
||||||
|
|
@ -3692,7 +3762,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
const stats = this.searchCache.getStats()
|
const stats = this.searchCache.getStats()
|
||||||
const memoryUsage = this.searchCache.getMemoryUsage()
|
const memoryUsage = this.searchCache.getMemoryUsage()
|
||||||
const currentConfig = this.searchCache.getConfig()
|
const currentConfig = this.searchCache.getConfig()
|
||||||
|
|
||||||
// Prepare performance metrics for adaptation
|
// Prepare performance metrics for adaptation
|
||||||
const performanceMetrics = {
|
const performanceMetrics = {
|
||||||
hitRate: stats.hitRate,
|
hitRate: stats.hitRate,
|
||||||
|
|
@ -3701,27 +3771,29 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
externalChangesDetected: 0, // Would be tracked from real-time updates
|
externalChangesDetected: 0, // Would be tracked from real-time updates
|
||||||
timeSinceLastChange: Date.now() - this.lastUpdateTime
|
timeSinceLastChange: Date.now() - this.lastUpdateTime
|
||||||
}
|
}
|
||||||
|
|
||||||
// Try to adapt configuration
|
// Try to adapt configuration
|
||||||
const newConfig = this.cacheAutoConfigurator.adaptConfiguration(
|
const newConfig = this.cacheAutoConfigurator.adaptConfiguration(
|
||||||
currentConfig,
|
currentConfig,
|
||||||
performanceMetrics
|
performanceMetrics
|
||||||
)
|
)
|
||||||
|
|
||||||
if (newConfig) {
|
if (newConfig) {
|
||||||
// Apply new cache configuration
|
// Apply new cache configuration
|
||||||
this.searchCache.updateConfig(newConfig.cacheConfig)
|
this.searchCache.updateConfig(newConfig.cacheConfig)
|
||||||
|
|
||||||
// Apply new real-time update configuration if needed
|
// Apply new real-time update configuration if needed
|
||||||
if (newConfig.realtimeConfig.enabled !== this.realtimeUpdateConfig.enabled ||
|
if (
|
||||||
newConfig.realtimeConfig.interval !== this.realtimeUpdateConfig.interval) {
|
newConfig.realtimeConfig.enabled !==
|
||||||
|
this.realtimeUpdateConfig.enabled ||
|
||||||
|
newConfig.realtimeConfig.interval !== this.realtimeUpdateConfig.interval
|
||||||
|
) {
|
||||||
const wasEnabled = this.realtimeUpdateConfig.enabled
|
const wasEnabled = this.realtimeUpdateConfig.enabled
|
||||||
this.realtimeUpdateConfig = {
|
this.realtimeUpdateConfig = {
|
||||||
...this.realtimeUpdateConfig,
|
...this.realtimeUpdateConfig,
|
||||||
...newConfig.realtimeConfig
|
...newConfig.realtimeConfig
|
||||||
}
|
}
|
||||||
|
|
||||||
// Restart real-time updates with new configuration
|
// Restart real-time updates with new configuration
|
||||||
if (wasEnabled) {
|
if (wasEnabled) {
|
||||||
this.stopRealtimeUpdates()
|
this.stopRealtimeUpdates()
|
||||||
|
|
@ -3730,7 +3802,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
this.startRealtimeUpdates()
|
this.startRealtimeUpdates()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (this.loggingConfig?.verbose) {
|
if (this.loggingConfig?.verbose) {
|
||||||
console.log('🔧 Auto-adapted cache configuration:')
|
console.log('🔧 Auto-adapted cache configuration:')
|
||||||
console.log(this.cacheAutoConfigurator.getConfigExplanation(newConfig))
|
console.log(this.cacheAutoConfigurator.getConfigExplanation(newConfig))
|
||||||
|
|
@ -3823,6 +3895,55 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
await this.storage.flushStatisticsToStorage()
|
await this.storage.flushStatisticsToStorage()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Update storage sizes if needed (called periodically for performance)
|
||||||
|
*/
|
||||||
|
private async updateStorageSizesIfNeeded(): Promise<void> {
|
||||||
|
// Only update every minute to avoid performance impact
|
||||||
|
const now = Date.now()
|
||||||
|
const lastUpdate = (this as any).lastStorageSizeUpdate || 0
|
||||||
|
|
||||||
|
if (now - lastUpdate < 60000) {
|
||||||
|
return // Skip if updated recently
|
||||||
|
}
|
||||||
|
|
||||||
|
;(this as any).lastStorageSizeUpdate = now
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Estimate sizes based on counts and average sizes
|
||||||
|
const stats = await this.storage!.getStatistics()
|
||||||
|
if (stats) {
|
||||||
|
const avgNounSize = 2048 // ~2KB per noun (vector + metadata)
|
||||||
|
const avgVerbSize = 512 // ~0.5KB per verb
|
||||||
|
const avgMetadataSize = 256 // ~0.25KB per metadata entry
|
||||||
|
const avgIndexEntrySize = 128 // ~128 bytes per index entry
|
||||||
|
|
||||||
|
// Calculate total counts
|
||||||
|
const totalNouns = Object.values(stats.nounCount).reduce(
|
||||||
|
(a, b) => a + b,
|
||||||
|
0
|
||||||
|
)
|
||||||
|
const totalVerbs = Object.values(stats.verbCount).reduce(
|
||||||
|
(a, b) => a + b,
|
||||||
|
0
|
||||||
|
)
|
||||||
|
const totalMetadata = Object.values(stats.metadataCount).reduce(
|
||||||
|
(a, b) => a + b,
|
||||||
|
0
|
||||||
|
)
|
||||||
|
|
||||||
|
this.statisticsCollector.updateStorageSizes({
|
||||||
|
nouns: totalNouns * avgNounSize,
|
||||||
|
verbs: totalVerbs * avgVerbSize,
|
||||||
|
metadata: totalMetadata * avgMetadataSize,
|
||||||
|
index: stats.hnswIndexSize * avgIndexEntrySize
|
||||||
|
})
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
// Ignore errors in size calculation
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get statistics about the current state of the database
|
* Get statistics about the current state of the database
|
||||||
* @param options Additional options for retrieving statistics
|
* @param options Additional options for retrieving statistics
|
||||||
|
|
@ -3941,6 +4062,42 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
total: result.nounCount + result.verbCount + result.metadataCount
|
total: result.nounCount + result.verbCount + result.metadataCount
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Add extended statistics if requested
|
||||||
|
if (true) {
|
||||||
|
// Always include for now
|
||||||
|
// Add index health metrics
|
||||||
|
try {
|
||||||
|
const indexHealth = this.index.getIndexHealth()
|
||||||
|
;(result as any).indexHealth = indexHealth
|
||||||
|
} catch (e) {
|
||||||
|
// Index health not available
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add cache metrics
|
||||||
|
try {
|
||||||
|
const cacheStats = this.searchCache.getStats()
|
||||||
|
;(result as any).cacheMetrics = cacheStats
|
||||||
|
} catch (e) {
|
||||||
|
// Cache stats not available
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add memory usage
|
||||||
|
if (typeof process !== 'undefined' && process.memoryUsage) {
|
||||||
|
;(result as any).memoryUsage = process.memoryUsage().heapUsed
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add last updated timestamp
|
||||||
|
;(result as any).lastUpdated =
|
||||||
|
stats.lastUpdated || new Date().toISOString()
|
||||||
|
|
||||||
|
// Add enhanced statistics from collector
|
||||||
|
const collectorStats = this.statisticsCollector.getStatistics()
|
||||||
|
Object.assign(result as any, collectorStats)
|
||||||
|
|
||||||
|
// Update storage sizes if needed (only periodically for performance)
|
||||||
|
await this.updateStorageSizesIfNeeded()
|
||||||
|
}
|
||||||
|
|
||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -4398,16 +4555,24 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
// Check if database is in write-only mode
|
// Check if database is in write-only mode
|
||||||
this.checkWriteOnly()
|
this.checkWriteOnly()
|
||||||
|
|
||||||
|
const searchStartTime = Date.now()
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// Embed the query text
|
// Embed the query text
|
||||||
const queryVector = await this.embed(query)
|
const queryVector = await this.embed(query)
|
||||||
|
|
||||||
// Search using the embedded vector
|
// Search using the embedded vector
|
||||||
return await this.search(queryVector, k, {
|
const results = await this.search(queryVector, k, {
|
||||||
nounTypes: options.nounTypes,
|
nounTypes: options.nounTypes,
|
||||||
includeVerbs: options.includeVerbs,
|
includeVerbs: options.includeVerbs,
|
||||||
searchMode: options.searchMode
|
searchMode: options.searchMode
|
||||||
})
|
})
|
||||||
|
|
||||||
|
// Track search performance
|
||||||
|
const duration = Date.now() - searchStartTime
|
||||||
|
this.statisticsCollector.trackSearch(query, duration)
|
||||||
|
|
||||||
|
return results
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('Failed to search with text query:', error)
|
console.error('Failed to search with text query:', error)
|
||||||
throw new Error(`Failed to search with text query: ${error}`)
|
throw new Error(`Failed to search with text query: ${error}`)
|
||||||
|
|
@ -4466,7 +4631,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
// When using offset, fetch more results and slice
|
// When using offset, fetch more results and slice
|
||||||
const offset = options.offset || 0
|
const offset = options.offset || 0
|
||||||
const totalNeeded = k + offset
|
const totalNeeded = k + offset
|
||||||
|
|
||||||
// Search the remote server for totalNeeded results
|
// Search the remote server for totalNeeded results
|
||||||
const searchResult = await this.serverSearchConduit.searchServer(
|
const searchResult = await this.serverSearchConduit.searchServer(
|
||||||
this.serverConnection.connectionId,
|
this.serverConnection.connectionId,
|
||||||
|
|
@ -5348,7 +5513,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
// Sort by score and limit to k results
|
// Sort by score and limit to k results
|
||||||
return allResults.sort((a, b) => b.score - a.score).slice(0, k)
|
return allResults.sort((a, b) => b.score - a.score).slice(0, k)
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Cleanup distributed resources
|
* Cleanup distributed resources
|
||||||
* Should be called when shutting down the instance
|
* Should be called when shutting down the instance
|
||||||
|
|
@ -5359,16 +5524,16 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
clearInterval(this.updateTimerId)
|
clearInterval(this.updateTimerId)
|
||||||
this.updateTimerId = null
|
this.updateTimerId = null
|
||||||
}
|
}
|
||||||
|
|
||||||
// Clean up distributed mode resources
|
// Clean up distributed mode resources
|
||||||
if (this.healthMonitor) {
|
if (this.healthMonitor) {
|
||||||
this.healthMonitor.stop()
|
this.healthMonitor.stop()
|
||||||
}
|
}
|
||||||
|
|
||||||
if (this.configManager) {
|
if (this.configManager) {
|
||||||
await this.configManager.cleanup()
|
await this.configManager.cleanup()
|
||||||
}
|
}
|
||||||
|
|
||||||
// Clean up worker pools
|
// Clean up worker pools
|
||||||
await cleanupWorkerPools()
|
await cleanupWorkerPools()
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -82,6 +82,7 @@ export interface HNSWNoun {
|
||||||
id: string
|
id: string
|
||||||
vector: Vector
|
vector: Vector
|
||||||
connections: Map<number, Set<string>> // level -> set of connected noun ids
|
connections: Map<number, Set<string>> // level -> set of connected noun ids
|
||||||
|
level: number // The highest layer this noun appears in
|
||||||
metadata?: any // Optional metadata for the noun
|
metadata?: any // Optional metadata for the noun
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -199,6 +200,58 @@ export interface StatisticsData {
|
||||||
*/
|
*/
|
||||||
standardFieldMappings?: Record<string, Record<string, string[]>>
|
standardFieldMappings?: Record<string, Record<string, string[]>>
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Content type breakdown (e.g., Person, Repository, Issue, etc.)
|
||||||
|
*/
|
||||||
|
contentTypes?: Record<string, number>
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Data freshness metrics
|
||||||
|
*/
|
||||||
|
dataFreshness?: {
|
||||||
|
oldestEntry: string
|
||||||
|
newestEntry: string
|
||||||
|
updatesLastHour: number
|
||||||
|
updatesLastDay: number
|
||||||
|
ageDistribution: {
|
||||||
|
last24h: number
|
||||||
|
last7d: number
|
||||||
|
last30d: number
|
||||||
|
older: number
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Storage utilization metrics
|
||||||
|
*/
|
||||||
|
storageMetrics?: {
|
||||||
|
totalSizeBytes: number
|
||||||
|
nounsSizeBytes: number
|
||||||
|
verbsSizeBytes: number
|
||||||
|
metadataSizeBytes: number
|
||||||
|
indexSizeBytes: number
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Search performance metrics
|
||||||
|
*/
|
||||||
|
searchMetrics?: {
|
||||||
|
totalSearches: number
|
||||||
|
averageSearchTimeMs: number
|
||||||
|
searchesLastHour: number
|
||||||
|
searchesLastDay: number
|
||||||
|
topSearchTerms?: string[]
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verb statistics similar to nouns
|
||||||
|
*/
|
||||||
|
verbStatistics?: {
|
||||||
|
totalVerbs: number
|
||||||
|
verbTypes: Record<string, number>
|
||||||
|
averageConnectionsPerVerb: number
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Last updated timestamp
|
* Last updated timestamp
|
||||||
*/
|
*/
|
||||||
|
|
|
||||||
|
|
@ -140,7 +140,8 @@ export class HNSWIndex {
|
||||||
const noun: HNSWNoun = {
|
const noun: HNSWNoun = {
|
||||||
id,
|
id,
|
||||||
vector,
|
vector,
|
||||||
connections: new Map()
|
connections: new Map(),
|
||||||
|
level: nounLevel
|
||||||
}
|
}
|
||||||
|
|
||||||
// Initialize empty connection sets for each level
|
// Initialize empty connection sets for each level
|
||||||
|
|
@ -558,6 +559,38 @@ export class HNSWIndex {
|
||||||
return { ...this.config }
|
return { ...this.config }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get index health metrics
|
||||||
|
*/
|
||||||
|
public getIndexHealth(): {
|
||||||
|
averageConnections: number
|
||||||
|
layerDistribution: number[]
|
||||||
|
maxLayer: number
|
||||||
|
totalNodes: number
|
||||||
|
} {
|
||||||
|
let totalConnections = 0
|
||||||
|
const layerCounts = new Array(this.maxLevel + 1).fill(0)
|
||||||
|
|
||||||
|
// Count connections and layer distribution
|
||||||
|
this.nouns.forEach(noun => {
|
||||||
|
// Count connections at each layer
|
||||||
|
for (let level = 0; level <= noun.level; level++) {
|
||||||
|
totalConnections += noun.connections.get(level)?.size || 0
|
||||||
|
layerCounts[level]++
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
const totalNodes = this.nouns.size
|
||||||
|
const averageConnections = totalNodes > 0 ? totalConnections / totalNodes : 0
|
||||||
|
|
||||||
|
return {
|
||||||
|
averageConnections,
|
||||||
|
layerDistribution: layerCounts,
|
||||||
|
maxLayer: this.maxLevel,
|
||||||
|
totalNodes
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Search within a specific layer
|
* Search within a specific layer
|
||||||
* Returns a map of noun IDs to distances, sorted by distance
|
* Returns a map of noun IDs to distances, sorted by distance
|
||||||
|
|
|
||||||
|
|
@ -411,7 +411,8 @@ export class HNSWIndexOptimized extends HNSWIndex {
|
||||||
const noun: HNSWNoun = {
|
const noun: HNSWNoun = {
|
||||||
id,
|
id,
|
||||||
vector,
|
vector,
|
||||||
connections: new Map()
|
connections: new Map(),
|
||||||
|
level: 0
|
||||||
}
|
}
|
||||||
|
|
||||||
// Store the noun
|
// Store the noun
|
||||||
|
|
|
||||||
|
|
@ -193,7 +193,8 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
return {
|
return {
|
||||||
id: parsedNode.id,
|
id: parsedNode.id,
|
||||||
vector: parsedNode.vector,
|
vector: parsedNode.vector,
|
||||||
connections
|
connections,
|
||||||
|
level: parsedNode.level || 0
|
||||||
}
|
}
|
||||||
} catch (error: any) {
|
} catch (error: any) {
|
||||||
if (error.code !== 'ENOENT') {
|
if (error.code !== 'ENOENT') {
|
||||||
|
|
@ -229,7 +230,8 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
allNodes.push({
|
allNodes.push({
|
||||||
id: parsedNode.id,
|
id: parsedNode.id,
|
||||||
vector: parsedNode.vector,
|
vector: parsedNode.vector,
|
||||||
connections
|
connections,
|
||||||
|
level: parsedNode.level || 0
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -273,7 +275,8 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
nouns.push({
|
nouns.push({
|
||||||
id: parsedNode.id,
|
id: parsedNode.id,
|
||||||
vector: parsedNode.vector,
|
vector: parsedNode.vector,
|
||||||
connections
|
connections,
|
||||||
|
level: parsedNode.level || 0
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -575,6 +578,10 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
|
|
||||||
// Remove all files in the index directory
|
// Remove all files in the index directory
|
||||||
await removeDirectoryContents(this.indexDir)
|
await removeDirectoryContents(this.indexDir)
|
||||||
|
|
||||||
|
// Clear the statistics cache
|
||||||
|
this.statisticsCache = null
|
||||||
|
this.statisticsModified = false
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
||||||
|
|
@ -42,7 +42,8 @@ export class MemoryStorage extends BaseStorage {
|
||||||
const nounCopy: HNSWNoun = {
|
const nounCopy: HNSWNoun = {
|
||||||
id: noun.id,
|
id: noun.id,
|
||||||
vector: [...noun.vector],
|
vector: [...noun.vector],
|
||||||
connections: new Map()
|
connections: new Map(),
|
||||||
|
level: noun.level || 0
|
||||||
}
|
}
|
||||||
|
|
||||||
// Copy connections
|
// Copy connections
|
||||||
|
|
@ -70,7 +71,8 @@ export class MemoryStorage extends BaseStorage {
|
||||||
const nounCopy: HNSWNoun = {
|
const nounCopy: HNSWNoun = {
|
||||||
id: noun.id,
|
id: noun.id,
|
||||||
vector: [...noun.vector],
|
vector: [...noun.vector],
|
||||||
connections: new Map()
|
connections: new Map(),
|
||||||
|
level: noun.level || 0
|
||||||
}
|
}
|
||||||
|
|
||||||
// Copy connections
|
// Copy connections
|
||||||
|
|
@ -93,7 +95,8 @@ export class MemoryStorage extends BaseStorage {
|
||||||
const nounCopy: HNSWNoun = {
|
const nounCopy: HNSWNoun = {
|
||||||
id: noun.id,
|
id: noun.id,
|
||||||
vector: [...noun.vector],
|
vector: [...noun.vector],
|
||||||
connections: new Map()
|
connections: new Map(),
|
||||||
|
level: noun.level || 0
|
||||||
}
|
}
|
||||||
|
|
||||||
// Copy connections
|
// Copy connections
|
||||||
|
|
@ -193,7 +196,8 @@ export class MemoryStorage extends BaseStorage {
|
||||||
const nounCopy: HNSWNoun = {
|
const nounCopy: HNSWNoun = {
|
||||||
id: noun.id,
|
id: noun.id,
|
||||||
vector: [...noun.vector],
|
vector: [...noun.vector],
|
||||||
connections: new Map()
|
connections: new Map(),
|
||||||
|
level: noun.level || 0
|
||||||
}
|
}
|
||||||
|
|
||||||
// Copy connections
|
// Copy connections
|
||||||
|
|
@ -578,6 +582,10 @@ export class MemoryStorage extends BaseStorage {
|
||||||
this.nounMetadata.clear()
|
this.nounMetadata.clear()
|
||||||
this.verbMetadata.clear()
|
this.verbMetadata.clear()
|
||||||
this.statistics = null
|
this.statistics = null
|
||||||
|
|
||||||
|
// Clear the statistics cache
|
||||||
|
this.statisticsCache = null
|
||||||
|
this.statisticsModified = false
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
||||||
|
|
@ -224,7 +224,8 @@ export class OPFSStorage extends BaseStorage {
|
||||||
return {
|
return {
|
||||||
id: data.id,
|
id: data.id,
|
||||||
vector: data.vector,
|
vector: data.vector,
|
||||||
connections
|
connections,
|
||||||
|
level: data.level || 0
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
// Noun not found or other error
|
// Noun not found or other error
|
||||||
|
|
@ -258,7 +259,8 @@ export class OPFSStorage extends BaseStorage {
|
||||||
allNouns.push({
|
allNouns.push({
|
||||||
id: data.id,
|
id: data.id,
|
||||||
vector: data.vector,
|
vector: data.vector,
|
||||||
connections
|
connections,
|
||||||
|
level: data.level || 0
|
||||||
})
|
})
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error(`Error reading noun file ${name}:`, error)
|
console.error(`Error reading noun file ${name}:`, error)
|
||||||
|
|
@ -315,7 +317,8 @@ export class OPFSStorage extends BaseStorage {
|
||||||
nodes.push({
|
nodes.push({
|
||||||
id: data.id,
|
id: data.id,
|
||||||
vector: data.vector,
|
vector: data.vector,
|
||||||
connections
|
connections,
|
||||||
|
level: data.level || 0
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
|
@ -732,6 +735,10 @@ export class OPFSStorage extends BaseStorage {
|
||||||
|
|
||||||
// Remove all files in the index directory
|
// Remove all files in the index directory
|
||||||
await removeDirectoryContents(this.indexDir!)
|
await removeDirectoryContents(this.indexDir!)
|
||||||
|
|
||||||
|
// Clear the statistics cache
|
||||||
|
this.statisticsCache = null
|
||||||
|
this.statisticsModified = false
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('Error clearing storage:', error)
|
console.error('Error clearing storage:', error)
|
||||||
throw error
|
throw error
|
||||||
|
|
|
||||||
|
|
@ -440,7 +440,8 @@ export class S3CompatibleStorage extends BaseStorage {
|
||||||
const node = {
|
const node = {
|
||||||
id: parsedNode.id,
|
id: parsedNode.id,
|
||||||
vector: parsedNode.vector,
|
vector: parsedNode.vector,
|
||||||
connections
|
connections,
|
||||||
|
level: parsedNode.level || 0
|
||||||
}
|
}
|
||||||
|
|
||||||
console.log(`Successfully retrieved node ${id}:`, node)
|
console.log(`Successfully retrieved node ${id}:`, node)
|
||||||
|
|
@ -1585,6 +1586,10 @@ export class S3CompatibleStorage extends BaseStorage {
|
||||||
|
|
||||||
// Delete all objects in the index directory
|
// Delete all objects in the index directory
|
||||||
await deleteObjectsWithPrefix(this.indexPrefix)
|
await deleteObjectsWithPrefix(this.indexPrefix)
|
||||||
|
|
||||||
|
// Clear the statistics cache
|
||||||
|
this.statisticsCache = null
|
||||||
|
this.statisticsModified = false
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('Failed to clear storage:', error)
|
console.error('Failed to clear storage:', error)
|
||||||
throw new Error(`Failed to clear storage: ${error}`)
|
throw new Error(`Failed to clear storage: ${error}`)
|
||||||
|
|
|
||||||
|
|
@ -7,7 +7,7 @@
|
||||||
* - Level 3: Cold storage (all nodes) - OPFS, Filesystem or S3 depending on environment
|
* - Level 3: Cold storage (all nodes) - OPFS, Filesystem or S3 depending on environment
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { HNSWNoun, GraphVerb } from '../coreTypes.js'
|
import { HNSWNoun, GraphVerb, HNSWVerb } from '../coreTypes.js'
|
||||||
import { BrainyError } from '../errors/brainyError.js'
|
import { BrainyError } from '../errors/brainyError.js'
|
||||||
|
|
||||||
// Extend Navigator interface to include deviceMemory property
|
// Extend Navigator interface to include deviceMemory property
|
||||||
|
|
@ -71,7 +71,7 @@ enum StorageType {
|
||||||
/**
|
/**
|
||||||
* Multi-level cache manager for efficient data access
|
* Multi-level cache manager for efficient data access
|
||||||
*/
|
*/
|
||||||
export class CacheManager<T extends HNSWNode | Edge> {
|
export class CacheManager<T extends HNSWNode | Edge | HNSWVerb> {
|
||||||
// Hot cache (RAM)
|
// Hot cache (RAM)
|
||||||
private hotCache = new Map<string, CacheEntry<T>>()
|
private hotCache = new Map<string, CacheEntry<T>>()
|
||||||
|
|
||||||
|
|
|
||||||
269
src/utils/statisticsCollector.ts
Normal file
269
src/utils/statisticsCollector.ts
Normal file
|
|
@ -0,0 +1,269 @@
|
||||||
|
/**
|
||||||
|
* Lightweight statistics collector for Brainy
|
||||||
|
* Designed to have minimal performance impact even with millions of entries
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { StatisticsData } from '../coreTypes.js'
|
||||||
|
|
||||||
|
interface TimeSeriesData {
|
||||||
|
timestamp: number
|
||||||
|
count: number
|
||||||
|
}
|
||||||
|
|
||||||
|
export class StatisticsCollector {
|
||||||
|
// Content type tracking (lightweight counters)
|
||||||
|
private contentTypes: Map<string, number> = new Map()
|
||||||
|
|
||||||
|
// Data freshness tracking (only track timestamps, not full data)
|
||||||
|
private oldestTimestamp: number = Date.now()
|
||||||
|
private newestTimestamp: number = Date.now()
|
||||||
|
private updateTimestamps: TimeSeriesData[] = []
|
||||||
|
|
||||||
|
// Search performance tracking (rolling window)
|
||||||
|
private searchMetrics = {
|
||||||
|
totalSearches: 0,
|
||||||
|
totalSearchTimeMs: 0,
|
||||||
|
searchTimestamps: [] as TimeSeriesData[],
|
||||||
|
topSearchTerms: new Map<string, number>()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verb type tracking
|
||||||
|
private verbTypes: Map<string, number> = new Map()
|
||||||
|
|
||||||
|
// Storage size estimates (updated periodically, not on every operation)
|
||||||
|
private storageSizeCache = {
|
||||||
|
lastUpdated: 0,
|
||||||
|
sizes: {
|
||||||
|
nouns: 0,
|
||||||
|
verbs: 0,
|
||||||
|
metadata: 0,
|
||||||
|
index: 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private readonly MAX_TIMESTAMPS = 1000 // Keep last 1000 timestamps
|
||||||
|
private readonly MAX_SEARCH_TERMS = 100 // Track top 100 search terms
|
||||||
|
private readonly SIZE_UPDATE_INTERVAL = 60000 // Update sizes every minute
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Track content type (very lightweight)
|
||||||
|
*/
|
||||||
|
trackContentType(type: string): void {
|
||||||
|
this.contentTypes.set(type, (this.contentTypes.get(type) || 0) + 1)
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Track data update timestamp (lightweight)
|
||||||
|
*/
|
||||||
|
trackUpdate(timestamp?: number): void {
|
||||||
|
const ts = timestamp || Date.now()
|
||||||
|
|
||||||
|
// Update oldest/newest
|
||||||
|
if (ts < this.oldestTimestamp) this.oldestTimestamp = ts
|
||||||
|
if (ts > this.newestTimestamp) this.newestTimestamp = ts
|
||||||
|
|
||||||
|
// Add to rolling window
|
||||||
|
this.updateTimestamps.push({ timestamp: ts, count: 1 })
|
||||||
|
|
||||||
|
// Keep window size limited
|
||||||
|
if (this.updateTimestamps.length > this.MAX_TIMESTAMPS) {
|
||||||
|
this.updateTimestamps.shift()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Track search performance (lightweight)
|
||||||
|
*/
|
||||||
|
trackSearch(searchTerm: string, durationMs: number): void {
|
||||||
|
this.searchMetrics.totalSearches++
|
||||||
|
this.searchMetrics.totalSearchTimeMs += durationMs
|
||||||
|
|
||||||
|
// Add to rolling window
|
||||||
|
this.searchMetrics.searchTimestamps.push({
|
||||||
|
timestamp: Date.now(),
|
||||||
|
count: 1
|
||||||
|
})
|
||||||
|
|
||||||
|
// Keep window size limited
|
||||||
|
if (this.searchMetrics.searchTimestamps.length > this.MAX_TIMESTAMPS) {
|
||||||
|
this.searchMetrics.searchTimestamps.shift()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Track search term (limit to top N)
|
||||||
|
const termCount = (this.searchMetrics.topSearchTerms.get(searchTerm) || 0) + 1
|
||||||
|
this.searchMetrics.topSearchTerms.set(searchTerm, termCount)
|
||||||
|
|
||||||
|
// Prune if too many terms
|
||||||
|
if (this.searchMetrics.topSearchTerms.size > this.MAX_SEARCH_TERMS * 2) {
|
||||||
|
this.pruneSearchTerms()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Track verb type (lightweight)
|
||||||
|
*/
|
||||||
|
trackVerbType(type: string): void {
|
||||||
|
this.verbTypes.set(type, (this.verbTypes.get(type) || 0) + 1)
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Update storage size estimates (called periodically, not on every operation)
|
||||||
|
*/
|
||||||
|
updateStorageSizes(sizes: {
|
||||||
|
nouns: number
|
||||||
|
verbs: number
|
||||||
|
metadata: number
|
||||||
|
index: number
|
||||||
|
}): void {
|
||||||
|
this.storageSizeCache = {
|
||||||
|
lastUpdated: Date.now(),
|
||||||
|
sizes
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get comprehensive statistics
|
||||||
|
*/
|
||||||
|
getStatistics(): Partial<StatisticsData> {
|
||||||
|
const now = Date.now()
|
||||||
|
const hourAgo = now - 3600000
|
||||||
|
const dayAgo = now - 86400000
|
||||||
|
const weekAgo = now - 604800000
|
||||||
|
const monthAgo = now - 2592000000
|
||||||
|
|
||||||
|
// Calculate data freshness
|
||||||
|
const updatesLastHour = this.updateTimestamps.filter(t => t.timestamp > hourAgo).length
|
||||||
|
const updatesLastDay = this.updateTimestamps.filter(t => t.timestamp > dayAgo).length
|
||||||
|
|
||||||
|
// Calculate age distribution
|
||||||
|
const ageDistribution = {
|
||||||
|
last24h: 0,
|
||||||
|
last7d: 0,
|
||||||
|
last30d: 0,
|
||||||
|
older: 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// Estimate based on update patterns (not scanning all data)
|
||||||
|
const totalUpdates = this.updateTimestamps.length
|
||||||
|
if (totalUpdates > 0) {
|
||||||
|
const recentUpdates = this.updateTimestamps.filter(t => t.timestamp > dayAgo).length
|
||||||
|
const weekUpdates = this.updateTimestamps.filter(t => t.timestamp > weekAgo).length
|
||||||
|
const monthUpdates = this.updateTimestamps.filter(t => t.timestamp > monthAgo).length
|
||||||
|
|
||||||
|
ageDistribution.last24h = Math.round((recentUpdates / totalUpdates) * 100)
|
||||||
|
ageDistribution.last7d = Math.round(((weekUpdates - recentUpdates) / totalUpdates) * 100)
|
||||||
|
ageDistribution.last30d = Math.round(((monthUpdates - weekUpdates) / totalUpdates) * 100)
|
||||||
|
ageDistribution.older = 100 - ageDistribution.last24h - ageDistribution.last7d - ageDistribution.last30d
|
||||||
|
}
|
||||||
|
|
||||||
|
// Calculate search metrics
|
||||||
|
const searchesLastHour = this.searchMetrics.searchTimestamps.filter(t => t.timestamp > hourAgo).length
|
||||||
|
const searchesLastDay = this.searchMetrics.searchTimestamps.filter(t => t.timestamp > dayAgo).length
|
||||||
|
const avgSearchTime = this.searchMetrics.totalSearches > 0
|
||||||
|
? this.searchMetrics.totalSearchTimeMs / this.searchMetrics.totalSearches
|
||||||
|
: 0
|
||||||
|
|
||||||
|
// Get top search terms
|
||||||
|
const topSearchTerms = Array.from(this.searchMetrics.topSearchTerms.entries())
|
||||||
|
.sort((a, b) => b[1] - a[1])
|
||||||
|
.slice(0, 10)
|
||||||
|
.map(([term]) => term)
|
||||||
|
|
||||||
|
// Calculate storage metrics
|
||||||
|
const totalSize = Object.values(this.storageSizeCache.sizes).reduce((a, b) => a + b, 0)
|
||||||
|
|
||||||
|
return {
|
||||||
|
contentTypes: Object.fromEntries(this.contentTypes),
|
||||||
|
|
||||||
|
dataFreshness: {
|
||||||
|
oldestEntry: new Date(this.oldestTimestamp).toISOString(),
|
||||||
|
newestEntry: new Date(this.newestTimestamp).toISOString(),
|
||||||
|
updatesLastHour,
|
||||||
|
updatesLastDay,
|
||||||
|
ageDistribution
|
||||||
|
},
|
||||||
|
|
||||||
|
storageMetrics: {
|
||||||
|
totalSizeBytes: totalSize,
|
||||||
|
nounsSizeBytes: this.storageSizeCache.sizes.nouns,
|
||||||
|
verbsSizeBytes: this.storageSizeCache.sizes.verbs,
|
||||||
|
metadataSizeBytes: this.storageSizeCache.sizes.metadata,
|
||||||
|
indexSizeBytes: this.storageSizeCache.sizes.index
|
||||||
|
},
|
||||||
|
|
||||||
|
searchMetrics: {
|
||||||
|
totalSearches: this.searchMetrics.totalSearches,
|
||||||
|
averageSearchTimeMs: avgSearchTime,
|
||||||
|
searchesLastHour,
|
||||||
|
searchesLastDay,
|
||||||
|
topSearchTerms
|
||||||
|
},
|
||||||
|
|
||||||
|
verbStatistics: {
|
||||||
|
totalVerbs: Array.from(this.verbTypes.values()).reduce((a, b) => a + b, 0),
|
||||||
|
verbTypes: Object.fromEntries(this.verbTypes),
|
||||||
|
averageConnectionsPerVerb: 2 // Verbs connect 2 nouns
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Merge statistics from storage (for distributed systems)
|
||||||
|
*/
|
||||||
|
mergeFromStorage(stored: Partial<StatisticsData>): void {
|
||||||
|
// Merge content types
|
||||||
|
if (stored.contentTypes) {
|
||||||
|
for (const [type, count] of Object.entries(stored.contentTypes)) {
|
||||||
|
this.contentTypes.set(type, count)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Merge verb types
|
||||||
|
if (stored.verbStatistics?.verbTypes) {
|
||||||
|
for (const [type, count] of Object.entries(stored.verbStatistics.verbTypes)) {
|
||||||
|
this.verbTypes.set(type, count)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Merge search metrics
|
||||||
|
if (stored.searchMetrics) {
|
||||||
|
this.searchMetrics.totalSearches = stored.searchMetrics.totalSearches || 0
|
||||||
|
this.searchMetrics.totalSearchTimeMs = (stored.searchMetrics.averageSearchTimeMs || 0) * this.searchMetrics.totalSearches
|
||||||
|
}
|
||||||
|
|
||||||
|
// Merge data freshness
|
||||||
|
if (stored.dataFreshness) {
|
||||||
|
this.oldestTimestamp = new Date(stored.dataFreshness.oldestEntry).getTime()
|
||||||
|
this.newestTimestamp = new Date(stored.dataFreshness.newestEntry).getTime()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reset statistics (for testing)
|
||||||
|
*/
|
||||||
|
reset(): void {
|
||||||
|
this.contentTypes.clear()
|
||||||
|
this.verbTypes.clear()
|
||||||
|
this.updateTimestamps = []
|
||||||
|
this.searchMetrics = {
|
||||||
|
totalSearches: 0,
|
||||||
|
totalSearchTimeMs: 0,
|
||||||
|
searchTimestamps: [],
|
||||||
|
topSearchTerms: new Map()
|
||||||
|
}
|
||||||
|
this.oldestTimestamp = Date.now()
|
||||||
|
this.newestTimestamp = Date.now()
|
||||||
|
}
|
||||||
|
|
||||||
|
private pruneSearchTerms(): void {
|
||||||
|
// Keep only top N search terms
|
||||||
|
const sorted = Array.from(this.searchMetrics.topSearchTerms.entries())
|
||||||
|
.sort((a, b) => b[1] - a[1])
|
||||||
|
.slice(0, this.MAX_SEARCH_TERMS)
|
||||||
|
|
||||||
|
this.searchMetrics.topSearchTerms.clear()
|
||||||
|
for (const [term, count] of sorted) {
|
||||||
|
this.searchMetrics.topSearchTerms.set(term, count)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -389,7 +389,7 @@ describe('Brainy Core Functionality', () => {
|
||||||
// Verify counts
|
// Verify counts
|
||||||
expect(stats.nounCount).toBe(3)
|
expect(stats.nounCount).toBe(3)
|
||||||
expect(stats.verbCount).toBe(2)
|
expect(stats.verbCount).toBe(2)
|
||||||
expect(stats.hnswIndexSize).toBe(1)
|
expect(stats.hnswIndexSize).toBe(5)
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
|
||||||
|
|
@ -57,7 +57,7 @@ describe('Brainy Statistics Functionality', () => {
|
||||||
expect(stats.nounCount).toBe(3)
|
expect(stats.nounCount).toBe(3)
|
||||||
expect(stats.verbCount).toBe(1)
|
expect(stats.verbCount).toBe(1)
|
||||||
expect(stats.metadataCount).toBe(3) // Each noun has metadata
|
expect(stats.metadataCount).toBe(3) // Each noun has metadata
|
||||||
expect(stats.hnswIndexSize).toBe(2)
|
expect(stats.hnswIndexSize).toBe(4) // 3 nouns + 1 verb (verbs are also added to HNSW index)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should throw an error when no instance is provided', async () => {
|
it('should throw an error when no instance is provided', async () => {
|
||||||
|
|
@ -77,8 +77,16 @@ describe('Brainy Statistics Functionality', () => {
|
||||||
const instanceStats = await data.getStatistics()
|
const instanceStats = await data.getStatistics()
|
||||||
const functionStats = await brainy.getStatistics(data)
|
const functionStats = await brainy.getStatistics(data)
|
||||||
|
|
||||||
// Verify they match
|
// Verify core statistics match (ignoring volatile fields like memoryUsage and timestamps)
|
||||||
expect(functionStats).toEqual(instanceStats)
|
expect(functionStats.nounCount).toBe(instanceStats.nounCount)
|
||||||
|
expect(functionStats.verbCount).toBe(instanceStats.verbCount)
|
||||||
|
expect(functionStats.metadataCount).toBe(instanceStats.metadataCount)
|
||||||
|
expect(functionStats.hnswIndexSize).toBe(instanceStats.hnswIndexSize)
|
||||||
|
|
||||||
|
// If serviceBreakdown exists, verify it matches
|
||||||
|
if (instanceStats.serviceBreakdown) {
|
||||||
|
expect(functionStats.serviceBreakdown).toEqual(instanceStats.serviceBreakdown)
|
||||||
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should track statistics by service', async () => {
|
it('should track statistics by service', async () => {
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue