chore(8.0): final pre-RC1 sweep — API consistency, named errors, orphans, zero-cast codebase
This commit is contained in:
parent
970e08c466
commit
1f7e365a4e
237 changed files with 1951 additions and 49413 deletions
|
|
@ -38,7 +38,7 @@ export function findCallerLocation(extraSkipPatterns: string[] = []): string | n
|
|||
for (const raw of lines) {
|
||||
const line = raw.trim()
|
||||
// Always skip Brainy's own source + compiled output. Consumers need their
|
||||
// own call site, not `brainy.ts:XXXX` or `dist/brainy.js:XXXX`.
|
||||
// own call site, not a line inside `brainy.ts` or `dist/brainy.js`.
|
||||
if (line.includes('/src/brainy.ts') || line.includes('/dist/brainy.js')) continue
|
||||
// Skip the validation + diagnostic helpers regardless of which file they
|
||||
// live in — they're plumbing between the public API and the consumer.
|
||||
|
|
|
|||
|
|
@ -120,7 +120,7 @@ export function createTransformerEmbedding(options: TransformerEmbeddingOptions
|
|||
* Convenience function to detect best device (always returns 'wasm')
|
||||
*/
|
||||
export async function detectBestDevice(): Promise<'cpu' | 'webgpu' | 'cuda' | 'wasm'> {
|
||||
return 'wasm' as any
|
||||
return 'wasm'
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -1,296 +0,0 @@
|
|||
/**
|
||||
* Enhanced logging system that bridges old logger with new structured logger
|
||||
* Provides backward compatibility while enabling gradual migration
|
||||
*/
|
||||
|
||||
import {
|
||||
structuredLogger,
|
||||
createModuleLogger as createStructuredModuleLogger,
|
||||
LogLevel as StructuredLogLevel,
|
||||
type LogContext,
|
||||
type ModuleLogger
|
||||
} from './structuredLogger.js'
|
||||
import { isProductionEnvironment, getLogLevel } from './environment.js'
|
||||
|
||||
// Re-export LogLevel for compatibility
|
||||
export enum LogLevel {
|
||||
ERROR = 0,
|
||||
WARN = 1,
|
||||
INFO = 2,
|
||||
DEBUG = 3,
|
||||
TRACE = 4
|
||||
}
|
||||
|
||||
// Map old LogLevel to new StructuredLogLevel
|
||||
function mapLogLevel(level: LogLevel): StructuredLogLevel {
|
||||
switch (level) {
|
||||
case LogLevel.ERROR: return StructuredLogLevel.ERROR
|
||||
case LogLevel.WARN: return StructuredLogLevel.WARN
|
||||
case LogLevel.INFO: return StructuredLogLevel.INFO
|
||||
case LogLevel.DEBUG: return StructuredLogLevel.DEBUG
|
||||
case LogLevel.TRACE: return StructuredLogLevel.TRACE
|
||||
default: return StructuredLogLevel.INFO
|
||||
}
|
||||
}
|
||||
|
||||
export interface LoggerConfig {
|
||||
level: LogLevel
|
||||
modules?: {
|
||||
[moduleName: string]: LogLevel
|
||||
}
|
||||
timestamps?: boolean
|
||||
includeModule?: boolean
|
||||
handler?: (level: LogLevel, module: string, message: string, ...args: any[]) => void
|
||||
}
|
||||
|
||||
/**
|
||||
* Enhanced Logger that uses structured logger internally
|
||||
* Maintains backward compatibility with existing code
|
||||
*/
|
||||
class EnhancedLogger {
|
||||
private static instance: EnhancedLogger
|
||||
private config: LoggerConfig = {
|
||||
level: LogLevel.ERROR,
|
||||
timestamps: false,
|
||||
includeModule: true
|
||||
}
|
||||
|
||||
private constructor() {
|
||||
this.applyEnvironmentDefaults()
|
||||
|
||||
// Sync with structured logger
|
||||
this.syncWithStructuredLogger()
|
||||
|
||||
// Set log level from environment variable if available
|
||||
const envLogLevel = process.env.BRAINY_LOG_LEVEL
|
||||
if (envLogLevel) {
|
||||
const level = LogLevel[envLogLevel.toUpperCase() as keyof typeof LogLevel]
|
||||
if (level !== undefined) {
|
||||
this.config.level = level
|
||||
this.syncWithStructuredLogger()
|
||||
}
|
||||
}
|
||||
|
||||
// Parse module-specific log levels
|
||||
const moduleLogLevels = process.env.BRAINY_MODULE_LOG_LEVELS
|
||||
if (moduleLogLevels) {
|
||||
try {
|
||||
this.config.modules = JSON.parse(moduleLogLevels)
|
||||
this.syncWithStructuredLogger()
|
||||
} catch (e) {
|
||||
// Ignore parsing errors
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private applyEnvironmentDefaults(): void {
|
||||
const envLogLevel = getLogLevel()
|
||||
|
||||
switch (envLogLevel) {
|
||||
case 'silent':
|
||||
this.config.level = -1 as LogLevel
|
||||
break
|
||||
case 'error':
|
||||
this.config.level = LogLevel.ERROR
|
||||
this.config.timestamps = false
|
||||
break
|
||||
case 'warn':
|
||||
this.config.level = LogLevel.WARN
|
||||
this.config.timestamps = false
|
||||
break
|
||||
case 'info':
|
||||
this.config.level = LogLevel.INFO
|
||||
this.config.timestamps = true
|
||||
break
|
||||
case 'verbose':
|
||||
this.config.level = LogLevel.DEBUG
|
||||
this.config.timestamps = true
|
||||
break
|
||||
}
|
||||
|
||||
if (isProductionEnvironment()) {
|
||||
this.config.level = Math.min(this.config.level, LogLevel.ERROR)
|
||||
this.config.timestamps = false
|
||||
this.config.includeModule = false
|
||||
}
|
||||
}
|
||||
|
||||
private syncWithStructuredLogger(): void {
|
||||
// Convert modules config
|
||||
const modules: Record<string, StructuredLogLevel> = {}
|
||||
if (this.config.modules) {
|
||||
for (const [module, level] of Object.entries(this.config.modules)) {
|
||||
modules[module] = mapLogLevel(level)
|
||||
}
|
||||
}
|
||||
|
||||
// Configure structured logger
|
||||
structuredLogger.configure({
|
||||
level: mapLogLevel(this.config.level),
|
||||
modules,
|
||||
format: this.config.timestamps ? 'pretty' : 'simple'
|
||||
})
|
||||
}
|
||||
|
||||
static getInstance(): EnhancedLogger {
|
||||
if (!EnhancedLogger.instance) {
|
||||
EnhancedLogger.instance = new EnhancedLogger()
|
||||
}
|
||||
return EnhancedLogger.instance
|
||||
}
|
||||
|
||||
configure(config: Partial<LoggerConfig>): void {
|
||||
this.config = { ...this.config, ...config }
|
||||
this.syncWithStructuredLogger()
|
||||
}
|
||||
|
||||
private shouldLog(level: LogLevel, module: string): boolean {
|
||||
if (this.config.modules && this.config.modules[module] !== undefined) {
|
||||
return level <= this.config.modules[module]
|
||||
}
|
||||
return level <= this.config.level
|
||||
}
|
||||
|
||||
error(module: string, message: string, ...args: any[]): void {
|
||||
const logger = createStructuredModuleLogger(module)
|
||||
logger.error(message, { data: args })
|
||||
}
|
||||
|
||||
warn(module: string, message: string, ...args: any[]): void {
|
||||
const logger = createStructuredModuleLogger(module)
|
||||
logger.warn(message, { data: args })
|
||||
}
|
||||
|
||||
info(module: string, message: string, ...args: any[]): void {
|
||||
const logger = createStructuredModuleLogger(module)
|
||||
logger.info(message, { data: args })
|
||||
}
|
||||
|
||||
debug(module: string, message: string, ...args: any[]): void {
|
||||
const logger = createStructuredModuleLogger(module)
|
||||
logger.debug(message, { data: args })
|
||||
}
|
||||
|
||||
trace(module: string, message: string, ...args: any[]): void {
|
||||
const logger = createStructuredModuleLogger(module)
|
||||
logger.trace(message, { data: args })
|
||||
}
|
||||
|
||||
createModuleLogger(module: string) {
|
||||
const structuredLogger = createStructuredModuleLogger(module)
|
||||
|
||||
// Return a backward-compatible interface
|
||||
return {
|
||||
error: (message: string, ...args: any[]) =>
|
||||
structuredLogger.error(message, { data: args }),
|
||||
warn: (message: string, ...args: any[]) =>
|
||||
structuredLogger.warn(message, { data: args }),
|
||||
info: (message: string, ...args: any[]) =>
|
||||
structuredLogger.info(message, { data: args }),
|
||||
debug: (message: string, ...args: any[]) =>
|
||||
structuredLogger.debug(message, { data: args }),
|
||||
trace: (message: string, ...args: any[]) =>
|
||||
structuredLogger.trace(message, { data: args }),
|
||||
// New structured logging methods
|
||||
withContext: (context: LogContext) =>
|
||||
structuredLogger.withContext(context),
|
||||
startTimer: (label: string) =>
|
||||
structuredLogger.startTimer(label),
|
||||
endTimer: (label: string) =>
|
||||
structuredLogger.endTimer(label)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Export singleton instance
|
||||
export const logger = EnhancedLogger.getInstance()
|
||||
|
||||
// Export convenience function for creating module loggers
|
||||
export function createModuleLogger(module: string) {
|
||||
return logger.createModuleLogger(module)
|
||||
}
|
||||
|
||||
// Export function to configure logger
|
||||
export function configureLogger(config: Partial<LoggerConfig>) {
|
||||
logger.configure(config)
|
||||
}
|
||||
|
||||
/**
|
||||
* Smart console replacement that uses structured logger
|
||||
*/
|
||||
export const smartConsole = {
|
||||
log: (message?: any, ...args: any[]) => {
|
||||
const logger = createStructuredModuleLogger('console')
|
||||
logger.info(typeof message === 'string' ? message : JSON.stringify(message), { data: args })
|
||||
},
|
||||
|
||||
info: (message?: any, ...args: any[]) => {
|
||||
const logger = createStructuredModuleLogger('console')
|
||||
logger.info(typeof message === 'string' ? message : JSON.stringify(message), { data: args })
|
||||
},
|
||||
|
||||
warn: (message?: any, ...args: any[]) => {
|
||||
const logger = createStructuredModuleLogger('console')
|
||||
logger.warn(typeof message === 'string' ? message : JSON.stringify(message), { data: args })
|
||||
},
|
||||
|
||||
error: (message?: any, ...args: any[]) => {
|
||||
const logger = createStructuredModuleLogger('console')
|
||||
logger.error(typeof message === 'string' ? message : JSON.stringify(message), { data: args })
|
||||
},
|
||||
|
||||
debug: (message?: any, ...args: any[]) => {
|
||||
const logger = createStructuredModuleLogger('console')
|
||||
logger.debug(typeof message === 'string' ? message : JSON.stringify(message), { data: args })
|
||||
},
|
||||
|
||||
trace: (message?: any, ...args: any[]) => {
|
||||
const logger = createStructuredModuleLogger('console')
|
||||
logger.trace(typeof message === 'string' ? message : JSON.stringify(message), { data: args })
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Production-optimized logging functions
|
||||
*/
|
||||
export const prodLog = {
|
||||
error: (message?: any, ...args: any[]) => {
|
||||
const logger = createStructuredModuleLogger('prod')
|
||||
logger.error(typeof message === 'string' ? message : JSON.stringify(message), { data: args })
|
||||
},
|
||||
|
||||
warn: (message?: any, ...args: any[]) => {
|
||||
if (!isProductionEnvironment() || process.env.BRAINY_LOG_LEVEL) {
|
||||
const logger = createStructuredModuleLogger('prod')
|
||||
logger.warn(typeof message === 'string' ? message : JSON.stringify(message), { data: args })
|
||||
}
|
||||
},
|
||||
|
||||
info: (message?: any, ...args: any[]) => {
|
||||
if (!isProductionEnvironment() || process.env.BRAINY_LOG_LEVEL) {
|
||||
const logger = createStructuredModuleLogger('prod')
|
||||
logger.info(typeof message === 'string' ? message : JSON.stringify(message), { data: args })
|
||||
}
|
||||
},
|
||||
|
||||
debug: (message?: any, ...args: any[]) => {
|
||||
if (!isProductionEnvironment() || process.env.BRAINY_LOG_LEVEL) {
|
||||
const logger = createStructuredModuleLogger('prod')
|
||||
logger.debug(typeof message === 'string' ? message : JSON.stringify(message), { data: args })
|
||||
}
|
||||
},
|
||||
|
||||
log: (message?: any, ...args: any[]) => {
|
||||
if (!isProductionEnvironment() || process.env.BRAINY_LOG_LEVEL) {
|
||||
const logger = createStructuredModuleLogger('prod')
|
||||
logger.info(typeof message === 'string' ? message : JSON.stringify(message), { data: args })
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Re-export structured logger utilities for new code
|
||||
export {
|
||||
createModuleLogger as createStructuredModuleLogger,
|
||||
type LogContext,
|
||||
type ModuleLogger
|
||||
} from './structuredLogger.js'
|
||||
|
|
@ -124,8 +124,11 @@ export class EntityIdMapper implements EntityIdMapperProvider {
|
|||
try {
|
||||
const metadata = await this.storage.getMetadata(this.storageKey)
|
||||
// metadata IS the data (no nested 'data' property)
|
||||
if (metadata && (metadata as any).nextId !== undefined) {
|
||||
const data = metadata as any as EntityIdMapperData
|
||||
if (metadata && metadata.nextId !== undefined) {
|
||||
// Typed boundary: mapper state round-trips through the storage
|
||||
// metadata channel as plain JSON; the `nextId` probe above identifies
|
||||
// the persisted EntityIdMapperData shape.
|
||||
const data = metadata as unknown as EntityIdMapperData
|
||||
this.nextId = data.nextId
|
||||
|
||||
// Rebuild maps from serialized data
|
||||
|
|
@ -305,7 +308,7 @@ export class EntityIdMapper implements EntityIdMapperProvider {
|
|||
intToUuid: Object.fromEntries(this.intToUuid)
|
||||
}
|
||||
|
||||
await this.storage.saveMetadata(this.storageKey, data as any)
|
||||
await this.storage.saveMetadata(this.storageKey, data)
|
||||
this.dirty = false
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@
|
|||
* 4. Store result for future queries
|
||||
*/
|
||||
|
||||
import { StorageAdapter } from '../coreTypes.js'
|
||||
import { StorageAdapter, NounMetadata } from '../coreTypes.js'
|
||||
import { prodLog } from './logger.js'
|
||||
|
||||
/**
|
||||
|
|
@ -412,7 +412,7 @@ export class FieldTypeInference {
|
|||
noun: 'FieldTypeCache',
|
||||
...typeInfo
|
||||
}
|
||||
await this.storage.saveMetadata(cacheKey, metadataObj as any).catch(error => {
|
||||
await this.storage.saveMetadata(cacheKey, metadataObj).catch(error => {
|
||||
prodLog.warn(`Failed to save field type cache for '${field}':`, error)
|
||||
})
|
||||
}
|
||||
|
|
@ -487,8 +487,10 @@ export class FieldTypeInference {
|
|||
if (field) {
|
||||
this.typeCache.delete(field)
|
||||
const cacheKey = `${this.CACHE_STORAGE_PREFIX}${field}`
|
||||
// null signals deletion to storage adapter
|
||||
await this.storage.saveMetadata(cacheKey, null as any)
|
||||
// Typed boundary: the storage metadata channel doubles as the delete
|
||||
// path — writing a JSON `null` tombstone clears the cached entry, but
|
||||
// the adapter signature only models real payloads.
|
||||
await this.storage.saveMetadata(cacheKey, null as unknown as NounMetadata)
|
||||
} else {
|
||||
this.typeCache.clear()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,393 +0,0 @@
|
|||
/**
|
||||
* Intelligent Type Mapper
|
||||
* Maps generic/invalid type names to specific semantic types based on data analysis
|
||||
* Prevents semantic degradation from overuse of generic types
|
||||
*/
|
||||
|
||||
import { NounType, VerbType } from '../types/graphTypes.js'
|
||||
|
||||
/**
|
||||
* Common aliases that users might use
|
||||
*/
|
||||
const GENERIC_ALIASES = new Set([
|
||||
'entity',
|
||||
'item',
|
||||
'object',
|
||||
'node',
|
||||
'record',
|
||||
'entry',
|
||||
'data',
|
||||
'resource'
|
||||
])
|
||||
|
||||
/**
|
||||
* Field signatures for type inference
|
||||
*/
|
||||
const TYPE_SIGNATURES = {
|
||||
// Person indicators
|
||||
person: {
|
||||
required: [],
|
||||
indicators: ['email', 'firstName', 'lastName', 'name', 'phone', 'username', 'userId'],
|
||||
patterns: [/@/, /^[a-z0-9._%+-]+@[a-z0-9.-]+\.[a-z]{2,}$/i],
|
||||
weight: 10
|
||||
},
|
||||
|
||||
// User account indicators
|
||||
user: {
|
||||
required: [],
|
||||
indicators: ['username', 'password', 'accountId', 'loginTime', 'permissions', 'role'],
|
||||
patterns: [],
|
||||
weight: 9
|
||||
},
|
||||
|
||||
// Organization indicators
|
||||
organization: {
|
||||
required: [],
|
||||
indicators: ['companyName', 'orgName', 'ein', 'vatNumber', 'employees', 'headquarters'],
|
||||
patterns: [],
|
||||
weight: 8
|
||||
},
|
||||
|
||||
// Product indicators
|
||||
product: {
|
||||
required: [],
|
||||
indicators: ['price', 'sku', 'barcode', 'inventory', 'cost', 'productId', 'inStock'],
|
||||
patterns: [/^\$?\d+\.?\d*$/, /^[A-Z0-9-]+$/],
|
||||
weight: 8
|
||||
},
|
||||
|
||||
// Document indicators
|
||||
document: {
|
||||
required: [],
|
||||
indicators: ['content', 'text', 'body', 'title', 'author', 'markdown', 'html'],
|
||||
patterns: [],
|
||||
weight: 7
|
||||
},
|
||||
|
||||
// Message indicators
|
||||
message: {
|
||||
required: [],
|
||||
indicators: ['from', 'to', 'subject', 'body', 'sentAt', 'messageId', 'threadId'],
|
||||
patterns: [],
|
||||
weight: 7
|
||||
},
|
||||
|
||||
// Task indicators
|
||||
task: {
|
||||
required: [],
|
||||
indicators: ['dueDate', 'assignee', 'status', 'priority', 'completed', 'taskId'],
|
||||
patterns: [],
|
||||
weight: 7
|
||||
},
|
||||
|
||||
// Event indicators
|
||||
event: {
|
||||
required: [],
|
||||
indicators: ['startTime', 'endTime', 'date', 'location', 'attendees', 'eventType'],
|
||||
patterns: [],
|
||||
weight: 6
|
||||
},
|
||||
|
||||
// Location indicators
|
||||
location: {
|
||||
required: [],
|
||||
indicators: ['latitude', 'longitude', 'address', 'city', 'country', 'zipCode', 'coordinates'],
|
||||
patterns: [/^-?\d+\.\d+$/, /^\d{5}(-\d{4})?$/],
|
||||
weight: 6
|
||||
},
|
||||
|
||||
// File indicators
|
||||
file: {
|
||||
required: [],
|
||||
indicators: ['filename', 'filepath', 'extension', 'mimeType', 'fileSize', 'checksum'],
|
||||
patterns: [/\.[a-z0-9]+$/i],
|
||||
weight: 5
|
||||
},
|
||||
|
||||
// Dataset indicators
|
||||
dataset: {
|
||||
required: [],
|
||||
indicators: ['schema', 'rows', 'columns', 'records', 'dataType', 'format'],
|
||||
patterns: [],
|
||||
weight: 5
|
||||
},
|
||||
|
||||
// Media indicators
|
||||
media: {
|
||||
required: [],
|
||||
indicators: ['url', 'thumbnail', 'duration', 'resolution', 'codec', 'bitrate'],
|
||||
patterns: [/\.(jpg|jpeg|png|gif|mp4|mp3|wav|avi)$/i],
|
||||
weight: 5
|
||||
},
|
||||
|
||||
// Project indicators
|
||||
project: {
|
||||
required: [],
|
||||
indicators: ['deadline', 'budget', 'team', 'milestones', 'deliverables', 'projectId'],
|
||||
patterns: [],
|
||||
weight: 5
|
||||
},
|
||||
|
||||
// Service indicators
|
||||
service: {
|
||||
required: [],
|
||||
indicators: ['endpoint', 'apiKey', 'serviceUrl', 'port', 'protocol', 'healthCheck'],
|
||||
patterns: [/^https?:\/\//, /:\d+$/],
|
||||
weight: 4
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Intelligent Type Mapper
|
||||
*/
|
||||
export class IntelligentTypeMapper {
|
||||
private typeCache: Map<string, string> = new Map()
|
||||
private inferenceStats = {
|
||||
total: 0,
|
||||
inferred: 0,
|
||||
defaulted: 0,
|
||||
cached: 0
|
||||
}
|
||||
|
||||
/**
|
||||
* Map a noun type, with intelligent inference for generic types
|
||||
*/
|
||||
mapNounType(inputType: string, data?: any): string {
|
||||
// Check if it's already a valid type
|
||||
if (this.isValidNounType(inputType)) {
|
||||
return inputType
|
||||
}
|
||||
|
||||
// Check cache for this exact input
|
||||
const cacheKey = `${inputType}-${JSON.stringify(data || {}).substring(0, 100)}`
|
||||
if (this.typeCache.has(cacheKey)) {
|
||||
this.inferenceStats.cached++
|
||||
return this.typeCache.get(cacheKey)!
|
||||
}
|
||||
|
||||
this.inferenceStats.total++
|
||||
|
||||
// If it's a generic alias and we have data, try to infer
|
||||
if (GENERIC_ALIASES.has(inputType.toLowerCase()) && data) {
|
||||
const inferred = this.inferTypeFromData(data)
|
||||
if (inferred) {
|
||||
this.inferenceStats.inferred++
|
||||
this.typeCache.set(cacheKey, inferred)
|
||||
return inferred
|
||||
}
|
||||
}
|
||||
|
||||
// Handle specific common mappings
|
||||
const directMapping = this.getDirectMapping(inputType)
|
||||
if (directMapping) {
|
||||
this.typeCache.set(cacheKey, directMapping)
|
||||
return directMapping
|
||||
}
|
||||
|
||||
// Default to 'thing' for truly unknown types
|
||||
this.inferenceStats.defaulted++
|
||||
const defaultType = NounType.Thing
|
||||
this.typeCache.set(cacheKey, defaultType)
|
||||
return defaultType
|
||||
}
|
||||
|
||||
/**
|
||||
* Map a verb type
|
||||
*/
|
||||
mapVerbType(inputType: string): string {
|
||||
// Check if it's already valid
|
||||
if (this.isValidVerbType(inputType)) {
|
||||
return inputType
|
||||
}
|
||||
|
||||
// Common verb mappings
|
||||
const verbMappings: Record<string, string> = {
|
||||
'related': VerbType.RelatedTo,
|
||||
'relates': VerbType.RelatedTo,
|
||||
'has': VerbType.Contains,
|
||||
'includes': VerbType.Contains,
|
||||
'belongsTo': VerbType.PartOf,
|
||||
'in': VerbType.LocatedAt,
|
||||
'at': VerbType.LocatedAt,
|
||||
'references': VerbType.References,
|
||||
'cites': VerbType.References,
|
||||
'before': VerbType.Precedes,
|
||||
'after': VerbType.Precedes,
|
||||
'causes': VerbType.Causes,
|
||||
'needs': VerbType.Requires,
|
||||
'requires': VerbType.Requires,
|
||||
'makes': VerbType.Creates,
|
||||
'produces': VerbType.Creates,
|
||||
'changes': VerbType.Modifies,
|
||||
'updates': VerbType.Modifies,
|
||||
'owns': VerbType.Owns,
|
||||
'ownedBy': VerbType.Owns, // Use BelongsTo for reverse ownership
|
||||
'uses': VerbType.Uses,
|
||||
'usedBy': VerbType.Uses // Same relationship, just interpret direction
|
||||
}
|
||||
|
||||
const normalized = inputType.toLowerCase()
|
||||
return verbMappings[normalized] || VerbType.RelatedTo
|
||||
}
|
||||
|
||||
/**
|
||||
* Infer type from data structure
|
||||
*/
|
||||
private inferTypeFromData(data: any): string | null {
|
||||
if (!data || typeof data !== 'object') return null
|
||||
|
||||
const scores: Map<string, number> = new Map()
|
||||
const fields = Object.keys(data)
|
||||
const values = Object.values(data)
|
||||
|
||||
// Calculate scores for each type based on field matches
|
||||
for (const [type, signature] of Object.entries(TYPE_SIGNATURES)) {
|
||||
let score = 0
|
||||
|
||||
// Check required fields
|
||||
if (signature.required.length > 0) {
|
||||
const hasRequired = signature.required.every(field => fields.includes(field))
|
||||
if (!hasRequired) continue
|
||||
score += signature.weight * 2
|
||||
}
|
||||
|
||||
// Check indicator fields
|
||||
for (const field of fields) {
|
||||
if (signature.indicators.some(indicator =>
|
||||
field.toLowerCase().includes(indicator.toLowerCase())
|
||||
)) {
|
||||
score += signature.weight
|
||||
}
|
||||
}
|
||||
|
||||
// Check value patterns
|
||||
for (const value of values) {
|
||||
if (typeof value === 'string' && signature.patterns.length > 0) {
|
||||
for (const pattern of signature.patterns) {
|
||||
if (pattern.test(value)) {
|
||||
score += signature.weight / 2
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (score > 0) {
|
||||
scores.set(type, score)
|
||||
}
|
||||
}
|
||||
|
||||
// Return the type with highest score
|
||||
if (scores.size > 0) {
|
||||
const sorted = Array.from(scores.entries()).sort((a, b) => b[1] - a[1])
|
||||
return sorted[0][0]
|
||||
}
|
||||
|
||||
// Fallback inference based on data structure
|
||||
if (fields.includes('url') || fields.includes('href')) {
|
||||
return NounType.Document
|
||||
}
|
||||
|
||||
if (Array.isArray(data) || fields.includes('items') || fields.includes('elements')) {
|
||||
return NounType.Collection
|
||||
}
|
||||
|
||||
// Check if it looks like a process/workflow
|
||||
if (fields.includes('steps') || fields.includes('stages')) {
|
||||
return NounType.Process
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
|
||||
/**
|
||||
* Get direct mapping for common aliases
|
||||
*/
|
||||
private getDirectMapping(inputType: string): string | null {
|
||||
const mappings: Record<string, string> = {
|
||||
// Specific mappings that aren't generic
|
||||
'company': NounType.Organization,
|
||||
'corp': NounType.Organization,
|
||||
'business': NounType.Organization,
|
||||
'employee': NounType.Person,
|
||||
'staff': NounType.Person,
|
||||
'customer': NounType.Person,
|
||||
'client': NounType.Person,
|
||||
'article': NounType.Document,
|
||||
'post': NounType.Document,
|
||||
'page': NounType.Document,
|
||||
'image': NounType.Media,
|
||||
'video': NounType.Media,
|
||||
'audio': NounType.Media,
|
||||
'photo': NounType.Media,
|
||||
'place': NounType.Location,
|
||||
'address': NounType.Location,
|
||||
'country': NounType.Location,
|
||||
'city': NounType.Location,
|
||||
'todo': NounType.Task,
|
||||
'job': NounType.Task,
|
||||
'work': NounType.Task,
|
||||
'meeting': NounType.Event,
|
||||
'appointment': NounType.Event,
|
||||
'conference': NounType.Event,
|
||||
'folder': NounType.Collection,
|
||||
'group': NounType.Collection,
|
||||
'list': NounType.Collection,
|
||||
'category': NounType.Collection
|
||||
}
|
||||
|
||||
const normalized = inputType.toLowerCase()
|
||||
return mappings[normalized] || null
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a type is valid
|
||||
*/
|
||||
private isValidNounType(type: string): boolean {
|
||||
return Object.values(NounType).includes(type as any)
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a verb type is valid
|
||||
*/
|
||||
private isValidVerbType(type: string): boolean {
|
||||
return Object.values(VerbType).includes(type as any)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get inference statistics
|
||||
*/
|
||||
getStats() {
|
||||
return {
|
||||
...this.inferenceStats,
|
||||
cacheSize: this.typeCache.size,
|
||||
inferenceRate: this.inferenceStats.total > 0
|
||||
? (this.inferenceStats.inferred / this.inferenceStats.total)
|
||||
: 0
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear the type cache
|
||||
*/
|
||||
clearCache(): void {
|
||||
this.typeCache.clear()
|
||||
}
|
||||
}
|
||||
|
||||
// Singleton instance
|
||||
export const typeMapper = new IntelligentTypeMapper()
|
||||
|
||||
/**
|
||||
* Helper function for easy type mapping
|
||||
*/
|
||||
export function mapNounType(inputType: string, data?: any): string {
|
||||
return typeMapper.mapNounType(inputType, data)
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper function for verb mapping
|
||||
*/
|
||||
export function mapVerbType(inputType: string): string {
|
||||
return typeMapper.mapVerbType(inputType)
|
||||
}
|
||||
|
|
@ -4,7 +4,7 @@
|
|||
* Automatically updates indexes when data changes
|
||||
*/
|
||||
|
||||
import { StorageAdapter, resolveEntityField } from '../coreTypes.js'
|
||||
import { StorageAdapter, resolveEntityField, NounMetadata, VerbMetadata } from '../coreTypes.js'
|
||||
import { ColumnStore } from '../indexes/columnStore/ColumnStore.js'
|
||||
import type { MetadataIndexProvider } from '../plugin.js'
|
||||
import { MetadataIndexCache, MetadataIndexCacheConfig } from './metadataIndexCache.js'
|
||||
|
|
@ -1520,13 +1520,13 @@ export class MetadataIndexManager implements MetadataIndexProvider {
|
|||
const allIds = new Set<string>()
|
||||
|
||||
// Storage.getNouns() is the definitive source of all entity IDs
|
||||
if (this.storage && typeof (this.storage as any).getNouns === 'function') {
|
||||
if (this.storage && typeof this.storage.getNouns === 'function') {
|
||||
try {
|
||||
const result = await (this.storage as any).getNouns({
|
||||
const result = await this.storage.getNouns({
|
||||
pagination: { limit: 100000 }
|
||||
})
|
||||
if (result && result.items) {
|
||||
result.items.forEach((item: any) => {
|
||||
result.items.forEach((item) => {
|
||||
if (item.id) allIds.add(item.id)
|
||||
})
|
||||
}
|
||||
|
|
@ -2446,7 +2446,7 @@ export class MetadataIndexManager implements MetadataIndexProvider {
|
|||
noun: 'MetadataFieldIndex',
|
||||
values: fieldIndex.values,
|
||||
lastUpdated: fieldIndex.lastUpdated
|
||||
} as any)
|
||||
})
|
||||
|
||||
// Update unified cache
|
||||
const size = JSON.stringify(fieldIndex).length
|
||||
|
|
@ -2582,8 +2582,11 @@ export class MetadataIndexManager implements MetadataIndexProvider {
|
|||
await this.chunkManager.deleteChunk(field, chunkId)
|
||||
}
|
||||
|
||||
// Delete the sparse index file itself
|
||||
await this.storage.saveMetadata(indexPath, null as any)
|
||||
// Delete the sparse index file itself.
|
||||
// Typed boundary: the storage metadata channel doubles as the delete
|
||||
// path — writing a JSON `null` tombstone clears the entry, but the
|
||||
// adapter signature only models real payloads.
|
||||
await this.storage.saveMetadata(indexPath, null as unknown as NounMetadata)
|
||||
}
|
||||
} catch (error) {
|
||||
// Silent failure - if we can't delete old chunks, rebuild will still work
|
||||
|
|
@ -2612,9 +2615,11 @@ export class MetadataIndexManager implements MetadataIndexProvider {
|
|||
deletedCount++
|
||||
}
|
||||
|
||||
// Delete field registry
|
||||
// Delete field registry.
|
||||
// Typed boundary: writing a JSON `null` tombstone clears the entry, but
|
||||
// the adapter signature only models real payloads.
|
||||
try {
|
||||
await this.storage.saveMetadata('__metadata_field_registry__', null as any)
|
||||
await this.storage.saveMetadata('__metadata_field_registry__', null as unknown as NounMetadata)
|
||||
} catch (error) {
|
||||
prodLog.debug('Could not delete field registry:', error)
|
||||
}
|
||||
|
|
@ -3104,10 +3109,15 @@ export class MetadataIndexManager implements MetadataIndexProvider {
|
|||
prodLog.info(`📦 Loading ${result.items.length} verbs with metadata...`)
|
||||
|
||||
const verbIds = result.items.map(verb => verb.id)
|
||||
let verbMetadataBatch: Map<string, any>
|
||||
let verbMetadataBatch: Map<string, VerbMetadata>
|
||||
|
||||
if ((this.storage as any).getVerbMetadataBatch) {
|
||||
verbMetadataBatch = await (this.storage as any).getVerbMetadataBatch(verbIds)
|
||||
// Optional adapter capability: batched verb-metadata reads. Not part of
|
||||
// the StorageAdapter contract, so it is probed structurally.
|
||||
const batchCapableStorage = this.storage as StorageAdapter & {
|
||||
getVerbMetadataBatch?: (ids: string[]) => Promise<Map<string, VerbMetadata>>
|
||||
}
|
||||
if (batchCapableStorage.getVerbMetadataBatch) {
|
||||
verbMetadataBatch = await batchCapableStorage.getVerbMetadataBatch(verbIds)
|
||||
prodLog.info(`✅ Loaded ${verbMetadataBatch.size}/${verbIds.length} verb metadata objects`)
|
||||
} else {
|
||||
verbMetadataBatch = new Map()
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@
|
|||
* - EntityIdMapper handles UUID ↔ integer conversion
|
||||
*/
|
||||
|
||||
import { StorageAdapter } from '../coreTypes.js'
|
||||
import { StorageAdapter, NounMetadata } from '../coreTypes.js'
|
||||
import { prodLog } from './logger.js'
|
||||
import { RoaringBitmap32 } from './roaring/index.js'
|
||||
import type { EntityIdMapper } from './entityIdMapper.js'
|
||||
|
|
@ -109,6 +109,29 @@ export interface ChunkData {
|
|||
lastUpdated: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Storage representation of a roaring bitmap inside a serialized chunk.
|
||||
* Produced by `ChunkManager.saveChunk()` — portable-format bytes as a plain
|
||||
* number array so the payload survives JSON round-trips.
|
||||
*/
|
||||
type SerializedRoaringBitmap = {
|
||||
buffer: number[]
|
||||
size: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Storage representation of a chunk as persisted by `ChunkManager.saveChunk()`
|
||||
* and re-hydrated by `ChunkManager.loadChunk()`. A type alias (not an
|
||||
* interface) so it carries an implicit index signature and converts cleanly
|
||||
* to/from the `NounMetadata` shape used by the storage metadata channel.
|
||||
*/
|
||||
type SerializedChunkData = {
|
||||
chunkId: number
|
||||
field: string
|
||||
entries: Record<string, SerializedRoaringBitmap | undefined>
|
||||
lastUpdated: number
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// BloomFilter - Production-Ready Implementation
|
||||
// ============================================================================
|
||||
|
|
@ -609,25 +632,26 @@ export class ChunkManager {
|
|||
const data = await this.storage.getMetadata(chunkPath)
|
||||
|
||||
if (data) {
|
||||
// Cast NounMetadata to chunk data structure
|
||||
const chunkData = data as unknown as any
|
||||
// Chunks round-trip through the storage metadata channel; re-type the
|
||||
// JSON payload to the serialized chunk shape written by saveChunk()
|
||||
const chunkData = data as SerializedChunkData
|
||||
|
||||
// Deserialize: convert serialized roaring bitmaps back to RoaringBitmap32 objects
|
||||
const chunk: ChunkData = {
|
||||
chunkId: chunkData.chunkId as number,
|
||||
field: chunkData.field as string,
|
||||
chunkId: chunkData.chunkId,
|
||||
field: chunkData.field,
|
||||
entries: new Map(
|
||||
Object.entries(chunkData.entries).map(([value, serializedBitmap]) => {
|
||||
// Deserialize roaring bitmap from portable format
|
||||
const bitmap = new RoaringBitmap32()
|
||||
if (serializedBitmap && typeof serializedBitmap === 'object' && (serializedBitmap as any).buffer) {
|
||||
if (serializedBitmap && typeof serializedBitmap === 'object' && serializedBitmap.buffer) {
|
||||
// Deserialize from Buffer
|
||||
bitmap.deserialize(Buffer.from((serializedBitmap as any).buffer), 'portable')
|
||||
bitmap.deserialize(Buffer.from(serializedBitmap.buffer), 'portable')
|
||||
}
|
||||
return [value, bitmap]
|
||||
return [value, bitmap] as const
|
||||
})
|
||||
),
|
||||
lastUpdated: chunkData.lastUpdated as number
|
||||
lastUpdated: chunkData.lastUpdated
|
||||
}
|
||||
|
||||
this.chunkCache.set(cacheKey, chunk)
|
||||
|
|
@ -668,7 +692,7 @@ export class ChunkManager {
|
|||
}
|
||||
|
||||
const chunkPath = this.getChunkPath(chunk.field, chunk.chunkId)
|
||||
await this.storage.saveMetadata(chunkPath, serializable as any)
|
||||
await this.storage.saveMetadata(chunkPath, serializable)
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -842,8 +866,10 @@ export class ChunkManager {
|
|||
this.chunkCache.delete(cacheKey)
|
||||
|
||||
const chunkPath = this.getChunkPath(field, chunkId)
|
||||
// null signals deletion to storage adapter
|
||||
await this.storage.saveMetadata(chunkPath, null as any)
|
||||
// Typed boundary: the storage metadata channel doubles as the delete path —
|
||||
// writing a JSON `null` tombstone clears the chunk. The adapter signature
|
||||
// only models real payloads, so the null must be re-typed here.
|
||||
await this.storage.saveMetadata(chunkPath, null as unknown as NounMetadata)
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -278,6 +278,8 @@ export function getGlobalMutex(): MutexInterface {
|
|||
*/
|
||||
export async function cleanupMutexes(): Promise<void> {
|
||||
if (globalMutex && 'cleanup' in globalMutex) {
|
||||
await (globalMutex as any).cleanup()
|
||||
// Only FileMutex (defined above) carries a cleanup() method; the 'in'
|
||||
// check above guarantees we hold that implementation.
|
||||
await (globalMutex as FileMutex).cleanup()
|
||||
}
|
||||
}
|
||||
|
|
@ -1,104 +0,0 @@
|
|||
/**
|
||||
* Node.js Version Compatibility Check
|
||||
*
|
||||
* Brainy requires Node.js 22.x LTS for maximum stability with ONNX Runtime.
|
||||
* This prevents V8 HandleScope locking issues in worker threads.
|
||||
*/
|
||||
|
||||
import { isNode } from './environment.js'
|
||||
|
||||
export interface VersionInfo {
|
||||
current: string
|
||||
major: number
|
||||
isSupported: boolean
|
||||
recommendation: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the current Node.js version is supported
|
||||
*/
|
||||
export function checkNodeVersion(): VersionInfo {
|
||||
// In browser environment, skip version check
|
||||
if (!isNode()) {
|
||||
return {
|
||||
current: 'browser',
|
||||
major: 0,
|
||||
isSupported: true, // Always supported in browser
|
||||
recommendation: 'Browser environment'
|
||||
}
|
||||
}
|
||||
|
||||
// Only access process.version in Node.js environment
|
||||
const nodeVersion = process.version
|
||||
const majorVersion = parseInt(nodeVersion.split('.')[0].substring(1))
|
||||
|
||||
const versionInfo: VersionInfo = {
|
||||
current: nodeVersion,
|
||||
major: majorVersion,
|
||||
isSupported: majorVersion === 22,
|
||||
recommendation: 'Node.js 22.x LTS'
|
||||
}
|
||||
|
||||
return versionInfo
|
||||
}
|
||||
|
||||
/**
|
||||
* Enforce Node.js version requirement with helpful error messaging
|
||||
*/
|
||||
export function enforceNodeVersion(): void {
|
||||
const versionInfo = checkNodeVersion()
|
||||
|
||||
// Skip enforcement in browser environment
|
||||
if (!isNode()) {
|
||||
return
|
||||
}
|
||||
|
||||
if (!versionInfo.isSupported) {
|
||||
const errorMessage = [
|
||||
'🚨 BRAINY COMPATIBILITY ERROR',
|
||||
'━'.repeat(50),
|
||||
`❌ Current Node.js: ${versionInfo.current}`,
|
||||
`✅ Required: ${versionInfo.recommendation}`,
|
||||
'',
|
||||
'💡 Quick Fix:',
|
||||
' nvm install 22 && nvm use 22',
|
||||
' npm install',
|
||||
'',
|
||||
'📖 Why Node.js 22?',
|
||||
' • Maximum ONNX Runtime stability',
|
||||
' • Prevents V8 threading crashes',
|
||||
' • Optimal zero-config performance',
|
||||
'',
|
||||
'🔗 More info: https://github.com/soulcraftlabs/brainy#node-version',
|
||||
'━'.repeat(50)
|
||||
].join('\n')
|
||||
|
||||
throw new Error(errorMessage)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Soft warning for version issues (non-blocking)
|
||||
*/
|
||||
export function warnNodeVersion(): boolean {
|
||||
const versionInfo = checkNodeVersion()
|
||||
|
||||
// Skip warning in browser environment
|
||||
if (!isNode()) {
|
||||
return true
|
||||
}
|
||||
|
||||
if (!versionInfo.isSupported) {
|
||||
console.warn([
|
||||
'⚠️ BRAINY VERSION WARNING',
|
||||
` Current: ${versionInfo.current}`,
|
||||
` Recommended: ${versionInfo.recommendation}`,
|
||||
' Consider upgrading for best stability',
|
||||
''
|
||||
].join('\n'))
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
|
@ -173,7 +173,7 @@ export interface ValidationConfigOptions {
|
|||
* These adapt to available memory and observed performance
|
||||
*/
|
||||
export class ValidationConfig {
|
||||
private static instance: ValidationConfig
|
||||
private static instance: ValidationConfig | null = null
|
||||
|
||||
// Dynamic limits based on system
|
||||
public maxLimit: number
|
||||
|
|
@ -265,7 +265,7 @@ export class ValidationConfig {
|
|||
* Reset singleton (for testing or reconfiguration)
|
||||
*/
|
||||
static reset(): void {
|
||||
ValidationConfig.instance = null as any
|
||||
ValidationConfig.instance = null
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -1,293 +0,0 @@
|
|||
/**
|
||||
* Periodic Cleanup for Soft-Deleted Items
|
||||
*
|
||||
* SAFETY-FIRST APPROACH:
|
||||
* - Maintains durability guarantees (storage-first)
|
||||
* - Coordinates HNSW and metadata index consistency
|
||||
* - Isolated from live operations
|
||||
* - Graceful failure handling
|
||||
*/
|
||||
|
||||
import { prodLog } from './logger.js'
|
||||
import { DELETED_FIELD, isDeleted } from './metadataNamespace.js'
|
||||
import type { StorageAdapter } from '../coreTypes.js'
|
||||
import type { JsHnswVectorIndex } from '../hnsw/hnswIndex.js'
|
||||
import type { MetadataIndexManager } from './metadataIndex.js'
|
||||
|
||||
export interface CleanupConfig {
|
||||
/** Age in milliseconds after which soft-deleted items are eligible for cleanup */
|
||||
maxAge: number
|
||||
/** Maximum number of items to clean up in one batch */
|
||||
batchSize: number
|
||||
/** Interval between cleanup runs (milliseconds) */
|
||||
cleanupInterval: number
|
||||
/** Whether to run cleanup automatically */
|
||||
enabled: boolean
|
||||
}
|
||||
|
||||
export interface CleanupStats {
|
||||
itemsProcessed: number
|
||||
itemsDeleted: number
|
||||
errors: number
|
||||
lastRun: number
|
||||
nextRun: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Coordinates safe cleanup of old soft-deleted items across all indexes
|
||||
*
|
||||
* CRITICAL SAFETY FEATURES:
|
||||
* 1. Storage-first deletion (durability)
|
||||
* 2. Index consistency coordination
|
||||
* 3. Batch processing with limits
|
||||
* 4. Error isolation and recovery
|
||||
*/
|
||||
export class PeriodicCleanup {
|
||||
private storage: StorageAdapter
|
||||
private hnswIndex: JsHnswVectorIndex
|
||||
private metadataIndex: MetadataIndexManager | null
|
||||
private config: CleanupConfig
|
||||
private stats: CleanupStats
|
||||
private cleanupTimer: NodeJS.Timeout | null = null
|
||||
private running = false
|
||||
|
||||
constructor(
|
||||
storage: StorageAdapter,
|
||||
hnswIndex: JsHnswVectorIndex,
|
||||
metadataIndex: MetadataIndexManager | null,
|
||||
config: Partial<CleanupConfig> = {}
|
||||
) {
|
||||
this.storage = storage
|
||||
this.hnswIndex = hnswIndex
|
||||
this.metadataIndex = metadataIndex
|
||||
|
||||
// Default: clean up items deleted more than 1 hour ago
|
||||
this.config = {
|
||||
maxAge: config.maxAge ?? 60 * 60 * 1000, // 1 hour
|
||||
batchSize: config.batchSize ?? 100, // 100 items max per batch
|
||||
cleanupInterval: config.cleanupInterval ?? 15 * 60 * 1000, // Every 15 minutes
|
||||
enabled: config.enabled ?? true
|
||||
}
|
||||
|
||||
this.stats = {
|
||||
itemsProcessed: 0,
|
||||
itemsDeleted: 0,
|
||||
errors: 0,
|
||||
lastRun: 0,
|
||||
nextRun: 0
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Start periodic cleanup
|
||||
*/
|
||||
start(): void {
|
||||
if (!this.config.enabled || this.cleanupTimer) {
|
||||
return
|
||||
}
|
||||
|
||||
prodLog.info(`Starting periodic cleanup: maxAge=${this.config.maxAge}, batchSize=${this.config.batchSize}, interval=${this.config.cleanupInterval}`)
|
||||
|
||||
this.scheduleNext()
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop periodic cleanup
|
||||
*/
|
||||
stop(): void {
|
||||
if (this.cleanupTimer) {
|
||||
clearTimeout(this.cleanupTimer)
|
||||
this.cleanupTimer = null
|
||||
}
|
||||
|
||||
prodLog.info('Stopped periodic cleanup')
|
||||
}
|
||||
|
||||
/**
|
||||
* Run cleanup manually
|
||||
*/
|
||||
async runNow(): Promise<CleanupStats> {
|
||||
if (this.running) {
|
||||
throw new Error('Cleanup already running')
|
||||
}
|
||||
|
||||
return this.performCleanup()
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current cleanup statistics
|
||||
*/
|
||||
getStats(): CleanupStats {
|
||||
return { ...this.stats }
|
||||
}
|
||||
|
||||
private scheduleNext(): void {
|
||||
const nextRun = Date.now() + this.config.cleanupInterval
|
||||
this.stats.nextRun = nextRun
|
||||
|
||||
this.cleanupTimer = setTimeout(async () => {
|
||||
await this.performCleanup()
|
||||
this.scheduleNext()
|
||||
}, this.config.cleanupInterval)
|
||||
}
|
||||
|
||||
/**
|
||||
* CRITICAL: Coordinated cleanup across all indexes
|
||||
*
|
||||
* SAFETY PROTOCOL:
|
||||
* 1. Find eligible items (old + soft-deleted)
|
||||
* 2. Remove from storage FIRST (durability)
|
||||
* 3. Remove from HNSW (graph consistency)
|
||||
* 4. Remove from metadata index (search consistency)
|
||||
* 5. Track stats and errors
|
||||
*/
|
||||
private async performCleanup(): Promise<CleanupStats> {
|
||||
if (this.running) {
|
||||
prodLog.warn('Cleanup already running, skipping')
|
||||
return this.stats
|
||||
}
|
||||
|
||||
this.running = true
|
||||
const startTime = Date.now()
|
||||
this.stats.lastRun = startTime
|
||||
|
||||
try {
|
||||
prodLog.debug(`Starting cleanup run: maxAge=${this.config.maxAge}, cutoffTime=${startTime - this.config.maxAge}`)
|
||||
|
||||
// Step 1: Find eligible items for cleanup
|
||||
const eligibleItems = await this.findEligibleItems(startTime)
|
||||
|
||||
if (eligibleItems.length === 0) {
|
||||
prodLog.debug('No items eligible for cleanup')
|
||||
return this.stats
|
||||
}
|
||||
|
||||
prodLog.info(`Found ${eligibleItems.length} items eligible for cleanup`)
|
||||
|
||||
// Step 2: Process in batches for safety
|
||||
let processed = 0
|
||||
let deleted = 0
|
||||
let errors = 0
|
||||
|
||||
for (let i = 0; i < eligibleItems.length; i += this.config.batchSize) {
|
||||
const batch = eligibleItems.slice(i, i + this.config.batchSize)
|
||||
|
||||
const batchResult = await this.processBatch(batch)
|
||||
processed += batchResult.processed
|
||||
deleted += batchResult.deleted
|
||||
errors += batchResult.errors
|
||||
|
||||
// Small delay between batches to avoid overwhelming the system
|
||||
if (i + this.config.batchSize < eligibleItems.length) {
|
||||
await new Promise(resolve => setTimeout(resolve, 10))
|
||||
}
|
||||
}
|
||||
|
||||
// Update stats
|
||||
this.stats.itemsProcessed += processed
|
||||
this.stats.itemsDeleted += deleted
|
||||
this.stats.errors += errors
|
||||
|
||||
prodLog.info(`Cleanup run completed: processed=${processed}, deleted=${deleted}, errors=${errors}, duration=${Date.now() - startTime}ms`)
|
||||
|
||||
} catch (error) {
|
||||
prodLog.error(`Cleanup run failed: ${error}`)
|
||||
this.stats.errors++
|
||||
} finally {
|
||||
this.running = false
|
||||
}
|
||||
|
||||
return this.stats
|
||||
}
|
||||
|
||||
/**
|
||||
* Find items eligible for cleanup (old + soft-deleted)
|
||||
*/
|
||||
private async findEligibleItems(currentTime: number): Promise<string[]> {
|
||||
const cutoffTime = currentTime - this.config.maxAge
|
||||
const eligibleItems: string[] = []
|
||||
|
||||
try {
|
||||
// Get all nouns from storage (using pagination to avoid memory issues)
|
||||
const nounsResult = await this.storage.getNouns({
|
||||
pagination: { limit: 1000 } // Process in chunks
|
||||
})
|
||||
|
||||
for (const noun of nounsResult.items) {
|
||||
try {
|
||||
// Cast NounMetadata to NamespacedMetadata for isDeleted check
|
||||
if (!noun.metadata || !isDeleted(noun.metadata as any)) {
|
||||
continue // Not deleted, skip
|
||||
}
|
||||
|
||||
// Check if old enough for cleanup
|
||||
const deletedTime = (noun.metadata as any)._brainy?.updated || 0
|
||||
if (deletedTime && (currentTime - deletedTime) > this.config.maxAge) {
|
||||
eligibleItems.push(noun.id)
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
prodLog.warn(`Failed to check item ${noun.id} for cleanup eligibility: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
prodLog.error(`Failed to find eligible items: ${error}`)
|
||||
throw error
|
||||
}
|
||||
|
||||
return eligibleItems
|
||||
}
|
||||
|
||||
/**
|
||||
* Process a batch of items for cleanup
|
||||
*
|
||||
* CRITICAL: This maintains the durability-first approach:
|
||||
* Storage → HNSW → Metadata Index
|
||||
*/
|
||||
private async processBatch(itemIds: string[]): Promise<{
|
||||
processed: number
|
||||
deleted: number
|
||||
errors: number
|
||||
}> {
|
||||
let processed = 0
|
||||
let deleted = 0
|
||||
let errors = 0
|
||||
|
||||
for (const id of itemIds) {
|
||||
processed++
|
||||
|
||||
try {
|
||||
// STEP 1: Remove from storage FIRST (durability guarantee)
|
||||
try {
|
||||
await this.storage.deleteNoun(id)
|
||||
} catch (storageError) {
|
||||
prodLog.warn(`Failed to delete ${id} from storage: ${storageError}`)
|
||||
errors++
|
||||
continue
|
||||
}
|
||||
|
||||
// STEP 2: Remove from HNSW index (vector search consistency)
|
||||
const hnswResult = this.hnswIndex.removeItem(id)
|
||||
if (!hnswResult) {
|
||||
prodLog.warn(`Failed to remove ${id} from HNSW index (may not have been indexed)`)
|
||||
// Not a critical error - item might not have been in vector index
|
||||
}
|
||||
|
||||
// STEP 3: Remove from metadata index (faceted search consistency)
|
||||
if (this.metadataIndex) {
|
||||
await this.metadataIndex.removeFromIndex(id)
|
||||
}
|
||||
|
||||
deleted++
|
||||
prodLog.debug(`Successfully cleaned up item ${id}`)
|
||||
|
||||
} catch (error) {
|
||||
errors++
|
||||
prodLog.error(`Failed to cleanup item ${id}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
return { processed, deleted, errors }
|
||||
}
|
||||
}
|
||||
|
|
@ -1,363 +0,0 @@
|
|||
/**
|
||||
* Rate Limiter for Brainy API
|
||||
*
|
||||
* Provides rate limiting without external dependencies like Redis.
|
||||
* - Uses in-memory storage for single instances
|
||||
* - Can use S3/R2 for distributed rate limiting
|
||||
*
|
||||
* @module rateLimiter
|
||||
*/
|
||||
|
||||
import { BaseStorageAdapter } from '../storage/adapters/baseStorageAdapter.js'
|
||||
|
||||
export interface RateLimitConfig {
|
||||
/**
|
||||
* Maximum number of requests allowed
|
||||
*/
|
||||
maxRequests: number
|
||||
|
||||
/**
|
||||
* Time window in milliseconds
|
||||
*/
|
||||
windowMs: number
|
||||
|
||||
/**
|
||||
* Optional message to return when rate limit is exceeded
|
||||
*/
|
||||
message?: string
|
||||
|
||||
/**
|
||||
* Whether to use distributed storage (S3/R2) for rate limiting
|
||||
*/
|
||||
distributed?: boolean
|
||||
|
||||
/**
|
||||
* Storage adapter for distributed rate limiting
|
||||
*/
|
||||
storage?: BaseStorageAdapter
|
||||
|
||||
/**
|
||||
* Key prefix for distributed storage
|
||||
*/
|
||||
keyPrefix?: string
|
||||
}
|
||||
|
||||
export interface RateLimitResult {
|
||||
/**
|
||||
* Whether the request is allowed
|
||||
*/
|
||||
allowed: boolean
|
||||
|
||||
/**
|
||||
* Number of requests remaining in the current window
|
||||
*/
|
||||
remaining: number
|
||||
|
||||
/**
|
||||
* Time when the rate limit window resets (Unix timestamp)
|
||||
*/
|
||||
resetTime: number
|
||||
|
||||
/**
|
||||
* Total limit for the window
|
||||
*/
|
||||
limit: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Simple in-memory rate limiter
|
||||
*/
|
||||
export class RateLimiter {
|
||||
private requests: Map<string, { count: number; resetTime: number }> = new Map()
|
||||
private cleanupInterval: NodeJS.Timeout | null = null
|
||||
|
||||
constructor(private config: RateLimitConfig) {
|
||||
// Start cleanup interval to remove expired entries
|
||||
this.startCleanup()
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a request is allowed and update the rate limit
|
||||
*/
|
||||
async checkLimit(identifier: string): Promise<RateLimitResult> {
|
||||
const now = Date.now()
|
||||
|
||||
if (this.config.distributed && this.config.storage) {
|
||||
return this.checkDistributedLimit(identifier, now)
|
||||
}
|
||||
|
||||
return this.checkMemoryLimit(identifier, now)
|
||||
}
|
||||
|
||||
/**
|
||||
* Check rate limit using in-memory storage
|
||||
*/
|
||||
private checkMemoryLimit(identifier: string, now: number): RateLimitResult {
|
||||
const entry = this.requests.get(identifier)
|
||||
const resetTime = now + this.config.windowMs
|
||||
|
||||
if (!entry || entry.resetTime <= now) {
|
||||
// New window or expired window
|
||||
this.requests.set(identifier, {
|
||||
count: 1,
|
||||
resetTime
|
||||
})
|
||||
|
||||
return {
|
||||
allowed: true,
|
||||
remaining: this.config.maxRequests - 1,
|
||||
resetTime,
|
||||
limit: this.config.maxRequests
|
||||
}
|
||||
}
|
||||
|
||||
// Existing window
|
||||
if (entry.count < this.config.maxRequests) {
|
||||
entry.count++
|
||||
|
||||
return {
|
||||
allowed: true,
|
||||
remaining: this.config.maxRequests - entry.count,
|
||||
resetTime: entry.resetTime,
|
||||
limit: this.config.maxRequests
|
||||
}
|
||||
}
|
||||
|
||||
// Rate limit exceeded
|
||||
return {
|
||||
allowed: false,
|
||||
remaining: 0,
|
||||
resetTime: entry.resetTime,
|
||||
limit: this.config.maxRequests
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check rate limit using distributed storage (S3/R2)
|
||||
*/
|
||||
private async checkDistributedLimit(
|
||||
identifier: string,
|
||||
now: number
|
||||
): Promise<RateLimitResult> {
|
||||
const storage = this.config.storage!
|
||||
const key = `ratelimit_${identifier}`
|
||||
const resetTime = now + this.config.windowMs
|
||||
|
||||
try {
|
||||
// Try to get existing rate limit data from metadata storage
|
||||
const existing = await storage.getMetadata(key)
|
||||
|
||||
if (!existing || !existing.resetTime ||
|
||||
Number(existing.resetTime) <= now) {
|
||||
// New window or expired window
|
||||
await storage.saveMetadata(key, {
|
||||
count: 1,
|
||||
resetTime: resetTime,
|
||||
identifier
|
||||
})
|
||||
|
||||
return {
|
||||
allowed: true,
|
||||
remaining: this.config.maxRequests - 1,
|
||||
resetTime,
|
||||
limit: this.config.maxRequests
|
||||
}
|
||||
}
|
||||
|
||||
const count = Number(existing.count || 0)
|
||||
|
||||
if (count < this.config.maxRequests) {
|
||||
// Update count
|
||||
await storage.saveMetadata(key, {
|
||||
count: count + 1,
|
||||
resetTime: existing.resetTime,
|
||||
identifier
|
||||
})
|
||||
|
||||
return {
|
||||
allowed: true,
|
||||
remaining: this.config.maxRequests - count - 1,
|
||||
resetTime: Number(existing.resetTime),
|
||||
limit: this.config.maxRequests
|
||||
}
|
||||
}
|
||||
|
||||
// Rate limit exceeded
|
||||
return {
|
||||
allowed: false,
|
||||
remaining: 0,
|
||||
resetTime: Number(existing.resetTime),
|
||||
limit: this.config.maxRequests
|
||||
}
|
||||
} catch (error) {
|
||||
// On error, fail open (allow the request)
|
||||
console.warn('Rate limiter error, failing open:', error)
|
||||
return {
|
||||
allowed: true,
|
||||
remaining: this.config.maxRequests,
|
||||
resetTime,
|
||||
limit: this.config.maxRequests
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset rate limit for a specific identifier
|
||||
*/
|
||||
async reset(identifier: string): Promise<void> {
|
||||
if (this.config.distributed && this.config.storage) {
|
||||
const key = `ratelimit_${identifier}`
|
||||
// Reset by setting count to 0 and expired time
|
||||
await this.config.storage.saveMetadata(key, {
|
||||
count: 0,
|
||||
resetTime: 0,
|
||||
identifier
|
||||
})
|
||||
} else {
|
||||
this.requests.delete(identifier)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Start cleanup interval to remove expired entries
|
||||
*/
|
||||
private startCleanup(): void {
|
||||
// Run cleanup every minute
|
||||
this.cleanupInterval = setInterval(() => {
|
||||
const now = Date.now()
|
||||
const expired: string[] = []
|
||||
|
||||
for (const [key, entry] of this.requests) {
|
||||
if (entry.resetTime <= now) {
|
||||
expired.push(key)
|
||||
}
|
||||
}
|
||||
|
||||
for (const key of expired) {
|
||||
this.requests.delete(key)
|
||||
}
|
||||
}, 60000) // 1 minute
|
||||
|
||||
// Don't keep Node.js process alive just for cleanup
|
||||
if (this.cleanupInterval.unref) {
|
||||
this.cleanupInterval.unref()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop the rate limiter and cleanup
|
||||
*/
|
||||
destroy(): void {
|
||||
if (this.cleanupInterval) {
|
||||
clearInterval(this.cleanupInterval)
|
||||
this.cleanupInterval = null
|
||||
}
|
||||
this.requests.clear()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Express/Connect middleware for rate limiting
|
||||
*/
|
||||
export function rateLimitMiddleware(config: RateLimitConfig) {
|
||||
const limiter = new RateLimiter(config)
|
||||
|
||||
return async (req: any, res: any, next: any) => {
|
||||
// Use IP address as identifier (can be customized)
|
||||
const identifier = req.ip || req.connection?.remoteAddress || 'unknown'
|
||||
|
||||
const result = await limiter.checkLimit(identifier)
|
||||
|
||||
// Set rate limit headers
|
||||
res.setHeader('X-RateLimit-Limit', result.limit)
|
||||
res.setHeader('X-RateLimit-Remaining', result.remaining)
|
||||
res.setHeader('X-RateLimit-Reset', result.resetTime)
|
||||
|
||||
if (!result.allowed) {
|
||||
res.status(429).json({
|
||||
error: config.message || 'Too many requests, please try again later.',
|
||||
retryAfter: Math.ceil((result.resetTime - Date.now()) / 1000)
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
next()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a rate limiter for use with Brainy
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* // For single instance (in-memory)
|
||||
* const limiter = createRateLimiter({
|
||||
* maxRequests: 100,
|
||||
* windowMs: 15 * 60 * 1000 // 15 minutes
|
||||
* })
|
||||
*
|
||||
* // For distributed (using S3/R2)
|
||||
* const limiter = createRateLimiter({
|
||||
* maxRequests: 100,
|
||||
* windowMs: 15 * 60 * 1000,
|
||||
* distributed: true,
|
||||
* storage: myS3Adapter
|
||||
* })
|
||||
*
|
||||
* // Check rate limit
|
||||
* const result = await limiter.checkLimit('user-123')
|
||||
* if (!result.allowed) {
|
||||
* throw new Error('Rate limit exceeded')
|
||||
* }
|
||||
* ```
|
||||
*/
|
||||
export function createRateLimiter(config: RateLimitConfig): RateLimiter {
|
||||
return new RateLimiter(config)
|
||||
}
|
||||
|
||||
/**
|
||||
* Preset configurations for common use cases
|
||||
*/
|
||||
export const RateLimitPresets = {
|
||||
/**
|
||||
* Default API rate limit: 100 requests per 15 minutes
|
||||
*/
|
||||
default: {
|
||||
maxRequests: 100,
|
||||
windowMs: 15 * 60 * 1000
|
||||
},
|
||||
|
||||
/**
|
||||
* Strict rate limit: 10 requests per minute
|
||||
*/
|
||||
strict: {
|
||||
maxRequests: 10,
|
||||
windowMs: 60 * 1000
|
||||
},
|
||||
|
||||
/**
|
||||
* Lenient rate limit: 1000 requests per hour
|
||||
*/
|
||||
lenient: {
|
||||
maxRequests: 1000,
|
||||
windowMs: 60 * 60 * 1000
|
||||
},
|
||||
|
||||
/**
|
||||
* Search endpoint: 30 requests per minute
|
||||
*/
|
||||
search: {
|
||||
maxRequests: 30,
|
||||
windowMs: 60 * 1000,
|
||||
message: 'Search rate limit exceeded. Please wait before searching again.'
|
||||
},
|
||||
|
||||
/**
|
||||
* Write operations: 20 requests per minute
|
||||
*/
|
||||
write: {
|
||||
maxRequests: 20,
|
||||
windowMs: 60 * 1000,
|
||||
message: 'Write rate limit exceeded. Please slow down your write operations.'
|
||||
}
|
||||
}
|
||||
|
|
@ -8,6 +8,45 @@
|
|||
*/
|
||||
|
||||
import type { BaseStorage } from '../storage/baseStorage.js'
|
||||
import type { HNSWNounWithMetadata, HNSWVerbWithMetadata } from '../coreTypes.js'
|
||||
|
||||
/**
|
||||
* Result page shape returned by the adapter offset-paginated readers.
|
||||
*/
|
||||
interface PaginatedScanResult<TItem> {
|
||||
items: TItem[]
|
||||
totalCount?: number
|
||||
hasMore: boolean
|
||||
nextCursor?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Structural view of the count bookkeeping this utility repairs.
|
||||
*
|
||||
* The count fields are `protected` on `BaseStorageAdapter`; this recovery
|
||||
* utility deliberately reaches past that visibility to overwrite
|
||||
* desynchronized counters, so the cast below is a visibility boundary rather
|
||||
* than a shape mismatch. The paginated readers are optional adapter
|
||||
* capabilities (probed at runtime) whose implementations accept an `offset`
|
||||
* option not modeled on the cursor-based `BaseStorageAdapter` declaration.
|
||||
*/
|
||||
interface CountRebuildTarget {
|
||||
getNounsWithPagination?: (options: {
|
||||
limit?: number
|
||||
offset?: number
|
||||
}) => Promise<PaginatedScanResult<HNSWNounWithMetadata>>
|
||||
getVerbsWithPagination?: (options: {
|
||||
limit?: number
|
||||
offset?: number
|
||||
}) => Promise<PaginatedScanResult<HNSWVerbWithMetadata>>
|
||||
totalNounCount: number
|
||||
totalVerbCount: number
|
||||
entityCounts: Map<string, number>
|
||||
verbCounts: Map<string, number>
|
||||
pendingCountPersist: boolean
|
||||
pendingCountOperations: number
|
||||
flushCounts(): Promise<void>
|
||||
}
|
||||
|
||||
export interface RebuildCountsResult {
|
||||
/** Total number of entities (nouns) found */
|
||||
|
|
@ -57,8 +96,10 @@ export async function rebuildCounts(storage: BaseStorage): Promise<RebuildCounts
|
|||
// Scan all nouns using pagination
|
||||
console.log('📊 Scanning entities...')
|
||||
|
||||
// Check if pagination method exists
|
||||
const storageWithPagination = storage as any
|
||||
// Check if pagination method exists.
|
||||
// Typed boundary: see CountRebuildTarget — protected count bookkeeping plus
|
||||
// optional offset-paginated readers, neither visible on the public type.
|
||||
const storageWithPagination = storage as unknown as CountRebuildTarget
|
||||
if (typeof storageWithPagination.getNounsWithPagination !== 'function') {
|
||||
throw new Error('Storage adapter does not support getNounsWithPagination')
|
||||
}
|
||||
|
|
@ -67,7 +108,7 @@ export async function rebuildCounts(storage: BaseStorage): Promise<RebuildCounts
|
|||
let offset = 0 // Use offset-based pagination instead of cursor (bug fix for infinite loop)
|
||||
|
||||
while (hasMore) {
|
||||
const result: any = await storageWithPagination.getNounsWithPagination({
|
||||
const result = await storageWithPagination.getNounsWithPagination({
|
||||
limit: 100,
|
||||
offset // Pass offset for proper pagination (previously passed cursor which was ignored)
|
||||
})
|
||||
|
|
@ -98,7 +139,7 @@ export async function rebuildCounts(storage: BaseStorage): Promise<RebuildCounts
|
|||
offset = 0 // Reset offset for verbs pagination
|
||||
|
||||
while (hasMore) {
|
||||
const result: any = await storageWithPagination.getVerbsWithPagination({
|
||||
const result = await storageWithPagination.getVerbsWithPagination({
|
||||
limit: 100,
|
||||
offset // Pass offset for proper pagination (previously passed cursor which was ignored)
|
||||
})
|
||||
|
|
|
|||
|
|
@ -1,398 +0,0 @@
|
|||
/**
|
||||
* Request Coalescer
|
||||
* Batches and deduplicates operations to reduce S3 API calls
|
||||
* Automatically flushes based on size, time, or pressure
|
||||
*/
|
||||
|
||||
import { createModuleLogger } from './logger.js'
|
||||
|
||||
interface CoalescedOperation {
|
||||
type: 'write' | 'read' | 'delete'
|
||||
key: string
|
||||
data?: any
|
||||
resolve: (value: any) => void
|
||||
reject: (error: any) => void
|
||||
timestamp: number
|
||||
}
|
||||
|
||||
interface BatchStats {
|
||||
totalOperations: number
|
||||
coalescedOperations: number
|
||||
deduplicated: number
|
||||
batchesProcessed: number
|
||||
averageBatchSize: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Coalesces multiple operations into efficient batches
|
||||
*/
|
||||
export class RequestCoalescer {
|
||||
private logger = createModuleLogger('RequestCoalescer')
|
||||
|
||||
// Operation queues by type
|
||||
private writeQueue = new Map<string, CoalescedOperation[]>()
|
||||
private readQueue = new Map<string, CoalescedOperation[]>()
|
||||
private deleteQueue = new Map<string, CoalescedOperation[]>()
|
||||
|
||||
// Batch configuration
|
||||
private maxBatchSize = 100
|
||||
private maxBatchAge = 100 // ms - flush quickly under load
|
||||
private minBatchSize = 10 // Don't flush until we have enough
|
||||
|
||||
// Flush timers
|
||||
private flushTimer: NodeJS.Timeout | null = null
|
||||
private lastFlush = Date.now()
|
||||
|
||||
// Statistics
|
||||
private stats: BatchStats = {
|
||||
totalOperations: 0,
|
||||
coalescedOperations: 0,
|
||||
deduplicated: 0,
|
||||
batchesProcessed: 0,
|
||||
averageBatchSize: 0
|
||||
}
|
||||
|
||||
// Processor function
|
||||
private processor: (batch: CoalescedOperation[]) => Promise<void>
|
||||
|
||||
constructor(
|
||||
processor: (batch: CoalescedOperation[]) => Promise<void>,
|
||||
options?: {
|
||||
maxBatchSize?: number
|
||||
maxBatchAge?: number
|
||||
minBatchSize?: number
|
||||
}
|
||||
) {
|
||||
this.processor = processor
|
||||
|
||||
if (options) {
|
||||
this.maxBatchSize = options.maxBatchSize || this.maxBatchSize
|
||||
this.maxBatchAge = options.maxBatchAge || this.maxBatchAge
|
||||
this.minBatchSize = options.minBatchSize || this.minBatchSize
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a write operation to be coalesced
|
||||
*/
|
||||
public async write(key: string, data: any): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
// Check if we already have a pending write for this key
|
||||
const existing = this.writeQueue.get(key)
|
||||
|
||||
if (existing && existing.length > 0) {
|
||||
// Replace the data but resolve all promises
|
||||
const last = existing[existing.length - 1]
|
||||
last.data = data // Use latest data
|
||||
|
||||
// Add this promise to be resolved
|
||||
existing.push({
|
||||
type: 'write',
|
||||
key,
|
||||
data,
|
||||
resolve,
|
||||
reject,
|
||||
timestamp: Date.now()
|
||||
})
|
||||
|
||||
this.stats.deduplicated++
|
||||
} else {
|
||||
// New write operation
|
||||
this.writeQueue.set(key, [{
|
||||
type: 'write',
|
||||
key,
|
||||
data,
|
||||
resolve,
|
||||
reject,
|
||||
timestamp: Date.now()
|
||||
}])
|
||||
}
|
||||
|
||||
this.stats.totalOperations++
|
||||
this.checkFlush()
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a read operation to be coalesced
|
||||
*/
|
||||
public async read(key: string): Promise<any> {
|
||||
return new Promise((resolve, reject) => {
|
||||
// Check if we already have a pending read for this key
|
||||
const existing = this.readQueue.get(key)
|
||||
|
||||
if (existing && existing.length > 0) {
|
||||
// Coalesce with existing read
|
||||
existing.push({
|
||||
type: 'read',
|
||||
key,
|
||||
resolve,
|
||||
reject,
|
||||
timestamp: Date.now()
|
||||
})
|
||||
|
||||
this.stats.deduplicated++
|
||||
} else {
|
||||
// New read operation
|
||||
this.readQueue.set(key, [{
|
||||
type: 'read',
|
||||
key,
|
||||
resolve,
|
||||
reject,
|
||||
timestamp: Date.now()
|
||||
}])
|
||||
}
|
||||
|
||||
this.stats.totalOperations++
|
||||
this.checkFlush()
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a delete operation to be coalesced
|
||||
*/
|
||||
public async delete(key: string): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
// Cancel any pending writes for this key
|
||||
if (this.writeQueue.has(key)) {
|
||||
const writes = this.writeQueue.get(key)!
|
||||
writes.forEach(op => op.reject(new Error('Cancelled by delete')))
|
||||
this.writeQueue.delete(key)
|
||||
this.stats.deduplicated += writes.length
|
||||
}
|
||||
|
||||
// Cancel any pending reads for this key
|
||||
if (this.readQueue.has(key)) {
|
||||
const reads = this.readQueue.get(key)!
|
||||
reads.forEach(op => op.resolve(null)) // Return null for deleted items
|
||||
this.readQueue.delete(key)
|
||||
this.stats.deduplicated += reads.length
|
||||
}
|
||||
|
||||
// Check if we already have a pending delete
|
||||
const existing = this.deleteQueue.get(key)
|
||||
|
||||
if (existing && existing.length > 0) {
|
||||
// Coalesce with existing delete
|
||||
existing.push({
|
||||
type: 'delete',
|
||||
key,
|
||||
resolve,
|
||||
reject,
|
||||
timestamp: Date.now()
|
||||
})
|
||||
|
||||
this.stats.deduplicated++
|
||||
} else {
|
||||
// New delete operation
|
||||
this.deleteQueue.set(key, [{
|
||||
type: 'delete',
|
||||
key,
|
||||
resolve,
|
||||
reject,
|
||||
timestamp: Date.now()
|
||||
}])
|
||||
}
|
||||
|
||||
this.stats.totalOperations++
|
||||
this.checkFlush()
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if we should flush the queues
|
||||
*/
|
||||
private checkFlush(): void {
|
||||
const totalSize = this.writeQueue.size + this.readQueue.size + this.deleteQueue.size
|
||||
const now = Date.now()
|
||||
const age = now - this.lastFlush
|
||||
|
||||
// Immediate flush conditions
|
||||
if (totalSize >= this.maxBatchSize) {
|
||||
this.flush('size_limit')
|
||||
return
|
||||
}
|
||||
|
||||
// Age-based flush
|
||||
if (age >= this.maxBatchAge && totalSize >= this.minBatchSize) {
|
||||
this.flush('age_limit')
|
||||
return
|
||||
}
|
||||
|
||||
// Schedule a flush if not already scheduled
|
||||
if (!this.flushTimer && totalSize > 0) {
|
||||
const delay = Math.max(10, this.maxBatchAge - age)
|
||||
this.flushTimer = setTimeout(() => {
|
||||
this.flush('timer')
|
||||
}, delay)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Flush all queued operations
|
||||
*/
|
||||
public async flush(reason: string = 'manual'): Promise<void> {
|
||||
// Clear timer
|
||||
if (this.flushTimer) {
|
||||
clearTimeout(this.flushTimer)
|
||||
this.flushTimer = null
|
||||
}
|
||||
|
||||
// Collect all operations into a single batch
|
||||
const batch: CoalescedOperation[] = []
|
||||
|
||||
// Process deletes first (highest priority)
|
||||
this.deleteQueue.forEach((ops) => {
|
||||
// Only take the first operation per key (others are duplicates)
|
||||
if (ops.length > 0) {
|
||||
batch.push(ops[0])
|
||||
this.stats.coalescedOperations += ops.length
|
||||
}
|
||||
})
|
||||
|
||||
// Then writes
|
||||
this.writeQueue.forEach((ops) => {
|
||||
if (ops.length > 0) {
|
||||
// Use the last write (most recent data)
|
||||
const lastWrite = ops[ops.length - 1]
|
||||
batch.push(lastWrite)
|
||||
this.stats.coalescedOperations += ops.length
|
||||
}
|
||||
})
|
||||
|
||||
// Then reads
|
||||
this.readQueue.forEach((ops) => {
|
||||
if (ops.length > 0) {
|
||||
batch.push(ops[0])
|
||||
this.stats.coalescedOperations += ops.length
|
||||
}
|
||||
})
|
||||
|
||||
// Clear queues
|
||||
const allOps = [
|
||||
...Array.from(this.deleteQueue.values()).flat(),
|
||||
...Array.from(this.writeQueue.values()).flat(),
|
||||
...Array.from(this.readQueue.values()).flat()
|
||||
]
|
||||
|
||||
this.deleteQueue.clear()
|
||||
this.writeQueue.clear()
|
||||
this.readQueue.clear()
|
||||
|
||||
if (batch.length === 0) {
|
||||
return
|
||||
}
|
||||
|
||||
// Update stats
|
||||
this.stats.batchesProcessed++
|
||||
this.stats.averageBatchSize =
|
||||
(this.stats.averageBatchSize * (this.stats.batchesProcessed - 1) + batch.length) /
|
||||
this.stats.batchesProcessed
|
||||
|
||||
this.logger.debug(`Flushing batch of ${batch.length} operations (${allOps.length} total) - reason: ${reason}`)
|
||||
|
||||
// Process the batch
|
||||
try {
|
||||
await this.processor(batch)
|
||||
|
||||
// Resolve all promises
|
||||
allOps.forEach(op => {
|
||||
if (op.type === 'read') {
|
||||
// Find the result for this read
|
||||
const result = batch.find(b => b.key === op.key && b.type === 'read')
|
||||
op.resolve(result?.data || null)
|
||||
} else {
|
||||
op.resolve(undefined)
|
||||
}
|
||||
})
|
||||
} catch (error) {
|
||||
// Reject all promises
|
||||
allOps.forEach(op => op.reject(error))
|
||||
|
||||
this.logger.error('Batch processing failed:', error)
|
||||
}
|
||||
|
||||
this.lastFlush = Date.now()
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current statistics
|
||||
*/
|
||||
public getStats(): BatchStats {
|
||||
return { ...this.stats }
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current queue sizes
|
||||
*/
|
||||
public getQueueSizes(): {
|
||||
writes: number
|
||||
reads: number
|
||||
deletes: number
|
||||
total: number
|
||||
} {
|
||||
return {
|
||||
writes: this.writeQueue.size,
|
||||
reads: this.readQueue.size,
|
||||
deletes: this.deleteQueue.size,
|
||||
total: this.writeQueue.size + this.readQueue.size + this.deleteQueue.size
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Adjust batch parameters based on load
|
||||
*/
|
||||
public adjustParameters(pending: number): void {
|
||||
if (pending > 10000) {
|
||||
// Extreme load - batch aggressively
|
||||
this.maxBatchSize = 500
|
||||
this.maxBatchAge = 50
|
||||
this.minBatchSize = 50
|
||||
} else if (pending > 1000) {
|
||||
// High load - larger batches
|
||||
this.maxBatchSize = 200
|
||||
this.maxBatchAge = 100
|
||||
this.minBatchSize = 20
|
||||
} else if (pending > 100) {
|
||||
// Moderate load
|
||||
this.maxBatchSize = 100
|
||||
this.maxBatchAge = 200
|
||||
this.minBatchSize = 10
|
||||
} else {
|
||||
// Low load - optimize for latency
|
||||
this.maxBatchSize = 50
|
||||
this.maxBatchAge = 500
|
||||
this.minBatchSize = 5
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Force immediate flush of all operations
|
||||
*/
|
||||
public async forceFlush(): Promise<void> {
|
||||
await this.flush('force')
|
||||
}
|
||||
}
|
||||
|
||||
// Global coalescer instances by storage type
|
||||
const coalescers = new Map<string, RequestCoalescer>()
|
||||
|
||||
/**
|
||||
* Get or create a coalescer for a storage instance
|
||||
*/
|
||||
export function getCoalescer(
|
||||
storageId: string,
|
||||
processor: (batch: any[]) => Promise<void>
|
||||
): RequestCoalescer {
|
||||
if (!coalescers.has(storageId)) {
|
||||
coalescers.set(storageId, new RequestCoalescer(processor))
|
||||
}
|
||||
return coalescers.get(storageId)!
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear all coalescers
|
||||
*/
|
||||
export function clearCoalescers(): void {
|
||||
coalescers.clear()
|
||||
}
|
||||
|
|
@ -1,29 +0,0 @@
|
|||
/**
|
||||
* Request Deduplicator Utility
|
||||
* Provides key generation for request deduplication
|
||||
*/
|
||||
|
||||
export class RequestDeduplicator {
|
||||
/**
|
||||
* Generate a unique key for search requests to enable deduplication
|
||||
*/
|
||||
static getSearchKey(
|
||||
query: string,
|
||||
k: number,
|
||||
options: any
|
||||
): string {
|
||||
// Create a consistent key from search parameters
|
||||
const optionsKey = options ? JSON.stringify({
|
||||
metadata: options.metadata,
|
||||
service: options.service,
|
||||
searchMode: options.searchMode,
|
||||
threshold: options.threshold,
|
||||
includeVectors: options.includeVectors,
|
||||
includeMetadata: options.includeMetadata,
|
||||
sortBy: options.sortBy,
|
||||
cursor: options.cursor
|
||||
}) : '{}'
|
||||
|
||||
return `search:${query}:${k}:${optionsKey}`
|
||||
}
|
||||
}
|
||||
|
|
@ -1,308 +0,0 @@
|
|||
/**
|
||||
* SearchCache - Caches search results for improved performance
|
||||
*/
|
||||
|
||||
import { SearchResult } from '../coreTypes.js'
|
||||
|
||||
export interface CacheEntry<T = any> {
|
||||
results: SearchResult<T>[]
|
||||
timestamp: number
|
||||
hits: number
|
||||
}
|
||||
|
||||
export interface SearchCacheConfig {
|
||||
maxAge?: number // Maximum age in milliseconds (default: 5 minutes)
|
||||
maxSize?: number // Maximum number of cached queries (default: 100)
|
||||
enabled?: boolean // Whether caching is enabled (default: true)
|
||||
hitCountWeight?: number // Weight for hit count in eviction policy (default: 0.3)
|
||||
}
|
||||
|
||||
export class SearchCache<T = any> {
|
||||
private cache = new Map<string, CacheEntry<T>>()
|
||||
private maxAge: number
|
||||
private maxSize: number
|
||||
private enabled: boolean
|
||||
private hitCountWeight: number
|
||||
|
||||
// Cache statistics
|
||||
private hits = 0
|
||||
private misses = 0
|
||||
private evictions = 0
|
||||
|
||||
constructor(config: SearchCacheConfig = {}) {
|
||||
this.maxAge = config.maxAge ?? 5 * 60 * 1000 // 5 minutes
|
||||
this.maxSize = config.maxSize ?? 100
|
||||
this.enabled = config.enabled ?? true
|
||||
this.hitCountWeight = config.hitCountWeight ?? 0.3
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate cache key from search parameters
|
||||
*/
|
||||
getCacheKey(
|
||||
query: any,
|
||||
k: number,
|
||||
options: Record<string, any> = {}
|
||||
): string {
|
||||
// Create a normalized key that ignores order of options
|
||||
const normalizedOptions = Object.keys(options)
|
||||
.sort()
|
||||
.reduce((acc, key) => {
|
||||
// Skip cache-related options
|
||||
if (key === 'skipCache' || key === 'useStreaming') return acc
|
||||
acc[key] = options[key]
|
||||
return acc
|
||||
}, {} as Record<string, any>)
|
||||
|
||||
return JSON.stringify({
|
||||
query: typeof query === 'object' ? JSON.stringify(query) : query,
|
||||
k,
|
||||
...normalizedOptions
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Get cached results if available and not expired
|
||||
*/
|
||||
get(key: string): SearchResult<T>[] | null {
|
||||
if (!this.enabled) return null
|
||||
|
||||
const entry = this.cache.get(key)
|
||||
if (!entry) {
|
||||
this.misses++
|
||||
return null
|
||||
}
|
||||
|
||||
// Check if expired
|
||||
if (Date.now() - entry.timestamp > this.maxAge) {
|
||||
this.cache.delete(key)
|
||||
this.misses++
|
||||
return null
|
||||
}
|
||||
|
||||
// Update hit count and statistics
|
||||
entry.hits++
|
||||
this.hits++
|
||||
return entry.results
|
||||
}
|
||||
|
||||
/**
|
||||
* Cache search results
|
||||
*/
|
||||
set(key: string, results: SearchResult<T>[]): void {
|
||||
if (!this.enabled) return
|
||||
|
||||
// Evict if cache is full
|
||||
if (this.cache.size >= this.maxSize) {
|
||||
this.evictOldest()
|
||||
}
|
||||
|
||||
this.cache.set(key, {
|
||||
results: [...results], // Deep copy to prevent mutations
|
||||
timestamp: Date.now(),
|
||||
hits: 0
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Evict the oldest entry based on timestamp and hit count
|
||||
*/
|
||||
private evictOldest(): void {
|
||||
let oldestKey: string | null = null
|
||||
let oldestScore = Infinity
|
||||
|
||||
const now = Date.now()
|
||||
|
||||
for (const [key, entry] of this.cache.entries()) {
|
||||
// Score combines age and inverse hit count
|
||||
const age = now - entry.timestamp
|
||||
const hitScore = entry.hits > 0 ? 1 / entry.hits : 1
|
||||
const score = age + (hitScore * this.hitCountWeight * this.maxAge)
|
||||
|
||||
if (score < oldestScore) {
|
||||
oldestScore = score
|
||||
oldestKey = key
|
||||
}
|
||||
}
|
||||
|
||||
if (oldestKey) {
|
||||
this.cache.delete(oldestKey)
|
||||
this.evictions++
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear all cached results
|
||||
*/
|
||||
clear(): void {
|
||||
this.cache.clear()
|
||||
this.hits = 0
|
||||
this.misses = 0
|
||||
this.evictions = 0
|
||||
}
|
||||
|
||||
/**
|
||||
* Invalidate cache entries that might be affected by data changes
|
||||
*/
|
||||
invalidate(pattern?: string | RegExp): void {
|
||||
if (!pattern) {
|
||||
this.clear()
|
||||
return
|
||||
}
|
||||
|
||||
const keysToDelete: string[] = []
|
||||
|
||||
for (const key of this.cache.keys()) {
|
||||
const shouldDelete = typeof pattern === 'string'
|
||||
? key.includes(pattern)
|
||||
: pattern.test(key)
|
||||
|
||||
if (shouldDelete) {
|
||||
keysToDelete.push(key)
|
||||
}
|
||||
}
|
||||
|
||||
keysToDelete.forEach(key => this.cache.delete(key))
|
||||
}
|
||||
|
||||
/**
|
||||
* Smart invalidation for real-time data updates
|
||||
* Only clears cache if it's getting stale or if data changes significantly
|
||||
*/
|
||||
invalidateOnDataChange(changeType?: 'add' | 'update' | 'delete'): void {
|
||||
// For now, clear all caches on data changes to ensure consistency
|
||||
// In the future, we could implement more sophisticated invalidation
|
||||
// based on the type of change and affected data
|
||||
this.clear()
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if cache entries have expired and remove them
|
||||
* This is especially important in distributed scenarios where
|
||||
* real-time updates might be delayed or missed
|
||||
*/
|
||||
cleanupExpiredEntries(): number {
|
||||
const now = Date.now()
|
||||
const keysToDelete: string[] = []
|
||||
|
||||
for (const [key, entry] of this.cache.entries()) {
|
||||
if (now - entry.timestamp > this.maxAge) {
|
||||
keysToDelete.push(key)
|
||||
}
|
||||
}
|
||||
|
||||
keysToDelete.forEach(key => this.cache.delete(key))
|
||||
return keysToDelete.length
|
||||
}
|
||||
|
||||
/**
|
||||
* Get cache statistics
|
||||
*/
|
||||
getStats() {
|
||||
const total = this.hits + this.misses
|
||||
return {
|
||||
hits: this.hits,
|
||||
misses: this.misses,
|
||||
evictions: this.evictions,
|
||||
hitRate: total > 0 ? this.hits / total : 0,
|
||||
size: this.cache.size,
|
||||
maxSize: this.maxSize,
|
||||
enabled: this.enabled
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Enable or disable caching
|
||||
*/
|
||||
setEnabled(enabled: boolean): void {
|
||||
Object.defineProperty(this, 'enabled', { value: enabled, writable: false })
|
||||
if (!enabled) {
|
||||
this.clear()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get memory usage estimate in bytes
|
||||
*/
|
||||
getMemoryUsage(): number {
|
||||
let totalSize = 0
|
||||
|
||||
for (const [key, entry] of this.cache.entries()) {
|
||||
// Estimate key size
|
||||
totalSize += key.length * 2 // UTF-16 characters
|
||||
|
||||
// Estimate entry size
|
||||
totalSize += JSON.stringify(entry.results).length * 2
|
||||
totalSize += 16 // timestamp + hits (8 bytes each)
|
||||
}
|
||||
|
||||
return totalSize
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current cache configuration
|
||||
*/
|
||||
getConfig(): SearchCacheConfig {
|
||||
return {
|
||||
enabled: this.enabled,
|
||||
maxSize: this.maxSize,
|
||||
maxAge: this.maxAge,
|
||||
hitCountWeight: this.hitCountWeight
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update cache configuration dynamically
|
||||
*/
|
||||
updateConfig(newConfig: Partial<SearchCacheConfig>): void {
|
||||
if (newConfig.enabled !== undefined) {
|
||||
this.enabled = newConfig.enabled
|
||||
}
|
||||
if (newConfig.maxSize !== undefined) {
|
||||
this.maxSize = newConfig.maxSize
|
||||
// Trigger eviction if current size exceeds new limit
|
||||
this.evictIfNeeded()
|
||||
}
|
||||
if (newConfig.maxAge !== undefined) {
|
||||
this.maxAge = newConfig.maxAge
|
||||
// Clean up entries that are now expired with new TTL
|
||||
this.cleanupExpiredEntries()
|
||||
}
|
||||
if (newConfig.hitCountWeight !== undefined) {
|
||||
this.hitCountWeight = newConfig.hitCountWeight
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Evict entries if cache exceeds maxSize
|
||||
*/
|
||||
private evictIfNeeded(): void {
|
||||
if (this.cache.size <= this.maxSize) {
|
||||
return
|
||||
}
|
||||
|
||||
// Calculate eviction score for each entry (same logic as existing eviction)
|
||||
const entries = Array.from(this.cache.entries()).map(([key, entry]) => {
|
||||
const age = Date.now() - entry.timestamp
|
||||
const hitCount = entry.hits
|
||||
|
||||
// Eviction score: lower is more likely to be evicted
|
||||
// Combines age and hit count (weighted by hitCountWeight)
|
||||
const ageScore = age / this.maxAge
|
||||
const hitScore = 1 / (hitCount + 1) // Inverse of hits (more hits = lower score)
|
||||
const score = ageScore * (1 - this.hitCountWeight) + hitScore * this.hitCountWeight
|
||||
|
||||
return { key, entry, score }
|
||||
})
|
||||
|
||||
// Sort by score (lowest first - these will be evicted)
|
||||
entries.sort((a, b) => a.score - b.score)
|
||||
|
||||
// Evict entries until we're under the limit
|
||||
const toEvict = entries.slice(0, this.cache.size - this.maxSize)
|
||||
toEvict.forEach(({ key }) => {
|
||||
this.cache.delete(key)
|
||||
this.evictions++
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -306,7 +306,7 @@ export class StructuredLogger {
|
|||
name: error.name,
|
||||
message: error.message,
|
||||
stack: error.stack,
|
||||
code: (error as any).code
|
||||
code: (error as Error & { code?: string }).code
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -51,7 +51,7 @@ export function validateGraphNoun(noun: unknown): ValidatedGraphNoun {
|
|||
if (!noun || typeof noun !== 'object') {
|
||||
throw new Error('Invalid noun: must be an object')
|
||||
}
|
||||
const n = noun as any
|
||||
const n = noun as Record<string, unknown>
|
||||
if (!n.noun) {
|
||||
throw new Error('Invalid noun: missing required "noun" type field')
|
||||
}
|
||||
|
|
@ -63,7 +63,7 @@ export function validateGraphVerb(verb: unknown): ValidatedGraphVerb {
|
|||
if (!verb || typeof verb !== 'object') {
|
||||
throw new Error('Invalid verb: must be an object')
|
||||
}
|
||||
const v = verb as any
|
||||
const v = verb as Record<string, unknown>
|
||||
if (!v.verb) {
|
||||
throw new Error('Invalid verb: missing required "verb" type field')
|
||||
}
|
||||
|
|
@ -346,8 +346,8 @@ export function validateSearchOptions(options: unknown, paramName = 'options'):
|
|||
throw new ValidationError(paramName, options, 'must be an object')
|
||||
}
|
||||
|
||||
const opts = options as any
|
||||
|
||||
const opts = options as Record<string, unknown>
|
||||
|
||||
// Validate limit
|
||||
if ('limit' in opts) {
|
||||
const limit = opts.limit
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue