/** * Memory Storage Adapter * In-memory storage adapter for environments where persistent storage is not available or needed */ import { GraphVerb, HNSWNoun, HNSWVerb, NounMetadata, VerbMetadata, HNSWNounWithMetadata, HNSWVerbWithMetadata, StatisticsData, NounType } from '../../coreTypes.js' import { BaseStorage, STATISTICS_KEY } from '../baseStorage.js' import { PaginatedResult } from '../../types/paginationTypes.js' // No type aliases needed - using the original types directly /** * In-memory storage adapter * Uses Maps to store data in memory */ export class MemoryStorage extends BaseStorage { // Single map of noun ID to noun private nouns: Map = new Map() private verbs: Map = new Map() private statistics: StatisticsData | null = null // Unified object store for primitive operations (replaces metadata, nounMetadata, verbMetadata) private objectStore: Map = new Map() // Backward compatibility aliases private get metadata(): Map { return this.objectStore } private get nounMetadata(): Map { return this.objectStore } private get verbMetadata(): Map { return this.objectStore } constructor() { super() } /** * Initialize the storage adapter * Nothing to initialize for in-memory storage */ public async init(): Promise { this.isInitialized = true } /** * Save a noun to storage (v4.0.0: pure vector only, no metadata) */ protected async saveNoun_internal(noun: HNSWNoun): Promise { const isNew = !this.nouns.has(noun.id) // Create a deep copy to avoid reference issues // v4.0.0: Store ONLY vector data (no metadata field) // Metadata is saved separately via saveNounMetadata() by base class const nounCopy: HNSWNoun = { id: noun.id, vector: [...noun.vector], connections: new Map(), level: noun.level || 0 // ✅ NO metadata field in v4.0.0 } // Copy connections for (const [level, connections] of noun.connections.entries()) { nounCopy.connections.set(level, new Set(connections)) } // Save the noun directly in the nouns map this.nouns.set(noun.id, nounCopy) // Count tracking happens in baseStorage.saveNounMetadata_internal (v4.1.2) // This fixes the race condition where metadata didn't exist yet } /** * Get a noun from storage (v4.0.0: returns pure vector only) * Base class handles combining with metadata */ protected async getNoun_internal(id: string): Promise { // Get the noun directly from the nouns map const noun = this.nouns.get(id) // If not found, return null if (!noun) { return null } // Return a deep copy to avoid reference issues // v4.0.0: Return ONLY vector data (no metadata field) const nounCopy: HNSWNoun = { id: noun.id, vector: [...noun.vector], connections: new Map(), level: noun.level || 0 // ✅ NO metadata field in v4.0.0 } // Copy connections for (const [level, connections] of noun.connections.entries()) { nounCopy.connections.set(level, new Set(connections)) } return nounCopy } /** * Get nouns with pagination and filtering * v4.0.0: Returns HNSWNounWithMetadata[] (includes metadata field) * @param options Pagination and filtering options * @returns Promise that resolves to a paginated result of nouns with metadata */ public async getNouns(options: { pagination?: { offset?: number limit?: number cursor?: string } filter?: { nounType?: string | string[] service?: string | string[] metadata?: Record } } = {}): Promise<{ items: HNSWNounWithMetadata[]; totalCount?: number; hasMore: boolean; nextCursor?: string }> { const pagination = options.pagination || {} const filter = options.filter || {} // Default values const offset = pagination.offset || 0 const limit = pagination.limit || 100 // Convert string types to arrays for consistent handling const nounTypes = filter.nounType ? Array.isArray(filter.nounType) ? filter.nounType : [filter.nounType] : undefined const services = filter.service ? Array.isArray(filter.service) ? filter.service : [filter.service] : undefined // First, collect all noun IDs that match the filter criteria const matchingIds: string[] = [] // Iterate through all nouns to find matches // v4.0.0: Load metadata from separate storage (no embedded metadata field) for (const [nounId, noun] of this.nouns.entries()) { // Get metadata from separate storage const metadata = await this.getNounMetadata(nounId) // Skip if no metadata (shouldn't happen in v4.0.0 but be defensive) if (!metadata) { continue } // Filter by noun type if specified if (nounTypes && metadata.noun && !nounTypes.includes(metadata.noun)) { continue } // Filter by service if specified if (services && metadata.service && !services.includes(metadata.service)) { continue } // Filter by metadata fields if specified if (filter.metadata) { let metadataMatch = true for (const [key, value] of Object.entries(filter.metadata)) { if (metadata[key] !== value) { metadataMatch = false break } } if (!metadataMatch) continue } // If we got here, the noun matches all filters matchingIds.push(nounId) } // Calculate pagination const totalCount = matchingIds.length const paginatedIds = matchingIds.slice(offset, offset + limit) const hasMore = offset + limit < totalCount // Create cursor for next page if there are more results const nextCursor = hasMore ? `${offset + limit}` : undefined // Fetch the actual nouns for the current page // v4.0.0: Return HNSWNounWithMetadata (includes metadata field) const items: HNSWNounWithMetadata[] = [] for (const id of paginatedIds) { const noun = this.nouns.get(id) if (!noun) continue // Get metadata from separate storage // FIX v4.7.4: Don't skip nouns without metadata - metadata is optional in v4.0.0 const metadata = await this.getNounMetadata(id) // v4.8.0: Extract standard fields from metadata to top-level const metadataObj = (metadata || {}) as NounMetadata const { noun: nounType, createdAt, updatedAt, confidence, weight, service, data, createdBy, ...customMetadata } = metadataObj // v4.8.0: Create HNSWNounWithMetadata with standard fields at top-level const nounWithMetadata: HNSWNounWithMetadata = { id: noun.id, vector: [...noun.vector], connections: new Map(), level: noun.level || 0, // v4.8.0: Standard fields at top-level type: (nounType as NounType) || NounType.Thing, createdAt: (createdAt as number) || Date.now(), updatedAt: (updatedAt as number) || Date.now(), confidence: confidence as number | undefined, weight: weight as number | undefined, service: service as string | undefined, data: data as Record | undefined, createdBy, // Only custom user fields in metadata metadata: customMetadata } // Copy connections for (const [level, connections] of noun.connections.entries()) { nounWithMetadata.connections.set(level, new Set(connections)) } items.push(nounWithMetadata) } return { items, totalCount, hasMore, nextCursor } } /** * Get nouns with pagination - simplified interface for compatibility * v4.0.0: Returns HNSWNounWithMetadata[] (includes metadata field) */ public async getNounsWithPagination(options: { limit?: number cursor?: string filter?: any } = {}): Promise<{ items: HNSWNounWithMetadata[] totalCount: number hasMore: boolean nextCursor?: string }> { // Convert to the getNouns format const result = await this.getNouns({ pagination: { offset: options.cursor ? parseInt(options.cursor) : 0, limit: options.limit || 100 }, filter: options.filter }) return { items: result.items, totalCount: result.totalCount || 0, hasMore: result.hasMore, nextCursor: result.nextCursor } } /** * Get nouns by noun type * @param nounType The noun type to filter by * @returns Promise that resolves to an array of nouns of the specified noun type * @deprecated Use getNouns() with filter.nounType instead */ protected async getNounsByNounType_internal(nounType: string): Promise { const result = await this.getNouns({ filter: { nounType } }) return result.items } /** * Delete a noun from storage (v4.0.0) */ protected async deleteNoun_internal(id: string): Promise { // v4.0.0: Get type from separate metadata storage const metadata = await this.getNounMetadata(id) if (metadata) { const type = metadata.noun || 'default' this.decrementEntityCount(type) } this.nouns.delete(id) } /** * Save a verb to storage (v4.0.0: pure vector + core fields, no metadata) */ protected async saveVerb_internal(verb: HNSWVerb): Promise { const isNew = !this.verbs.has(verb.id) // Create a deep copy to avoid reference issues // v4.0.0: Include core relational fields but NO metadata field const verbCopy: HNSWVerb = { id: verb.id, vector: [...verb.vector], connections: new Map(), // CORE RELATIONAL DATA (part of HNSWVerb in v4.0.0) verb: verb.verb, sourceId: verb.sourceId, targetId: verb.targetId // ✅ NO metadata field in v4.0.0 } // Copy connections for (const [level, connections] of verb.connections.entries()) { verbCopy.connections.set(level, new Set(connections)) } // Save the verb directly in the verbs map this.verbs.set(verb.id, verbCopy) // Note: Count tracking happens in saveVerbMetadata since metadata is separate } /** * Get a verb from storage (v4.0.0: returns pure vector + core fields) * Base class handles combining with metadata */ protected async getVerb_internal(id: string): Promise { // Get the verb directly from the verbs map const verb = this.verbs.get(id) // If not found, return null if (!verb) { return null } // Return a deep copy of the HNSWVerb // v4.0.0: Include core relational fields but NO metadata field const verbCopy: HNSWVerb = { id: verb.id, vector: [...verb.vector], connections: new Map(), // CORE RELATIONAL DATA (part of HNSWVerb in v4.0.0) verb: verb.verb, sourceId: verb.sourceId, targetId: verb.targetId // ✅ NO metadata field in v4.0.0 } // Copy connections for (const [level, connections] of verb.connections.entries()) { verbCopy.connections.set(level, new Set(connections)) } return verbCopy } /** * Get verbs with pagination and filtering * v4.0.0: Returns HNSWVerbWithMetadata[] (includes metadata field) * @param options Pagination and filtering options * @returns Promise that resolves to a paginated result of verbs with metadata */ public async getVerbs(options: { pagination?: { offset?: number limit?: number cursor?: string } filter?: { verbType?: string | string[] sourceId?: string | string[] targetId?: string | string[] service?: string | string[] metadata?: Record } } = {}): Promise<{ items: HNSWVerbWithMetadata[]; totalCount?: number; hasMore: boolean; nextCursor?: string }> { const pagination = options.pagination || {} const filter = options.filter || {} // Default values const offset = pagination.offset || 0 const limit = pagination.limit || 100 // Convert string types to arrays for consistent handling const verbTypes = filter.verbType ? Array.isArray(filter.verbType) ? filter.verbType : [filter.verbType] : undefined const sourceIds = filter.sourceId ? Array.isArray(filter.sourceId) ? filter.sourceId : [filter.sourceId] : undefined const targetIds = filter.targetId ? Array.isArray(filter.targetId) ? filter.targetId : [filter.targetId] : undefined const services = filter.service ? Array.isArray(filter.service) ? filter.service : [filter.service] : undefined // First, collect all verb IDs that match the filter criteria const matchingIds: string[] = [] // Iterate through all verbs to find matches // v4.0.0: Core fields (verb, sourceId, targetId) are in HNSWVerb, not metadata for (const [verbId, hnswVerb] of this.verbs.entries()) { // Get the metadata for service/data filtering const metadata = await this.getVerbMetadata(verbId) // Filter by verb type if specified // v4.0.0: verb type is in HNSWVerb.verb if (verbTypes && !verbTypes.includes(hnswVerb.verb || '')) { continue } // Filter by source ID if specified // v4.0.0: sourceId is in HNSWVerb.sourceId if (sourceIds && !sourceIds.includes(hnswVerb.sourceId || '')) { continue } // Filter by target ID if specified // v4.0.0: targetId is in HNSWVerb.targetId if (targetIds && !targetIds.includes(hnswVerb.targetId || '')) { continue } // Filter by metadata fields if specified if (filter.metadata && metadata) { let metadataMatch = true for (const [key, value] of Object.entries(filter.metadata)) { const metadataValue = (metadata as any)[key] if (metadataValue !== value) { metadataMatch = false break } } if (!metadataMatch) continue } // Filter by service if specified if (services && metadata && metadata.service && !services.includes(metadata.service)) { continue } // If we got here, the verb matches all filters matchingIds.push(verbId) } // Calculate pagination const totalCount = matchingIds.length const paginatedIds = matchingIds.slice(offset, offset + limit) const hasMore = offset + limit < totalCount // Create cursor for next page if there are more results const nextCursor = hasMore ? `${offset + limit}` : undefined // Fetch the actual verbs for the current page // v4.0.0: Return HNSWVerbWithMetadata (includes metadata field) const items: HNSWVerbWithMetadata[] = [] for (const id of paginatedIds) { const hnswVerb = this.verbs.get(id) if (!hnswVerb) continue // Get metadata from separate storage // FIX v4.7.4: Don't skip verbs without metadata - metadata is optional in v4.0.0 // Core fields (verb, sourceId, targetId) are in HNSWVerb itself const metadata = await this.getVerbMetadata(id) // v4.8.0: Extract standard fields from metadata to top-level const metadataObj = metadata || {} const { createdAt, updatedAt, confidence, weight, service, data, createdBy, ...customMetadata } = metadataObj // v4.8.0: Create HNSWVerbWithMetadata with standard fields at top-level const verbWithMetadata: HNSWVerbWithMetadata = { id: hnswVerb.id, vector: [...hnswVerb.vector], connections: new Map(), // Core relational fields (part of HNSWVerb) verb: hnswVerb.verb, sourceId: hnswVerb.sourceId, targetId: hnswVerb.targetId, // v4.8.0: Standard fields at top-level createdAt: (createdAt as number) || Date.now(), updatedAt: (updatedAt as number) || Date.now(), confidence: confidence as number | undefined, weight: weight as number | undefined, service: service as string | undefined, data: data as Record | undefined, createdBy, // Only custom user fields in metadata metadata: customMetadata } // Copy connections for (const [level, connections] of hnswVerb.connections.entries()) { verbWithMetadata.connections.set(level, new Set(connections)) } items.push(verbWithMetadata) } return { items, totalCount, hasMore, nextCursor } } /** * Get verbs by source * @deprecated Use getVerbs() with filter.sourceId instead */ protected async getVerbsBySource_internal(sourceId: string): Promise { const result = await this.getVerbs({ filter: { sourceId } }) return result.items } /** * Get verbs by target * @deprecated Use getVerbs() with filter.targetId instead */ protected async getVerbsByTarget_internal(targetId: string): Promise { const result = await this.getVerbs({ filter: { targetId } }) return result.items } /** * Get verbs by type * @deprecated Use getVerbs() with filter.verbType instead */ protected async getVerbsByType_internal(type: string): Promise { const result = await this.getVerbs({ filter: { verbType: type } }) return result.items } /** * Delete a verb from storage */ protected async deleteVerb_internal(id: string): Promise { // Delete the HNSWVerb from the verbs map this.verbs.delete(id) // CRITICAL: Also delete verb metadata - this is what getVerbs() uses to find verbs // Without this, getVerbsBySource() will still find "deleted" verbs via their metadata const metadata = await this.getVerbMetadata(id) if (metadata) { const verbType = metadata.verb || metadata.type || 'default' this.decrementVerbCount(verbType as string) // Delete the metadata using the base storage method await this.deleteVerbMetadata(id) } } /** * Primitive operation: Write object to path * All metadata operations use this internally via base class routing */ protected async writeObjectToPath(path: string, data: any): Promise { // Store in unified object store using path as key this.objectStore.set(path, JSON.parse(JSON.stringify(data))) } /** * Primitive operation: Read object from path * All metadata operations use this internally via base class routing */ protected async readObjectFromPath(path: string): Promise { const data = this.objectStore.get(path) if (!data) { return null } return JSON.parse(JSON.stringify(data)) } /** * Primitive operation: Delete object from path * All metadata operations use this internally via base class routing */ protected async deleteObjectFromPath(path: string): Promise { this.objectStore.delete(path) } /** * Primitive operation: List objects under path prefix * All metadata operations use this internally via base class routing */ protected async listObjectsUnderPath(prefix: string): Promise { const paths: string[] = [] for (const key of this.objectStore.keys()) { if (key.startsWith(prefix)) { paths.push(key) } } return paths.sort() } /** * Get multiple metadata objects in batches (CRITICAL: Prevents socket exhaustion) * Memory storage implementation is simple since all data is already in memory */ public async getMetadataBatch(ids: string[]): Promise> { const results = new Map() // Memory storage can handle all IDs at once since it's in-memory for (const id of ids) { // CRITICAL: Use getNounMetadata() instead of deprecated getMetadata() // This ensures we fetch from the correct noun metadata store (2-file system) const metadata = await this.getNounMetadata(id) if (metadata) { results.set(id, metadata) } } return results } /** * Clear all data from storage */ public async clear(): Promise { this.nouns.clear() this.verbs.clear() this.objectStore.clear() this.statistics = null // Clear the statistics cache this.statisticsCache = null this.statisticsModified = false } /** * Get information about storage usage and capacity */ public async getStorageStatus(): Promise<{ type: string used: number quota: number | null details?: Record }> { return { type: 'memory', used: 0, // In-memory storage doesn't have a meaningful size quota: null, // In-memory storage doesn't have a quota details: { nodeCount: this.nouns.size, edgeCount: this.verbs.size, metadataCount: this.objectStore.size } } } /** * Save statistics data to storage * @param statistics The statistics data to save */ protected async saveStatisticsData(statistics: StatisticsData): Promise { // For memory storage, we just need to store the statistics in memory // Create a deep copy to avoid reference issues this.statistics = { nounCount: {...statistics.nounCount}, verbCount: {...statistics.verbCount}, metadataCount: {...statistics.metadataCount}, hnswIndexSize: statistics.hnswIndexSize, lastUpdated: statistics.lastUpdated, // Include serviceActivity if present ...(statistics.serviceActivity && { serviceActivity: Object.fromEntries( Object.entries(statistics.serviceActivity).map(([k, v]) => [k, {...v}]) ) }), // Include services if present ...(statistics.services && { services: statistics.services.map(s => ({...s})) }), // Include distributedConfig if present ...(statistics.distributedConfig && { distributedConfig: JSON.parse(JSON.stringify(statistics.distributedConfig)) }) } // Since this is in-memory, there's no need for time-based partitioning // or legacy file handling } /** * Get statistics data from storage * @returns Promise that resolves to the statistics data or null if not found */ protected async getStatisticsData(): Promise { if (!this.statistics) { // CRITICAL FIX (v3.37.4): Statistics don't exist yet (first init) // Return minimal stats with counts instead of null // This prevents HNSW from seeing entityCount=0 during index rebuild return { nounCount: {}, verbCount: {}, metadataCount: {}, hnswIndexSize: 0, totalNodes: this.totalNounCount, totalEdges: this.totalVerbCount, totalMetadata: 0, lastUpdated: new Date().toISOString() } } // Return a deep copy to avoid reference issues return { nounCount: {...this.statistics.nounCount}, verbCount: {...this.statistics.verbCount}, metadataCount: {...this.statistics.metadataCount}, hnswIndexSize: this.statistics.hnswIndexSize, // CRITICAL FIX: Populate totalNodes and totalEdges from in-memory counts // HNSW rebuild depends on these fields to determine entity count totalNodes: this.totalNounCount, totalEdges: this.totalVerbCount, lastUpdated: this.statistics.lastUpdated, // Include serviceActivity if present ...(this.statistics.serviceActivity && { serviceActivity: Object.fromEntries( Object.entries(this.statistics.serviceActivity).map(([k, v]) => [k, {...v}]) ) }), // Include services if present ...(this.statistics.services && { services: this.statistics.services.map(s => ({...s})) }), // Include distributedConfig if present ...(this.statistics.distributedConfig && { distributedConfig: JSON.parse(JSON.stringify(this.statistics.distributedConfig)) }) } // Since this is in-memory, there's no need for fallback mechanisms // to check multiple storage locations } /** * Initialize counts from in-memory storage - O(1) operation (v4.0.0) */ protected async initializeCounts(): Promise { // For memory storage, initialize counts from current in-memory state this.totalNounCount = this.nouns.size this.totalVerbCount = this.verbs.size // Initialize type-based counts by scanning metadata storage (v4.0.0) this.entityCounts.clear() this.verbCounts.clear() // Count nouns by loading metadata for each for (const [nounId, noun] of this.nouns.entries()) { const metadata = await this.getNounMetadata(nounId) if (metadata) { const type = metadata.noun || 'default' this.entityCounts.set(type, (this.entityCounts.get(type) || 0) + 1) } } // Count verbs by loading metadata for each for (const [verbId, verb] of this.verbs.entries()) { const metadata = await this.getVerbMetadata(verbId) if (metadata) { // VerbMetadata doesn't have verb type - that's in HNSWVerb now // Use the verb's type from the HNSWVerb itself const type = verb.verb || 'default' this.verbCounts.set(type, (this.verbCounts.get(type) || 0) + 1) } } } /** * Persist counts to storage - no-op for memory storage */ protected async persistCounts(): Promise { // No persistence needed for in-memory storage // Counts are always accurate from the live data structures } // ============================================= // HNSW Index Persistence (v3.35.0+) // ============================================= /** * Get vector for a noun */ public async getNounVector(id: string): Promise { const noun = this.nouns.get(id) return noun ? [...noun.vector] : null } // CRITICAL FIX (v4.10.1): Mutex locks for HNSW concurrency control // Even in-memory operations need serialization to prevent async race conditions private hnswLocks = new Map>() /** * Save HNSW graph data for a noun * * CRITICAL FIX (v4.10.1): Mutex locking to prevent race conditions during concurrent HNSW updates * Even in-memory operations can race due to async/await interleaving * Prevents data corruption when multiple entities connect to same neighbor simultaneously */ public async saveHNSWData(nounId: string, hnswData: { level: number connections: Record }): Promise { const path = `hnsw/${nounId}.json` // MUTEX LOCK: Wait for any pending operations on this entity while (this.hnswLocks.has(path)) { await this.hnswLocks.get(path) } // Acquire lock by creating a promise that we'll resolve when done let releaseLock!: () => void const lockPromise = new Promise(resolve => { releaseLock = resolve }) this.hnswLocks.set(path, lockPromise) try { // Read existing data (if exists) let existingNode: any = {} const existing = this.objectStore.get(path) if (existing) { existingNode = existing } // Preserve id and vector, update only HNSW graph metadata const updatedNode = { ...existingNode, // Preserve all existing fields level: hnswData.level, connections: hnswData.connections } // Write atomically (in-memory, but now serialized by mutex) this.objectStore.set(path, JSON.parse(JSON.stringify(updatedNode))) } finally { // Release lock this.hnswLocks.delete(path) releaseLock() } } /** * Get HNSW graph data for a noun */ public async getHNSWData(nounId: string): Promise<{ level: number connections: Record } | null> { const path = `hnsw/${nounId}.json` const data = await this.readObjectFromPath(path) return data || null } /** * Save HNSW system data (entry point, max level) * * CRITICAL FIX (v4.10.1): Mutex locking to prevent race conditions */ public async saveHNSWSystem(systemData: { entryPointId: string | null maxLevel: number }): Promise { const path = 'system/hnsw-system.json' // MUTEX LOCK: Wait for any pending operations while (this.hnswLocks.has(path)) { await this.hnswLocks.get(path) } // Acquire lock let releaseLock!: () => void const lockPromise = new Promise(resolve => { releaseLock = resolve }) this.hnswLocks.set(path, lockPromise) try { // Write atomically (serialized by mutex) this.objectStore.set(path, JSON.parse(JSON.stringify(systemData))) } finally { // Release lock this.hnswLocks.delete(path) releaseLock() } } /** * Get HNSW system data */ public async getHNSWSystem(): Promise<{ entryPointId: string | null maxLevel: number } | null> { const path = 'system/hnsw-system.json' const data = await this.readObjectFromPath(path) return data || null } }