From 9905a5dc35f4585e63a0f1b4c132e4db7cff33c2 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Sun, 3 Aug 2025 10:47:47 -0700 Subject: [PATCH] feat: refactor verb storage to use HNSWVerb for improved performance - Updated MemoryStorage and BaseStorage to handle HNSWVerb instead of GraphVerb. - Introduced methods to save and retrieve verb metadata separately. - Enhanced getVerb and getAllVerbs methods to convert HNSWVerb to GraphVerb with metadata. - Improved data handling and filtering in various storage methods. --- src/brainyData.ts | 119 ++++++++++++++++++---- src/coreTypes.ts | 36 ++++++- src/index.ts | 2 + src/storage/adapters/memoryStorage.ts | 141 ++++++++++---------------- src/storage/baseStorage.ts | 124 ++++++++++++++++++++-- 5 files changed, 303 insertions(+), 119 deletions(-) diff --git a/src/brainyData.ts b/src/brainyData.ts index d2c2d961..a9bae5f7 100644 --- a/src/brainyData.ts +++ b/src/brainyData.ts @@ -13,6 +13,7 @@ import { createStorage } from './storage/storageFactory.js' import { DistanceFunction, GraphVerb, + HNSWVerb, EmbeddingFunction, HNSWConfig, HNSWNoun, @@ -2907,22 +2908,22 @@ export class BrainyData implements BrainyDataInterface { nanoseconds: (now.getTime() % 1000) * 1000000 } - // Create verb data (without metadata fields) - const verb: GraphVerb = { + // Create lightweight verb for HNSW index storage + const hnswVerb: HNSWVerb = { id, vector: verbVector, - connections: new Map(), + connections: new Map() + } + + // Create complete verb metadata separately + const verbMetadata = { sourceId: sourceId, targetId: targetId, source: sourceId, target: targetId, verb: verbType as VerbType, type: verbType, // Set the type property to match the verb type - weight: options.weight - } - - // Create verb metadata separately - const verbMetadata = { + weight: options.weight, createdAt: timestamp, updatedAt: timestamp, createdBy: { @@ -2945,10 +2946,30 @@ export class BrainyData implements BrainyDataInterface { } // Update verb connections from index - verb.connections = indexNoun.connections + hnswVerb.connections = indexNoun.connections - // Save verb to storage - await this.storage!.saveVerb(verb) + // Combine HNSWVerb and metadata into a GraphVerb for storage + const fullVerb: GraphVerb = { + id: hnswVerb.id, + vector: hnswVerb.vector, + connections: hnswVerb.connections, + sourceId: verbMetadata.sourceId, + targetId: verbMetadata.targetId, + source: verbMetadata.source, + target: verbMetadata.target, + verb: verbMetadata.verb, + type: verbMetadata.type, + weight: verbMetadata.weight, + createdAt: verbMetadata.createdAt, + updatedAt: verbMetadata.updatedAt, + createdBy: verbMetadata.createdBy, + metadata: verbMetadata.data, + data: verbMetadata.data, + embedding: hnswVerb.vector + } + + // Save the complete verb (BaseStorage will handle the separation) + await this.storage!.saveVerb(fullVerb) // Track verb statistics const serviceForStats = this.getServiceName(options) @@ -2971,7 +2992,44 @@ export class BrainyData implements BrainyDataInterface { await this.ensureInitialized() try { - return await this.storage!.getVerb(id) + // Get the lightweight verb from storage + const hnswVerb = await this.storage!.getVerb(id) + if (!hnswVerb) { + return null + } + + // Get the verb metadata + const metadata = await this.storage!.getVerbMetadata(id) + if (!metadata) { + console.warn(`Verb ${id} found but no metadata - creating minimal GraphVerb`) + // Return minimal GraphVerb if metadata is missing + return { + id: hnswVerb.id, + vector: hnswVerb.vector, + sourceId: '', + targetId: '' + } + } + + // Combine into a complete GraphVerb + const graphVerb: GraphVerb = { + id: hnswVerb.id, + vector: hnswVerb.vector, + sourceId: metadata.sourceId, + targetId: metadata.targetId, + source: metadata.source, + target: metadata.target, + verb: metadata.verb, + type: metadata.type, + weight: metadata.weight, + createdAt: metadata.createdAt, + updatedAt: metadata.updatedAt, + createdBy: metadata.createdBy, + data: metadata.data, + metadata: metadata.data // Alias for backward compatibility + } + + return graphVerb } catch (error) { console.error(`Failed to get verb ${id}:`, error) throw new Error(`Failed to get verb ${id}: ${error}`) @@ -2986,14 +3044,37 @@ export class BrainyData implements BrainyDataInterface { await this.ensureInitialized() try { - // Use getVerbs with no pagination to get all verbs - const result = await this.getVerbs({ - pagination: { - limit: Number.MAX_SAFE_INTEGER // Request all verbs + // Get all lightweight verbs from storage + const hnswVerbs = await this.storage!.getAllVerbs() + + // Convert each HNSWVerb to GraphVerb by loading metadata + const graphVerbs: GraphVerb[] = [] + for (const hnswVerb of hnswVerbs) { + const metadata = await this.storage!.getVerbMetadata(hnswVerb.id) + if (metadata) { + const graphVerb: GraphVerb = { + id: hnswVerb.id, + vector: hnswVerb.vector, + sourceId: metadata.sourceId, + targetId: metadata.targetId, + source: metadata.source, + target: metadata.target, + verb: metadata.verb, + type: metadata.type, + weight: metadata.weight, + createdAt: metadata.createdAt, + updatedAt: metadata.updatedAt, + createdBy: metadata.createdBy, + data: metadata.data, + metadata: metadata.data // Alias for backward compatibility + } + graphVerbs.push(graphVerb) + } else { + console.warn(`Verb ${hnswVerb.id} found but no metadata - skipping`) } - }) - - return result.items + } + + return graphVerbs } catch (error) { console.error('Failed to get all verbs:', error) throw new Error(`Failed to get all verbs: ${error}`) diff --git a/src/coreTypes.ts b/src/coreTypes.ts index 509d748e..1e4a9b38 100644 --- a/src/coreTypes.ts +++ b/src/coreTypes.ts @@ -67,12 +67,25 @@ export interface HNSWNoun { } /** - * Verb representing a relationship between nouns - * Extends HNSWNoun to allow verbs to be first-class entities in the data model + * Lightweight verb for HNSW index storage + * Contains only essential data needed for vector operations */ -export interface GraphVerb extends HNSWNoun { +export interface HNSWVerb { + id: string + vector: Vector + connections: Map> // level -> set of connected verb ids +} + +/** + * Verb representing a relationship between nouns + * Stored separately from HNSW index for lightweight performance + */ +export interface GraphVerb { + id: string // Unique identifier for the verb sourceId: string // ID of the source noun targetId: string // ID of the target noun + vector: Vector // Vector representation of the relationship + connections?: Map> // Optional connections from HNSW index type?: string // Optional type of the relationship weight?: number // Optional weight of the relationship metadata?: any // Optional metadata for the verb @@ -82,7 +95,7 @@ export interface GraphVerb extends HNSWNoun { target?: string // Alias for targetId verb?: string // Alias for type data?: Record // Additional flexible data storage - embedding?: Vector // Vector representation of the relationship + embedding?: Vector // Alias for vector // Timestamp and creator properties createdAt?: { seconds: number; nanoseconds: number } // When the verb was created @@ -286,6 +299,21 @@ export interface StorageAdapter { getMetadata(id: string): Promise + /** + * Save verb metadata to storage + * @param id The ID of the verb + * @param metadata The metadata to save + * @returns Promise that resolves when the metadata is saved + */ + saveVerbMetadata(id: string, metadata: any): Promise + + /** + * Get verb metadata from storage + * @param id The ID of the verb + * @returns Promise that resolves to the metadata or null if not found + */ + getVerbMetadata(id: string): Promise + clear(): Promise /** diff --git a/src/index.ts b/src/index.ts index 29b8fe84..62644d2d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -250,6 +250,7 @@ import type { EmbeddingFunction, EmbeddingModel, HNSWNoun, + HNSWVerb, HNSWConfig, StorageAdapter } from './coreTypes.js' @@ -271,6 +272,7 @@ export type { EmbeddingFunction, EmbeddingModel, HNSWNoun, + HNSWVerb, HNSWConfig, HNSWOptimizedConfig, StorageAdapter diff --git a/src/storage/adapters/memoryStorage.ts b/src/storage/adapters/memoryStorage.ts index cc0c763b..9f43e803 100644 --- a/src/storage/adapters/memoryStorage.ts +++ b/src/storage/adapters/memoryStorage.ts @@ -3,7 +3,7 @@ * In-memory storage adapter for environments where persistent storage is not available or needed */ -import { GraphVerb, HNSWNoun, StatisticsData } from '../../coreTypes.js' +import { GraphVerb, HNSWNoun, HNSWVerb, StatisticsData } from '../../coreTypes.js' import { BaseStorage, STATISTICS_KEY } from '../baseStorage.js' import { PaginatedResult } from '../../types/paginationTypes.js' @@ -16,7 +16,7 @@ import { PaginatedResult } from '../../types/paginationTypes.js' export class MemoryStorage extends BaseStorage { // Single map of noun ID to noun private nouns: Map = new Map() - private verbs: Map = new Map() + private verbs: Map = new Map() private metadata: Map = new Map() private nounMetadata: Map = new Map() private verbMetadata: Map = new Map() @@ -237,20 +237,12 @@ export class MemoryStorage extends BaseStorage { /** * Save a verb to storage */ - protected async saveVerb_internal(verb: GraphVerb): Promise { + protected async saveVerb_internal(verb: HNSWVerb): Promise { // Create a deep copy to avoid reference issues - const verbCopy: GraphVerb = { + const verbCopy: HNSWVerb = { id: verb.id, vector: [...verb.vector], - connections: new Map(), - sourceId: verb.sourceId, - targetId: verb.targetId, - source: verb.sourceId || verb.source, - target: verb.targetId || verb.target, - verb: verb.type || verb.verb, - type: verb.type || verb.verb, - weight: verb.weight, - metadata: verb.metadata + connections: new Map() } // Copy connections @@ -265,7 +257,7 @@ export class MemoryStorage extends BaseStorage { /** * Get a verb from storage */ - protected async getVerb_internal(id: string): Promise { + protected async getVerb_internal(id: string): Promise { // Get the verb directly from the verbs map const verb = this.verbs.get(id) @@ -286,22 +278,11 @@ export class MemoryStorage extends BaseStorage { version: '1.0' } - // Return a deep copy to avoid reference issues - const verbCopy: GraphVerb = { + // Return a deep copy of the HNSWVerb + const verbCopy: HNSWVerb = { id: verb.id, vector: [...verb.vector], - connections: new Map(), - sourceId: (verb.sourceId || verb.source || ""), - targetId: (verb.targetId || verb.target || ""), - source: (verb.sourceId || verb.source || ""), - target: (verb.targetId || verb.target || ""), - verb: verb.type || verb.verb, - type: verb.type || verb.verb, // Ensure type is also set - weight: verb.weight, - metadata: verb.metadata, - createdAt: verb.createdAt || defaultTimestamp, - updatedAt: verb.updatedAt || defaultTimestamp, - createdBy: verb.createdBy || defaultCreatedBy + connections: new Map() } // Copy connections @@ -315,38 +296,16 @@ export class MemoryStorage extends BaseStorage { /** * Get all verbs from storage */ - protected async getAllVerbs_internal(): Promise { - const allVerbs: GraphVerb[] = [] + protected async getAllVerbs_internal(): Promise { + const allVerbs: HNSWVerb[] = [] // Iterate through all verbs in the verbs map for (const [verbId, verb] of this.verbs.entries()) { - // Create default timestamp if not present - const defaultTimestamp = { - seconds: Math.floor(Date.now() / 1000), - nanoseconds: (Date.now() % 1000) * 1000000 - } - - // Create default createdBy if not present - const defaultCreatedBy = { - augmentation: 'unknown', - version: '1.0' - } - - // Return a deep copy to avoid reference issues - const verbCopy: GraphVerb = { + // Create a deep copy of the HNSWVerb + const verbCopy: HNSWVerb = { id: verb.id, vector: [...verb.vector], - connections: new Map(), - sourceId: (verb.sourceId || verb.source || ""), - targetId: (verb.targetId || verb.target || ""), - source: (verb.sourceId || verb.source || ""), - target: (verb.targetId || verb.target || ""), - verb: verb.type || verb.verb, - weight: verb.weight, - metadata: verb.metadata, - createdAt: verb.createdAt || defaultTimestamp, - updatedAt: verb.updatedAt || defaultTimestamp, - createdBy: verb.createdBy || defaultCreatedBy + connections: new Map() } // Copy connections @@ -407,27 +366,30 @@ export class MemoryStorage extends BaseStorage { const matchingIds: string[] = [] // Iterate through all verbs to find matches - for (const [verbId, verb] of this.verbs.entries()) { + for (const [verbId, hnswVerb] of this.verbs.entries()) { + // Get the metadata for this verb to do filtering + const metadata = this.verbMetadata.get(verbId) + // Filter by verb type if specified - if (verbTypes && !verbTypes.includes(verb.type || verb.verb || '')) { + if (verbTypes && metadata && !verbTypes.includes(metadata.type || metadata.verb || '')) { continue } // Filter by source ID if specified - if (sourceIds && !sourceIds.includes(verb.sourceId || verb.source || '')) { + if (sourceIds && metadata && !sourceIds.includes(metadata.sourceId || metadata.source || '')) { continue } // Filter by target ID if specified - if (targetIds && !targetIds.includes(verb.targetId || verb.target || '')) { + if (targetIds && metadata && !targetIds.includes(metadata.targetId || metadata.target || '')) { continue } // Filter by metadata fields if specified - if (filter.metadata && verb.metadata) { + if (filter.metadata && metadata && metadata.data) { let metadataMatch = true for (const [key, value] of Object.entries(filter.metadata)) { - if (verb.metadata[key] !== value) { + if (metadata.data[key] !== value) { metadataMatch = false break } @@ -436,8 +398,8 @@ export class MemoryStorage extends BaseStorage { } // Filter by service if specified - if (services && verb.metadata && verb.metadata.service && - !services.includes(verb.metadata.service)) { + if (services && metadata && metadata.createdBy && metadata.createdBy.augmentation && + !services.includes(metadata.createdBy.augmentation)) { continue } @@ -456,33 +418,42 @@ export class MemoryStorage extends BaseStorage { // Fetch the actual verbs for the current page const items: GraphVerb[] = [] for (const id of paginatedIds) { - const verb = this.verbs.get(id) - if (!verb) continue + const hnswVerb = this.verbs.get(id) + const metadata = this.verbMetadata.get(id) - // Create a deep copy to avoid reference issues - const verbCopy: GraphVerb = { - id: verb.id, - vector: [...verb.vector], - connections: new Map(), - sourceId: verb.sourceId || verb.source || '', - targetId: verb.targetId || verb.target || '', - source: verb.sourceId || verb.source || '', - target: verb.targetId || verb.target || '', - verb: verb.type || verb.verb, - type: verb.type || verb.verb, - weight: verb.weight, - metadata: verb.metadata ? JSON.parse(JSON.stringify(verb.metadata)) : undefined, - createdAt: verb.createdAt ? { ...verb.createdAt } : undefined, - updatedAt: verb.updatedAt ? { ...verb.updatedAt } : undefined, - createdBy: verb.createdBy ? { ...verb.createdBy } : undefined + if (!hnswVerb) continue + + if (!metadata) { + console.warn(`Verb ${id} found but no metadata - creating minimal GraphVerb`) + // Return minimal GraphVerb if metadata is missing + items.push({ + id: hnswVerb.id, + vector: hnswVerb.vector, + sourceId: '', + targetId: '' + }) + continue } - // Copy connections - for (const [level, connections] of verb.connections.entries()) { - verbCopy.connections.set(level, new Set(connections)) + // Create a complete GraphVerb by combining HNSWVerb with metadata + const graphVerb: GraphVerb = { + id: hnswVerb.id, + vector: [...hnswVerb.vector], + sourceId: metadata.sourceId, + targetId: metadata.targetId, + source: metadata.source, + target: metadata.target, + verb: metadata.verb, + type: metadata.type, + weight: metadata.weight, + createdAt: metadata.createdAt, + updatedAt: metadata.updatedAt, + createdBy: metadata.createdBy, + data: metadata.data, + metadata: metadata.data // Alias for backward compatibility } - items.push(verbCopy) + items.push(graphVerb) } return { diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts index 8a6c3480..3c0b1d5f 100644 --- a/src/storage/baseStorage.ts +++ b/src/storage/baseStorage.ts @@ -3,7 +3,7 @@ * Provides common functionality for all storage adapters */ -import { GraphVerb, HNSWNoun, StatisticsData } from '../coreTypes.js' +import { GraphVerb, HNSWNoun, HNSWVerb, StatisticsData } from '../coreTypes.js' import { BaseStorageAdapter } from './adapters/baseStorageAdapter.js' // Common directory/prefix names @@ -87,7 +87,34 @@ export abstract class BaseStorage extends BaseStorageAdapter { */ public async saveVerb(verb: GraphVerb): Promise { await this.ensureInitialized() - return this.saveVerb_internal(verb) + + // Extract the lightweight HNSWVerb data + const hnswVerb: HNSWVerb = { + id: verb.id, + vector: verb.vector, + connections: verb.connections || new Map() + } + + // Extract and save the metadata separately + const metadata = { + sourceId: verb.sourceId || verb.source, + targetId: verb.targetId || verb.target, + source: verb.source || verb.sourceId, + target: verb.target || verb.targetId, + type: verb.type || verb.verb, + verb: verb.verb || verb.type, + weight: verb.weight, + metadata: verb.metadata, + data: verb.data, + createdAt: verb.createdAt, + updatedAt: verb.updatedAt, + createdBy: verb.createdBy, + embedding: verb.embedding + } + + // Save both the HNSWVerb and metadata + await this.saveVerb_internal(hnswVerb) + await this.saveVerbMetadata(verb.id, metadata) } /** @@ -95,7 +122,56 @@ export abstract class BaseStorage extends BaseStorageAdapter { */ public async getVerb(id: string): Promise { await this.ensureInitialized() - return this.getVerb_internal(id) + const hnswVerb = await this.getVerb_internal(id) + if (!hnswVerb) { + return null + } + return this.convertHNSWVerbToGraphVerb(hnswVerb) + } + + /** + * Convert HNSWVerb to GraphVerb by combining with metadata + */ + protected async convertHNSWVerbToGraphVerb(hnswVerb: HNSWVerb): Promise { + try { + const metadata = await this.getVerbMetadata(hnswVerb.id) + if (!metadata) { + return null + } + + // Create default timestamp if not present + const defaultTimestamp = { + seconds: Math.floor(Date.now() / 1000), + nanoseconds: (Date.now() % 1000) * 1000000 + } + + // Create default createdBy if not present + const defaultCreatedBy = { + augmentation: 'unknown', + version: '1.0' + } + + return { + id: hnswVerb.id, + vector: hnswVerb.vector, + sourceId: metadata.sourceId, + targetId: metadata.targetId, + source: metadata.source, + target: metadata.target, + verb: metadata.verb, + type: metadata.type, + weight: metadata.weight || 1.0, + metadata: metadata.metadata || {}, + createdAt: metadata.createdAt || defaultTimestamp, + updatedAt: metadata.updatedAt || defaultTimestamp, + createdBy: metadata.createdBy || defaultCreatedBy, + data: metadata.data, + embedding: hnswVerb.vector + } + } catch (error) { + console.error(`Failed to convert HNSWVerb to GraphVerb for ${hnswVerb.id}:`, error) + return null + } } /** @@ -106,7 +182,18 @@ export abstract class BaseStorage extends BaseStorageAdapter { public async getAllVerbs(): Promise { await this.ensureInitialized() console.warn('WARNING: getAllVerbs() is deprecated and will be removed in a future version. Use getVerbs() with pagination instead.') - return this.getAllVerbs_internal() + + const hnswVerbs = await this.getAllVerbs_internal() + const graphVerbs: GraphVerb[] = [] + + for (const hnswVerb of hnswVerbs) { + const graphVerb = await this.convertHNSWVerbToGraphVerb(hnswVerb) + if (graphVerb) { + graphVerbs.push(graphVerb) + } + } + + return graphVerbs } /** @@ -114,7 +201,12 @@ export abstract class BaseStorage extends BaseStorageAdapter { */ public async getVerbsBySource(sourceId: string): Promise { await this.ensureInitialized() - return this.getVerbsBySource_internal(sourceId) + + // Get all verbs and filter by source + const allVerbs = await this.getAllVerbs() + return allVerbs.filter(verb => + verb.sourceId === sourceId || verb.source === sourceId + ) } /** @@ -122,7 +214,12 @@ export abstract class BaseStorage extends BaseStorageAdapter { */ public async getVerbsByTarget(targetId: string): Promise { await this.ensureInitialized() - return this.getVerbsByTarget_internal(targetId) + + // Get all verbs and filter by target + const allVerbs = await this.getAllVerbs() + return allVerbs.filter(verb => + verb.targetId === targetId || verb.target === targetId + ) } /** @@ -130,7 +227,12 @@ export abstract class BaseStorage extends BaseStorageAdapter { */ public async getVerbsByType(type: string): Promise { await this.ensureInitialized() - return this.getVerbsByType_internal(type) + + // Get all verbs and filter by type + const allVerbs = await this.getAllVerbs() + return allVerbs.filter(verb => + verb.type === type || verb.verb === type + ) } /** @@ -522,7 +624,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { try { // Try to get only the verbs we need - allVerbs = await this.getAllVerbs_internal() + allVerbs = await this.getAllVerbs() // If we have too many verbs, truncate the array to avoid memory issues if (allVerbs.length > maxVerbs) { @@ -734,19 +836,19 @@ export abstract class BaseStorage extends BaseStorageAdapter { * Save a verb to storage * This method should be implemented by each specific adapter */ - protected abstract saveVerb_internal(verb: GraphVerb): Promise + protected abstract saveVerb_internal(verb: HNSWVerb): Promise /** * Get a verb from storage * This method should be implemented by each specific adapter */ - protected abstract getVerb_internal(id: string): Promise + protected abstract getVerb_internal(id: string): Promise /** * Get all verbs from storage * This method should be implemented by each specific adapter */ - protected abstract getAllVerbs_internal(): Promise + protected abstract getAllVerbs_internal(): Promise /** * Get verbs by source