soulcraft/brainy
2
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

feat: refactor verb storage to use HNSWVerb for improved performance

Browse source 
- Updated MemoryStorage and BaseStorage to handle HNSWVerb instead of GraphVerb.
- Introduced methods to save and retrieve verb metadata separately.
- Enhanced getVerb and getAllVerbs methods to convert HNSWVerb to GraphVerb with metadata.
- Improved data handling and filtering in various storage methods.
This commit is contained in:
David Snelling 2025-08-03 10:47:47 -07:00
parent abd98a9f37
commit 9905a5dc35
5 changed files with 303 additions and 119 deletions
Download patch file Download diff file Expand all files Collapse all files

119
src/brainyData.ts
View file

@ -13,6 +13,7 @@ import { createStorage } from './storage/storageFactory.js'
import {
DistanceFunction,
GraphVerb,
HNSWVerb,
EmbeddingFunction,
HNSWConfig,
HNSWNoun,
@ -2907,22 +2908,22 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
nanoseconds: (now.getTime() % 1000) * 1000000
}
// Create verb data (without metadata fields)
const verb: GraphVerb = {
// Create lightweight verb for HNSW index storage
const hnswVerb: HNSWVerb = {
id,
vector: verbVector,
connections: new Map(),
connections: new Map()
}
// Create complete verb metadata separately
const verbMetadata = {
sourceId: sourceId,
targetId: targetId,
source: sourceId,
target: targetId,
verb: verbType as VerbType,
type: verbType, // Set the type property to match the verb type
weight: options.weight
}
// Create verb metadata separately
const verbMetadata = {
weight: options.weight,
createdAt: timestamp,
updatedAt: timestamp,
createdBy: {
@ -2945,10 +2946,30 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
}
// Update verb connections from index
verb.connections = indexNoun.connections
hnswVerb.connections = indexNoun.connections
// Save verb to storage
await this.storage!.saveVerb(verb)
// Combine HNSWVerb and metadata into a GraphVerb for storage
const fullVerb: GraphVerb = {
id: hnswVerb.id,
vector: hnswVerb.vector,
connections: hnswVerb.connections,
sourceId: verbMetadata.sourceId,
targetId: verbMetadata.targetId,
source: verbMetadata.source,
target: verbMetadata.target,
verb: verbMetadata.verb,
type: verbMetadata.type,
weight: verbMetadata.weight,
createdAt: verbMetadata.createdAt,
updatedAt: verbMetadata.updatedAt,
createdBy: verbMetadata.createdBy,
metadata: verbMetadata.data,
data: verbMetadata.data,
embedding: hnswVerb.vector
}
// Save the complete verb (BaseStorage will handle the separation)
await this.storage!.saveVerb(fullVerb)
// Track verb statistics
const serviceForStats = this.getServiceName(options)
@ -2971,7 +2992,44 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
await this.ensureInitialized()
try {
return await this.storage!.getVerb(id)
// Get the lightweight verb from storage
const hnswVerb = await this.storage!.getVerb(id)
if (!hnswVerb) {
return null
}
// Get the verb metadata
const metadata = await this.storage!.getVerbMetadata(id)
if (!metadata) {
console.warn(`Verb ${id} found but no metadata - creating minimal GraphVerb`)
// Return minimal GraphVerb if metadata is missing
return {
id: hnswVerb.id,
vector: hnswVerb.vector,
sourceId: '',
targetId: ''
}
}
// Combine into a complete GraphVerb
const graphVerb: GraphVerb = {
id: hnswVerb.id,
vector: hnswVerb.vector,
sourceId: metadata.sourceId,
targetId: metadata.targetId,
source: metadata.source,
target: metadata.target,
verb: metadata.verb,
type: metadata.type,
weight: metadata.weight,
createdAt: metadata.createdAt,
updatedAt: metadata.updatedAt,
createdBy: metadata.createdBy,
data: metadata.data,
metadata: metadata.data // Alias for backward compatibility
}
return graphVerb
} catch (error) {
console.error(`Failed to get verb ${id}:`, error)
throw new Error(`Failed to get verb ${id}: ${error}`)
@ -2986,14 +3044,37 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
await this.ensureInitialized()
try {
// Use getVerbs with no pagination to get all verbs
const result = await this.getVerbs({
pagination: {
limit: Number.MAX_SAFE_INTEGER // Request all verbs
// Get all lightweight verbs from storage
const hnswVerbs = await this.storage!.getAllVerbs()
// Convert each HNSWVerb to GraphVerb by loading metadata
const graphVerbs: GraphVerb[] = []
for (const hnswVerb of hnswVerbs) {
const metadata = await this.storage!.getVerbMetadata(hnswVerb.id)
if (metadata) {
const graphVerb: GraphVerb = {
id: hnswVerb.id,
vector: hnswVerb.vector,
sourceId: metadata.sourceId,
targetId: metadata.targetId,
source: metadata.source,
target: metadata.target,
verb: metadata.verb,
type: metadata.type,
weight: metadata.weight,
createdAt: metadata.createdAt,
updatedAt: metadata.updatedAt,
createdBy: metadata.createdBy,
data: metadata.data,
metadata: metadata.data // Alias for backward compatibility
}
graphVerbs.push(graphVerb)
} else {
console.warn(`Verb ${hnswVerb.id} found but no metadata - skipping`)
}
})
return result.items
}
return graphVerbs
} catch (error) {
console.error('Failed to get all verbs:', error)
throw new Error(`Failed to get all verbs: ${error}`)

36
src/coreTypes.ts
View file

@ -67,12 +67,25 @@ export interface HNSWNoun {
}
/**
* Verb representing a relationship between nouns
* Extends HNSWNoun to allow verbs to be first-class entities in the data model
* Lightweight verb for HNSW index storage
* Contains only essential data needed for vector operations
*/
export interface GraphVerb extends HNSWNoun {
export interface HNSWVerb {
id: string
vector: Vector
connections: Map<number, Set<string>> // level -> set of connected verb ids
}
/**
* Verb representing a relationship between nouns
* Stored separately from HNSW index for lightweight performance
*/
export interface GraphVerb {
id: string // Unique identifier for the verb
sourceId: string // ID of the source noun
targetId: string // ID of the target noun
vector: Vector // Vector representation of the relationship
connections?: Map<number, Set<string>> // Optional connections from HNSW index
type?: string // Optional type of the relationship
weight?: number // Optional weight of the relationship
metadata?: any // Optional metadata for the verb
@ -82,7 +95,7 @@ export interface GraphVerb extends HNSWNoun {
target?: string // Alias for targetId
verb?: string // Alias for type
data?: Record<string, any> // Additional flexible data storage
embedding?: Vector // Vector representation of the relationship
embedding?: Vector // Alias for vector
// Timestamp and creator properties
createdAt?: { seconds: number; nanoseconds: number } // When the verb was created
@ -286,6 +299,21 @@ export interface StorageAdapter {
getMetadata(id: string): Promise<any | null>
/**
* Save verb metadata to storage
* @param id The ID of the verb
* @param metadata The metadata to save
* @returns Promise that resolves when the metadata is saved
*/
saveVerbMetadata(id: string, metadata: any): Promise<void>
/**
* Get verb metadata from storage
* @param id The ID of the verb
* @returns Promise that resolves to the metadata or null if not found
*/
getVerbMetadata(id: string): Promise<any | null>
clear(): Promise<void>
/**

2
src/index.ts
View file

@ -250,6 +250,7 @@ import type {
EmbeddingFunction,
EmbeddingModel,
HNSWNoun,
HNSWVerb,
HNSWConfig,
StorageAdapter
} from './coreTypes.js'
@ -271,6 +272,7 @@ export type {
EmbeddingFunction,
EmbeddingModel,
HNSWNoun,
HNSWVerb,
HNSWConfig,
HNSWOptimizedConfig,
StorageAdapter

141
src/storage/adapters/memoryStorage.ts
View file

@ -3,7 +3,7 @@
* In-memory storage adapter for environments where persistent storage is not available or needed
*/
import { GraphVerb, HNSWNoun, StatisticsData } from '../../coreTypes.js'
import { GraphVerb, HNSWNoun, HNSWVerb, StatisticsData } from '../../coreTypes.js'
import { BaseStorage, STATISTICS_KEY } from '../baseStorage.js'
import { PaginatedResult } from '../../types/paginationTypes.js'
@ -16,7 +16,7 @@ import { PaginatedResult } from '../../types/paginationTypes.js'
export class MemoryStorage extends BaseStorage {
// Single map of noun ID to noun
private nouns: Map<string, HNSWNoun> = new Map()
private verbs: Map<string, GraphVerb> = new Map()
private verbs: Map<string, HNSWVerb> = new Map()
private metadata: Map<string, any> = new Map()
private nounMetadata: Map<string, any> = new Map()
private verbMetadata: Map<string, any> = new Map()
@ -237,20 +237,12 @@ export class MemoryStorage extends BaseStorage {
/**
* Save a verb to storage
*/
protected async saveVerb_internal(verb: GraphVerb): Promise<void> {
protected async saveVerb_internal(verb: HNSWVerb): Promise<void> {
// Create a deep copy to avoid reference issues
const verbCopy: GraphVerb = {
const verbCopy: HNSWVerb = {
id: verb.id,
vector: [...verb.vector],
connections: new Map(),
sourceId: verb.sourceId,
targetId: verb.targetId,
source: verb.sourceId || verb.source,
target: verb.targetId || verb.target,
verb: verb.type || verb.verb,
type: verb.type || verb.verb,
weight: verb.weight,
metadata: verb.metadata
connections: new Map()
}
// Copy connections
@ -265,7 +257,7 @@ export class MemoryStorage extends BaseStorage {
/**
* Get a verb from storage
*/
protected async getVerb_internal(id: string): Promise<GraphVerb | null> {
protected async getVerb_internal(id: string): Promise<HNSWVerb | null> {
// Get the verb directly from the verbs map
const verb = this.verbs.get(id)
@ -286,22 +278,11 @@ export class MemoryStorage extends BaseStorage {
version: '1.0'
}
// Return a deep copy to avoid reference issues
const verbCopy: GraphVerb = {
// Return a deep copy of the HNSWVerb
const verbCopy: HNSWVerb = {
id: verb.id,
vector: [...verb.vector],
connections: new Map(),
sourceId: (verb.sourceId || verb.source || ""),
targetId: (verb.targetId || verb.target || ""),
source: (verb.sourceId || verb.source || ""),
target: (verb.targetId || verb.target || ""),
verb: verb.type || verb.verb,
type: verb.type || verb.verb, // Ensure type is also set
weight: verb.weight,
metadata: verb.metadata,
createdAt: verb.createdAt || defaultTimestamp,
updatedAt: verb.updatedAt || defaultTimestamp,
createdBy: verb.createdBy || defaultCreatedBy
connections: new Map()
}
// Copy connections
@ -315,38 +296,16 @@ export class MemoryStorage extends BaseStorage {
/**
* Get all verbs from storage
*/
protected async getAllVerbs_internal(): Promise<GraphVerb[]> {
const allVerbs: GraphVerb[] = []
protected async getAllVerbs_internal(): Promise<HNSWVerb[]> {
const allVerbs: HNSWVerb[] = []
// Iterate through all verbs in the verbs map
for (const [verbId, verb] of this.verbs.entries()) {
// Create default timestamp if not present
const defaultTimestamp = {
seconds: Math.floor(Date.now() / 1000),
nanoseconds: (Date.now() % 1000) * 1000000
}
// Create default createdBy if not present
const defaultCreatedBy = {
augmentation: 'unknown',
version: '1.0'
}
// Return a deep copy to avoid reference issues
const verbCopy: GraphVerb = {
// Create a deep copy of the HNSWVerb
const verbCopy: HNSWVerb = {
id: verb.id,
vector: [...verb.vector],
connections: new Map(),
sourceId: (verb.sourceId || verb.source || ""),
targetId: (verb.targetId || verb.target || ""),
source: (verb.sourceId || verb.source || ""),
target: (verb.targetId || verb.target || ""),
verb: verb.type || verb.verb,
weight: verb.weight,
metadata: verb.metadata,
createdAt: verb.createdAt || defaultTimestamp,
updatedAt: verb.updatedAt || defaultTimestamp,
createdBy: verb.createdBy || defaultCreatedBy
connections: new Map()
}
// Copy connections
@ -407,27 +366,30 @@ export class MemoryStorage extends BaseStorage {
const matchingIds: string[] = []
// Iterate through all verbs to find matches
for (const [verbId, verb] of this.verbs.entries()) {
for (const [verbId, hnswVerb] of this.verbs.entries()) {
// Get the metadata for this verb to do filtering
const metadata = this.verbMetadata.get(verbId)
// Filter by verb type if specified
if (verbTypes && !verbTypes.includes(verb.type || verb.verb || '')) {
if (verbTypes && metadata && !verbTypes.includes(metadata.type || metadata.verb || '')) {
continue
}
// Filter by source ID if specified
if (sourceIds && !sourceIds.includes(verb.sourceId || verb.source || '')) {
if (sourceIds && metadata && !sourceIds.includes(metadata.sourceId || metadata.source || '')) {
continue
}
// Filter by target ID if specified
if (targetIds && !targetIds.includes(verb.targetId || verb.target || '')) {
if (targetIds && metadata && !targetIds.includes(metadata.targetId || metadata.target || '')) {
continue
}
// Filter by metadata fields if specified
if (filter.metadata && verb.metadata) {
if (filter.metadata && metadata && metadata.data) {
let metadataMatch = true
for (const [key, value] of Object.entries(filter.metadata)) {
if (verb.metadata[key] !== value) {
if (metadata.data[key] !== value) {
metadataMatch = false
break
}
@ -436,8 +398,8 @@ export class MemoryStorage extends BaseStorage {
}
// Filter by service if specified
if (services && verb.metadata && verb.metadata.service &&
!services.includes(verb.metadata.service)) {
if (services && metadata && metadata.createdBy && metadata.createdBy.augmentation &&
!services.includes(metadata.createdBy.augmentation)) {
continue
}
@ -456,33 +418,42 @@ export class MemoryStorage extends BaseStorage {
// Fetch the actual verbs for the current page
const items: GraphVerb[] = []
for (const id of paginatedIds) {
const verb = this.verbs.get(id)
if (!verb) continue
const hnswVerb = this.verbs.get(id)
const metadata = this.verbMetadata.get(id)
// Create a deep copy to avoid reference issues
const verbCopy: GraphVerb = {
id: verb.id,
vector: [...verb.vector],
connections: new Map(),
sourceId: verb.sourceId || verb.source || '',
targetId: verb.targetId || verb.target || '',
source: verb.sourceId || verb.source || '',
target: verb.targetId || verb.target || '',
verb: verb.type || verb.verb,
type: verb.type || verb.verb,
weight: verb.weight,
metadata: verb.metadata ? JSON.parse(JSON.stringify(verb.metadata)) : undefined,
createdAt: verb.createdAt ? { ...verb.createdAt } : undefined,
updatedAt: verb.updatedAt ? { ...verb.updatedAt } : undefined,
createdBy: verb.createdBy ? { ...verb.createdBy } : undefined
if (!hnswVerb) continue
if (!metadata) {
console.warn(`Verb ${id} found but no metadata - creating minimal GraphVerb`)
// Return minimal GraphVerb if metadata is missing
items.push({
id: hnswVerb.id,
vector: hnswVerb.vector,
sourceId: '',
targetId: ''
})
continue
}
// Copy connections
for (const [level, connections] of verb.connections.entries()) {
verbCopy.connections.set(level, new Set(connections))
// Create a complete GraphVerb by combining HNSWVerb with metadata
const graphVerb: GraphVerb = {
id: hnswVerb.id,
vector: [...hnswVerb.vector],
sourceId: metadata.sourceId,
targetId: metadata.targetId,
source: metadata.source,
target: metadata.target,
verb: metadata.verb,
type: metadata.type,
weight: metadata.weight,
createdAt: metadata.createdAt,
updatedAt: metadata.updatedAt,
createdBy: metadata.createdBy,
data: metadata.data,
metadata: metadata.data // Alias for backward compatibility
}
items.push(verbCopy)
items.push(graphVerb)
}
return {

124
src/storage/baseStorage.ts
View file

@ -3,7 +3,7 @@
* Provides common functionality for all storage adapters
*/
import { GraphVerb, HNSWNoun, StatisticsData } from '../coreTypes.js'
import { GraphVerb, HNSWNoun, HNSWVerb, StatisticsData } from '../coreTypes.js'
import { BaseStorageAdapter } from './adapters/baseStorageAdapter.js'
// Common directory/prefix names
@ -87,7 +87,34 @@ export abstract class BaseStorage extends BaseStorageAdapter {
*/
public async saveVerb(verb: GraphVerb): Promise<void> {
await this.ensureInitialized()
return this.saveVerb_internal(verb)
// Extract the lightweight HNSWVerb data
const hnswVerb: HNSWVerb = {
id: verb.id,
vector: verb.vector,
connections: verb.connections || new Map()
}
// Extract and save the metadata separately
const metadata = {
sourceId: verb.sourceId || verb.source,
targetId: verb.targetId || verb.target,
source: verb.source || verb.sourceId,
target: verb.target || verb.targetId,
type: verb.type || verb.verb,
verb: verb.verb || verb.type,
weight: verb.weight,
metadata: verb.metadata,
data: verb.data,
createdAt: verb.createdAt,
updatedAt: verb.updatedAt,
createdBy: verb.createdBy,
embedding: verb.embedding
}
// Save both the HNSWVerb and metadata
await this.saveVerb_internal(hnswVerb)
await this.saveVerbMetadata(verb.id, metadata)
}
/**
@ -95,7 +122,56 @@ export abstract class BaseStorage extends BaseStorageAdapter {
*/
public async getVerb(id: string): Promise<GraphVerb | null> {
await this.ensureInitialized()
return this.getVerb_internal(id)
const hnswVerb = await this.getVerb_internal(id)
if (!hnswVerb) {
return null
}
return this.convertHNSWVerbToGraphVerb(hnswVerb)
}
/**
* Convert HNSWVerb to GraphVerb by combining with metadata
*/
protected async convertHNSWVerbToGraphVerb(hnswVerb: HNSWVerb): Promise<GraphVerb | null> {
try {
const metadata = await this.getVerbMetadata(hnswVerb.id)
if (!metadata) {
return null
}
// Create default timestamp if not present
const defaultTimestamp = {
seconds: Math.floor(Date.now() / 1000),
nanoseconds: (Date.now() % 1000) * 1000000
}
// Create default createdBy if not present
const defaultCreatedBy = {
augmentation: 'unknown',
version: '1.0'
}
return {
id: hnswVerb.id,
vector: hnswVerb.vector,
sourceId: metadata.sourceId,
targetId: metadata.targetId,
source: metadata.source,
target: metadata.target,
verb: metadata.verb,
type: metadata.type,
weight: metadata.weight || 1.0,
metadata: metadata.metadata || {},
createdAt: metadata.createdAt || defaultTimestamp,
updatedAt: metadata.updatedAt || defaultTimestamp,
createdBy: metadata.createdBy || defaultCreatedBy,
data: metadata.data,
embedding: hnswVerb.vector
}
} catch (error) {
console.error(`Failed to convert HNSWVerb to GraphVerb for ${hnswVerb.id}:`, error)
return null
}
}
/**
@ -106,7 +182,18 @@ export abstract class BaseStorage extends BaseStorageAdapter {
public async getAllVerbs(): Promise<GraphVerb[]> {
await this.ensureInitialized()
console.warn('WARNING: getAllVerbs() is deprecated and will be removed in a future version. Use getVerbs() with pagination instead.')
return this.getAllVerbs_internal()
const hnswVerbs = await this.getAllVerbs_internal()
const graphVerbs: GraphVerb[] = []
for (const hnswVerb of hnswVerbs) {
const graphVerb = await this.convertHNSWVerbToGraphVerb(hnswVerb)
if (graphVerb) {
graphVerbs.push(graphVerb)
}
}
return graphVerbs
}
/**
@ -114,7 +201,12 @@ export abstract class BaseStorage extends BaseStorageAdapter {
*/
public async getVerbsBySource(sourceId: string): Promise<GraphVerb[]> {
await this.ensureInitialized()
return this.getVerbsBySource_internal(sourceId)
// Get all verbs and filter by source
const allVerbs = await this.getAllVerbs()
return allVerbs.filter(verb =>
verb.sourceId === sourceId || verb.source === sourceId
)
}
/**
@ -122,7 +214,12 @@ export abstract class BaseStorage extends BaseStorageAdapter {
*/
public async getVerbsByTarget(targetId: string): Promise<GraphVerb[]> {
await this.ensureInitialized()
return this.getVerbsByTarget_internal(targetId)
// Get all verbs and filter by target
const allVerbs = await this.getAllVerbs()
return allVerbs.filter(verb =>
verb.targetId === targetId || verb.target === targetId
)
}
/**
@ -130,7 +227,12 @@ export abstract class BaseStorage extends BaseStorageAdapter {
*/
public async getVerbsByType(type: string): Promise<GraphVerb[]> {
await this.ensureInitialized()
return this.getVerbsByType_internal(type)
// Get all verbs and filter by type
const allVerbs = await this.getAllVerbs()
return allVerbs.filter(verb =>
verb.type === type || verb.verb === type
)
}
/**
@ -522,7 +624,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
try {
// Try to get only the verbs we need
allVerbs = await this.getAllVerbs_internal()
allVerbs = await this.getAllVerbs()
// If we have too many verbs, truncate the array to avoid memory issues
if (allVerbs.length > maxVerbs) {
@ -734,19 +836,19 @@ export abstract class BaseStorage extends BaseStorageAdapter {
* Save a verb to storage
* This method should be implemented by each specific adapter
*/
protected abstract saveVerb_internal(verb: GraphVerb): Promise<void>
protected abstract saveVerb_internal(verb: HNSWVerb): Promise<void>
/**
* Get a verb from storage
* This method should be implemented by each specific adapter
*/
protected abstract getVerb_internal(id: string): Promise<GraphVerb | null>
protected abstract getVerb_internal(id: string): Promise<HNSWVerb | null>
/**
* Get all verbs from storage
* This method should be implemented by each specific adapter
*/
protected abstract getAllVerbs_internal(): Promise<GraphVerb[]>
protected abstract getAllVerbs_internal(): Promise<HNSWVerb[]>
/**
* Get verbs by source