From 6f6f787bf14475a90d4ff6e492a1a0d88134c4fa Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 19 Jun 2025 15:02:05 -0700 Subject: [PATCH] **feat: introduce Optimized HNSW Index with product quantization and disk-based storage support** ### Changes: - Added `HNSWIndexOptimized` class in `src/hnsw/hnswIndexOptimized.ts`: - Supports large datasets with product quantization for dimensionality reduction. - Enables disk-based index storage via a configurable storage adapter. - Introduces `HNSWOptimizedConfig` interface for advanced configuration: - Includes memory thresholds, product quantization (subvectors, centroids), and disk-based indexing options. - Implements methods for adding, searching, and managing indexed vectors with enhanced scalability. - Introduced `ProductQuantizer` utility: - Handles vector quantization with training and reconstruction capabilities. - Reduces vector dimensions, improving memory and performance efficiency. - Added accessor methods in `src/hnsw/hnswIndex.ts` for configuration and index metadata (e.g., dimension, max level, entry point). ### Purpose: Enhanced the HNSW implementation to support optimized indexing and search for large datasets, addressing scalability challenges via product quantization and optional disk-based storage. This update elevates the library's performance and applicability for memory-constrained environments. --- src/coreTypes.ts | 92 ++++++++++++++++++------------------ src/types/fileSystemTypes.ts | 2 + src/types/graphTypes.ts | 83 ++++++++++++++++---------------- 3 files changed, 89 insertions(+), 88 deletions(-) diff --git a/src/coreTypes.ts b/src/coreTypes.ts index 2dedf051..450c38b8 100644 --- a/src/coreTypes.ts +++ b/src/coreTypes.ts @@ -5,36 +5,36 @@ /** * Vector representation - an array of numbers */ -export type Vector = number[]; +export type Vector = number[] /** * A document with a vector embedding and optional metadata */ export interface VectorDocument { - id: string; - vector: Vector; - metadata?: T; + id: string + vector: Vector + metadata?: T } /** * Search result with similarity score */ export interface SearchResult { - id: string; - score: number; - vector: Vector; - metadata?: T; + id: string + score: number + vector: Vector + metadata?: T } /** * Distance function for comparing vectors */ -export type DistanceFunction = (a: Vector, b: Vector) => number; +export type DistanceFunction = (a: Vector, b: Vector) => number /** * Embedding function for converting data to vectors */ -export type EmbeddingFunction = (data: any) => Promise; +export type EmbeddingFunction = (data: any) => Promise /** * Embedding model interface @@ -43,26 +43,26 @@ export interface EmbeddingModel { /** * Initialize the embedding model */ - init(): Promise; + init(): Promise /** * Embed data into a vector */ - embed(data: any): Promise; + embed(data: any): Promise /** * Dispose of the model resources */ - dispose(): Promise; + dispose(): Promise } /** * HNSW graph noun */ export interface HNSWNoun { - id: string; - vector: Vector; - connections: Map>; // level -> set of connected noun ids + id: string + vector: Vector + connections: Map> // level -> set of connected noun ids } /** @@ -70,63 +70,63 @@ export interface HNSWNoun { * Extends HNSWNoun to allow verbs to be first-class entities in the data model */ export interface GraphVerb extends HNSWNoun { - sourceId: string; // ID of the source noun - targetId: string; // ID of the target noun - type?: string; // Optional type of the relationship - weight?: number; // Optional weight of the relationship - metadata?: any; // Optional metadata for the verb + sourceId: string // ID of the source noun + targetId: string // ID of the target noun + type?: string // Optional type of the relationship + weight?: number // Optional weight of the relationship + metadata?: any // Optional metadata for the verb } /** * HNSW index configuration */ export interface HNSWConfig { - M: number; // Maximum number of connections per noun - efConstruction: number; // Size of the dynamic candidate list during construction - efSearch: number; // Size of the dynamic candidate list during search - ml: number; // Maximum level + M: number // Maximum number of connections per noun + efConstruction: number // Size of the dynamic candidate list during construction + efSearch: number // Size of the dynamic candidate list during search + ml: number // Maximum level } /** * Storage interface for persistence */ export interface StorageAdapter { - init(): Promise; + init(): Promise - saveNoun(noun: HNSWNoun): Promise; + saveNoun(noun: HNSWNoun): Promise - getNoun(id: string): Promise; + getNoun(id: string): Promise - getAllNouns(): Promise; + getAllNouns(): Promise /** * Get nouns by noun type * @param nounType The noun type to filter by * @returns Promise that resolves to an array of nouns of the specified noun type */ - getNounsByNounType(nounType: string): Promise; + getNounsByNounType(nounType: string): Promise - deleteNoun(id: string): Promise; + deleteNoun(id: string): Promise - saveVerb(verb: GraphVerb): Promise; + saveVerb(verb: GraphVerb): Promise - getVerb(id: string): Promise; + getVerb(id: string): Promise - getAllVerbs(): Promise; + getAllVerbs(): Promise - getVerbsBySource(sourceId: string): Promise; + getVerbsBySource(sourceId: string): Promise - getVerbsByTarget(targetId: string): Promise; + getVerbsByTarget(targetId: string): Promise - getVerbsByType(type: string): Promise; + getVerbsByType(type: string): Promise - deleteVerb(id: string): Promise; + deleteVerb(id: string): Promise - saveMetadata(id: string, metadata: any): Promise; + saveMetadata(id: string, metadata: any): Promise - getMetadata(id: string): Promise; + getMetadata(id: string): Promise - clear(): Promise; + clear(): Promise /** * Get information about storage usage and capacity @@ -136,21 +136,21 @@ export interface StorageAdapter { /** * The type of storage being used (e.g., 'filesystem', 'opfs', 'memory') */ - type: string; + type: string /** * The amount of storage being used in bytes */ - used: number; + used: number /** * The total amount of storage available in bytes, or null if unknown */ - quota: number | null; + quota: number | null /** * Additional storage-specific information */ - details?: Record; - }>; + details?: Record + }> } diff --git a/src/types/fileSystemTypes.ts b/src/types/fileSystemTypes.ts index 59ff80cf..3b488d83 100644 --- a/src/types/fileSystemTypes.ts +++ b/src/types/fileSystemTypes.ts @@ -6,6 +6,8 @@ // Extend the FileSystemDirectoryHandle interface interface FileSystemDirectoryHandle { [Symbol.asyncIterator](): AsyncIterableIterator<[string, FileSystemHandle]>; + keys(): AsyncIterableIterator; + entries(): AsyncIterableIterator<[string, FileSystemHandle]>; } // Export something to make this a module diff --git a/src/types/graphTypes.ts b/src/types/graphTypes.ts index ff9afc0c..8bd46ac8 100644 --- a/src/types/graphTypes.ts +++ b/src/types/graphTypes.ts @@ -13,8 +13,8 @@ interface Timestamp { * Tracks which augmentation and model created the element */ interface CreatorMetadata { - augmentation: string // Name of the augmentation that created this element - version: string // Version of the augmentation + augmentation: string // Name of the augmentation that created this element + version: string // Version of the augmentation } /** @@ -22,15 +22,15 @@ interface CreatorMetadata { * Represents entities like people, places, things, etc. */ export interface GraphNoun { - id: string // Unique identifier for the noun - createdBy: CreatorMetadata // Information about what created this noun - noun: NounType // Type classification of the noun - createdAt: Timestamp // When the noun was created - updatedAt: Timestamp // When the noun was last updated - label?: string // Optional descriptive label - data?: Record // Additional flexible data storage - embeddedVerbs?: EmbeddedGraphVerb[] // Optional embedded relationships - embedding?: number[] // Vector representation of the noun + id: string // Unique identifier for the noun + createdBy: CreatorMetadata // Information about what created this noun + noun: NounType // Type classification of the noun + createdAt: Timestamp // When the noun was created + updatedAt: Timestamp // When the noun was last updated + label?: string // Optional descriptive label + data?: Record // Additional flexible data storage + embeddedVerbs?: EmbeddedGraphVerb[] // Optional embedded relationships + embedding?: number[] // Vector representation of the noun } /** @@ -38,17 +38,17 @@ export interface GraphNoun { * Represents relationships between nouns */ export interface GraphVerb { - id: string // Unique identifier for the verb - source: string // ID of the source noun - target: string // ID of the target noun - label?: string // Optional descriptive label - verb: VerbType // Type of relationship - createdAt: Timestamp // When the verb was created - updatedAt: Timestamp // When the verb was last updated - data?: Record // Additional flexible data storage - embedding?: number[] // Vector representation of the relationship - confidence?: number // Confidence score (0-1) - weight?: number // Strength/importance of the relationship + id: string // Unique identifier for the verb + source: string // ID of the source noun + target: string // ID of the target noun + label?: string // Optional descriptive label + verb: VerbType // Type of relationship + createdAt: Timestamp // When the verb was created + updatedAt: Timestamp // When the verb was last updated + data?: Record // Additional flexible data storage + embedding?: number[] // Vector representation of the relationship + confidence?: number // Confidence score (0-1) + weight?: number // Strength/importance of the relationship } /** @@ -94,7 +94,6 @@ export interface Concept extends GraphNoun { noun: typeof NounType.Concept } - export interface Group extends GraphNoun { noun: typeof NounType.Group } @@ -116,14 +115,14 @@ export interface Content extends GraphNoun { */ export const NounType = { - Person: 'person', // Person entities - Place: 'place', // Physical locations - Thing: 'thing', // Physical or virtual objects - Event: 'event', // Events or occurrences - Concept: 'concept', // Abstract concepts or ideas + Person: 'person', // Person entities + Place: 'place', // Physical locations + Thing: 'thing', // Physical or virtual objects + Event: 'event', // Events or occurrences + Concept: 'concept', // Abstract concepts or ideas Content: 'content', // Content items - Group: 'group', // Groups of related entities - List: 'list', // Ordered collections of entities + Group: 'group', // Groups of related entities + List: 'list', // Ordered collections of entities Category: 'category' // Categories for content items including tags } as const export type NounType = (typeof NounType)[keyof typeof NounType] @@ -133,17 +132,17 @@ export type NounType = (typeof NounType)[keyof typeof NounType] * Used for categorizing different types of connections */ export const VerbType = { - AttributedTo: 'attributedTo', // Indicates attribution or authorship - Controls: 'controls', // Indicates control or ownership - Created: 'created', // Indicates creation or authorship - Earned: 'earned', // Indicates achievement or acquisition - Owns: 'owns', // Indicates ownership - MemberOf: 'memberOf', // Indicates membership or affiliation - RelatedTo: 'relatedTo', // Indicates family relationship - WorksWith: 'worksWith', // Indicates professional relationship - FriendOf: 'friendOf', // Indicates friendship - ReportsTo: 'reportsTo', // Indicates reporting relationship - Supervises: 'supervises', // Indicates supervisory relationship - Mentors: 'mentors' // Indicates mentorship relationship + AttributedTo: 'attributedTo', // Indicates attribution or authorship + Controls: 'controls', // Indicates control or ownership + Created: 'created', // Indicates creation or authorship + Earned: 'earned', // Indicates achievement or acquisition + Owns: 'owns', // Indicates ownership + MemberOf: 'memberOf', // Indicates membership or affiliation + RelatedTo: 'relatedTo', // Indicates family relationship + WorksWith: 'worksWith', // Indicates professional relationship + FriendOf: 'friendOf', // Indicates friendship + ReportsTo: 'reportsTo', // Indicates reporting relationship + Supervises: 'supervises', // Indicates supervisory relationship + Mentors: 'mentors' // Indicates mentorship relationship } as const export type VerbType = (typeof VerbType)[keyof typeof VerbType]