**feat: introduce Optimized HNSW Index with product quantization and disk-based storage support**
### Changes:
- Added `HNSWIndexOptimized` class in `src/hnsw/hnswIndexOptimized.ts`:
- Supports large datasets with product quantization for dimensionality reduction.
- Enables disk-based index storage via a configurable storage adapter.
- Introduces `HNSWOptimizedConfig` interface for advanced configuration:
- Includes memory thresholds, product quantization (subvectors, centroids), and disk-based indexing options.
- Implements methods for adding, searching, and managing indexed vectors with enhanced scalability.
- Introduced `ProductQuantizer` utility:
- Handles vector quantization with training and reconstruction capabilities.
- Reduces vector dimensions, improving memory and performance efficiency.
- Added accessor methods in `src/hnsw/hnswIndex.ts` for configuration and index metadata (e.g., dimension, max level, entry point).
### Purpose:
Enhanced the HNSW implementation to support optimized indexing and search for large datasets, addressing scalability challenges via product quantization and optional disk-based storage. This update elevates the library's performance and applicability for memory-constrained environments.
This commit is contained in:
parent
57880cfd06
commit
6f6f787bf1
3 changed files with 89 additions and 88 deletions
|
|
@ -5,36 +5,36 @@
|
|||
/**
|
||||
* Vector representation - an array of numbers
|
||||
*/
|
||||
export type Vector = number[];
|
||||
export type Vector = number[]
|
||||
|
||||
/**
|
||||
* A document with a vector embedding and optional metadata
|
||||
*/
|
||||
export interface VectorDocument<T = any> {
|
||||
id: string;
|
||||
vector: Vector;
|
||||
metadata?: T;
|
||||
id: string
|
||||
vector: Vector
|
||||
metadata?: T
|
||||
}
|
||||
|
||||
/**
|
||||
* Search result with similarity score
|
||||
*/
|
||||
export interface SearchResult<T = any> {
|
||||
id: string;
|
||||
score: number;
|
||||
vector: Vector;
|
||||
metadata?: T;
|
||||
id: string
|
||||
score: number
|
||||
vector: Vector
|
||||
metadata?: T
|
||||
}
|
||||
|
||||
/**
|
||||
* Distance function for comparing vectors
|
||||
*/
|
||||
export type DistanceFunction = (a: Vector, b: Vector) => number;
|
||||
export type DistanceFunction = (a: Vector, b: Vector) => number
|
||||
|
||||
/**
|
||||
* Embedding function for converting data to vectors
|
||||
*/
|
||||
export type EmbeddingFunction = (data: any) => Promise<Vector>;
|
||||
export type EmbeddingFunction = (data: any) => Promise<Vector>
|
||||
|
||||
/**
|
||||
* Embedding model interface
|
||||
|
|
@ -43,26 +43,26 @@ export interface EmbeddingModel {
|
|||
/**
|
||||
* Initialize the embedding model
|
||||
*/
|
||||
init(): Promise<void>;
|
||||
init(): Promise<void>
|
||||
|
||||
/**
|
||||
* Embed data into a vector
|
||||
*/
|
||||
embed(data: any): Promise<Vector>;
|
||||
embed(data: any): Promise<Vector>
|
||||
|
||||
/**
|
||||
* Dispose of the model resources
|
||||
*/
|
||||
dispose(): Promise<void>;
|
||||
dispose(): Promise<void>
|
||||
}
|
||||
|
||||
/**
|
||||
* HNSW graph noun
|
||||
*/
|
||||
export interface HNSWNoun {
|
||||
id: string;
|
||||
vector: Vector;
|
||||
connections: Map<number, Set<string>>; // level -> set of connected noun ids
|
||||
id: string
|
||||
vector: Vector
|
||||
connections: Map<number, Set<string>> // level -> set of connected noun ids
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -70,63 +70,63 @@ export interface HNSWNoun {
|
|||
* Extends HNSWNoun to allow verbs to be first-class entities in the data model
|
||||
*/
|
||||
export interface GraphVerb extends HNSWNoun {
|
||||
sourceId: string; // ID of the source noun
|
||||
targetId: string; // ID of the target noun
|
||||
type?: string; // Optional type of the relationship
|
||||
weight?: number; // Optional weight of the relationship
|
||||
metadata?: any; // Optional metadata for the verb
|
||||
sourceId: string // ID of the source noun
|
||||
targetId: string // ID of the target noun
|
||||
type?: string // Optional type of the relationship
|
||||
weight?: number // Optional weight of the relationship
|
||||
metadata?: any // Optional metadata for the verb
|
||||
}
|
||||
|
||||
/**
|
||||
* HNSW index configuration
|
||||
*/
|
||||
export interface HNSWConfig {
|
||||
M: number; // Maximum number of connections per noun
|
||||
efConstruction: number; // Size of the dynamic candidate list during construction
|
||||
efSearch: number; // Size of the dynamic candidate list during search
|
||||
ml: number; // Maximum level
|
||||
M: number // Maximum number of connections per noun
|
||||
efConstruction: number // Size of the dynamic candidate list during construction
|
||||
efSearch: number // Size of the dynamic candidate list during search
|
||||
ml: number // Maximum level
|
||||
}
|
||||
|
||||
/**
|
||||
* Storage interface for persistence
|
||||
*/
|
||||
export interface StorageAdapter {
|
||||
init(): Promise<void>;
|
||||
init(): Promise<void>
|
||||
|
||||
saveNoun(noun: HNSWNoun): Promise<void>;
|
||||
saveNoun(noun: HNSWNoun): Promise<void>
|
||||
|
||||
getNoun(id: string): Promise<HNSWNoun | null>;
|
||||
getNoun(id: string): Promise<HNSWNoun | null>
|
||||
|
||||
getAllNouns(): Promise<HNSWNoun[]>;
|
||||
getAllNouns(): Promise<HNSWNoun[]>
|
||||
|
||||
/**
|
||||
* Get nouns by noun type
|
||||
* @param nounType The noun type to filter by
|
||||
* @returns Promise that resolves to an array of nouns of the specified noun type
|
||||
*/
|
||||
getNounsByNounType(nounType: string): Promise<HNSWNoun[]>;
|
||||
getNounsByNounType(nounType: string): Promise<HNSWNoun[]>
|
||||
|
||||
deleteNoun(id: string): Promise<void>;
|
||||
deleteNoun(id: string): Promise<void>
|
||||
|
||||
saveVerb(verb: GraphVerb): Promise<void>;
|
||||
saveVerb(verb: GraphVerb): Promise<void>
|
||||
|
||||
getVerb(id: string): Promise<GraphVerb | null>;
|
||||
getVerb(id: string): Promise<GraphVerb | null>
|
||||
|
||||
getAllVerbs(): Promise<GraphVerb[]>;
|
||||
getAllVerbs(): Promise<GraphVerb[]>
|
||||
|
||||
getVerbsBySource(sourceId: string): Promise<GraphVerb[]>;
|
||||
getVerbsBySource(sourceId: string): Promise<GraphVerb[]>
|
||||
|
||||
getVerbsByTarget(targetId: string): Promise<GraphVerb[]>;
|
||||
getVerbsByTarget(targetId: string): Promise<GraphVerb[]>
|
||||
|
||||
getVerbsByType(type: string): Promise<GraphVerb[]>;
|
||||
getVerbsByType(type: string): Promise<GraphVerb[]>
|
||||
|
||||
deleteVerb(id: string): Promise<void>;
|
||||
deleteVerb(id: string): Promise<void>
|
||||
|
||||
saveMetadata(id: string, metadata: any): Promise<void>;
|
||||
saveMetadata(id: string, metadata: any): Promise<void>
|
||||
|
||||
getMetadata(id: string): Promise<any | null>;
|
||||
getMetadata(id: string): Promise<any | null>
|
||||
|
||||
clear(): Promise<void>;
|
||||
clear(): Promise<void>
|
||||
|
||||
/**
|
||||
* Get information about storage usage and capacity
|
||||
|
|
@ -136,21 +136,21 @@ export interface StorageAdapter {
|
|||
/**
|
||||
* The type of storage being used (e.g., 'filesystem', 'opfs', 'memory')
|
||||
*/
|
||||
type: string;
|
||||
type: string
|
||||
|
||||
/**
|
||||
* The amount of storage being used in bytes
|
||||
*/
|
||||
used: number;
|
||||
used: number
|
||||
|
||||
/**
|
||||
* The total amount of storage available in bytes, or null if unknown
|
||||
*/
|
||||
quota: number | null;
|
||||
quota: number | null
|
||||
|
||||
/**
|
||||
* Additional storage-specific information
|
||||
*/
|
||||
details?: Record<string, any>;
|
||||
}>;
|
||||
details?: Record<string, any>
|
||||
}>
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,6 +6,8 @@
|
|||
// Extend the FileSystemDirectoryHandle interface
|
||||
interface FileSystemDirectoryHandle {
|
||||
[Symbol.asyncIterator](): AsyncIterableIterator<[string, FileSystemHandle]>;
|
||||
keys(): AsyncIterableIterator<string>;
|
||||
entries(): AsyncIterableIterator<[string, FileSystemHandle]>;
|
||||
}
|
||||
|
||||
// Export something to make this a module
|
||||
|
|
|
|||
|
|
@ -13,8 +13,8 @@ interface Timestamp {
|
|||
* Tracks which augmentation and model created the element
|
||||
*/
|
||||
interface CreatorMetadata {
|
||||
augmentation: string // Name of the augmentation that created this element
|
||||
version: string // Version of the augmentation
|
||||
augmentation: string // Name of the augmentation that created this element
|
||||
version: string // Version of the augmentation
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -22,15 +22,15 @@ interface CreatorMetadata {
|
|||
* Represents entities like people, places, things, etc.
|
||||
*/
|
||||
export interface GraphNoun {
|
||||
id: string // Unique identifier for the noun
|
||||
createdBy: CreatorMetadata // Information about what created this noun
|
||||
noun: NounType // Type classification of the noun
|
||||
createdAt: Timestamp // When the noun was created
|
||||
updatedAt: Timestamp // When the noun was last updated
|
||||
label?: string // Optional descriptive label
|
||||
data?: Record<string, unknown> // Additional flexible data storage
|
||||
embeddedVerbs?: EmbeddedGraphVerb[] // Optional embedded relationships
|
||||
embedding?: number[] // Vector representation of the noun
|
||||
id: string // Unique identifier for the noun
|
||||
createdBy: CreatorMetadata // Information about what created this noun
|
||||
noun: NounType // Type classification of the noun
|
||||
createdAt: Timestamp // When the noun was created
|
||||
updatedAt: Timestamp // When the noun was last updated
|
||||
label?: string // Optional descriptive label
|
||||
data?: Record<string, any> // Additional flexible data storage
|
||||
embeddedVerbs?: EmbeddedGraphVerb[] // Optional embedded relationships
|
||||
embedding?: number[] // Vector representation of the noun
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -38,17 +38,17 @@ export interface GraphNoun {
|
|||
* Represents relationships between nouns
|
||||
*/
|
||||
export interface GraphVerb {
|
||||
id: string // Unique identifier for the verb
|
||||
source: string // ID of the source noun
|
||||
target: string // ID of the target noun
|
||||
label?: string // Optional descriptive label
|
||||
verb: VerbType // Type of relationship
|
||||
createdAt: Timestamp // When the verb was created
|
||||
updatedAt: Timestamp // When the verb was last updated
|
||||
data?: Record<string, unknown> // Additional flexible data storage
|
||||
embedding?: number[] // Vector representation of the relationship
|
||||
confidence?: number // Confidence score (0-1)
|
||||
weight?: number // Strength/importance of the relationship
|
||||
id: string // Unique identifier for the verb
|
||||
source: string // ID of the source noun
|
||||
target: string // ID of the target noun
|
||||
label?: string // Optional descriptive label
|
||||
verb: VerbType // Type of relationship
|
||||
createdAt: Timestamp // When the verb was created
|
||||
updatedAt: Timestamp // When the verb was last updated
|
||||
data?: Record<string, any> // Additional flexible data storage
|
||||
embedding?: number[] // Vector representation of the relationship
|
||||
confidence?: number // Confidence score (0-1)
|
||||
weight?: number // Strength/importance of the relationship
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -94,7 +94,6 @@ export interface Concept extends GraphNoun {
|
|||
noun: typeof NounType.Concept
|
||||
}
|
||||
|
||||
|
||||
export interface Group extends GraphNoun {
|
||||
noun: typeof NounType.Group
|
||||
}
|
||||
|
|
@ -116,14 +115,14 @@ export interface Content extends GraphNoun {
|
|||
*/
|
||||
|
||||
export const NounType = {
|
||||
Person: 'person', // Person entities
|
||||
Place: 'place', // Physical locations
|
||||
Thing: 'thing', // Physical or virtual objects
|
||||
Event: 'event', // Events or occurrences
|
||||
Concept: 'concept', // Abstract concepts or ideas
|
||||
Person: 'person', // Person entities
|
||||
Place: 'place', // Physical locations
|
||||
Thing: 'thing', // Physical or virtual objects
|
||||
Event: 'event', // Events or occurrences
|
||||
Concept: 'concept', // Abstract concepts or ideas
|
||||
Content: 'content', // Content items
|
||||
Group: 'group', // Groups of related entities
|
||||
List: 'list', // Ordered collections of entities
|
||||
Group: 'group', // Groups of related entities
|
||||
List: 'list', // Ordered collections of entities
|
||||
Category: 'category' // Categories for content items including tags
|
||||
} as const
|
||||
export type NounType = (typeof NounType)[keyof typeof NounType]
|
||||
|
|
@ -133,17 +132,17 @@ export type NounType = (typeof NounType)[keyof typeof NounType]
|
|||
* Used for categorizing different types of connections
|
||||
*/
|
||||
export const VerbType = {
|
||||
AttributedTo: 'attributedTo', // Indicates attribution or authorship
|
||||
Controls: 'controls', // Indicates control or ownership
|
||||
Created: 'created', // Indicates creation or authorship
|
||||
Earned: 'earned', // Indicates achievement or acquisition
|
||||
Owns: 'owns', // Indicates ownership
|
||||
MemberOf: 'memberOf', // Indicates membership or affiliation
|
||||
RelatedTo: 'relatedTo', // Indicates family relationship
|
||||
WorksWith: 'worksWith', // Indicates professional relationship
|
||||
FriendOf: 'friendOf', // Indicates friendship
|
||||
ReportsTo: 'reportsTo', // Indicates reporting relationship
|
||||
Supervises: 'supervises', // Indicates supervisory relationship
|
||||
Mentors: 'mentors' // Indicates mentorship relationship
|
||||
AttributedTo: 'attributedTo', // Indicates attribution or authorship
|
||||
Controls: 'controls', // Indicates control or ownership
|
||||
Created: 'created', // Indicates creation or authorship
|
||||
Earned: 'earned', // Indicates achievement or acquisition
|
||||
Owns: 'owns', // Indicates ownership
|
||||
MemberOf: 'memberOf', // Indicates membership or affiliation
|
||||
RelatedTo: 'relatedTo', // Indicates family relationship
|
||||
WorksWith: 'worksWith', // Indicates professional relationship
|
||||
FriendOf: 'friendOf', // Indicates friendship
|
||||
ReportsTo: 'reportsTo', // Indicates reporting relationship
|
||||
Supervises: 'supervises', // Indicates supervisory relationship
|
||||
Mentors: 'mentors' // Indicates mentorship relationship
|
||||
} as const
|
||||
export type VerbType = (typeof VerbType)[keyof typeof VerbType]
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue