soulcraft/brainy
2
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

**feat: introduce Optimized HNSW Index with product quantization and disk-based storage support**

Browse source 
### Changes:
- Added `HNSWIndexOptimized` class in `src/hnsw/hnswIndexOptimized.ts`:
  - Supports large datasets with product quantization for dimensionality reduction.
  - Enables disk-based index storage via a configurable storage adapter.
  - Introduces `HNSWOptimizedConfig` interface for advanced configuration:
    - Includes memory thresholds, product quantization (subvectors, centroids), and disk-based indexing options.
  - Implements methods for adding, searching, and managing indexed vectors with enhanced scalability.
- Introduced `ProductQuantizer` utility:
  - Handles vector quantization with training and reconstruction capabilities.
  - Reduces vector dimensions, improving memory and performance efficiency.
- Added accessor methods in `src/hnsw/hnswIndex.ts` for configuration and index metadata (e.g., dimension, max level, entry point).

### Purpose:
Enhanced the HNSW implementation to support optimized indexing and search for large datasets, addressing scalability challenges via product quantization and optional disk-based storage. This update elevates the library's performance and applicability for memory-constrained environments.
This commit is contained in:
David Snelling 2025-06-19 15:02:05 -07:00
parent 57880cfd06
commit 6f6f787bf1
3 changed files with 89 additions and 88 deletions
Download patch file Download diff file Expand all files Collapse all files

92
src/coreTypes.ts
View file

@ -5,36 +5,36 @@
/** /**
* Vector representation - an array of numbers * Vector representation - an array of numbers
*/ */
export type Vector = number[]; export type Vector = number[]
/** /**
* A document with a vector embedding and optional metadata * A document with a vector embedding and optional metadata
*/ */
export interface VectorDocument<T = any> { export interface VectorDocument<T = any> {
id: string; id: string
vector: Vector; vector: Vector
metadata?: T; metadata?: T
} }
/** /**
* Search result with similarity score * Search result with similarity score
*/ */
export interface SearchResult<T = any> { export interface SearchResult<T = any> {
id: string; id: string
score: number; score: number
vector: Vector; vector: Vector
metadata?: T; metadata?: T
} }
/** /**
* Distance function for comparing vectors * Distance function for comparing vectors
*/ */
export type DistanceFunction = (a: Vector, b: Vector) => number; export type DistanceFunction = (a: Vector, b: Vector) => number
/** /**
* Embedding function for converting data to vectors * Embedding function for converting data to vectors
*/ */
export type EmbeddingFunction = (data: any) => Promise<Vector>; export type EmbeddingFunction = (data: any) => Promise<Vector>
/** /**
* Embedding model interface * Embedding model interface
@ -43,26 +43,26 @@ export interface EmbeddingModel {
/** /**
* Initialize the embedding model * Initialize the embedding model
*/ */
init(): Promise<void>; init(): Promise<void>
/** /**
* Embed data into a vector * Embed data into a vector
*/ */
embed(data: any): Promise<Vector>; embed(data: any): Promise<Vector>
/** /**
* Dispose of the model resources * Dispose of the model resources
*/ */
dispose(): Promise<void>; dispose(): Promise<void>
} }
/** /**
* HNSW graph noun * HNSW graph noun
*/ */
export interface HNSWNoun { export interface HNSWNoun {
id: string; id: string
vector: Vector; vector: Vector
connections: Map<number, Set<string>>; // level -> set of connected noun ids connections: Map<number, Set<string>> // level -> set of connected noun ids
} }
/** /**
@ -70,63 +70,63 @@ export interface HNSWNoun {
* Extends HNSWNoun to allow verbs to be first-class entities in the data model * Extends HNSWNoun to allow verbs to be first-class entities in the data model
*/ */
export interface GraphVerb extends HNSWNoun { export interface GraphVerb extends HNSWNoun {
sourceId: string; // ID of the source noun sourceId: string // ID of the source noun
targetId: string; // ID of the target noun targetId: string // ID of the target noun
type?: string; // Optional type of the relationship type?: string // Optional type of the relationship
weight?: number; // Optional weight of the relationship weight?: number // Optional weight of the relationship
metadata?: any; // Optional metadata for the verb metadata?: any // Optional metadata for the verb
} }
/** /**
* HNSW index configuration * HNSW index configuration
*/ */
export interface HNSWConfig { export interface HNSWConfig {
M: number; // Maximum number of connections per noun M: number // Maximum number of connections per noun
efConstruction: number; // Size of the dynamic candidate list during construction efConstruction: number // Size of the dynamic candidate list during construction
efSearch: number; // Size of the dynamic candidate list during search efSearch: number // Size of the dynamic candidate list during search
ml: number; // Maximum level ml: number // Maximum level
} }
/** /**
* Storage interface for persistence * Storage interface for persistence
*/ */
export interface StorageAdapter { export interface StorageAdapter {
init(): Promise<void>; init(): Promise<void>
saveNoun(noun: HNSWNoun): Promise<void>; saveNoun(noun: HNSWNoun): Promise<void>
getNoun(id: string): Promise<HNSWNoun | null>; getNoun(id: string): Promise<HNSWNoun | null>
getAllNouns(): Promise<HNSWNoun[]>; getAllNouns(): Promise<HNSWNoun[]>
/** /**
* Get nouns by noun type * Get nouns by noun type
* @param nounType The noun type to filter by * @param nounType The noun type to filter by
* @returns Promise that resolves to an array of nouns of the specified noun type * @returns Promise that resolves to an array of nouns of the specified noun type
*/ */
getNounsByNounType(nounType: string): Promise<HNSWNoun[]>; getNounsByNounType(nounType: string): Promise<HNSWNoun[]>
deleteNoun(id: string): Promise<void>; deleteNoun(id: string): Promise<void>
saveVerb(verb: GraphVerb): Promise<void>; saveVerb(verb: GraphVerb): Promise<void>
getVerb(id: string): Promise<GraphVerb | null>; getVerb(id: string): Promise<GraphVerb | null>
getAllVerbs(): Promise<GraphVerb[]>; getAllVerbs(): Promise<GraphVerb[]>
getVerbsBySource(sourceId: string): Promise<GraphVerb[]>; getVerbsBySource(sourceId: string): Promise<GraphVerb[]>
getVerbsByTarget(targetId: string): Promise<GraphVerb[]>; getVerbsByTarget(targetId: string): Promise<GraphVerb[]>
getVerbsByType(type: string): Promise<GraphVerb[]>; getVerbsByType(type: string): Promise<GraphVerb[]>
deleteVerb(id: string): Promise<void>; deleteVerb(id: string): Promise<void>
saveMetadata(id: string, metadata: any): Promise<void>; saveMetadata(id: string, metadata: any): Promise<void>
getMetadata(id: string): Promise<any | null>; getMetadata(id: string): Promise<any | null>
clear(): Promise<void>; clear(): Promise<void>
/** /**
* Get information about storage usage and capacity * Get information about storage usage and capacity
@ -136,21 +136,21 @@ export interface StorageAdapter {
/** /**
* The type of storage being used (e.g., 'filesystem', 'opfs', 'memory') * The type of storage being used (e.g., 'filesystem', 'opfs', 'memory')
*/ */
type: string; type: string
/** /**
* The amount of storage being used in bytes * The amount of storage being used in bytes
*/ */
used: number; used: number
/** /**
* The total amount of storage available in bytes, or null if unknown * The total amount of storage available in bytes, or null if unknown
*/ */
quota: number | null; quota: number | null
/** /**
* Additional storage-specific information * Additional storage-specific information
*/ */
details?: Record<string, any>; details?: Record<string, any>
}>; }>
} }

2
src/types/fileSystemTypes.ts
View file

@ -6,6 +6,8 @@
// Extend the FileSystemDirectoryHandle interface // Extend the FileSystemDirectoryHandle interface
interface FileSystemDirectoryHandle { interface FileSystemDirectoryHandle {
[Symbol.asyncIterator](): AsyncIterableIterator<[string, FileSystemHandle]>; [Symbol.asyncIterator](): AsyncIterableIterator<[string, FileSystemHandle]>;
keys(): AsyncIterableIterator<string>;
entries(): AsyncIterableIterator<[string, FileSystemHandle]>;
} }
// Export something to make this a module // Export something to make this a module

83
src/types/graphTypes.ts
View file

@ -13,8 +13,8 @@ interface Timestamp {
* Tracks which augmentation and model created the element * Tracks which augmentation and model created the element
*/ */
interface CreatorMetadata { interface CreatorMetadata {
augmentation: string // Name of the augmentation that created this element augmentation: string // Name of the augmentation that created this element
version: string // Version of the augmentation version: string // Version of the augmentation
} }
/** /**
@ -22,15 +22,15 @@ interface CreatorMetadata {
* Represents entities like people, places, things, etc. * Represents entities like people, places, things, etc.
*/ */
export interface GraphNoun { export interface GraphNoun {
id: string // Unique identifier for the noun id: string // Unique identifier for the noun
createdBy: CreatorMetadata // Information about what created this noun createdBy: CreatorMetadata // Information about what created this noun
noun: NounType // Type classification of the noun noun: NounType // Type classification of the noun
createdAt: Timestamp // When the noun was created createdAt: Timestamp // When the noun was created
updatedAt: Timestamp // When the noun was last updated updatedAt: Timestamp // When the noun was last updated
label?: string // Optional descriptive label label?: string // Optional descriptive label
data?: Record<string, unknown> // Additional flexible data storage data?: Record<string, any> // Additional flexible data storage
embeddedVerbs?: EmbeddedGraphVerb[] // Optional embedded relationships embeddedVerbs?: EmbeddedGraphVerb[] // Optional embedded relationships
embedding?: number[] // Vector representation of the noun embedding?: number[] // Vector representation of the noun
} }
/** /**
@ -38,17 +38,17 @@ export interface GraphNoun {
* Represents relationships between nouns * Represents relationships between nouns
*/ */
export interface GraphVerb { export interface GraphVerb {
id: string // Unique identifier for the verb id: string // Unique identifier for the verb
source: string // ID of the source noun source: string // ID of the source noun
target: string // ID of the target noun target: string // ID of the target noun
label?: string // Optional descriptive label label?: string // Optional descriptive label
verb: VerbType // Type of relationship verb: VerbType // Type of relationship
createdAt: Timestamp // When the verb was created createdAt: Timestamp // When the verb was created
updatedAt: Timestamp // When the verb was last updated updatedAt: Timestamp // When the verb was last updated
data?: Record<string, unknown> // Additional flexible data storage data?: Record<string, any> // Additional flexible data storage
embedding?: number[] // Vector representation of the relationship embedding?: number[] // Vector representation of the relationship
confidence?: number // Confidence score (0-1) confidence?: number // Confidence score (0-1)
weight?: number // Strength/importance of the relationship weight?: number // Strength/importance of the relationship
} }
/** /**
@ -94,7 +94,6 @@ export interface Concept extends GraphNoun {
noun: typeof NounType.Concept noun: typeof NounType.Concept
} }
export interface Group extends GraphNoun { export interface Group extends GraphNoun {
noun: typeof NounType.Group noun: typeof NounType.Group
} }
@ -116,14 +115,14 @@ export interface Content extends GraphNoun {
*/ */
export const NounType = { export const NounType = {
Person: 'person', // Person entities Person: 'person', // Person entities
Place: 'place', // Physical locations Place: 'place', // Physical locations
Thing: 'thing', // Physical or virtual objects Thing: 'thing', // Physical or virtual objects
Event: 'event', // Events or occurrences Event: 'event', // Events or occurrences
Concept: 'concept', // Abstract concepts or ideas Concept: 'concept', // Abstract concepts or ideas
Content: 'content', // Content items Content: 'content', // Content items
Group: 'group', // Groups of related entities Group: 'group', // Groups of related entities
List: 'list', // Ordered collections of entities List: 'list', // Ordered collections of entities
Category: 'category' // Categories for content items including tags Category: 'category' // Categories for content items including tags
} as const } as const
export type NounType = (typeof NounType)[keyof typeof NounType] export type NounType = (typeof NounType)[keyof typeof NounType]
@ -133,17 +132,17 @@ export type NounType = (typeof NounType)[keyof typeof NounType]
* Used for categorizing different types of connections * Used for categorizing different types of connections
*/ */
export const VerbType = { export const VerbType = {
AttributedTo: 'attributedTo', // Indicates attribution or authorship AttributedTo: 'attributedTo', // Indicates attribution or authorship
Controls: 'controls', // Indicates control or ownership Controls: 'controls', // Indicates control or ownership
Created: 'created', // Indicates creation or authorship Created: 'created', // Indicates creation or authorship
Earned: 'earned', // Indicates achievement or acquisition Earned: 'earned', // Indicates achievement or acquisition
Owns: 'owns', // Indicates ownership Owns: 'owns', // Indicates ownership
MemberOf: 'memberOf', // Indicates membership or affiliation MemberOf: 'memberOf', // Indicates membership or affiliation
RelatedTo: 'relatedTo', // Indicates family relationship RelatedTo: 'relatedTo', // Indicates family relationship
WorksWith: 'worksWith', // Indicates professional relationship WorksWith: 'worksWith', // Indicates professional relationship
FriendOf: 'friendOf', // Indicates friendship FriendOf: 'friendOf', // Indicates friendship
ReportsTo: 'reportsTo', // Indicates reporting relationship ReportsTo: 'reportsTo', // Indicates reporting relationship
Supervises: 'supervises', // Indicates supervisory relationship Supervises: 'supervises', // Indicates supervisory relationship
Mentors: 'mentors' // Indicates mentorship relationship Mentors: 'mentors' // Indicates mentorship relationship
} as const } as const
export type VerbType = (typeof VerbType)[keyof typeof VerbType] export type VerbType = (typeof VerbType)[keyof typeof VerbType]