soulcraft/brainy
2
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

**feat: introduce Optimized HNSW Index with product quantization and disk-based storage support**

Browse source 
### Changes:
- Added `HNSWIndexOptimized` class in `src/hnsw/hnswIndexOptimized.ts`:
  - Supports large datasets with product quantization for dimensionality reduction.
  - Enables disk-based index storage via a configurable storage adapter.
  - Introduces `HNSWOptimizedConfig` interface for advanced configuration:
    - Includes memory thresholds, product quantization (subvectors, centroids), and disk-based indexing options.
  - Implements methods for adding, searching, and managing indexed vectors with enhanced scalability.
- Introduced `ProductQuantizer` utility:
  - Handles vector quantization with training and reconstruction capabilities.
  - Reduces vector dimensions, improving memory and performance efficiency.
- Added accessor methods in `src/hnsw/hnswIndex.ts` for configuration and index metadata (e.g., dimension, max level, entry point).

### Purpose:
Enhanced the HNSW implementation to support optimized indexing and search for large datasets, addressing scalability challenges via product quantization and optional disk-based storage. This update elevates the library's performance and applicability for memory-constrained environments.
This commit is contained in:
David Snelling 2025-06-19 15:02:05 -07:00
parent 57880cfd06
commit 6f6f787bf1
3 changed files with 89 additions and 88 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/types/fileSystemTypes.ts
View file

@ -6,6 +6,8 @@
// Extend the FileSystemDirectoryHandle interface
interface FileSystemDirectoryHandle {
[Symbol.asyncIterator](): AsyncIterableIterator<[string, FileSystemHandle]>;
keys(): AsyncIterableIterator<string>;
entries(): AsyncIterableIterator<[string, FileSystemHandle]>;
}
// Export something to make this a module

83
src/types/graphTypes.ts
View file

@ -13,8 +13,8 @@ interface Timestamp {
* Tracks which augmentation and model created the element
*/
interface CreatorMetadata {
augmentation: string // Name of the augmentation that created this element
version: string // Version of the augmentation
augmentation: string // Name of the augmentation that created this element
version: string // Version of the augmentation
}
/**
@ -22,15 +22,15 @@ interface CreatorMetadata {
* Represents entities like people, places, things, etc.
*/
export interface GraphNoun {
id: string // Unique identifier for the noun
createdBy: CreatorMetadata // Information about what created this noun
noun: NounType // Type classification of the noun
createdAt: Timestamp // When the noun was created
updatedAt: Timestamp // When the noun was last updated
label?: string // Optional descriptive label
data?: Record<string, unknown> // Additional flexible data storage
embeddedVerbs?: EmbeddedGraphVerb[] // Optional embedded relationships
embedding?: number[] // Vector representation of the noun
id: string // Unique identifier for the noun
createdBy: CreatorMetadata // Information about what created this noun
noun: NounType // Type classification of the noun
createdAt: Timestamp // When the noun was created
updatedAt: Timestamp // When the noun was last updated
label?: string // Optional descriptive label
data?: Record<string, any> // Additional flexible data storage
embeddedVerbs?: EmbeddedGraphVerb[] // Optional embedded relationships
embedding?: number[] // Vector representation of the noun
}
/**
@ -38,17 +38,17 @@ export interface GraphNoun {
* Represents relationships between nouns
*/
export interface GraphVerb {
id: string // Unique identifier for the verb
source: string // ID of the source noun
target: string // ID of the target noun
label?: string // Optional descriptive label
verb: VerbType // Type of relationship
createdAt: Timestamp // When the verb was created
updatedAt: Timestamp // When the verb was last updated
data?: Record<string, unknown> // Additional flexible data storage
embedding?: number[] // Vector representation of the relationship
confidence?: number // Confidence score (0-1)
weight?: number // Strength/importance of the relationship
id: string // Unique identifier for the verb
source: string // ID of the source noun
target: string // ID of the target noun
label?: string // Optional descriptive label
verb: VerbType // Type of relationship
createdAt: Timestamp // When the verb was created
updatedAt: Timestamp // When the verb was last updated
data?: Record<string, any> // Additional flexible data storage
embedding?: number[] // Vector representation of the relationship
confidence?: number // Confidence score (0-1)
weight?: number // Strength/importance of the relationship
}
/**
@ -94,7 +94,6 @@ export interface Concept extends GraphNoun {
noun: typeof NounType.Concept
}
export interface Group extends GraphNoun {
noun: typeof NounType.Group
}
@ -116,14 +115,14 @@ export interface Content extends GraphNoun {
*/
export const NounType = {
Person: 'person', // Person entities
Place: 'place', // Physical locations
Thing: 'thing', // Physical or virtual objects
Event: 'event', // Events or occurrences
Concept: 'concept', // Abstract concepts or ideas
Person: 'person', // Person entities
Place: 'place', // Physical locations
Thing: 'thing', // Physical or virtual objects
Event: 'event', // Events or occurrences
Concept: 'concept', // Abstract concepts or ideas
Content: 'content', // Content items
Group: 'group', // Groups of related entities
List: 'list', // Ordered collections of entities
Group: 'group', // Groups of related entities
List: 'list', // Ordered collections of entities
Category: 'category' // Categories for content items including tags
} as const
export type NounType = (typeof NounType)[keyof typeof NounType]
@ -133,17 +132,17 @@ export type NounType = (typeof NounType)[keyof typeof NounType]
* Used for categorizing different types of connections
*/
export const VerbType = {
AttributedTo: 'attributedTo', // Indicates attribution or authorship
Controls: 'controls', // Indicates control or ownership
Created: 'created', // Indicates creation or authorship
Earned: 'earned', // Indicates achievement or acquisition
Owns: 'owns', // Indicates ownership
MemberOf: 'memberOf', // Indicates membership or affiliation
RelatedTo: 'relatedTo', // Indicates family relationship
WorksWith: 'worksWith', // Indicates professional relationship
FriendOf: 'friendOf', // Indicates friendship
ReportsTo: 'reportsTo', // Indicates reporting relationship
Supervises: 'supervises', // Indicates supervisory relationship
Mentors: 'mentors' // Indicates mentorship relationship
AttributedTo: 'attributedTo', // Indicates attribution or authorship
Controls: 'controls', // Indicates control or ownership
Created: 'created', // Indicates creation or authorship
Earned: 'earned', // Indicates achievement or acquisition
Owns: 'owns', // Indicates ownership
MemberOf: 'memberOf', // Indicates membership or affiliation
RelatedTo: 'relatedTo', // Indicates family relationship
WorksWith: 'worksWith', // Indicates professional relationship
FriendOf: 'friendOf', // Indicates friendship
ReportsTo: 'reportsTo', // Indicates reporting relationship
Supervises: 'supervises', // Indicates supervisory relationship
Mentors: 'mentors' // Indicates mentorship relationship
} as const
export type VerbType = (typeof VerbType)[keyof typeof VerbType]