feat: add distributed architecture with sharding and coordination
- Wire up distributed components (Coordinator, ShardManager, CacheSync) - Implement automatic sharding for S3 storage (256 shards) - Add read/write separation for operational modes - Zero-config automatic detection for distributed mode - Add mutex implementation for thread safety - Fix metadata filtering in find operations - Fix neural API vector similarity calculations - Improve batch operations performance - Add Bluesky distributed setup example BREAKING CHANGE: None - backward compatible
This commit is contained in:
parent
8aafc769a3
commit
ed64c266ec
30 changed files with 2084 additions and 439 deletions
|
|
@ -25,6 +25,9 @@ export class HNSWIndex {
|
|||
private nouns: Map<string, HNSWNoun> = new Map()
|
||||
private entryPointId: string | null = null
|
||||
private maxLevel = 0
|
||||
// Track high-level nodes for O(1) entry point selection
|
||||
private highLevelNodes = new Map<number, Set<string>>() // level -> node IDs
|
||||
private readonly MAX_TRACKED_LEVELS = 10 // Only track top levels for memory efficiency
|
||||
private config: HNSWConfig
|
||||
private distanceFunction: DistanceFunction
|
||||
private dimension: number | null = null
|
||||
|
|
@ -272,6 +275,15 @@ export class HNSWIndex {
|
|||
|
||||
// Add noun to the index
|
||||
this.nouns.set(id, noun)
|
||||
|
||||
// Track high-level nodes for O(1) entry point selection
|
||||
if (nounLevel >= 2 && nounLevel <= this.MAX_TRACKED_LEVELS) {
|
||||
if (!this.highLevelNodes.has(nounLevel)) {
|
||||
this.highLevelNodes.set(nounLevel, new Set())
|
||||
}
|
||||
this.highLevelNodes.get(nounLevel)!.add(id)
|
||||
}
|
||||
|
||||
return id
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue