feat: add distributed architecture with sharding and coordination

- Wire up distributed components (Coordinator, ShardManager, CacheSync)
- Implement automatic sharding for S3 storage (256 shards)
- Add read/write separation for operational modes
- Zero-config automatic detection for distributed mode
- Add mutex implementation for thread safety
- Fix metadata filtering in find operations
- Fix neural API vector similarity calculations
- Improve batch operations performance
- Add Bluesky distributed setup example

BREAKING CHANGE: None - backward compatible
This commit is contained in:
David Snelling 2025-09-22 15:45:35 -07:00
parent 8aafc769a3
commit ed64c266ec
30 changed files with 2084 additions and 439 deletions

View file

@ -25,6 +25,9 @@ export class HNSWIndex {
private nouns: Map<string, HNSWNoun> = new Map()
private entryPointId: string | null = null
private maxLevel = 0
// Track high-level nodes for O(1) entry point selection
private highLevelNodes = new Map<number, Set<string>>() // level -> node IDs
private readonly MAX_TRACKED_LEVELS = 10 // Only track top levels for memory efficiency
private config: HNSWConfig
private distanceFunction: DistanceFunction
private dimension: number | null = null
@ -272,6 +275,15 @@ export class HNSWIndex {
// Add noun to the index
this.nouns.set(id, noun)
// Track high-level nodes for O(1) entry point selection
if (nounLevel >= 2 && nounLevel <= this.MAX_TRACKED_LEVELS) {
if (!this.highLevelNodes.has(nounLevel)) {
this.highLevelNodes.set(nounLevel, new Set())
}
this.highLevelNodes.get(nounLevel)!.add(id)
}
return id
}