soulcraft/brainy
2
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

feat(partitioning): simplify partition strategies and enable auto-tuning of semantic clusters

Browse source 
- Removed unused partition strategies: 'random' and 'geographic'
- Defaulted to 'semantic' partitioning for improved performance
- Introduced auto-tuning for semantic clusters based on dataset size
- Enhanced configuration options for better adaptability
This commit is contained in:
David Snelling 2025-08-03 17:26:41 -07:00
parent 69ca8e1ced
commit fd3cfdda6b
2 changed files with 266 additions and 66 deletions
Download patch file Download diff file Expand all files Collapse all files

76
src/hnsw/partitionedHNSWIndex.ts
View file

@ -15,14 +15,9 @@ import { euclideanDistance } from '../utils/index.js'
export interface PartitionConfig {
maxNodesPerPartition: number
partitionStrategy: 'semantic' | 'random' | 'geographic' | 'hash'
semanticClusters?: number
geographicBounds?: {
minLat: number
maxLat: number
minLng: number
maxLng: number
}
partitionStrategy: 'semantic' | 'hash' // Simplified to focus on useful strategies
semanticClusters?: number // Auto-configured based on dataset size
autoTuneSemanticClusters?: boolean // Automatically adjust cluster count
}
export interface PartitionMetadata {
@ -57,8 +52,9 @@ export class PartitionedHNSWIndex {
) {
this.config = {
maxNodesPerPartition: 50000, // Optimal size for memory efficiency
partitionStrategy: 'hash',
semanticClusters: 10,
partitionStrategy: 'semantic', // Default to semantic for better performance
semanticClusters: 8, // Auto-tuned based on dataset
autoTuneSemanticClusters: true,
...partitionConfig
}
@ -111,8 +107,8 @@ export class PartitionedHNSWIndex {
const metadata = this.partitionMetadata.get(partitionId)!
metadata.nodeCount = partition.size()
// Update bounds for semantic/geographic strategies
if (this.config.partitionStrategy === 'semantic' || this.config.partitionStrategy === 'geographic') {
// Update bounds for semantic strategy
if (this.config.partitionStrategy === 'semantic') {
this.updatePartitionBounds(partitionId, item.vector)
}
@ -167,21 +163,19 @@ export class PartitionedHNSWIndex {
/**
* Select the appropriate partition for a new item
* Automatically chooses semantic partitioning when beneficial, falls back to hash
*/
private async selectPartition(item: VectorDocument): Promise<string> {
// Auto-tune semantic clusters based on current dataset size
if (this.config.autoTuneSemanticClusters && this.config.partitionStrategy === 'semantic') {
this.autoTuneSemanticClusters()
}
switch (this.config.partitionStrategy) {
case 'hash':
return this.hashPartition(item.id)
case 'semantic':
return await this.semanticPartition(item.vector)
case 'geographic':
return this.geographicPartition(item)
case 'random':
return this.randomPartition()
case 'hash':
default:
return this.hashPartition(item.id)
}
@ -234,30 +228,32 @@ export class PartitionedHNSWIndex {
}
/**
* Geographic partitioning (requires lat/lng in metadata)
* Auto-tune semantic clusters based on dataset size and performance
*/
private geographicPartition(item: VectorDocument): string {
// This would require geographic metadata in the item
// For now, fall back to hash partitioning
return this.hashPartition(item.id)
}
/**
* Random partitioning
*/
private randomPartition(): string {
const existingPartitions = Array.from(this.partitions.keys())
private autoTuneSemanticClusters(): void {
const totalNodes = this.size()
const currentPartitions = this.partitions.size
// Find partition with space
for (const partitionId of existingPartitions) {
const metadata = this.partitionMetadata.get(partitionId)
if (metadata && metadata.nodeCount < this.config.maxNodesPerPartition) {
return partitionId
// Optimal clusters based on dataset size
let optimalClusters = Math.max(4, Math.min(32, Math.floor(totalNodes / 10000)))
// Adjust based on current partition performance
if (currentPartitions > 0) {
const avgNodesPerPartition = totalNodes / currentPartitions
if (avgNodesPerPartition > this.config.maxNodesPerPartition * 0.8) {
// Partitions are getting full, increase clusters
optimalClusters = Math.min(32, this.config.semanticClusters! + 2)
} else if (avgNodesPerPartition < this.config.maxNodesPerPartition * 0.3 && currentPartitions > 4) {
// Partitions are underutilized, decrease clusters
optimalClusters = Math.max(4, this.config.semanticClusters! - 1)
}
}
// Create new partition
return `random_${this.nextPartitionId++}`
if (optimalClusters !== this.config.semanticClusters) {
console.log(`Auto-tuning semantic clusters: ${this.config.semanticClusters} → ${optimalClusters}`)
this.config.semanticClusters = optimalClusters
}
}
/**

256
src/hnsw/scaledHNSWSystem.ts
View file

@ -11,29 +11,34 @@ import { EnhancedCacheManager } from '../storage/enhancedCacheManager.js'
import { BatchS3Operations } from '../storage/adapters/batchS3Operations.js'
import { ReadOnlyOptimizations } from '../storage/readOnlyOptimizations.js'
import { euclideanDistance } from '../utils/index.js'
import { autoConfigureBrainy, AutoConfiguration } from '../utils/autoConfiguration.js'
export interface ScaledHNSWConfig {
// Dataset scale expectations
expectedDatasetSize: number
maxMemoryUsage: number // bytes
targetSearchLatency: number // ms
// Required: Basic dataset expectations (can be auto-detected if not provided)
expectedDatasetSize?: number // Auto-detected if not provided
maxMemoryUsage?: number // Auto-detected based on environment
targetSearchLatency?: number // Auto-configured based on environment
// Storage configuration
// Storage configuration (optional - auto-detects S3 availability)
s3Config?: {
bucketName: string
region: string
endpoint?: string
accessKeyId: string
secretAccessKey: string
accessKeyId?: string // Falls back to env vars
secretAccessKey?: string // Falls back to env vars
}
// Optimization strategies
// Auto-configuration options
autoConfigureEnvironment?: boolean // Default: true
learningEnabled?: boolean // Default: true - adapts to performance
// Manual overrides (optional - auto-configured if not provided)
enablePartitioning?: boolean
enableCompression?: boolean
enableDistributedSearch?: boolean
enablePredictiveCaching?: boolean
// Performance tuning
// Advanced manual tuning (optional)
partitionConfig?: Partial<PartitionConfig>
hnswConfig?: Partial<OptimizedHNSWConfig>
readOnlyMode?: boolean
@ -44,35 +49,52 @@ export interface ScaledHNSWConfig {
* Handles datasets from thousands to millions of vectors
*/
export class ScaledHNSWSystem {
private config: ScaledHNSWConfig
private config: ScaledHNSWConfig & {
expectedDatasetSize: number
maxMemoryUsage: number
targetSearchLatency: number
autoConfigureEnvironment: boolean
learningEnabled: boolean
enablePartitioning: boolean
enableCompression: boolean
enableDistributedSearch: boolean
enablePredictiveCaching: boolean
readOnlyMode: boolean
}
private autoConfig: AutoConfiguration
private partitionedIndex?: PartitionedHNSWIndex
private distributedSearch?: DistributedSearchSystem
private cacheManager?: EnhancedCacheManager<any>
private batchOperations?: BatchS3Operations
private readOnlyOptimizations?: ReadOnlyOptimizations
// Performance monitoring
// Performance monitoring and learning
private performanceMetrics = {
totalSearches: 0,
averageSearchTime: 0,
cacheHitRate: 0,
compressionRatio: 0,
memoryUsage: 0,
indexSize: 0
indexSize: 0,
lastLearningUpdate: Date.now()
}
constructor(config: ScaledHNSWConfig) {
constructor(config: ScaledHNSWConfig = {}) {
this.autoConfig = AutoConfiguration.getInstance()
// Set basic defaults - these will be overridden by auto-configuration
this.config = {
expectedDatasetSize: 100000,
maxMemoryUsage: 4 * 1024 * 1024 * 1024,
targetSearchLatency: 150,
autoConfigureEnvironment: true,
learningEnabled: true,
enablePartitioning: true,
enableCompression: true,
enableDistributedSearch: true,
enablePredictiveCaching: true,
readOnlyMode: false,
...config,
// Set defaults only if not provided
expectedDatasetSize: config.expectedDatasetSize || 100000,
maxMemoryUsage: config.maxMemoryUsage || (4 * 1024 * 1024 * 1024),
targetSearchLatency: config.targetSearchLatency || 100
...config
}
this.initializeOptimizedSystem()
@ -82,19 +104,48 @@ export class ScaledHNSWSystem {
* Initialize the optimized system based on configuration
*/
private async initializeOptimizedSystem(): Promise<void> {
console.log('Initializing Scaled HNSW System...')
console.log('Initializing Scaled HNSW System with auto-configuration...')
// Determine optimal configuration based on dataset size
// Auto-configure if enabled
if (this.config.autoConfigureEnvironment) {
const autoConfigResult = await this.autoConfig.detectAndConfigure({
expectedDataSize: this.config.expectedDatasetSize,
s3Available: !!this.config.s3Config,
memoryBudget: this.config.maxMemoryUsage
})
console.log(`Detected environment: ${autoConfigResult.environment}`)
console.log(`Available memory: ${(autoConfigResult.availableMemory / 1024 / 1024 / 1024).toFixed(1)}GB`)
console.log(`CPU cores: ${autoConfigResult.cpuCores}`)
// Override config with auto-detected values
this.config = {
...this.config,
expectedDatasetSize: autoConfigResult.recommendedConfig.expectedDatasetSize,
maxMemoryUsage: autoConfigResult.recommendedConfig.maxMemoryUsage,
targetSearchLatency: autoConfigResult.recommendedConfig.targetSearchLatency,
enablePartitioning: autoConfigResult.recommendedConfig.enablePartitioning,
enableCompression: autoConfigResult.recommendedConfig.enableCompression,
enableDistributedSearch: autoConfigResult.recommendedConfig.enableDistributedSearch,
enablePredictiveCaching: autoConfigResult.recommendedConfig.enablePredictiveCaching
}
}
// Determine optimal configuration
const optimizedConfig = this.calculateOptimalConfiguration()
// Initialize partitioned index
// Initialize partitioned index with semantic partitioning as default
if (this.config.enablePartitioning) {
this.partitionedIndex = new PartitionedHNSWIndex(
optimizedConfig.partitionConfig,
{
...optimizedConfig.partitionConfig,
partitionStrategy: 'semantic', // Always use semantic for better performance
autoTuneSemanticClusters: true // Enable auto-tuning
},
optimizedConfig.hnswConfig,
euclideanDistance
)
console.log('✓ Partitioned index initialized')
console.log('✓ Partitioned index initialized with semantic clustering')
}
// Initialize distributed search system
@ -355,9 +406,14 @@ export class ScaledHNSWSystem {
throw new Error('No search system available')
}
// Update performance metrics
// Update performance metrics and learn from performance
const searchTime = Date.now() - startTime
this.updateSearchMetrics(searchTime, results.length)
// Adaptive learning - adjust configuration based on performance
if (this.config.learningEnabled && this.shouldTriggerLearning()) {
await this.adaptivelyLearnFromPerformance()
}
return results
@ -511,6 +567,104 @@ System Status: ${this.getSystemStatus()}
}
}
/**
* Check if adaptive learning should be triggered
*/
private shouldTriggerLearning(): boolean {
const timeSinceLastLearning = Date.now() - this.performanceMetrics.lastLearningUpdate
const minLearningInterval = 30000 // 30 seconds
const minSearches = 20 // Minimum searches before learning
return timeSinceLastLearning > minLearningInterval &&
this.performanceMetrics.totalSearches > minSearches &&
this.performanceMetrics.totalSearches % 50 === 0 // Learn every 50 searches
}
/**
* Adaptively learn from performance and adjust configuration
*/
private async adaptivelyLearnFromPerformance(): Promise<void> {
try {
const currentMetrics = {
averageSearchTime: this.performanceMetrics.averageSearchTime,
memoryUsage: this.performanceMetrics.memoryUsage,
cacheHitRate: this.performanceMetrics.cacheHitRate,
errorRate: 0 // Could be tracked separately
}
const adjustments = await this.autoConfig.learnFromPerformance(currentMetrics)
if (Object.keys(adjustments).length > 0) {
console.log('🧠 Adaptive learning: Adjusting configuration based on performance')
// Apply learned adjustments
let configChanged = false
if (adjustments.enableDistributedSearch !== undefined &&
adjustments.enableDistributedSearch !== this.config.enableDistributedSearch) {
this.config.enableDistributedSearch = adjustments.enableDistributedSearch
configChanged = true
}
if (adjustments.enableCompression !== undefined &&
adjustments.enableCompression !== this.config.enableCompression) {
this.config.enableCompression = adjustments.enableCompression
configChanged = true
}
if (adjustments.enablePredictiveCaching !== undefined &&
adjustments.enablePredictiveCaching !== this.config.enablePredictiveCaching) {
this.config.enablePredictiveCaching = adjustments.enablePredictiveCaching
configChanged = true
}
// Apply partition adjustments
if (adjustments.maxNodesPerPartition &&
this.partitionedIndex &&
adjustments.maxNodesPerPartition !== this.partitionedIndex.getPartitionStats().averageNodesPerPartition) {
// This would require rebuilding the index in a real implementation
console.log(`Learning suggests partition size: ${adjustments.maxNodesPerPartition}`)
}
if (configChanged) {
console.log('✅ Configuration updated based on performance learning')
}
}
this.performanceMetrics.lastLearningUpdate = Date.now()
} catch (error) {
console.warn('Adaptive learning failed:', error)
}
}
/**
* Update dataset analysis for better auto-configuration
*/
public async updateDatasetAnalysis(vectorCount: number, vectorDimension?: number): Promise<void> {
if (this.config.autoConfigureEnvironment) {
const analysis = {
estimatedSize: vectorCount,
vectorDimension,
accessPatterns: this.inferAccessPatterns()
}
await this.autoConfig.adaptToDataset(analysis)
console.log(`📊 Dataset analysis updated: ${vectorCount} vectors${vectorDimension ? `, ${vectorDimension}D` : ''}`)
}
}
/**
* Infer access patterns from current metrics
*/
private inferAccessPatterns(): 'read-heavy' | 'write-heavy' | 'balanced' {
// Simple heuristic - in practice, this would track read/write ratios
if (this.performanceMetrics.totalSearches > 100) {
return 'read-heavy'
}
return 'balanced'
}
/**
* Cleanup system resources
*/
@ -519,12 +673,62 @@ System Status: ${this.getSystemStatus()}
this.cacheManager?.clear()
this.readOnlyOptimizations?.cleanup()
this.partitionedIndex?.clear()
this.autoConfig.resetCache()
console.log('Scaled HNSW System cleaned up')
}
}
// Export convenience factory function
export function createScaledHNSWSystem(config: ScaledHNSWConfig): ScaledHNSWSystem {
// Export convenience factory functions
/**
* Create a fully auto-configured Brainy system - minimal setup required!
* Just provide S3 config if you want persistence beyond the current session
*/
export function createAutoBrainy(s3Config?: {
bucketName: string
region?: string
accessKeyId?: string
secretAccessKey?: string
}): ScaledHNSWSystem {
return new ScaledHNSWSystem({
s3Config: s3Config ? {
bucketName: s3Config.bucketName,
region: s3Config.region || 'us-east-1',
accessKeyId: s3Config.accessKeyId,
secretAccessKey: s3Config.secretAccessKey
} : undefined,
autoConfigureEnvironment: true,
learningEnabled: true
})
}
/**
* Create a Brainy system optimized for specific scenarios
*/
export async function createQuickBrainy(
scenario: 'small' | 'medium' | 'large' | 'enterprise',
s3Config?: { bucketName: string; region?: string }
): Promise<ScaledHNSWSystem> {
const { getQuickSetup } = await import('../utils/autoConfiguration.js')
const quickConfig = await getQuickSetup(scenario)
return new ScaledHNSWSystem({
...quickConfig,
s3Config: s3Config && quickConfig.s3Required ? {
bucketName: s3Config.bucketName,
region: s3Config.region || 'us-east-1',
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY
} : undefined,
autoConfigureEnvironment: true,
learningEnabled: true
})
}
/**
* Legacy factory function - still works but consider using createAutoBrainy() instead
*/
export function createScaledHNSWSystem(config: ScaledHNSWConfig = {}): ScaledHNSWSystem {
return new ScaledHNSWSystem(config)
}