fix(emergency): drastically lower high-volume mode activation thresholds
EMERGENCY FIX for bluesky-package socket exhaustion despite v0.54.1 🚨 CRITICAL CHANGES: - Activation threshold: 100 → 1 pending operation (configurable) - Add socket utilization trigger: >10% usage activates buffering - Add environment variables: - BRAINY_BUFFER_THRESHOLD=1 (when to start buffering) - BRAINY_FORCE_BUFFERING=true (force enable) - Add comprehensive logging with emojis for visibility: - '🚨 HIGH-VOLUME MODE ACTIVATED 🚨' when buffering starts - '🚀 BATCH FLUSH: N items → 1 bulk S3 operation' for batch writes - '📈 BUFFER GROWTH: N items buffered' every 100 additions PROBLEM: v0.54.1 buffering not activating with 7,500 pending requests SOLUTION: Activate buffering at first sign of load (1+ pending operations) This should immediately activate buffering in bluesky-package production.
This commit is contained in:
parent
a0d736472c
commit
b989e72be4
5 changed files with 151 additions and 23 deletions
81
tests/emergency-high-volume-test.js
Normal file
81
tests/emergency-high-volume-test.js
Normal file
|
|
@ -0,0 +1,81 @@
|
|||
/**
|
||||
* Emergency test to verify high-volume buffering activates correctly
|
||||
* Simulates the exact bluesky-package scenario
|
||||
*/
|
||||
|
||||
const { BrainyData } = await import('../dist/index.js')
|
||||
|
||||
console.log('🧪 Testing high-volume buffering activation...')
|
||||
|
||||
// Create Brainy with memory storage for testing
|
||||
const brainy = new BrainyData({
|
||||
dimensions: 384,
|
||||
storage: {
|
||||
type: 'memory'
|
||||
}
|
||||
})
|
||||
|
||||
console.log('📊 Starting rapid fire operations (simulating bluesky firehose)...')
|
||||
|
||||
// Simulate the exact pattern that's failing
|
||||
let operationCount = 0
|
||||
const startTime = Date.now()
|
||||
|
||||
// Fire off 1000 operations without awaiting (like bluesky firehose)
|
||||
const promises = []
|
||||
for (let i = 0; i < 1000; i++) {
|
||||
const promise = brainy.add({
|
||||
id: `firehose-${i}`,
|
||||
data: {
|
||||
type: 'bluesky_post',
|
||||
content: `Message ${i} from firehose`,
|
||||
timestamp: Date.now()
|
||||
},
|
||||
metadata: {
|
||||
source: 'bluesky',
|
||||
batch: Math.floor(i / 100)
|
||||
}
|
||||
}).catch(error => {
|
||||
console.error(`Failed to add item ${i}:`, error.message)
|
||||
return null
|
||||
})
|
||||
|
||||
promises.push(promise)
|
||||
operationCount++
|
||||
|
||||
// Log every 100 operations
|
||||
if (i > 0 && i % 100 === 0) {
|
||||
console.log(`📈 Fired ${i} operations (not awaited)`)
|
||||
// Small delay to allow buffer detection
|
||||
await new Promise(resolve => setTimeout(resolve, 10))
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`🚀 All ${operationCount} operations fired! Now waiting for completion...`)
|
||||
|
||||
// Wait for all operations to complete
|
||||
const results = await Promise.all(promises)
|
||||
const successful = results.filter(r => r !== null).length
|
||||
const failed = results.filter(r => r === null).length
|
||||
|
||||
const duration = Date.now() - startTime
|
||||
console.log(`\n✅ Test completed in ${duration}ms`)
|
||||
console.log(` Successful: ${successful}/${operationCount}`)
|
||||
console.log(` Failed: ${failed}/${operationCount}`)
|
||||
|
||||
if (successful < operationCount * 0.9) {
|
||||
console.error('❌ Test failed - too many operations failed')
|
||||
process.exit(1)
|
||||
} else {
|
||||
console.log('✅ Test passed - high-volume operations completed successfully')
|
||||
}
|
||||
|
||||
// Check if we have data
|
||||
const searchResults = await brainy.search('bluesky', { k: 5 })
|
||||
console.log(`🔍 Search results: ${searchResults.length} items found`)
|
||||
|
||||
console.log('\n🎯 Key things to check in logs:')
|
||||
console.log(' 1. "🚨 HIGH-VOLUME MODE ACTIVATED 🚨" message')
|
||||
console.log(' 2. "📝 BUFFERING: Adding noun/verb to write buffer" messages')
|
||||
console.log(' 3. "🚀 BATCH FLUSH:" messages showing bulk operations')
|
||||
console.log(' 4. "📈 BUFFER GROWTH:" messages showing buffer accumulation')
|
||||
Loading…
Add table
Add a link
Reference in a new issue