soulcraft/brainy
2
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

fix: eliminate cloud storage write amplification and rate limiting

Browse source 
brain.add() was generating 26-40 immediate cloud writes per call, causing
HTTP 429 rate limit errors and high latency on GCS/S3/R2/Azure. Three-layer
fix: (1) deferred metadata writes with dirty-marking, (2) MetadataWriteBuffer
for write coalescing, (3) retry/backoff on all cloud storage adapters.
This commit is contained in:
David Snelling 2026-01-31 09:09:36 -08:00
parent 23e1c56ae0
commit 92d9420a5c
10 changed files with 674 additions and 75 deletions
Download patch file Download diff file Expand all files Collapse all files

54
src/storage/adapters/azureBlobStorage.ts
View file

@ -37,6 +37,7 @@ import {
import { BrainyError } from '../../errors/brainyError.js'
import { CacheManager } from '../cacheManager.js'
import { createModuleLogger, prodLog } from '../../utils/logger.js'
import { MetadataWriteBuffer } from '../../utils/metadataWriteBuffer.js'
import { getGlobalBackpressure } from '../../utils/adaptiveBackpressure.js'
import { getWriteBuffer, WriteBuffer } from '../../utils/writeBuffer.js'
import { getCoalescer, RequestCoalescer } from '../../utils/requestCoalescer.js'
@ -201,6 +202,12 @@ export class AzureBlobStorage extends BaseStorage {
this.verbCacheManager = new CacheManager<Edge>(options.cacheConfig)
// Write buffering always enabled - no env var check needed
// Initialize metadata write buffer for cloud rate limit protection
this.metadataWriteBuffer = new MetadataWriteBuffer(
(path, data) => this.writeObjectToPath(path, data),
{ maxBufferSize: 200, flushIntervalMs: 200, concurrencyLimit: 10 }
)
}
/**
@ -897,20 +904,45 @@ export class AzureBlobStorage extends BaseStorage {
// Lazy container validation for progressive init
await this.ensureValidatedForWrite()
try {
this.logger.trace(`Writing object to path: ${path}`)
const MAX_RETRIES = 5
let lastError: any
const blockBlobClient = this.containerClient!.getBlockBlobClient(path)
const content = JSON.stringify(data, null, 2)
await blockBlobClient.upload(content, content.length, {
blobHTTPHeaders: { blobContentType: 'application/json' }
})
for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
try {
this.logger.trace(`Writing object to path: ${path}`)
this.logger.trace(`Object written successfully to ${path}`)
} catch (error) {
this.logger.error(`Failed to write object to ${path}:`, error)
throw new Error(`Failed to write object to ${path}: ${error}`)
const blockBlobClient = this.containerClient!.getBlockBlobClient(path)
const content = JSON.stringify(data, null, 2)
await blockBlobClient.upload(content, content.length, {
blobHTTPHeaders: { blobContentType: 'application/json' }
})
this.logger.trace(`Object written successfully to ${path}`)
if (attempt > 0) {
this.clearThrottlingState()
}
return
} catch (error: any) {
lastError = error
if (this.isThrottlingError(error) && attempt < MAX_RETRIES) {
const baseDelay = Math.min(100 * Math.pow(2, attempt), 5000)
const jitter = baseDelay * 0.2 * (Math.random() - 0.5)
const delay = Math.round(baseDelay + jitter)
this.logger.warn(
`Throttled writing ${path} (attempt ${attempt + 1}/${MAX_RETRIES + 1}), retrying in ${delay}ms`
)
await this.handleThrottling(error)
await new Promise(resolve => setTimeout(resolve, delay))
continue
}
break
}
}
this.logger.error(`Failed to write object to ${path}:`, lastError)
throw new Error(`Failed to write object to ${path}: ${lastError}`)
}
/**

54
src/storage/adapters/gcsStorage.ts
View file

@ -34,6 +34,7 @@ import {
import { BrainyError } from '../../errors/brainyError.js'
import { CacheManager } from '../cacheManager.js'
import { createModuleLogger, prodLog } from '../../utils/logger.js'
import { MetadataWriteBuffer } from '../../utils/metadataWriteBuffer.js'
import { getGlobalSocketManager } from '../../utils/adaptiveSocketManager.js'
import { getGlobalBackpressure } from '../../utils/adaptiveBackpressure.js'
import { getWriteBuffer, WriteBuffer } from '../../utils/writeBuffer.js'
@ -240,6 +241,12 @@ export class GcsStorage extends BaseStorage {
this.verbCacheManager = new CacheManager<Edge>(options.cacheConfig)
// Write buffering always enabled - no env var check needed
// Initialize metadata write buffer for cloud rate limit protection
this.metadataWriteBuffer = new MetadataWriteBuffer(
(path, data) => this.writeObjectToPath(path, data),
{ maxBufferSize: 200, flushIntervalMs: 200, concurrencyLimit: 10 }
)
}
/**
@ -840,20 +847,45 @@ export class GcsStorage extends BaseStorage {
// Lazy bucket validation for progressive init
await this.ensureValidatedForWrite()
try {
this.logger.trace(`Writing object to path: ${path}`)
const MAX_RETRIES = 5
let lastError: any
const file = this.bucket!.file(path)
await file.save(JSON.stringify(data, null, 2), {
contentType: 'application/json',
resumable: false
})
for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
try {
this.logger.trace(`Writing object to path: ${path}`)
this.logger.trace(`Object written successfully to ${path}`)
} catch (error) {
this.logger.error(`Failed to write object to ${path}:`, error)
throw new Error(`Failed to write object to ${path}: ${error}`)
const file = this.bucket!.file(path)
await file.save(JSON.stringify(data, null, 2), {
contentType: 'application/json',
resumable: false
})
this.logger.trace(`Object written successfully to ${path}`)
if (attempt > 0) {
this.clearThrottlingState()
}
return
} catch (error: any) {
lastError = error
if (this.isThrottlingError(error) && attempt < MAX_RETRIES) {
const baseDelay = Math.min(100 * Math.pow(2, attempt), 5000)
const jitter = baseDelay * 0.2 * (Math.random() - 0.5)
const delay = Math.round(baseDelay + jitter)
this.logger.warn(
`Throttled writing ${path} (attempt ${attempt + 1}/${MAX_RETRIES + 1}), retrying in ${delay}ms`
)
await this.handleThrottling(error)
await new Promise(resolve => setTimeout(resolve, delay))
continue
}
break
}
}
this.logger.error(`Failed to write object to ${path}:`, lastError)
throw new Error(`Failed to write object to ${path}: ${lastError}`)
}
/**

62
src/storage/adapters/r2Storage.ts
View file

@ -37,6 +37,7 @@ import {
import { BrainyError } from '../../errors/brainyError.js'
import { CacheManager } from '../cacheManager.js'
import { createModuleLogger, prodLog } from '../../utils/logger.js'
import { MetadataWriteBuffer } from '../../utils/metadataWriteBuffer.js'
import { getGlobalSocketManager } from '../../utils/adaptiveSocketManager.js'
import { getGlobalBackpressure } from '../../utils/adaptiveBackpressure.js'
import { getWriteBuffer, WriteBuffer } from '../../utils/writeBuffer.js'
@ -169,6 +170,12 @@ export class R2Storage extends BaseStorage {
this.verbCacheManager = new CacheManager<Edge>(options.cacheConfig)
// Write buffering always enabled - no env var check needed
// Initialize metadata write buffer for cloud rate limit protection
this.metadataWriteBuffer = new MetadataWriteBuffer(
(path, data) => this.writeObjectToPath(path, data),
{ maxBufferSize: 200, flushIntervalMs: 200, concurrencyLimit: 10 }
)
}
/**
@ -611,24 +618,49 @@ export class R2Storage extends BaseStorage {
protected async writeObjectToPath(path: string, data: any): Promise<void> {
await this.ensureInitialized()
try {
this.logger.trace(`Writing object to path: ${path}`)
const MAX_RETRIES = 5
let lastError: any
const { PutObjectCommand } = await import('@aws-sdk/client-s3')
await this.s3Client!.send(
new PutObjectCommand({
Bucket: this.bucketName,
Key: path,
Body: JSON.stringify(data, null, 2),
ContentType: 'application/json'
})
)
for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
try {
this.logger.trace(`Writing object to path: ${path}`)
this.logger.trace(`Object written successfully to ${path}`)
} catch (error) {
this.logger.error(`Failed to write object to ${path}:`, error)
throw new Error(`Failed to write object to ${path}: ${error}`)
const { PutObjectCommand } = await import('@aws-sdk/client-s3')
await this.s3Client!.send(
new PutObjectCommand({
Bucket: this.bucketName,
Key: path,
Body: JSON.stringify(data, null, 2),
ContentType: 'application/json'
})
)
this.logger.trace(`Object written successfully to ${path}`)
if (attempt > 0) {
this.clearThrottlingState()
}
return
} catch (error: any) {
lastError = error
if (this.isThrottlingError(error) && attempt < MAX_RETRIES) {
const baseDelay = Math.min(100 * Math.pow(2, attempt), 5000)
const jitter = baseDelay * 0.2 * (Math.random() - 0.5)
const delay = Math.round(baseDelay + jitter)
this.logger.warn(
`Throttled writing ${path} (attempt ${attempt + 1}/${MAX_RETRIES + 1}), retrying in ${delay}ms`
)
await this.handleThrottling(error)
await new Promise(resolve => setTimeout(resolve, delay))
continue
}
break
}
}
this.logger.error(`Failed to write object to ${path}:`, lastError)
throw new Error(`Failed to write object to ${path}: ${lastError}`)
}
/**

90
src/storage/adapters/s3CompatibleStorage.ts
View file

@ -35,6 +35,7 @@ import {
import { BrainyError } from '../../errors/brainyError.js'
import { CacheManager } from '../cacheManager.js'
import { createModuleLogger, prodLog } from '../../utils/logger.js'
import { MetadataWriteBuffer } from '../../utils/metadataWriteBuffer.js'
import { getGlobalSocketManager } from '../../utils/adaptiveSocketManager.js'
import { getGlobalBackpressure } from '../../utils/adaptiveBackpressure.js'
import { getWriteBuffer, WriteBuffer } from '../../utils/writeBuffer.js'
@ -256,6 +257,12 @@ export class S3CompatibleStorage extends BaseStorage {
// Initialize cache managers
this.nounCacheManager = new CacheManager<HNSWNode>(options.cacheConfig)
this.verbCacheManager = new CacheManager<Edge>(options.cacheConfig)
// Initialize metadata write buffer for cloud rate limit protection
this.metadataWriteBuffer = new MetadataWriteBuffer(
(path, data) => this.writeObjectToPath(path, data),
{ maxBufferSize: 200, flushIntervalMs: 200, concurrencyLimit: 10 }
)
}
/**
@ -1801,43 +1808,68 @@ export class S3CompatibleStorage extends BaseStorage {
// Lazy bucket validation for progressive init
await this.ensureValidatedForWrite()
// Apply backpressure before starting operation
const requestId = await this.applyBackpressure()
const MAX_RETRIES = 5
let lastError: any
try {
const { PutObjectCommand } = await import('@aws-sdk/client-s3')
const body = JSON.stringify(data, null, 2)
for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
// Apply backpressure before each attempt
const requestId = await this.applyBackpressure()
this.logger.trace(`Writing object to path: ${path}`)
try {
const { PutObjectCommand } = await import('@aws-sdk/client-s3')
const body = JSON.stringify(data, null, 2)
await this.s3Client!.send(
new PutObjectCommand({
Bucket: this.bucketName,
Key: path,
Body: body,
ContentType: 'application/json'
this.logger.trace(`Writing object to path: ${path}`)
await this.s3Client!.send(
new PutObjectCommand({
Bucket: this.bucketName,
Key: path,
Body: body,
ContentType: 'application/json'
})
)
this.logger.debug(`Object written successfully to ${path}`)
// Log the change for efficient synchronization
await this.appendToChangeLog({
timestamp: Date.now(),
operation: 'add',
entityType: 'metadata',
entityId: path,
data: data
})
)
this.logger.debug(`Object written successfully to ${path}`)
// Release backpressure on success
this.releaseBackpressure(true, requestId)
if (attempt > 0) {
this.clearThrottlingState()
}
return
} catch (error: any) {
// Release backpressure on error
this.releaseBackpressure(false, requestId)
lastError = error
// Log the change for efficient synchronization
await this.appendToChangeLog({
timestamp: Date.now(),
operation: 'add',
entityType: 'metadata',
entityId: path,
data: data
})
if (this.isThrottlingError(error) && attempt < MAX_RETRIES) {
const baseDelay = Math.min(100 * Math.pow(2, attempt), 5000)
const jitter = baseDelay * 0.2 * (Math.random() - 0.5)
const delay = Math.round(baseDelay + jitter)
this.logger.warn(
`Throttled writing ${path} (attempt ${attempt + 1}/${MAX_RETRIES + 1}), retrying in ${delay}ms`
)
await this.handleThrottling(error)
await new Promise(resolve => setTimeout(resolve, delay))
continue
}
// Release backpressure on success
this.releaseBackpressure(true, requestId)
} catch (error) {
// Release backpressure on error
this.releaseBackpressure(false, requestId)
this.logger.error(`Failed to write object to ${path}:`, error)
throw new Error(`Failed to write object to ${path}: ${error}`)
break
}
}
this.logger.error(`Failed to write object to ${path}:`, lastError)
throw new Error(`Failed to write object to ${path}: ${lastError}`)
}
/**

14
src/storage/baseStorage.ts
View file

@ -30,6 +30,7 @@ import { BlobStorage, type COWStorageAdapter } from './cow/BlobStorage.js'
import { CommitLog } from './cow/CommitLog.js'
import { unwrapBinaryData, wrapBinaryData } from './cow/binaryDataCodec.js'
import { prodLog } from '../utils/logger.js'
import { MetadataWriteBuffer } from '../utils/metadataWriteBuffer.js'
/**
* Storage key analysis result
@ -217,6 +218,11 @@ export abstract class BaseStorage extends BaseStorageAdapter {
// Track if type counts have been rebuilt (prevent repeated rebuilds)
private typeCountsRebuilt = false
// Write buffer for cloud storage adapters — deduplicates rapid writes to the same path
// FileSystem adapter does NOT use this (local writes are already fast)
// Initialized by cloud adapters in their init() method
protected metadataWriteBuffer: MetadataWriteBuffer | null = null
/**
* Analyze a storage key to determine its routing and path
* @param id - The key to analyze (UUID or system key)
@ -585,8 +591,12 @@ export abstract class BaseStorage extends BaseStorageAdapter {
// This ensures readWithInheritance() returns data immediately, fixing "Source entity not found" bug
this.writeCache.set(branchPath, data)
// Write to storage (async)
await this.writeObjectToPath(branchPath, data)
// Use write buffer if available (cloud adapters), otherwise write directly (filesystem)
if (this.metadataWriteBuffer) {
await this.metadataWriteBuffer.write(branchPath, data)
} else {
await this.writeObjectToPath(branchPath, data)
}
// Cache is NOT cleared here anymore - persists until flush()
// This provides a safety net for immediate queries after batch writes