fix: eliminate cloud storage write amplification and rate limiting
brain.add() was generating 26-40 immediate cloud writes per call, causing HTTP 429 rate limit errors and high latency on GCS/S3/R2/Azure. Three-layer fix: (1) deferred metadata writes with dirty-marking, (2) MetadataWriteBuffer for write coalescing, (3) retry/backoff on all cloud storage adapters.
This commit is contained in:
parent
23e1c56ae0
commit
92d9420a5c
10 changed files with 674 additions and 75 deletions
|
|
@ -6530,6 +6530,14 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
this.originalConsole = undefined
|
||||
}
|
||||
|
||||
// Drain the metadata write buffer if the storage adapter has one
|
||||
if (this.storage && 'metadataWriteBuffer' in this.storage) {
|
||||
const buffer = (this.storage as any).metadataWriteBuffer
|
||||
if (buffer && typeof buffer.destroy === 'function') {
|
||||
await buffer.destroy()
|
||||
}
|
||||
}
|
||||
|
||||
// Storage doesn't have close in current interface
|
||||
// We'll just mark as not initialized
|
||||
this.initialized = false
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue