soulcraft/brainy
2
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

fix: eliminate cloud storage write amplification and rate limiting

Browse source 
brain.add() was generating 26-40 immediate cloud writes per call, causing
HTTP 429 rate limit errors and high latency on GCS/S3/R2/Azure. Three-layer
fix: (1) deferred metadata writes with dirty-marking, (2) MetadataWriteBuffer
for write coalescing, (3) retry/backoff on all cloud storage adapters.
This commit is contained in:
David Snelling 2026-01-31 09:09:36 -08:00
parent 23e1c56ae0
commit 92d9420a5c
10 changed files with 674 additions and 75 deletions
Download patch file Download diff file Expand all files Collapse all files

14
src/storage/baseStorage.ts
View file

@ -30,6 +30,7 @@ import { BlobStorage, type COWStorageAdapter } from './cow/BlobStorage.js'
import { CommitLog } from './cow/CommitLog.js'
import { unwrapBinaryData, wrapBinaryData } from './cow/binaryDataCodec.js'
import { prodLog } from '../utils/logger.js'
import { MetadataWriteBuffer } from '../utils/metadataWriteBuffer.js'
/**
* Storage key analysis result
@ -217,6 +218,11 @@ export abstract class BaseStorage extends BaseStorageAdapter {
// Track if type counts have been rebuilt (prevent repeated rebuilds)
private typeCountsRebuilt = false
// Write buffer for cloud storage adapters — deduplicates rapid writes to the same path
// FileSystem adapter does NOT use this (local writes are already fast)
// Initialized by cloud adapters in their init() method
protected metadataWriteBuffer: MetadataWriteBuffer | null = null
/**
* Analyze a storage key to determine its routing and path
* @param id - The key to analyze (UUID or system key)
@ -585,8 +591,12 @@ export abstract class BaseStorage extends BaseStorageAdapter {
// This ensures readWithInheritance() returns data immediately, fixing "Source entity not found" bug
this.writeCache.set(branchPath, data)
// Write to storage (async)
await this.writeObjectToPath(branchPath, data)
// Use write buffer if available (cloud adapters), otherwise write directly (filesystem)
if (this.metadataWriteBuffer) {
await this.metadataWriteBuffer.write(branchPath, data)
} else {
await this.writeObjectToPath(branchPath, data)
}
// Cache is NOT cleared here anymore - persists until flush()
// This provides a safety net for immediate queries after batch writes