soulcraft/brainy
2
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

feat: add automatic high-volume handling for S3 storage adapter

Browse source 
- Configure AWS SDK with 500 max sockets (up from default 50)
- Add intelligent backpressure with pending operation tracking
- Implement dynamic batch sizing based on memory pressure
- Auto-reduce operations when heap usage exceeds 80%
- Gradually recover throughput when system stabilizes
- Track and respond to consecutive error patterns
- Fix S3 mock to not add ID to metadata objects
- Add backpressure to metadata save operations
- All changes are transparent - no configuration required
This commit is contained in:
David Snelling 2025-08-07 06:09:00 -07:00
parent 7fa67b55ab
commit 34c517d878
4 changed files with 169 additions and 29 deletions
Download patch file Download diff file Expand all files Collapse all files

45
package-lock.json generated
View file

@ -11,6 +11,7 @@
"dependencies": {
"@aws-sdk/client-s3": "^3.540.0",
"@huggingface/transformers": "^3.1.0",
"@smithy/node-http-handler": "^4.1.1",
"buffer": "^6.0.3",
"dotenv": "^16.4.5",
"uuid": "^9.0.1"
@ -2926,12 +2927,12 @@
]
},
"node_modules/@smithy/abort-controller": {
"version": "4.0.4",
"resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-4.0.4.tgz",
"integrity": "sha512-gJnEjZMvigPDQWHrW3oPrFhQtkrgqBkyjj3pCIdF3A5M6vsZODG93KNlfJprv6bp4245bdT32fsHK4kkH3KYDA==",
"version": "4.0.5",
"resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-4.0.5.tgz",
"integrity": "sha512-jcrqdTQurIrBbUm4W2YdLVMQDoL0sA9DTxYd2s+R/y+2U9NLOP7Xf/YqfSg1FZhlZIYEnvk2mwbyvIfdLEPo8g==",
"license": "Apache-2.0",
"dependencies": {
"@smithy/types": "^4.3.1",
"@smithy/types": "^4.3.2",
"tslib": "^2.6.2"
},
"engines": {
@ -3280,15 +3281,15 @@
}
},
"node_modules/@smithy/node-http-handler": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.1.0.tgz",
"integrity": "sha512-vqfSiHz2v8b3TTTrdXi03vNz1KLYYS3bhHCDv36FYDqxT7jvTll1mMnCrkD+gOvgwybuunh/2VmvOMqwBegxEg==",
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.1.1.tgz",
"integrity": "sha512-RHnlHqFpoVdjSPPiYy/t40Zovf3BBHc2oemgD7VsVTFFZrU5erFFe0n52OANZZ/5sbshgD93sOh5r6I35Xmpaw==",
"license": "Apache-2.0",
"dependencies": {
"@smithy/abort-controller": "^4.0.4",
"@smithy/protocol-http": "^5.1.2",
"@smithy/querystring-builder": "^4.0.4",
"@smithy/types": "^4.3.1",
"@smithy/abort-controller": "^4.0.5",
"@smithy/protocol-http": "^5.1.3",
"@smithy/querystring-builder": "^4.0.5",
"@smithy/types": "^4.3.2",
"tslib": "^2.6.2"
},
"engines": {
@ -3309,12 +3310,12 @@
}
},
"node_modules/@smithy/protocol-http": {
"version": "5.1.2",
"resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-5.1.2.tgz",
"integrity": "sha512-rOG5cNLBXovxIrICSBm95dLqzfvxjEmuZx4KK3hWwPFHGdW3lxY0fZNXfv2zebfRO7sJZ5pKJYHScsqopeIWtQ==",
"version": "5.1.3",
"resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-5.1.3.tgz",
"integrity": "sha512-fCJd2ZR7D22XhDY0l+92pUag/7je2BztPRQ01gU5bMChcyI0rlly7QFibnYHzcxDvccMjlpM/Q1ev8ceRIb48w==",
"license": "Apache-2.0",
"dependencies": {
"@smithy/types": "^4.3.1",
"@smithy/types": "^4.3.2",
"tslib": "^2.6.2"
},
"engines": {
@ -3322,12 +3323,12 @@
}
},
"node_modules/@smithy/querystring-builder": {
"version": "4.0.4",
"resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-4.0.4.tgz",
"integrity": "sha512-SwREZcDnEYoh9tLNgMbpop+UTGq44Hl9tdj3rf+yeLcfH7+J8OXEBaMc2kDxtyRHu8BhSg9ADEx0gFHvpJgU8w==",
"version": "4.0.5",
"resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-4.0.5.tgz",
"integrity": "sha512-NJeSCU57piZ56c+/wY+AbAw6rxCCAOZLCIniRE7wqvndqxcKKDOXzwWjrY7wGKEISfhL9gBbAaWWgHsUGedk+A==",
"license": "Apache-2.0",
"dependencies": {
"@smithy/types": "^4.3.1",
"@smithy/types": "^4.3.2",
"@smithy/util-uri-escape": "^4.0.0",
"tslib": "^2.6.2"
},
@ -3411,9 +3412,9 @@
}
},
"node_modules/@smithy/types": {
"version": "4.3.1",
"resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.3.1.tgz",
"integrity": "sha512-UqKOQBL2x6+HWl3P+3QqFD4ncKq0I8Nuz9QItGv5WuKuMHuuwlhvqcZCoXGfc+P1QmfJE7VieykoYYmrOoFJxA==",
"version": "4.3.2",
"resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.3.2.tgz",
"integrity": "sha512-QO4zghLxiQ5W9UZmX2Lo0nta2PuE1sSrXUYDoaB6HMR762C0P7v/HEPHf6ZdglTVssJG1bsrSBxdc3quvDSihw==",
"license": "Apache-2.0",
"dependencies": {
"tslib": "^2.6.2"