soulcraft/brainy
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

fix: resolve 50-100x slower add() on cloud storage (GCS/S3/R2/Azure)

Browse source 
Root cause: Storage type detection at setupIndex() relied on
this.config.storage.type which was never set after createStorage()
auto-detected the storage type. This caused cloud storage to use
'immediate' persistence mode instead of 'deferred', resulting in
20-30 GCS writes per add() operation (7-12 seconds instead of 50-200ms).

Fix: Added getStorageType() helper that detects storage type from
the storage instance class name (e.g., GcsStorage → 'gcs'), used as
fallback when config.storage.type is not explicitly set.

Also added:
- Performance regression tests (10 new tests)
- test:perf npm script for running performance tests

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
David Snelling 2026-01-06 16:02:13 -08:00
parent 493b840527
commit 65703dbe59
3 changed files with 246 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

34
src/brainy.ts
View file

@ -5276,6 +5276,31 @@ export class Brainy<T = any> implements BrainyInterface<T> {
return storage as BaseStorage
}
/**
* Detect storage type from the storage instance class name
*
* v7.1.1: Fixes storage type detection for HNSW persistence mode.
* Previously relied on this.config.storage.type which was often not set
* after storage creation, causing cloud storage to use 'immediate' mode
* and resulting in 50-100x slower add() operations.
*
* @returns Storage type string ('gcs', 's3', 'memory', etc.)
*/
private getStorageType(): string {
if (!this.storage) return 'memory'
const className = this.storage.constructor.name
if (className.includes('Gcs') || className.includes('GCS')) return 'gcs'
if (className.includes('S3')) return 's3'
if (className.includes('R2')) return 'r2'
if (className.includes('Azure')) return 'azure'
if (className.includes('OPFS')) return 'opfs'
if (className.includes('FileSystem')) return 'filesystem'
if (className.includes('Memory')) return 'memory'
return 'unknown'
}
/**
* Setup index
*
@ -5295,11 +5320,15 @@ export class Brainy<T = any> implements BrainyInterface<T> {
}
// v6.2.8: Determine persist mode (user config > smart default)
// v7.1.1: Fixed to use getStorageType() for reliable detection
let persistMode: 'immediate' | 'deferred' = this.config.hnswPersistMode || 'immediate'
// Smart default: Use deferred mode for cloud storage adapters
if (!this.config.hnswPersistMode) {
const storageType = this.config.storage?.type || 'auto'
// v7.1.1 FIX: Use instance-based detection as fallback
// Previously this.config.storage.type was often undefined after storage creation,
// causing cloud storage to incorrectly use 'immediate' mode (50-100x slower)
const storageType = this.config.storage?.type || this.getStorageType()
const cloudStorageTypes = ['gcs', 's3', 'r2', 'azure']
if (cloudStorageTypes.includes(storageType)) {
persistMode = 'deferred'
@ -5307,7 +5336,8 @@ export class Brainy<T = any> implements BrainyInterface<T> {
}
// Phase 2: Use TypeAwareHNSWIndex for billion-scale optimization
if (this.config.storage?.type !== 'memory') {
const detectedStorageType = this.config.storage?.type || this.getStorageType()
if (detectedStorageType !== 'memory') {
return new TypeAwareHNSWIndex(indexConfig, this.distance, {
storage: this.storage,
useParallelization: true,