perf: optimize init() and rebuild performance

- Remove detectAndRepairCorruption from init() hot path — was loading all
  metadata chunks sequentially on startup. Now available via checkHealth()
  and repairIndex() methods.
- Short-circuit warmCache on empty workspace — skip 4-6 wasted storage reads.
- Parallelize MetadataIndex.init() and getGraphIndex() via Promise.all().
- Defer metadata writes during rebuild to batch boundaries (every 5000
  entities) instead of flushing per-entity.
- Skip pre-reads for new entities in transactions — saves 2 storage
  round-trips per add() on cloud storage.
This commit is contained in:
David Snelling 2026-01-31 09:14:51 -08:00
parent cd875294ad
commit 35cb674157
3 changed files with 113 additions and 40 deletions

View file

@ -270,15 +270,14 @@ export class Brainy<T = any> implements BrainyInterface<T> {
// Setup index now that we have storage
this.index = this.setupIndex()
// Initialize core metadata index
// Initialize metadata index and graph index in parallel
// Both only need storage — they are independent of each other
this.metadataIndex = new MetadataIndexManager(this.storage)
await this.metadataIndex.init()
// Get GraphAdjacencyIndex from storage (SINGLETON pattern)
// Storage owns the single instance, Brainy accesses it via getGraphIndex()
// This fixes the dual-ownership bug where Brainy and Storage had separate instances
// causing verbIdSet to be out of sync and VFS tree queries to fail
this.graphIndex = await (this.storage as any).getGraphIndex()
const [, graphIndex] = await Promise.all([
this.metadataIndex.init(),
(this.storage as any).getGraphIndex()
])
this.graphIndex = graphIndex
// Rebuild indexes if needed for existing data
await this.rebuildIndexesIfNeeded()
@ -688,18 +687,20 @@ export class Brainy<T = any> implements BrainyInterface<T> {
// All operations succeed or all rollback - prevents partial failures
await this.transactionManager.executeTransaction(async (tx) => {
// Operation 1: Save metadata FIRST (TypeAwareStorage caching)
// isNew=true: skip pre-read for rollback (entity doesn't exist yet)
tx.addOperation(
new SaveNounMetadataOperation(this.storage, id, storageMetadata)
new SaveNounMetadataOperation(this.storage, id, storageMetadata, true)
)
// Operation 2: Save vector data
// isNew=true: skip pre-read for rollback (entity doesn't exist yet)
tx.addOperation(
new SaveNounOperation(this.storage, {
id,
vector,
connections: new Map(),
level: 0
})
}, true)
)
// Operation 3: Add to HNSW index (after entity saved)
@ -6500,6 +6501,39 @@ export class Brainy<T = any> implements BrainyInterface<T> {
}
}
/**
* Check health of metadata indexes
*
* Returns validation result indicating whether indexes are healthy
* or corrupted (e.g., from the update() field asymmetry bug).
*
* This check was previously run on every init(), causing significant
* overhead on cloud storage (90+ sequential reads for 30-field datasets).
* Now available as an on-demand diagnostic method.
*/
async checkHealth(): Promise<{
healthy: boolean
avgEntriesPerEntity: number
entityCount: number
indexEntryCount: number
recommendation: string | null
}> {
await this.ensureInitialized()
return this.metadataIndex.validateConsistency()
}
/**
* Detect and repair corrupted metadata indexes
*
* Runs corruption detection and auto-rebuilds if corruption is found.
* This is the equivalent of the old init()-time corruption check,
* now available as an explicit operation.
*/
async repairIndex(): Promise<void> {
await this.ensureInitialized()
await this.metadataIndex.detectAndRepairCorruption()
}
/**
* Close and cleanup
*