feat: add storage-level batch operations to eliminate N+1 query patterns
Implements comprehensive batching infrastructure (brain.batchGet, storage.getNounMetadataBatch, storage.getVerbsBySourceBatch) with native cloud adapter APIs for GCS, S3, R2, and Azure. VFS operations now use parallel breadth-first traversal with batching, reducing directory reads from 22 sequential calls to 2-3 batched calls. Improves cloud storage performance by 90%+ (12.7s → <1s for 12 files). Fully compatible with type-aware storage, sharding, COW, fork(), and all indexes. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
parent
d624f39fce
commit
95cbab2e3f
10 changed files with 2162 additions and 81 deletions
|
|
@ -632,20 +632,40 @@ export class VirtualFileSystem implements IVirtualFileSystem {
|
|||
throw new VFSError(VFSErrorCode.ENOTDIR, `Not a directory: ${path}`, path, 'getTreeStructure')
|
||||
}
|
||||
|
||||
// Recursively gather all descendants
|
||||
// v5.12.0: Parallel breadth-first traversal for maximum cloud performance
|
||||
// OLD: Sequential depth-first → 12.7s for 12 files (22 sequential calls × 580ms)
|
||||
// NEW: Parallel breadth-first → <1s for 12 files (batched levels)
|
||||
const allEntities: VFSEntity[] = []
|
||||
const visited = new Set<string>()
|
||||
|
||||
const gatherDescendants = async (dirId: string) => {
|
||||
if (visited.has(dirId)) return // Prevent cycles
|
||||
visited.add(dirId)
|
||||
const gatherDescendants = async (rootId: string) => {
|
||||
visited.add(rootId) // Mark root as visited
|
||||
let currentLevel = [rootId]
|
||||
|
||||
const children = await this.pathResolver.getChildren(dirId)
|
||||
for (const child of children) {
|
||||
allEntities.push(child)
|
||||
if (child.metadata.vfsType === 'directory') {
|
||||
await gatherDescendants(child.id)
|
||||
while (currentLevel.length > 0) {
|
||||
// v5.12.0: Fetch all directories at this level IN PARALLEL
|
||||
// PathResolver.getChildren() uses brain.batchGet() internally - double win!
|
||||
const childrenArrays = await Promise.all(
|
||||
currentLevel.map(dirId => this.pathResolver.getChildren(dirId))
|
||||
)
|
||||
|
||||
const nextLevel: string[] = []
|
||||
|
||||
// Process all children from this level
|
||||
for (const children of childrenArrays) {
|
||||
for (const child of children) {
|
||||
allEntities.push(child)
|
||||
|
||||
// Queue subdirectories for next level (breadth-first)
|
||||
if (child.metadata.vfsType === 'directory' && !visited.has(child.id)) {
|
||||
visited.add(child.id)
|
||||
nextLevel.push(child.id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Move to next level
|
||||
currentLevel = nextLevel
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue