fix: implement unified UUID-based sharding for metadata across all storage adapters
Fixes critical scalability bottleneck where metadata was stored in non-sharded
directories, causing performance degradation at scale (1M+ entities).
Changes:
- Add UUID-based sharding to metadata operations in S3Compatible, FileSystem, and OpFS storage
- Implement complete UUID-based sharding for OpFS storage (nouns, verbs, metadata)
- Update pagination methods to iterate through all 256 UUID-based shards
- Add integration tests verifying sharding behavior across storage adapters
Impact:
- Metadata now scales to millions of entities without directory bottlenecks
- All storage adapters now use consistent UUID-based sharding (256 buckets: 00-ff)
- Improves GCS/S3/R2/OpFS performance at scale
- Path format: entities/{type}/{subtype}/{shard}/{id}.json
Breaking change: Requires data migration for existing S3/GCS/R2/OpFS deployments.
See .strategy/UNIFIED-UUID-SHARDING.md for migration guidance.
This commit is contained in:
parent
6d50fe4054
commit
2f3357132d
6 changed files with 1011 additions and 217 deletions
|
|
@ -627,7 +627,13 @@ export class FileSystemStorage extends BaseStorage {
|
|||
protected async saveNounMetadata_internal(id: string, metadata: any): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
const filePath = path.join(this.nounMetadataDir, `${id}.json`)
|
||||
// Use UUID-based sharding for metadata (consistent with noun vectors)
|
||||
const filePath = this.getShardedPath(this.nounMetadataDir, id)
|
||||
|
||||
// Ensure shard directory exists
|
||||
const shardDir = path.dirname(filePath)
|
||||
await fs.promises.mkdir(shardDir, { recursive: true })
|
||||
|
||||
await fs.promises.writeFile(filePath, JSON.stringify(metadata, null, 2))
|
||||
}
|
||||
|
||||
|
|
@ -637,7 +643,8 @@ export class FileSystemStorage extends BaseStorage {
|
|||
public async getNounMetadata(id: string): Promise<any | null> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
const filePath = path.join(this.nounMetadataDir, `${id}.json`)
|
||||
// Use UUID-based sharding for metadata (consistent with noun vectors)
|
||||
const filePath = this.getShardedPath(this.nounMetadataDir, id)
|
||||
try {
|
||||
const data = await fs.promises.readFile(filePath, 'utf-8')
|
||||
return JSON.parse(data)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue