feat: add distributed architecture with sharding and coordination
- Wire up distributed components (Coordinator, ShardManager, CacheSync) - Implement automatic sharding for S3 storage (256 shards) - Add read/write separation for operational modes - Zero-config automatic detection for distributed mode - Add mutex implementation for thread safety - Fix metadata filtering in find operations - Fix neural API vector similarity calculations - Improve batch operations performance - Add Bluesky distributed setup example BREAKING CHANGE: None - backward compatible
This commit is contained in:
parent
8aafc769a3
commit
ed64c266ec
30 changed files with 2084 additions and 439 deletions
|
|
@ -404,15 +404,16 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
|
||||
// Check if the adapter has a paginated method for getting nouns
|
||||
if (typeof (this as any).getNounsWithPagination === 'function') {
|
||||
// Use the adapter's paginated method
|
||||
// Use the adapter's paginated method - pass offset directly to adapter
|
||||
const result = await (this as any).getNounsWithPagination({
|
||||
limit,
|
||||
offset, // Let the adapter handle offset for O(1) operation
|
||||
cursor,
|
||||
filter: options?.filter
|
||||
})
|
||||
|
||||
// Apply offset if needed (some adapters might not support offset)
|
||||
const items = result.items.slice(offset)
|
||||
// Don't slice here - the adapter should handle offset efficiently
|
||||
const items = result.items
|
||||
|
||||
// CRITICAL SAFETY CHECK: Prevent infinite loops
|
||||
// If we have no items but hasMore is true, force hasMore to false
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue