soulcraft/brainy
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

fix: implement unified UUID-based sharding for metadata across all storage adapters

Browse source 
Fixes critical scalability bottleneck where metadata was stored in non-sharded
directories, causing performance degradation at scale (1M+ entities).

Changes:
- Add UUID-based sharding to metadata operations in S3Compatible, FileSystem, and OpFS storage
- Implement complete UUID-based sharding for OpFS storage (nouns, verbs, metadata)
- Update pagination methods to iterate through all 256 UUID-based shards
- Add integration tests verifying sharding behavior across storage adapters

Impact:
- Metadata now scales to millions of entities without directory bottlenecks
- All storage adapters now use consistent UUID-based sharding (256 buckets: 00-ff)
- Improves GCS/S3/R2/OpFS performance at scale
- Path format: entities/{type}/{subtype}/{shard}/{id}.json

Breaking change: Requires data migration for existing S3/GCS/R2/OpFS deployments.
See .strategy/UNIFIED-UUID-SHARDING.md for migration guidance.
This commit is contained in:
David Snelling 2025-10-08 13:26:35 -07:00
parent 6d50fe4054
commit 2f3357132d
6 changed files with 1011 additions and 217 deletions
Download patch file Download diff file Expand all files Collapse all files

11
src/storage/adapters/fileSystemStorage.ts
View file

@ -627,7 +627,13 @@ export class FileSystemStorage extends BaseStorage {
protected async saveNounMetadata_internal(id: string, metadata: any): Promise<void> {
await this.ensureInitialized()
const filePath = path.join(this.nounMetadataDir, `${id}.json`)
// Use UUID-based sharding for metadata (consistent with noun vectors)
const filePath = this.getShardedPath(this.nounMetadataDir, id)
// Ensure shard directory exists
const shardDir = path.dirname(filePath)
await fs.promises.mkdir(shardDir, { recursive: true })
await fs.promises.writeFile(filePath, JSON.stringify(metadata, null, 2))
}
@ -637,7 +643,8 @@ export class FileSystemStorage extends BaseStorage {
public async getNounMetadata(id: string): Promise<any | null> {
await this.ensureInitialized()
const filePath = path.join(this.nounMetadataDir, `${id}.json`)
// Use UUID-based sharding for metadata (consistent with noun vectors)
const filePath = this.getShardedPath(this.nounMetadataDir, id)
try {
const data = await fs.promises.readFile(filePath, 'utf-8')
return JSON.parse(data)

313
src/storage/adapters/opfsStorage.ts
View file

@ -19,6 +19,7 @@ import {
INDEX_DIR,
STATISTICS_KEY
} from '../baseStorage.js'
import { getShardIdFromUuid } from '../sharding.js'
import '../../types/fileSystemTypes.js'
// Type alias for HNSWNode
@ -205,8 +206,16 @@ export class OPFSStorage extends BaseStorage {
)
}
// Create or get the file for this noun
const fileHandle = await this.nounsDir!.getFileHandle(`${noun.id}.json`, {
// Use UUID-based sharding for nouns
const shardId = getShardIdFromUuid(noun.id)
// Get or create the shard directory
const shardDir = await this.nounsDir!.getDirectoryHandle(shardId, {
create: true
})
// Create or get the file in the shard directory
const fileHandle = await shardDir.getFileHandle(`${noun.id}.json`, {
create: true
})
@ -229,8 +238,14 @@ export class OPFSStorage extends BaseStorage {
await this.ensureInitialized()
try {
// Get the file handle for this noun
const fileHandle = await this.nounsDir!.getFileHandle(`${id}.json`)
// Use UUID-based sharding for nouns
const shardId = getShardIdFromUuid(id)
// Get the shard directory
const shardDir = await this.nounsDir!.getDirectoryHandle(shardId)
// Get the file handle from the shard directory
const fileHandle = await shardDir.getFileHandle(`${id}.json`)
// Read the noun data from the file
const file = await fileHandle.getFile()
@ -278,35 +293,42 @@ export class OPFSStorage extends BaseStorage {
const nodes: HNSWNode[] = []
try {
// Iterate through all files in the nouns directory
for await (const [name, handle] of this.nounsDir!.entries()) {
if (handle.kind === 'file') {
try {
// Read the node data from the file
const file = await safeGetFile(handle)
const text = await file.text()
const data = JSON.parse(text)
// Iterate through all shard directories
for await (const [shardName, shardHandle] of this.nounsDir!.entries()) {
if (shardHandle.kind === 'directory') {
const shardDir = shardHandle as FileSystemDirectoryHandle
// Get the metadata to check the noun type
const metadata = await this.getMetadata(data.id)
// Iterate through all files in this shard
for await (const [fileName, fileHandle] of shardDir.entries()) {
if (fileHandle.kind === 'file') {
try {
// Read the node data from the file
const file = await safeGetFile(fileHandle)
const text = await file.text()
const data = JSON.parse(text)
// Include the node if its noun type matches the requested type
if (metadata && metadata.noun === nounType) {
// Convert serialized connections back to Map<number, Set<string>>
const connections = new Map<number, Set<string>>()
for (const [level, nodeIds] of Object.entries(data.connections)) {
connections.set(Number(level), new Set(nodeIds as string[]))
// Get the metadata to check the noun type
const metadata = await this.getMetadata(data.id)
// Include the node if its noun type matches the requested type
if (metadata && metadata.noun === nounType) {
// Convert serialized connections back to Map<number, Set<string>>
const connections = new Map<number, Set<string>>()
for (const [level, nodeIds] of Object.entries(data.connections)) {
connections.set(Number(level), new Set(nodeIds as string[]))
}
nodes.push({
id: data.id,
vector: data.vector,
connections,
level: data.level || 0
})
}
} catch (error) {
console.error(`Error reading node file ${shardName}/${fileName}:`, error)
}
nodes.push({
id: data.id,
vector: data.vector,
connections,
level: data.level || 0
})
}
} catch (error) {
console.error(`Error reading node file ${name}:`, error)
}
}
}
@ -331,7 +353,14 @@ export class OPFSStorage extends BaseStorage {
await this.ensureInitialized()
try {
await this.nounsDir!.removeEntry(`${id}.json`)
// Use UUID-based sharding for nouns
const shardId = getShardIdFromUuid(id)
// Get the shard directory
const shardDir = await this.nounsDir!.getDirectoryHandle(shardId)
// Delete the file from the shard directory
await shardDir.removeEntry(`${id}.json`)
} catch (error: any) {
// Ignore NotFoundError, which means the file doesn't exist
if (error.name !== 'NotFoundError') {
@ -363,8 +392,16 @@ export class OPFSStorage extends BaseStorage {
)
}
// Create or get the file for this verb
const fileHandle = await this.verbsDir!.getFileHandle(`${edge.id}.json`, {
// Use UUID-based sharding for verbs
const shardId = getShardIdFromUuid(edge.id)
// Get or create the shard directory
const shardDir = await this.verbsDir!.getDirectoryHandle(shardId, {
create: true
})
// Create or get the file in the shard directory
const fileHandle = await shardDir.getFileHandle(`${edge.id}.json`, {
create: true
})
@ -392,8 +429,14 @@ export class OPFSStorage extends BaseStorage {
await this.ensureInitialized()
try {
// Get the file handle for this edge
const fileHandle = await this.verbsDir!.getFileHandle(`${id}.json`)
// Use UUID-based sharding for verbs
const shardId = getShardIdFromUuid(id)
// Get the shard directory
const shardDir = await this.verbsDir!.getDirectoryHandle(shardId)
// Get the file handle from the shard directory
const fileHandle = await shardDir.getFileHandle(`${id}.json`)
// Read the edge data from the file
const file = await fileHandle.getFile()
@ -438,40 +481,47 @@ export class OPFSStorage extends BaseStorage {
const allEdges: Edge[] = []
try {
// Iterate through all files in the verbs directory
for await (const [name, handle] of this.verbsDir!.entries()) {
if (handle.kind === 'file') {
try {
// Read the edge data from the file
const file = await safeGetFile(handle)
const text = await file.text()
const data = JSON.parse(text)
// Iterate through all shard directories
for await (const [shardName, shardHandle] of this.verbsDir!.entries()) {
if (shardHandle.kind === 'directory') {
const shardDir = shardHandle as FileSystemDirectoryHandle
// Convert serialized connections back to Map<number, Set<string>>
const connections = new Map<number, Set<string>>()
for (const [level, nodeIds] of Object.entries(data.connections)) {
connections.set(Number(level), new Set(nodeIds as string[]))
// Iterate through all files in this shard
for await (const [fileName, fileHandle] of shardDir.entries()) {
if (fileHandle.kind === 'file') {
try {
// Read the edge data from the file
const file = await safeGetFile(fileHandle)
const text = await file.text()
const data = JSON.parse(text)
// Convert serialized connections back to Map<number, Set<string>>
const connections = new Map<number, Set<string>>()
for (const [level, nodeIds] of Object.entries(data.connections)) {
connections.set(Number(level), new Set(nodeIds as string[]))
}
// Create default timestamp if not present
const defaultTimestamp = {
seconds: Math.floor(Date.now() / 1000),
nanoseconds: (Date.now() % 1000) * 1000000
}
// Create default createdBy if not present
const defaultCreatedBy = {
augmentation: 'unknown',
version: '1.0'
}
allEdges.push({
id: data.id,
vector: data.vector,
connections
})
} catch (error) {
console.error(`Error reading edge file ${shardName}/${fileName}:`, error)
}
}
// Create default timestamp if not present
const defaultTimestamp = {
seconds: Math.floor(Date.now() / 1000),
nanoseconds: (Date.now() % 1000) * 1000000
}
// Create default createdBy if not present
const defaultCreatedBy = {
augmentation: 'unknown',
version: '1.0'
}
allEdges.push({
id: data.id,
vector: data.vector,
connections
})
} catch (error) {
console.error(`Error reading edge file ${name}:`, error)
}
}
}
@ -572,7 +622,14 @@ export class OPFSStorage extends BaseStorage {
await this.ensureInitialized()
try {
await this.verbsDir!.removeEntry(`${id}.json`)
// Use UUID-based sharding for verbs
const shardId = getShardIdFromUuid(id)
// Get the shard directory
const shardDir = await this.verbsDir!.getDirectoryHandle(shardId)
// Delete the file from the shard directory
await shardDir.removeEntry(`${id}.json`)
} catch (error: any) {
// Ignore NotFoundError, which means the file doesn't exist
if (error.name !== 'NotFoundError') {
@ -669,10 +726,17 @@ export class OPFSStorage extends BaseStorage {
protected async saveVerbMetadata_internal(id: string, metadata: any): Promise<void> {
await this.ensureInitialized()
const fileName = `${id}.json`
const fileHandle = await (
// Use UUID-based sharding for metadata (consistent with verb vectors)
const shardId = getShardIdFromUuid(id)
// Get or create the shard directory
const shardDir = await (
this.verbMetadataDir as FileSystemDirectoryHandle
).getFileHandle(fileName, { create: true })
).getDirectoryHandle(shardId, { create: true })
// Create or get the file in the shard directory
const fileName = `${id}.json`
const fileHandle = await shardDir.getFileHandle(fileName, { create: true })
const writable = await (fileHandle as FileSystemFileHandle).createWritable()
await writable.write(JSON.stringify(metadata, null, 2))
await writable.close()
@ -684,11 +748,18 @@ export class OPFSStorage extends BaseStorage {
public async getVerbMetadata(id: string): Promise<any | null> {
await this.ensureInitialized()
// Use UUID-based sharding for metadata (consistent with verb vectors)
const shardId = getShardIdFromUuid(id)
const fileName = `${id}.json`
try {
const fileHandle = await (
// Get the shard directory
const shardDir = await (
this.verbMetadataDir as FileSystemDirectoryHandle
).getFileHandle(fileName)
).getDirectoryHandle(shardId)
// Get the file from the shard directory
const fileHandle = await shardDir.getFileHandle(fileName)
const file = await safeGetFile(fileHandle)
const text = await file.text()
return JSON.parse(text)
@ -706,10 +777,17 @@ export class OPFSStorage extends BaseStorage {
protected async saveNounMetadata_internal(id: string, metadata: any): Promise<void> {
await this.ensureInitialized()
const fileName = `${id}.json`
const fileHandle = await (
// Use UUID-based sharding for metadata (consistent with noun vectors)
const shardId = getShardIdFromUuid(id)
// Get or create the shard directory
const shardDir = await (
this.nounMetadataDir as FileSystemDirectoryHandle
).getFileHandle(fileName, { create: true })
).getDirectoryHandle(shardId, { create: true })
// Create or get the file in the shard directory
const fileName = `${id}.json`
const fileHandle = await shardDir.getFileHandle(fileName, { create: true })
const writable = await fileHandle.createWritable()
await writable.write(JSON.stringify(metadata, null, 2))
await writable.close()
@ -721,11 +799,18 @@ export class OPFSStorage extends BaseStorage {
public async getNounMetadata(id: string): Promise<any | null> {
await this.ensureInitialized()
// Use UUID-based sharding for metadata (consistent with noun vectors)
const shardId = getShardIdFromUuid(id)
const fileName = `${id}.json`
try {
const fileHandle = await (
// Get the shard directory
const shardDir = await (
this.nounMetadataDir as FileSystemDirectoryHandle
).getFileHandle(fileName)
).getDirectoryHandle(shardId)
// Get the file from the shard directory
const fileHandle = await shardDir.getFileHandle(fileName)
const file = await safeGetFile(fileHandle)
const text = await file.text()
return JSON.parse(text)
@ -1336,16 +1421,23 @@ export class OPFSStorage extends BaseStorage {
nextCursor?: string
}> {
await this.ensureInitialized()
const limit = options.limit || 100
const cursor = options.cursor
// Get all noun files
// Get all noun files from all shards
const nounFiles: string[] = []
if (this.nounsDir) {
for await (const [name, handle] of this.nounsDir.entries()) {
if (handle.kind === 'file' && name.endsWith('.json')) {
nounFiles.push(name)
// Iterate through all shard directories
for await (const [shardName, shardHandle] of this.nounsDir.entries()) {
if (shardHandle.kind === 'directory') {
// Iterate through files in this shard
const shardDir = shardHandle as FileSystemDirectoryHandle
for await (const [fileName, fileHandle] of shardDir.entries()) {
if (fileHandle.kind === 'file' && fileName.endsWith('.json')) {
nounFiles.push(`${shardName}/${fileName}`)
}
}
}
}
}
@ -1368,7 +1460,8 @@ export class OPFSStorage extends BaseStorage {
// Load nouns from files
const items: HNSWNoun[] = []
for (const fileName of pageFiles) {
const id = fileName.replace('.json', '')
// fileName is in format "shard/uuid.json", extract just the UUID
const id = fileName.split('/')[1].replace('.json', '')
const noun = await this.getNoun_internal(id)
if (noun) {
// Apply filters if provided
@ -1451,23 +1544,30 @@ export class OPFSStorage extends BaseStorage {
nextCursor?: string
}> {
await this.ensureInitialized()
const limit = options.limit || 100
const cursor = options.cursor
// Get all verb files
// Get all verb files from all shards
const verbFiles: string[] = []
if (this.verbsDir) {
for await (const [name, handle] of this.verbsDir.entries()) {
if (handle.kind === 'file' && name.endsWith('.json')) {
verbFiles.push(name)
// Iterate through all shard directories
for await (const [shardName, shardHandle] of this.verbsDir.entries()) {
if (shardHandle.kind === 'directory') {
// Iterate through files in this shard
const shardDir = shardHandle as FileSystemDirectoryHandle
for await (const [fileName, fileHandle] of shardDir.entries()) {
if (fileHandle.kind === 'file' && fileName.endsWith('.json')) {
verbFiles.push(`${shardName}/${fileName}`)
}
}
}
}
}
// Sort files for consistent ordering
verbFiles.sort()
// Apply cursor-based pagination
let startIndex = 0
if (cursor) {
@ -1476,14 +1576,15 @@ export class OPFSStorage extends BaseStorage {
startIndex = cursorIndex
}
}
// Get the subset of files for this page
const pageFiles = verbFiles.slice(startIndex, startIndex + limit)
// Load verbs from files and convert to GraphVerb
const items: GraphVerb[] = []
for (const fileName of pageFiles) {
const id = fileName.replace('.json', '')
// fileName is in format "shard/uuid.json", extract just the UUID
const id = fileName.split('/')[1].replace('.json', '')
const hnswVerb = await this.getVerb_internal(id)
if (hnswVerb) {
// Convert HNSWVerb to GraphVerb
@ -1593,17 +1694,27 @@ export class OPFSStorage extends BaseStorage {
*/
private async initializeCountsFromScan(): Promise<void> {
try {
// Count nouns
// Count nouns across all shards
let nounCount = 0
for await (const [, ] of this.nounsDir!.entries()) {
nounCount++
for await (const [shardName, shardHandle] of this.nounsDir!.entries()) {
if (shardHandle.kind === 'directory') {
const shardDir = shardHandle as FileSystemDirectoryHandle
for await (const [, ] of shardDir.entries()) {
nounCount++
}
}
}
this.totalNounCount = nounCount
// Count verbs
// Count verbs across all shards
let verbCount = 0
for await (const [, ] of this.verbsDir!.entries()) {
verbCount++
for await (const [shardName, shardHandle] of this.verbsDir!.entries()) {
if (shardHandle.kind === 'directory') {
const shardDir = shardHandle as FileSystemDirectoryHandle
for await (const [, ] of shardDir.entries()) {
verbCount++
}
}
}
this.totalVerbCount = verbCount

325
src/storage/adapters/s3CompatibleStorage.ts
View file

@ -27,6 +27,7 @@ import { getGlobalSocketManager } from '../../utils/adaptiveSocketManager.js'
import { getGlobalBackpressure } from '../../utils/adaptiveBackpressure.js'
import { getWriteBuffer, WriteBuffer } from '../../utils/writeBuffer.js'
import { getCoalescer, RequestCoalescer } from '../../utils/requestCoalescer.js'
import { getShardIdFromUuid, getAllShardIds, getShardIdByIndex, TOTAL_SHARDS } from '../sharding.js'
// Type aliases for better readability
type HNSWNode = HNSWNoun
@ -123,10 +124,12 @@ export class S3CompatibleStorage extends BaseStorage {
// Distributed components (optional)
private coordinator?: any // DistributedCoordinator
private shardManager?: any // ShardManager
private cacheSync?: any // CacheSync
private readWriteSeparation?: any // ReadWriteSeparation
// Note: Sharding is always enabled via UUID-based prefixes (00-ff)
// ShardManager is no longer used - sharding is deterministic
// Request coalescer for deduplication
private requestCoalescer: RequestCoalescer | null = null
@ -356,23 +359,22 @@ export class S3CompatibleStorage extends BaseStorage {
/**
* Set distributed components for multi-node coordination
* Zero-config: Automatically optimizes based on components provided
*
* Note: Sharding is always enabled via UUID-based prefixes (00-ff).
* ShardManager is no longer required - sharding is deterministic based on UUID.
*/
public setDistributedComponents(components: {
coordinator?: any
shardManager?: any
shardManager?: any // Deprecated - kept for backward compatibility
cacheSync?: any
readWriteSeparation?: any
}): void {
this.coordinator = components.coordinator
this.shardManager = components.shardManager
this.cacheSync = components.cacheSync
this.readWriteSeparation = components.readWriteSeparation
// Auto-configure based on what's available
if (this.shardManager) {
console.log(`🎯 S3 Storage: Sharding enabled with ${this.shardManager.config?.shardCount || 64} shards`)
}
// Note: UUID-based sharding is always active (256 shards: 00-ff)
console.log(`🎯 S3 Storage: UUID-based sharding active (256 shards: 00-ff)`)
if (this.coordinator) {
console.log(`🤝 S3 Storage: Distributed coordination active (node: ${this.coordinator.nodeId})`)
@ -388,25 +390,33 @@ export class S3CompatibleStorage extends BaseStorage {
}
/**
* Get the S3 key for a noun, using sharding if available
* Get the S3 key for a noun using UUID-based sharding
*
* Uses first 2 hex characters of UUID for consistent sharding.
* Path format: entities/nouns/vectors/{shardId}/{uuid}.json
*
* @example
* getNounKey('ab123456-1234-5678-9abc-def012345678')
* // returns 'entities/nouns/vectors/ab/ab123456-1234-5678-9abc-def012345678.json'
*/
private getNounKey(id: string): string {
if (this.shardManager) {
const shardId = this.shardManager.getShardForKey(id)
return `shards/${shardId}/${this.nounPrefix}${id}.json`
}
return `${this.nounPrefix}${id}.json`
const shardId = getShardIdFromUuid(id)
return `${this.nounPrefix}${shardId}/${id}.json`
}
/**
* Get the S3 key for a verb, using sharding if available
* Get the S3 key for a verb using UUID-based sharding
*
* Uses first 2 hex characters of UUID for consistent sharding.
* Path format: verbs/{shardId}/{uuid}.json
*
* @example
* getVerbKey('cd987654-4321-8765-cba9-fed543210987')
* // returns 'verbs/cd/cd987654-4321-8765-cba9-fed543210987.json'
*/
private getVerbKey(id: string): string {
if (this.shardManager) {
const shardId = this.shardManager.getShardForKey(id)
return `shards/${shardId}/${this.verbPrefix}${id}.json`
}
return `${this.verbPrefix}${id}.json`
const shardId = getShardIdFromUuid(id)
return `${this.verbPrefix}${shardId}/${id}.json`
}
/**
@ -1036,7 +1046,8 @@ export class S3CompatibleStorage extends BaseStorage {
// Import the GetObjectCommand only when needed
const { GetObjectCommand } = await import('@aws-sdk/client-s3')
const key = `${this.nounPrefix}${id}.json`
// Use getNounKey() to properly handle sharding
const key = this.getNounKey(id)
this.logger.trace(`Getting node ${id} from key: ${key}`)
// Try to get the node from the nouns directory
@ -1133,9 +1144,23 @@ export class S3CompatibleStorage extends BaseStorage {
}
/**
* Get nodes with pagination
* Get nodes with pagination using UUID-based sharding
*
* Iterates through 256 UUID-based shards (00-ff) to retrieve nodes.
* Cursor format: "shardIndex:s3ContinuationToken" to support pagination across shards.
*
* @param options Pagination options
* @returns Promise that resolves to a paginated result of nodes
*
* @example
* // First page
* const page1 = await getNodesWithPagination({ limit: 100 })
* // page1.nodes contains up to 100 nodes
* // page1.nextCursor might be "5:some-s3-token" (currently in shard 05)
*
* // Next page
* const page2 = await getNodesWithPagination({ limit: 100, cursor: page1.nextCursor })
* // Continues from where page1 left off
*/
protected async getNodesWithPagination(options: {
limit?: number
@ -1147,98 +1172,91 @@ export class S3CompatibleStorage extends BaseStorage {
nextCursor?: string
}> {
await this.ensureInitialized()
const limit = options.limit || 100
const useCache = options.useCache !== false
try {
// Import the ListObjectsV2Command and GetObjectCommand only when needed
const { ListObjectsV2Command } = await import('@aws-sdk/client-s3')
// List objects with pagination
const listResponse = await this.s3Client!.send(
new ListObjectsV2Command({
Bucket: this.bucketName,
Prefix: this.nounPrefix,
MaxKeys: limit,
ContinuationToken: options.cursor
})
)
// If listResponse is null/undefined or there are no objects, return an empty result
if (
!listResponse ||
!listResponse.Contents ||
listResponse.Contents.length === 0
) {
return {
nodes: [],
hasMore: false
}
}
// Extract node IDs from the keys
const nodeIds = listResponse.Contents
.filter((object: { Key?: string }) => object && object.Key)
.map((object: { Key?: string }) => object.Key!.replace(this.nounPrefix, '').replace('.json', ''))
// Use the cache manager to get nodes efficiently
const nodes: HNSWNode[] = []
if (useCache) {
// Get nodes from cache manager
const cachedNodes = await this.nounCacheManager.getMany(nodeIds)
// Add nodes to result in the same order as nodeIds
for (const id of nodeIds) {
const node = cachedNodes.get(id)
if (node) {
nodes.push(node)
// Parse cursor (format: "shardIndex:s3ContinuationToken")
let startShardIndex = 0
let s3ContinuationToken: string | undefined
if (options.cursor) {
const parts = options.cursor.split(':', 2)
startShardIndex = parseInt(parts[0]) || 0
s3ContinuationToken = parts[1] || undefined
}
// Iterate through shards starting from cursor position
for (let shardIndex = startShardIndex; shardIndex < TOTAL_SHARDS; shardIndex++) {
const shardId = getShardIdByIndex(shardIndex)
const shardPrefix = `${this.nounPrefix}${shardId}/`
// List objects in this shard
const listResponse = await this.s3Client!.send(
new ListObjectsV2Command({
Bucket: this.bucketName,
Prefix: shardPrefix,
MaxKeys: limit - nodes.length,
ContinuationToken: shardIndex === startShardIndex ? s3ContinuationToken : undefined
})
)
// Extract node IDs from keys
if (listResponse.Contents && listResponse.Contents.length > 0) {
const nodeIds = listResponse.Contents
.filter((obj: { Key?: string }) => obj && obj.Key)
.map((obj: { Key?: string }) => {
// Extract UUID from: entities/nouns/vectors/ab/ab123456-uuid.json
let key = obj.Key!
if (key.startsWith(shardPrefix)) {
key = key.substring(shardPrefix.length)
}
if (key.endsWith('.json')) {
key = key.substring(0, key.length - 5)
}
return key
})
// Load nodes for this shard (use direct loading for pagination scans)
const shardNodes = await this.loadNodesByIds(nodeIds, false)
nodes.push(...shardNodes)
}
// Check if we've reached the limit
if (nodes.length >= limit) {
const hasMore = !!listResponse.IsTruncated || shardIndex < TOTAL_SHARDS - 1
const nextCursor = listResponse.IsTruncated
? `${shardIndex}:${listResponse.NextContinuationToken}`
: shardIndex < TOTAL_SHARDS - 1
? `${shardIndex + 1}:`
: undefined
return {
nodes: nodes.slice(0, limit),
hasMore,
nextCursor
}
}
} else {
// Get nodes directly from S3 without using cache
// Process in smaller batches to reduce memory usage
const batchSize = 50
const batches: string[][] = []
// Split into batches
for (let i = 0; i < nodeIds.length; i += batchSize) {
const batch = nodeIds.slice(i, i + batchSize)
batches.push(batch)
}
// Process each batch sequentially
for (const batch of batches) {
const batchNodes = await Promise.all(
batch.map(async (id) => {
try {
return await this.getNoun_internal(id)
} catch (error) {
return null
}
})
)
// Add non-null nodes to result
for (const node of batchNodes) {
if (node) {
nodes.push(node)
}
// If this shard has more data but we haven't hit limit, continue to next shard
if (listResponse.IsTruncated) {
return {
nodes,
hasMore: true,
nextCursor: `${shardIndex}:${listResponse.NextContinuationToken}`
}
}
}
// Determine if there are more nodes
const hasMore = !!listResponse.IsTruncated
// Set next cursor if there are more nodes
const nextCursor = listResponse.NextContinuationToken
// All shards exhausted
return {
nodes,
hasMore,
nextCursor
hasMore: false,
nextCursor: undefined
}
} catch (error) {
this.logger.error('Failed to get nodes with pagination:', error)
@ -1249,6 +1267,47 @@ export class S3CompatibleStorage extends BaseStorage {
}
}
/**
* Load nodes by IDs efficiently using cache or direct fetch
*/
private async loadNodesByIds(nodeIds: string[], useCache: boolean): Promise<HNSWNode[]> {
const nodes: HNSWNode[] = []
if (useCache) {
const cachedNodes = await this.nounCacheManager.getMany(nodeIds)
for (const id of nodeIds) {
const node = cachedNodes.get(id)
if (node) {
nodes.push(node)
}
}
} else {
// Load directly in batches
const batchSize = 50
for (let i = 0; i < nodeIds.length; i += batchSize) {
const batch = nodeIds.slice(i, i + batchSize)
const batchNodes = await Promise.all(
batch.map(async (id) => {
try {
return await this.getNoun_internal(id)
} catch (error) {
this.logger.warn(`Failed to load node ${id}:`, error)
return null
}
})
)
for (const node of batchNodes) {
if (node) {
nodes.push(node)
}
}
}
}
return nodes
}
/**
* Get nouns by noun type (internal implementation)
* @param nounType The noun type to filter by
@ -1434,7 +1493,7 @@ export class S3CompatibleStorage extends BaseStorage {
// Import the GetObjectCommand only when needed
const { GetObjectCommand } = await import('@aws-sdk/client-s3')
const key = `${this.verbPrefix}${id}.json`
const key = this.getVerbKey(id)
this.logger.trace(`Getting edge ${id} from key: ${key}`)
// Try to get the edge from the verbs directory
@ -2039,7 +2098,9 @@ export class S3CompatibleStorage extends BaseStorage {
// Import the PutObjectCommand only when needed
const { PutObjectCommand } = await import('@aws-sdk/client-s3')
const key = `${this.metadataPrefix}${id}.json`
// Use UUID-based sharding for metadata (consistent with noun vectors)
const shardId = getShardIdFromUuid(id)
const key = `${this.metadataPrefix}${shardId}/${id}.json`
const body = JSON.stringify(metadata, null, 2)
this.logger.trace(`Saving noun metadata for ${id} to key: ${key}`)
@ -2186,7 +2247,9 @@ export class S3CompatibleStorage extends BaseStorage {
// Import the GetObjectCommand only when needed
const { GetObjectCommand } = await import('@aws-sdk/client-s3')
const key = `${this.metadataPrefix}${id}.json`
// Use UUID-based sharding for metadata (consistent with noun vectors)
const shardId = getShardIdFromUuid(id)
const key = `${this.metadataPrefix}${shardId}/${id}.json`
this.logger.trace(`Getting noun metadata for ${id} from key: ${key}`)
// Try to get the noun metadata
@ -3459,13 +3522,65 @@ export class S3CompatibleStorage extends BaseStorage {
}
}
// Calculate total count efficiently
// For the first page (no cursor), we can estimate total count
let totalCount: number | undefined
if (!cursor) {
try {
totalCount = await this.estimateTotalNounCount()
} catch (error) {
this.logger.warn('Failed to estimate total noun count:', error)
// totalCount remains undefined
}
}
return {
items: filteredNodes,
totalCount,
hasMore: result.hasMore,
nextCursor: result.nextCursor
}
}
/**
* Estimate total noun count by listing objects across all shards
* This is more efficient than loading all nouns
*/
private async estimateTotalNounCount(): Promise<number> {
const { ListObjectsV2Command } = await import('@aws-sdk/client-s3')
let totalCount = 0
// Count across all UUID-based shards (00-ff)
for (let shardIndex = 0; shardIndex < TOTAL_SHARDS; shardIndex++) {
const shardId = getShardIdByIndex(shardIndex)
const shardPrefix = `${this.nounPrefix}${shardId}/`
let shardCursor: string | undefined
let hasMore = true
while (hasMore) {
const listResponse = await this.s3Client!.send(
new ListObjectsV2Command({
Bucket: this.bucketName,
Prefix: shardPrefix,
MaxKeys: 1000,
ContinuationToken: shardCursor
})
)
if (listResponse.Contents) {
totalCount += listResponse.Contents.length
}
hasMore = !!listResponse.IsTruncated
shardCursor = listResponse.NextContinuationToken
}
}
return totalCount
}
/**
* Initialize counts from S3 storage
*/

150
src/storage/sharding.ts Normal file
View file

@ -0,0 +1,150 @@
/**
* Unified UUID-based sharding for all storage adapters
*
* Uses first 2 hex characters of UUID for consistent, predictable sharding
* that scales from hundreds to millions of entities without configuration.
*
* Sharding characteristics:
* - 256 buckets (00-ff)
* - Deterministic (same UUID always maps to same shard)
* - No configuration required
* - Works across all storage types (filesystem, S3, GCS, memory)
* - Efficient for list operations and pagination
*/
/**
* Extract shard ID from UUID
*
* Uses first 2 hex characters of the UUID as the shard ID.
* This provides 256 evenly-distributed buckets (00-ff).
*
* @param uuid - UUID string (with or without hyphens)
* @returns 2-character hex shard ID (00-ff)
*
* @example
* ```typescript
* getShardIdFromUuid('ab123456-1234-5678-9abc-def012345678') // returns 'ab'
* getShardIdFromUuid('cd987654-4321-8765-cba9-fed543210987') // returns 'cd'
* getShardIdFromUuid('00000000-0000-0000-0000-000000000000') // returns '00'
* ```
*/
export function getShardIdFromUuid(uuid: string): string {
if (!uuid) {
throw new Error('UUID is required for sharding')
}
// Remove hyphens and convert to lowercase
const normalized = uuid.toLowerCase().replace(/-/g, '')
// Validate UUID format (32 hex characters)
if (normalized.length !== 32) {
throw new Error(`Invalid UUID format: ${uuid} (expected 32 hex chars, got ${normalized.length})`)
}
// Extract first 2 characters
const shardId = normalized.substring(0, 2)
// Validate hex format
if (!/^[0-9a-f]{2}$/.test(shardId)) {
throw new Error(`Invalid UUID prefix: ${shardId} (expected 2 hex chars)`)
}
return shardId
}
/**
* Get all possible shard IDs (00-ff)
*
* Returns array of 256 shard IDs in ascending order.
* Useful for iterating through all shards during pagination.
*
* @returns Array of 256 shard IDs
*
* @example
* ```typescript
* const shards = getAllShardIds()
* // ['00', '01', '02', ..., 'fd', 'fe', 'ff']
*
* for (const shardId of shards) {
* const prefix = `entities/nouns/vectors/${shardId}/`
* // List objects with this prefix
* }
* ```
*/
export function getAllShardIds(): string[] {
const shards: string[] = []
for (let i = 0; i < 256; i++) {
shards.push(i.toString(16).padStart(2, '0'))
}
return shards
}
/**
* Get shard ID for a given index (0-255)
*
* @param index - Shard index (0-255)
* @returns 2-character hex shard ID
*
* @example
* ```typescript
* getShardIdByIndex(0) // '00'
* getShardIdByIndex(15) // '0f'
* getShardIdByIndex(255) // 'ff'
* ```
*/
export function getShardIdByIndex(index: number): string {
if (index < 0 || index > 255) {
throw new Error(`Shard index out of range: ${index} (expected 0-255)`)
}
return index.toString(16).padStart(2, '0')
}
/**
* Get shard index from shard ID (0-255)
*
* @param shardId - 2-character hex shard ID
* @returns Shard index (0-255)
*
* @example
* ```typescript
* getShardIndexFromId('00') // 0
* getShardIndexFromId('0f') // 15
* getShardIndexFromId('ff') // 255
* ```
*/
export function getShardIndexFromId(shardId: string): number {
if (!/^[0-9a-f]{2}$/.test(shardId)) {
throw new Error(`Invalid shard ID: ${shardId} (expected 2 hex chars)`)
}
return parseInt(shardId, 16)
}
/**
* Total number of shards in the system
*/
export const TOTAL_SHARDS = 256
/**
* Shard configuration (read-only)
*/
export const SHARD_CONFIG = {
/**
* Total number of shards (256)
*/
count: TOTAL_SHARDS,
/**
* Number of hex characters used for sharding (2)
*/
prefixLength: 2,
/**
* Sharding method description
*/
method: 'uuid-prefix',
/**
* Whether sharding is always enabled
*/
alwaysEnabled: true
} as const