/** * Unified id-based sharding for all storage adapters. * * Maps any entity id to one of 256 buckets (00-ff) for consistent, predictable * on-disk layout that scales from hundreds to millions of entities with no * configuration. * * Sharding characteristics: * - 256 buckets (00-ff) * - Deterministic (the same id always maps to the same shard) * - No configuration required * - Works across the filesystem and memory adapters * - Efficient for list operations and pagination * * Two id shapes are supported, by design: * - **UUID-format ids** (32 hex chars) shard by their first byte. This is the * original scheme, kept byte-for-byte so existing on-disk layouts never move. * - **Application ids** (`'user-123'`, slugs, emails — anything else) shard by a * stable FNV-1a hash. Requiring callers to mint UUIDs would break the * documented `add({ id: '…' })` contract; a shard is only a bucket, so any * deterministic, well-distributed mapping is correct. * * The hash is part of the on-disk contract: **never change it** — doing so would * relocate every application-id record. */ /** * 32-bit FNV-1a hash of a string, returned as a 2-hex-char shard bucket (00-ff). * Deterministic and uniformly distributed across the 256 buckets. * * @param value - The string to hash (an entity id that is not UUID-format). * @returns A 2-character hex shard id. */ function hashToShardId(value: string): string { let hash = 0x811c9dc5 // FNV offset basis for (let i = 0; i < value.length; i++) { hash ^= value.charCodeAt(i) hash = Math.imul(hash, 0x01000193) // FNV prime } return ((hash >>> 0) & 0xff).toString(16).padStart(2, '0') } /** * Resolve the shard bucket (00-ff) for an entity id. * * UUID-format ids (32 hex chars, with or without hyphens) shard by their first * byte — preserving the original on-disk layout. Any other id is hashed (FNV-1a) * into the same 256-bucket space, so application-supplied ids like `'user-123'` * work without forcing callers to mint UUIDs. * * @param id - The entity id (UUID-format or any application string). * @returns A 2-character hex shard id (00-ff). * @throws If `id` is empty. * * @example * ```typescript * getShardId('ab123456-1234-5678-9abc-def012345678') // 'ab' (UUID first byte) * getShardId('user-12345') // FNV-1a hash bucket * ``` */ export function getShardId(id: string): string { if (!id) { throw new Error('id is required for sharding') } // UUID-format ids (32 hex chars) keep the original first-byte bucketing so // existing on-disk data is never relocated. const normalized = id.toLowerCase().replace(/-/g, '') if (/^[0-9a-f]{32}$/.test(normalized)) { return normalized.substring(0, 2) } // Any other id (application keys, slugs, emails) is hashed into a bucket. return hashToShardId(id) } /** * Get all possible shard IDs (00-ff) * * Returns array of 256 shard IDs in ascending order. * Useful for iterating through all shards during pagination. * * @returns Array of 256 shard IDs * * @example * ```typescript * const shards = getAllShardIds() * // ['00', '01', '02', ..., 'fd', 'fe', 'ff'] * * for (const shardId of shards) { * const prefix = `entities/nouns/vectors/${shardId}/` * // List objects with this prefix * } * ``` */ export function getAllShardIds(): string[] { const shards: string[] = [] for (let i = 0; i < 256; i++) { shards.push(i.toString(16).padStart(2, '0')) } return shards } /** * Get shard ID for a given index (0-255) * * @param index - Shard index (0-255) * @returns 2-character hex shard ID * * @example * ```typescript * getShardIdByIndex(0) // '00' * getShardIdByIndex(15) // '0f' * getShardIdByIndex(255) // 'ff' * ``` */ export function getShardIdByIndex(index: number): string { if (index < 0 || index > 255) { throw new Error(`Shard index out of range: ${index} (expected 0-255)`) } return index.toString(16).padStart(2, '0') } /** * Get shard index from shard ID (0-255) * * @param shardId - 2-character hex shard ID * @returns Shard index (0-255) * * @example * ```typescript * getShardIndexFromId('00') // 0 * getShardIndexFromId('0f') // 15 * getShardIndexFromId('ff') // 255 * ``` */ export function getShardIndexFromId(shardId: string): number { if (!/^[0-9a-f]{2}$/.test(shardId)) { throw new Error(`Invalid shard ID: ${shardId} (expected 2 hex chars)`) } return parseInt(shardId, 16) } /** * Total number of shards in the system */ export const TOTAL_SHARDS = 256 /** * Shard configuration (read-only) */ export const SHARD_CONFIG = { /** * Total number of shards (256) */ count: TOTAL_SHARDS, /** * Number of hex characters used for sharding (2) */ prefixLength: 2, /** * Sharding method description */ method: 'uuid-prefix', /** * Whether sharding is always enabled */ alwaysEnabled: true } as const