brainy/src/storage/sharding.ts

/**
 * Unified id-based sharding for all storage adapters.
 *
 * Maps any entity id to one of 256 buckets (00-ff) for consistent, predictable
 * on-disk layout that scales from hundreds to millions of entities with no
 * configuration.
 *
 * Sharding characteristics:
 * - 256 buckets (00-ff)
 * - Deterministic (the same id always maps to the same shard)
 * - No configuration required
 * - Works across the filesystem and memory adapters
 * - Efficient for list operations and pagination
 *
 * Two id shapes are supported, by design:
 * - **UUID-format ids** (32 hex chars) shard by their first byte. This is the
 *   original scheme, kept byte-for-byte so existing on-disk layouts never move.
 * - **Application ids** (`'user-123'`, slugs, emails — anything else) shard by a
 *   stable FNV-1a hash. Requiring callers to mint UUIDs would break the
 *   documented `add({ id: '…' })` contract; a shard is only a bucket, so any
 *   deterministic, well-distributed mapping is correct.
 *
 * The hash is part of the on-disk contract: **never change it** — doing so would
 * relocate every application-id record.
 */

/**
 * 32-bit FNV-1a hash of a string, returned as a 2-hex-char shard bucket (00-ff).
 * Deterministic and uniformly distributed across the 256 buckets.
 *
 * @param value - The string to hash (an entity id that is not UUID-format).
 * @returns A 2-character hex shard id.
 */
function hashToShardId(value: string): string {
  let hash = 0x811c9dc5 // FNV offset basis
  for (let i = 0; i < value.length; i++) {
    hash ^= value.charCodeAt(i)
    hash = Math.imul(hash, 0x01000193) // FNV prime
  }
  return ((hash >>> 0) & 0xff).toString(16).padStart(2, '0')
}

/**
 * Resolve the shard bucket (00-ff) for an entity id.
 *
 * UUID-format ids (32 hex chars, with or without hyphens) shard by their first
 * byte — preserving the original on-disk layout. Any other id is hashed (FNV-1a)
 * into the same 256-bucket space, so application-supplied ids like `'user-123'`
 * work without forcing callers to mint UUIDs.
 *
 * @param id - The entity id (UUID-format or any application string).
 * @returns A 2-character hex shard id (00-ff).
 * @throws If `id` is empty.
 *
 * @example
 * ```typescript
 * getShardId('ab123456-1234-5678-9abc-def012345678') // 'ab' (UUID first byte)
 * getShardId('user-12345')                            // FNV-1a hash bucket
 * ```
 */
export function getShardId(id: string): string {
  if (!id) {
    throw new Error('id is required for sharding')
  }

  // UUID-format ids (32 hex chars) keep the original first-byte bucketing so
  // existing on-disk data is never relocated.
  const normalized = id.toLowerCase().replace(/-/g, '')
  if (/^[0-9a-f]{32}$/.test(normalized)) {
    return normalized.substring(0, 2)
  }

  // Any other id (application keys, slugs, emails) is hashed into a bucket.
  return hashToShardId(id)
}

/**
 * Get all possible shard IDs (00-ff)
 *
 * Returns array of 256 shard IDs in ascending order.
 * Useful for iterating through all shards during pagination.
 *
 * @returns Array of 256 shard IDs
 *
 * @example
 * ```typescript
 * const shards = getAllShardIds()
 * // ['00', '01', '02', ..., 'fd', 'fe', 'ff']
 *
 * for (const shardId of shards) {
 *   const prefix = `entities/nouns/vectors/${shardId}/`
 *   // List objects with this prefix
 * }
 * ```
 */
export function getAllShardIds(): string[] {
  const shards: string[] = []
  for (let i = 0; i < 256; i++) {
    shards.push(i.toString(16).padStart(2, '0'))
  }
  return shards
}

/**
 * Get shard ID for a given index (0-255)
 *
 * @param index - Shard index (0-255)
 * @returns 2-character hex shard ID
 *
 * @example
 * ```typescript
 * getShardIdByIndex(0)   // '00'
 * getShardIdByIndex(15)  // '0f'
 * getShardIdByIndex(255) // 'ff'
 * ```
 */
export function getShardIdByIndex(index: number): string {
  if (index < 0 || index > 255) {
    throw new Error(`Shard index out of range: ${index} (expected 0-255)`)
  }
  return index.toString(16).padStart(2, '0')
}

/**
 * Get shard index from shard ID (0-255)
 *
 * @param shardId - 2-character hex shard ID
 * @returns Shard index (0-255)
 *
 * @example
 * ```typescript
 * getShardIndexFromId('00')  // 0
 * getShardIndexFromId('0f')  // 15
 * getShardIndexFromId('ff')  // 255
 * ```
 */
export function getShardIndexFromId(shardId: string): number {
  if (!/^[0-9a-f]{2}$/.test(shardId)) {
    throw new Error(`Invalid shard ID: ${shardId} (expected 2 hex chars)`)
  }
  return parseInt(shardId, 16)
}

/**
 * Total number of shards in the system
 */
export const TOTAL_SHARDS = 256

/**
 * Shard configuration (read-only)
 */
export const SHARD_CONFIG = {
  /**
   * Total number of shards (256)
   */
  count: TOTAL_SHARDS,

  /**
   * Number of hex characters used for sharding (2)
   */
  prefixLength: 2,

  /**
   * Sharding method description
   */
  method: 'uuid-prefix',

  /**
   * Whether sharding is always enabled
   */
  alwaysEnabled: true
} as const