diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts index e83b6841..53dcd901 100644 --- a/src/storage/baseStorage.ts +++ b/src/storage/baseStorage.ts @@ -25,7 +25,7 @@ import { NOUN_TYPE_COUNT, VERB_TYPE_COUNT } from '../types/graphTypes.js' -import { getShardIdFromUuid } from './sharding.js' +import { getShardId } from './sharding.js' import { BlobStorage, type BlobStoreAdapter } from './blobStorage.js' import { unwrapBinaryData } from './binaryDataCodec.js' import { prodLog } from '../utils/logger.js' @@ -161,7 +161,7 @@ function isSingletonSystemKey(key: string): boolean { * No type parameter needed - direct O(1) lookup by ID */ function getNounVectorPath(id: string): string { - const shard = getShardIdFromUuid(id) + const shard = getShardId(id) return `entities/nouns/${shard}/${id}/vectors.json` } @@ -170,7 +170,7 @@ function getNounVectorPath(id: string): string { * No type parameter needed - direct O(1) lookup by ID */ function getNounMetadataPath(id: string): string { - const shard = getShardIdFromUuid(id) + const shard = getShardId(id) return `entities/nouns/${shard}/${id}/metadata.json` } @@ -179,7 +179,7 @@ function getNounMetadataPath(id: string): string { * No type parameter needed - direct O(1) lookup by ID */ function getVerbVectorPath(id: string): string { - const shard = getShardIdFromUuid(id) + const shard = getShardId(id) return `entities/verbs/${shard}/${id}/vectors.json` } @@ -188,7 +188,7 @@ function getVerbVectorPath(id: string): string { * No type parameter needed - direct O(1) lookup by ID */ function getVerbMetadataPath(id: string): string { - const shard = getShardIdFromUuid(id) + const shard = getShardId(id) return `entities/verbs/${shard}/${id}/metadata.json` } @@ -392,7 +392,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { } // Valid entity UUID - apply sharding - const shardId = getShardIdFromUuid(id) + const shardId = getShardId(id) if (context === 'noun-metadata') { return { diff --git a/src/storage/sharding.ts b/src/storage/sharding.ts index 96ff2c30..f13791dc 100644 --- a/src/storage/sharding.ts +++ b/src/storage/sharding.ts @@ -1,55 +1,77 @@ /** - * Unified UUID-based sharding for all storage adapters + * Unified id-based sharding for all storage adapters. * - * Uses first 2 hex characters of UUID for consistent, predictable sharding - * that scales from hundreds to millions of entities without configuration. + * Maps any entity id to one of 256 buckets (00-ff) for consistent, predictable + * on-disk layout that scales from hundreds to millions of entities with no + * configuration. * * Sharding characteristics: * - 256 buckets (00-ff) - * - Deterministic (same UUID always maps to same shard) + * - Deterministic (the same id always maps to the same shard) * - No configuration required - * - Works across all storage types (filesystem, S3, GCS, memory) + * - Works across the filesystem and memory adapters * - Efficient for list operations and pagination + * + * Two id shapes are supported, by design: + * - **UUID-format ids** (32 hex chars) shard by their first byte. This is the + * original scheme, kept byte-for-byte so existing on-disk layouts never move. + * - **Application ids** (`'user-123'`, slugs, emails — anything else) shard by a + * stable FNV-1a hash. Requiring callers to mint UUIDs would break the + * documented `add({ id: '…' })` contract; a shard is only a bucket, so any + * deterministic, well-distributed mapping is correct. + * + * The hash is part of the on-disk contract: **never change it** — doing so would + * relocate every application-id record. */ /** - * Extract shard ID from UUID + * 32-bit FNV-1a hash of a string, returned as a 2-hex-char shard bucket (00-ff). + * Deterministic and uniformly distributed across the 256 buckets. * - * Uses first 2 hex characters of the UUID as the shard ID. - * This provides 256 evenly-distributed buckets (00-ff). + * @param value - The string to hash (an entity id that is not UUID-format). + * @returns A 2-character hex shard id. + */ +function hashToShardId(value: string): string { + let hash = 0x811c9dc5 // FNV offset basis + for (let i = 0; i < value.length; i++) { + hash ^= value.charCodeAt(i) + hash = Math.imul(hash, 0x01000193) // FNV prime + } + return ((hash >>> 0) & 0xff).toString(16).padStart(2, '0') +} + +/** + * Resolve the shard bucket (00-ff) for an entity id. * - * @param uuid - UUID string (with or without hyphens) - * @returns 2-character hex shard ID (00-ff) + * UUID-format ids (32 hex chars, with or without hyphens) shard by their first + * byte — preserving the original on-disk layout. Any other id is hashed (FNV-1a) + * into the same 256-bucket space, so application-supplied ids like `'user-123'` + * work without forcing callers to mint UUIDs. + * + * @param id - The entity id (UUID-format or any application string). + * @returns A 2-character hex shard id (00-ff). + * @throws If `id` is empty. * * @example * ```typescript - * getShardIdFromUuid('ab123456-1234-5678-9abc-def012345678') // returns 'ab' - * getShardIdFromUuid('cd987654-4321-8765-cba9-fed543210987') // returns 'cd' - * getShardIdFromUuid('00000000-0000-0000-0000-000000000000') // returns '00' + * getShardId('ab123456-1234-5678-9abc-def012345678') // 'ab' (UUID first byte) + * getShardId('user-12345') // FNV-1a hash bucket * ``` */ -export function getShardIdFromUuid(uuid: string): string { - if (!uuid) { - throw new Error('UUID is required for sharding') +export function getShardId(id: string): string { + if (!id) { + throw new Error('id is required for sharding') } - // Remove hyphens and convert to lowercase - const normalized = uuid.toLowerCase().replace(/-/g, '') - - // Validate UUID format (32 hex characters) - if (normalized.length !== 32) { - throw new Error(`Invalid UUID format: ${uuid} (expected 32 hex chars, got ${normalized.length})`) + // UUID-format ids (32 hex chars) keep the original first-byte bucketing so + // existing on-disk data is never relocated. + const normalized = id.toLowerCase().replace(/-/g, '') + if (/^[0-9a-f]{32}$/.test(normalized)) { + return normalized.substring(0, 2) } - // Extract first 2 characters - const shardId = normalized.substring(0, 2) - - // Validate hex format - if (!/^[0-9a-f]{2}$/.test(shardId)) { - throw new Error(`Invalid UUID prefix: ${shardId} (expected 2 hex chars)`) - } - - return shardId + // Any other id (application keys, slugs, emails) is hashed into a bucket. + return hashToShardId(id) } /** diff --git a/tests/unit/storage/sharding.test.ts b/tests/unit/storage/sharding.test.ts new file mode 100644 index 00000000..3b85a695 --- /dev/null +++ b/tests/unit/storage/sharding.test.ts @@ -0,0 +1,52 @@ +/** + * Shard-bucket resolution for entity ids. + * + * `getShardId` buckets every entity id into one of 256 dirs (00-ff). It must + * accept BOTH UUID-format ids (bucketed by first byte — the original on-disk + * layout) AND arbitrary application ids like `'user-123'` (bucketed by a stable + * hash). A refactor once required UUID format and threw on everything else, + * which broke the documented `add({ id: '…' })` contract. These tests pin the + * dual scheme, determinism, and distribution. + */ +import { describe, it, expect } from 'vitest' +import { getShardId, getAllShardIds } from '../../../src/storage/sharding.js' + +describe('getShardId', () => { + it('buckets a UUID by its first byte (with or without hyphens)', () => { + expect(getShardId('ab123456-1234-5678-9abc-def012345678')).toBe('ab') + expect(getShardId('ab123456123456789abcdef012345678')).toBe('ab') + expect(getShardId('CD987654-4321-8765-cba9-fed543210987')).toBe('cd') // case-insensitive + expect(getShardId('00000000-0000-0000-0000-000000000000')).toBe('00') + }) + + it('buckets an arbitrary application id without throwing (the regression)', () => { + const shard = getShardId('user-12345') + expect(shard).toMatch(/^[0-9a-f]{2}$/) + }) + + it('is deterministic for application ids', () => { + expect(getShardId('order-42')).toBe(getShardId('order-42')) + expect(getShardId('alice@example.com')).toBe(getShardId('alice@example.com')) + }) + + it('distributes application ids across many buckets (not one hot bucket)', () => { + const buckets = new Set() + for (let i = 0; i < 1000; i++) { + buckets.add(getShardId(`entity-${i}`)) + } + // FNV-1a over 1000 sequential keys should touch a large fraction of the + // 256-bucket space; a degenerate hash would collapse to a handful. + expect(buckets.size).toBeGreaterThan(200) + }) + + it('every produced shard is a valid bucket id', () => { + const valid = new Set(getAllShardIds()) + for (const sample of ['user-1', 'café', '🚀', 'a', 'A_VERY_LONG_APPLICATION_KEY_0001']) { + expect(valid.has(getShardId(sample))).toBe(true) + } + }) + + it('throws on an empty id', () => { + expect(() => getShardId('')).toThrow(/id is required/) + }) +})