fix(8.0): accept application-supplied entity ids, not just UUIDs
Sharding required a 32-hex UUID and threw "Invalid UUID format" on every non-UUID id, even though add()'s own docs show `id: "user-12345"` and the u64 id-mapper happily assigns ints to any string. So custom ids mapped fine in memory then threw on save — breaking documented usage and any 7.x consumer using friendly ids (`'user-123'`, slugs, emails) on upgrade. getShardId() (renamed from getShardIdFromUuid) now buckets UUID-format ids by their first byte (on-disk layout unchanged, so existing data never moves) and hashes any other id via FNV-1a into the same 256-bucket space. The hash is part of the on-disk contract and must not change. Verbs stay Brainy-generated UUIDs by contract; only custom noun ids take the hash path. Adds getShardId unit coverage: dual scheme, determinism, even distribution across buckets, and the empty-id guard.
This commit is contained in:
parent
5096f90fbc
commit
36b7216929
3 changed files with 111 additions and 37 deletions
|
|
@ -25,7 +25,7 @@ import {
|
|||
NOUN_TYPE_COUNT,
|
||||
VERB_TYPE_COUNT
|
||||
} from '../types/graphTypes.js'
|
||||
import { getShardIdFromUuid } from './sharding.js'
|
||||
import { getShardId } from './sharding.js'
|
||||
import { BlobStorage, type BlobStoreAdapter } from './blobStorage.js'
|
||||
import { unwrapBinaryData } from './binaryDataCodec.js'
|
||||
import { prodLog } from '../utils/logger.js'
|
||||
|
|
@ -161,7 +161,7 @@ function isSingletonSystemKey(key: string): boolean {
|
|||
* No type parameter needed - direct O(1) lookup by ID
|
||||
*/
|
||||
function getNounVectorPath(id: string): string {
|
||||
const shard = getShardIdFromUuid(id)
|
||||
const shard = getShardId(id)
|
||||
return `entities/nouns/${shard}/${id}/vectors.json`
|
||||
}
|
||||
|
||||
|
|
@ -170,7 +170,7 @@ function getNounVectorPath(id: string): string {
|
|||
* No type parameter needed - direct O(1) lookup by ID
|
||||
*/
|
||||
function getNounMetadataPath(id: string): string {
|
||||
const shard = getShardIdFromUuid(id)
|
||||
const shard = getShardId(id)
|
||||
return `entities/nouns/${shard}/${id}/metadata.json`
|
||||
}
|
||||
|
||||
|
|
@ -179,7 +179,7 @@ function getNounMetadataPath(id: string): string {
|
|||
* No type parameter needed - direct O(1) lookup by ID
|
||||
*/
|
||||
function getVerbVectorPath(id: string): string {
|
||||
const shard = getShardIdFromUuid(id)
|
||||
const shard = getShardId(id)
|
||||
return `entities/verbs/${shard}/${id}/vectors.json`
|
||||
}
|
||||
|
||||
|
|
@ -188,7 +188,7 @@ function getVerbVectorPath(id: string): string {
|
|||
* No type parameter needed - direct O(1) lookup by ID
|
||||
*/
|
||||
function getVerbMetadataPath(id: string): string {
|
||||
const shard = getShardIdFromUuid(id)
|
||||
const shard = getShardId(id)
|
||||
return `entities/verbs/${shard}/${id}/metadata.json`
|
||||
}
|
||||
|
||||
|
|
@ -392,7 +392,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
}
|
||||
|
||||
// Valid entity UUID - apply sharding
|
||||
const shardId = getShardIdFromUuid(id)
|
||||
const shardId = getShardId(id)
|
||||
|
||||
if (context === 'noun-metadata') {
|
||||
return {
|
||||
|
|
|
|||
|
|
@ -1,55 +1,77 @@
|
|||
/**
|
||||
* Unified UUID-based sharding for all storage adapters
|
||||
* Unified id-based sharding for all storage adapters.
|
||||
*
|
||||
* Uses first 2 hex characters of UUID for consistent, predictable sharding
|
||||
* that scales from hundreds to millions of entities without configuration.
|
||||
* Maps any entity id to one of 256 buckets (00-ff) for consistent, predictable
|
||||
* on-disk layout that scales from hundreds to millions of entities with no
|
||||
* configuration.
|
||||
*
|
||||
* Sharding characteristics:
|
||||
* - 256 buckets (00-ff)
|
||||
* - Deterministic (same UUID always maps to same shard)
|
||||
* - Deterministic (the same id always maps to the same shard)
|
||||
* - No configuration required
|
||||
* - Works across all storage types (filesystem, S3, GCS, memory)
|
||||
* - Works across the filesystem and memory adapters
|
||||
* - Efficient for list operations and pagination
|
||||
*
|
||||
* Two id shapes are supported, by design:
|
||||
* - **UUID-format ids** (32 hex chars) shard by their first byte. This is the
|
||||
* original scheme, kept byte-for-byte so existing on-disk layouts never move.
|
||||
* - **Application ids** (`'user-123'`, slugs, emails — anything else) shard by a
|
||||
* stable FNV-1a hash. Requiring callers to mint UUIDs would break the
|
||||
* documented `add({ id: '…' })` contract; a shard is only a bucket, so any
|
||||
* deterministic, well-distributed mapping is correct.
|
||||
*
|
||||
* The hash is part of the on-disk contract: **never change it** — doing so would
|
||||
* relocate every application-id record.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Extract shard ID from UUID
|
||||
* 32-bit FNV-1a hash of a string, returned as a 2-hex-char shard bucket (00-ff).
|
||||
* Deterministic and uniformly distributed across the 256 buckets.
|
||||
*
|
||||
* Uses first 2 hex characters of the UUID as the shard ID.
|
||||
* This provides 256 evenly-distributed buckets (00-ff).
|
||||
* @param value - The string to hash (an entity id that is not UUID-format).
|
||||
* @returns A 2-character hex shard id.
|
||||
*/
|
||||
function hashToShardId(value: string): string {
|
||||
let hash = 0x811c9dc5 // FNV offset basis
|
||||
for (let i = 0; i < value.length; i++) {
|
||||
hash ^= value.charCodeAt(i)
|
||||
hash = Math.imul(hash, 0x01000193) // FNV prime
|
||||
}
|
||||
return ((hash >>> 0) & 0xff).toString(16).padStart(2, '0')
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the shard bucket (00-ff) for an entity id.
|
||||
*
|
||||
* @param uuid - UUID string (with or without hyphens)
|
||||
* @returns 2-character hex shard ID (00-ff)
|
||||
* UUID-format ids (32 hex chars, with or without hyphens) shard by their first
|
||||
* byte — preserving the original on-disk layout. Any other id is hashed (FNV-1a)
|
||||
* into the same 256-bucket space, so application-supplied ids like `'user-123'`
|
||||
* work without forcing callers to mint UUIDs.
|
||||
*
|
||||
* @param id - The entity id (UUID-format or any application string).
|
||||
* @returns A 2-character hex shard id (00-ff).
|
||||
* @throws If `id` is empty.
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* getShardIdFromUuid('ab123456-1234-5678-9abc-def012345678') // returns 'ab'
|
||||
* getShardIdFromUuid('cd987654-4321-8765-cba9-fed543210987') // returns 'cd'
|
||||
* getShardIdFromUuid('00000000-0000-0000-0000-000000000000') // returns '00'
|
||||
* getShardId('ab123456-1234-5678-9abc-def012345678') // 'ab' (UUID first byte)
|
||||
* getShardId('user-12345') // FNV-1a hash bucket
|
||||
* ```
|
||||
*/
|
||||
export function getShardIdFromUuid(uuid: string): string {
|
||||
if (!uuid) {
|
||||
throw new Error('UUID is required for sharding')
|
||||
export function getShardId(id: string): string {
|
||||
if (!id) {
|
||||
throw new Error('id is required for sharding')
|
||||
}
|
||||
|
||||
// Remove hyphens and convert to lowercase
|
||||
const normalized = uuid.toLowerCase().replace(/-/g, '')
|
||||
|
||||
// Validate UUID format (32 hex characters)
|
||||
if (normalized.length !== 32) {
|
||||
throw new Error(`Invalid UUID format: ${uuid} (expected 32 hex chars, got ${normalized.length})`)
|
||||
// UUID-format ids (32 hex chars) keep the original first-byte bucketing so
|
||||
// existing on-disk data is never relocated.
|
||||
const normalized = id.toLowerCase().replace(/-/g, '')
|
||||
if (/^[0-9a-f]{32}$/.test(normalized)) {
|
||||
return normalized.substring(0, 2)
|
||||
}
|
||||
|
||||
// Extract first 2 characters
|
||||
const shardId = normalized.substring(0, 2)
|
||||
|
||||
// Validate hex format
|
||||
if (!/^[0-9a-f]{2}$/.test(shardId)) {
|
||||
throw new Error(`Invalid UUID prefix: ${shardId} (expected 2 hex chars)`)
|
||||
}
|
||||
|
||||
return shardId
|
||||
// Any other id (application keys, slugs, emails) is hashed into a bucket.
|
||||
return hashToShardId(id)
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue