diff --git a/src/graph/lsm/SSTable.ts b/src/graph/lsm/SSTable.ts index da64db98..e7ce7c89 100644 --- a/src/graph/lsm/SSTable.ts +++ b/src/graph/lsm/SSTable.ts @@ -17,6 +17,7 @@ import { createHash } from 'node:crypto' import { encode as defaultEncode, decode as defaultDecode } from '@msgpack/msgpack' import { BloomFilter, SerializedBloomFilter } from './BloomFilter.js' +import { compareCodePoints } from '../../utils/collation.js' // Swappable msgpack implementation — defaults to @msgpack/msgpack JS, // can be replaced with native msgpack (e.g., cortex's Rust-backed encoder) @@ -181,8 +182,11 @@ export class SSTable { * @param id Unique ID for this SSTable */ constructor(entries: SSTableEntry[], level: number = 0, id?: string) { - // Sort entries by sourceId for binary search - this.entries = entries.sort((a, b) => a.sourceId.localeCompare(b.sourceId)) + // Sort entries by sourceId for binary search. Code-point (UTF-8 byte) order — + // deterministic across environments and identical to the native engine, unlike + // localeCompare. The zone map (isInRange) and binary search (get) below MUST use + // the same ordering or they will skip/miss entries. + this.entries = entries.sort((a, b) => compareCodePoints(a.sourceId, b.sourceId)) // Calculate statistics const relationshipCount = entries.reduce( @@ -243,8 +247,8 @@ export class SSTable { } return ( - sourceId >= this.metadata.minSourceId && - sourceId <= this.metadata.maxSourceId + compareCodePoints(sourceId, this.metadata.minSourceId) >= 0 && + compareCodePoints(sourceId, this.metadata.maxSourceId) <= 0 ) } @@ -271,7 +275,7 @@ export class SSTable { while (left <= right) { const mid = Math.floor((left + right) / 2) const entry = this.entries[mid] - const cmp = entry.sourceId.localeCompare(sourceId) + const cmp = compareCodePoints(entry.sourceId, sourceId) if (cmp === 0) { // Found it! diff --git a/src/storage/cow/RefManager.ts b/src/storage/cow/RefManager.ts index 69203029..2c096702 100644 --- a/src/storage/cow/RefManager.ts +++ b/src/storage/cow/RefManager.ts @@ -20,6 +20,7 @@ */ import type { COWStorageAdapter } from './BlobStorage.js' +import { compareCodePoints } from '../../utils/collation.js' /** * Reference type @@ -226,7 +227,7 @@ export class RefManager { // Mark cache as valid this.cacheValid = true - return refs.sort((a, b) => a.name.localeCompare(b.name)) + return refs.sort((a, b) => compareCodePoints(a.name, b.name)) } /** diff --git a/src/storage/cow/TreeObject.ts b/src/storage/cow/TreeObject.ts index 4f2c20e9..9f73b057 100644 --- a/src/storage/cow/TreeObject.ts +++ b/src/storage/cow/TreeObject.ts @@ -15,6 +15,7 @@ */ import { BlobStorage } from './BlobStorage.js' +import { compareCodePoints } from '../../utils/collation.js' /** * Tree entry: name → blob hash mapping @@ -98,7 +99,12 @@ export class TreeBuilder { */ async build(): Promise { const tree: TreeObject = { - entries: this.entries.sort((a, b) => a.name.localeCompare(b.name)), + // Code-point (UTF-8 byte) order so the serialized tree — and therefore its + // content hash — is reproducible across environments (localeCompare's default + // locale varies by OS/Node/ICU). Sorting happens only here at write time; + // deserialize() preserves stored order, so existing trees keep their hashes + // and stay valid — no migration needed. + entries: this.entries.sort((a, b) => compareCodePoints(a.name, b.name)), createdAt: Date.now() } diff --git a/src/utils/metadataIndex.ts b/src/utils/metadataIndex.ts index 33c9476c..7db9e5fa 100644 --- a/src/utils/metadataIndex.ts +++ b/src/utils/metadataIndex.ts @@ -7,6 +7,7 @@ import { StorageAdapter, resolveEntityField } from '../coreTypes.js' import { ColumnStore } from '../indexes/columnStore/ColumnStore.js' import { MetadataIndexCache, MetadataIndexCacheConfig } from './metadataIndexCache.js' +import { compareCodePoints } from './collation.js' import { prodLog } from './logger.js' import { getGlobalCache, UnifiedCache } from './unifiedCache.js' import { @@ -2109,7 +2110,16 @@ export class MetadataIndexManager { if (a.value == null) return order === 'asc' ? 1 : -1 if (b.value == null) return order === 'asc' ? -1 : 1 if (a.value === b.value) return 0 - const comparison = a.value < b.value ? -1 : 1 + // Numbers compare numerically; everything else by code-point (UTF-8 byte) order. + // This makes the JS fallback sort match cortex's native column store exactly + // (numeric i64/f64 vs code-point strings) and stay deterministic across + // environments, unlike the `<` operator's UTF-16 ordering for strings. + let comparison: number + if (typeof a.value === 'number' && typeof b.value === 'number') { + comparison = a.value < b.value ? -1 : 1 + } else { + comparison = compareCodePoints(String(a.value), String(b.value)) + } return order === 'asc' ? comparison : -comparison })