fix: code-point string collation in LSM SSTable, COW trees/refs, sorted queries

Replaces localeCompare / raw relational operators with compareCodePoints (UTF-8
byte order) in the remaining persisted or native-facing comparison sites:
- SSTable sourceId sort + zone-map range check + binary search (graph LSM)
- COW TreeObject + RefManager name sorts (reproducible content hashes and ref
  listings across environments)
- getSortedIdsForFilter (numeric-aware; code-point for strings) so the JS sort
  fallback matches the native column store exactly

Deterministic across OS/Node/ICU and byte-identical to the native engine. No
migration: COW serialize/deserialize preserve stored order, so existing trees
keep their hashes; only new writes adopt code-point order.

src/graph/lsm/SSTable.ts

@@ -17,6 +17,7 @@
 import { createHash } from 'node:crypto'
 import { encode as defaultEncode, decode as defaultDecode } from '@msgpack/msgpack'
 import { BloomFilter, SerializedBloomFilter } from './BloomFilter.js'
+import { compareCodePoints } from '../../utils/collation.js'
 
 // Swappable msgpack implementation — defaults to @msgpack/msgpack JS,
 // can be replaced with native msgpack (e.g., cortex's Rust-backed encoder)
@@ -181,8 +182,11 @@ export class SSTable {
    * @param id Unique ID for this SSTable
    */
   constructor(entries: SSTableEntry[], level: number = 0, id?: string) {
-    // Sort entries by sourceId for binary search
+    // Sort entries by sourceId for binary search. Code-point (UTF-8 byte) order —
-    this.entries = entries.sort((a, b) => a.sourceId.localeCompare(b.sourceId))
+    // deterministic across environments and identical to the native engine, unlike
+    // localeCompare. The zone map (isInRange) and binary search (get) below MUST use
+    // the same ordering or they will skip/miss entries.
+    this.entries = entries.sort((a, b) => compareCodePoints(a.sourceId, b.sourceId))
 
     // Calculate statistics
     const relationshipCount = entries.reduce(
@@ -243,8 +247,8 @@ export class SSTable {
     }
 
     return (
-      sourceId >= this.metadata.minSourceId &&
+      compareCodePoints(sourceId, this.metadata.minSourceId) >= 0 &&
-      sourceId <= this.metadata.maxSourceId
+      compareCodePoints(sourceId, this.metadata.maxSourceId) <= 0
     )
   }
 
@@ -271,7 +275,7 @@ export class SSTable {
     while (left <= right) {
       const mid = Math.floor((left + right) / 2)
       const entry = this.entries[mid]
-      const cmp = entry.sourceId.localeCompare(sourceId)
+      const cmp = compareCodePoints(entry.sourceId, sourceId)
 
       if (cmp === 0) {
         // Found it!

src/storage/cow/RefManager.ts

@@ -20,6 +20,7 @@
  */
 
 import type { COWStorageAdapter } from './BlobStorage.js'
+import { compareCodePoints } from '../../utils/collation.js'
 
 /**
  * Reference type
@@ -226,7 +227,7 @@ export class RefManager {
     // Mark cache as valid
     this.cacheValid = true
 
-    return refs.sort((a, b) => a.name.localeCompare(b.name))
+    return refs.sort((a, b) => compareCodePoints(a.name, b.name))
   }
 
   /**

src/storage/cow/TreeObject.ts

@@ -15,6 +15,7 @@
  */
 
 import { BlobStorage } from './BlobStorage.js'
+import { compareCodePoints } from '../../utils/collation.js'
 
 /**
  * Tree entry: name → blob hash mapping
@@ -98,7 +99,12 @@ export class TreeBuilder {
    */
   async build(): Promise<string> {
     const tree: TreeObject = {
-      entries: this.entries.sort((a, b) => a.name.localeCompare(b.name)),
+      // Code-point (UTF-8 byte) order so the serialized tree — and therefore its
+      // content hash — is reproducible across environments (localeCompare's default
+      // locale varies by OS/Node/ICU). Sorting happens only here at write time;
+      // deserialize() preserves stored order, so existing trees keep their hashes
+      // and stay valid — no migration needed.
+      entries: this.entries.sort((a, b) => compareCodePoints(a.name, b.name)),
       createdAt: Date.now()
     }
 

src/utils/metadataIndex.ts

@@ -7,6 +7,7 @@
 import { StorageAdapter, resolveEntityField } from '../coreTypes.js'
 import { ColumnStore } from '../indexes/columnStore/ColumnStore.js'
 import { MetadataIndexCache, MetadataIndexCacheConfig } from './metadataIndexCache.js'
+import { compareCodePoints } from './collation.js'
 import { prodLog } from './logger.js'
 import { getGlobalCache, UnifiedCache } from './unifiedCache.js'
 import {
@@ -2109,7 +2110,16 @@ export class MetadataIndexManager {
       if (a.value == null) return order === 'asc' ? 1 : -1
       if (b.value == null) return order === 'asc' ? -1 : 1
       if (a.value === b.value) return 0
-      const comparison = a.value < b.value ? -1 : 1
+      // Numbers compare numerically; everything else by code-point (UTF-8 byte) order.
+      // This makes the JS fallback sort match cortex's native column store exactly
+      // (numeric i64/f64 vs code-point strings) and stay deterministic across
+      // environments, unlike the `<` operator's UTF-16 ordering for strings.
+      let comparison: number
+      if (typeof a.value === 'number' && typeof b.value === 'number') {
+        comparison = a.value < b.value ? -1 : 1
+      } else {
+        comparison = compareCodePoints(String(a.value), String(b.value))
+      }
       return order === 'asc' ? comparison : -comparison
     })