fix: deterministic code-point string collation for column store + aggregation

Replace localeCompare (default-locale, non-deterministic across environments —
unsafe for a persisted sorted index) with UTF-8 byte / code-point order via a
shared compareCodePoints() helper. String ordering is now deterministic and
byte-identical to the native column store / aggregation sort, so results are the
same with or without the native accelerator. Covers the aggregate orderBy sort
and all 5 column-store comparison sites (tail buffer, merge sort, binary search).
Adds compareCodePoints unit tests.

src/aggregation/AggregationIndex.ts

@@ -26,6 +26,7 @@ import type {
   MetricState
 } from '../types/brainy.types.js'
 import { matchesMetadataFilter } from '../utils/metadataFilter.js'
+import { compareCodePoints } from '../utils/collation.js'
 import { bucketTimestamp } from './timeWindows.js'
 import { NounType } from '../types/graphTypes.js'
 
@@ -685,7 +686,7 @@ export class AggregationIndex {
         if (typeof aVal === 'number' && typeof bVal === 'number') {
           return (aVal - bVal) * dir
         }
-        return String(aVal).localeCompare(String(bVal)) * dir
+        return compareCodePoints(String(aVal), String(bVal)) * dir
       })
     }
 

src/indexes/columnStore/ColumnSegmentCursor.ts

@@ -14,6 +14,7 @@
 import type { SegmentHeader } from './types.js'
 import { ValueType } from './types.js'
 import type { RoaringBitmap32 } from '../../utils/roaring/index.js'
+import { compareCodePoints } from '../../utils/collation.js'
 
 /**
  * Binary search result.
@@ -100,7 +101,7 @@ export class ColumnSegmentCursor {
     while (lo < hi) {
       const mid = (lo + hi) >>> 1
       const cmp = isString
-        ? String(this.values[mid]).localeCompare(String(target))
+        ? compareCodePoints(String(this.values[mid]), String(target))
         : (this.values[mid] as number) - (target as number)
 
       if (cmp < 0) {
@@ -136,7 +137,7 @@ export class ColumnSegmentCursor {
     while (endLo < endHi) {
       const mid = (endLo + endHi) >>> 1
       const cmp = isString
-        ? String(this.values[mid]).localeCompare(String(hi))
+        ? compareCodePoints(String(this.values[mid]), String(hi))
         : (this.values[mid] as number) - (hi as number)
       if (cmp <= 0) {
         endLo = mid + 1

src/indexes/columnStore/ColumnStore.ts

@@ -30,6 +30,7 @@ import { ColumnManifest } from './ColumnManifest.js'
 import { ColumnSegmentCursor, TailBufferCursor, type CursorEntry } from './ColumnSegmentCursor.js'
 import { writeSegmentToBuffer, readSegmentFromBuffer } from './ColumnSegmentFormat.js'
 import { RoaringBitmap32 } from '../../utils/roaring/index.js'
+import { compareCodePoints } from '../../utils/collation.js'
 
 /**
  * Configuration for the ColumnStore.
@@ -554,7 +555,7 @@ export class ColumnStore implements ColumnStoreProvider {
     // Sort a copy for the cursor
     const sorted = entries.slice().sort((a, b) => {
       if (valueType === ValueType.String) {
-        return String(a.value).localeCompare(String(b.value))
+        return compareCodePoints(String(a.value), String(b.value))
       }
       return (a.value as number) - (b.value as number)
     })
@@ -615,7 +616,7 @@ export class ColumnStore implements ColumnStoreProvider {
     const compare = (a: HeapEntry, b: HeapEntry): number => {
       let cmp: number
       if (isString) {
-        cmp = String(a.value).localeCompare(String(b.value))
+        cmp = compareCodePoints(String(a.value), String(b.value))
       } else {
         cmp = (a.value as number) - (b.value as number)
       }

src/indexes/columnStore/ColumnTailBuffer.ts

@@ -12,6 +12,7 @@
  */
 
 import { ValueType, DEFAULT_FLUSH_THRESHOLD } from './types.js'
+import { compareCodePoints } from '../../utils/collation.js'
 
 /**
  * Entry in the tail buffer: a (value, entityIntId) pair.
@@ -131,7 +132,7 @@ export class ColumnTailBuffer {
     const sorted = this.entries.slice()
     if (this.valueType === ValueType.String) {
       sorted.sort((a, b) => {
-        const cmp = String(a.value).localeCompare(String(b.value))
+        const cmp = compareCodePoints(String(a.value), String(b.value))
         return cmp !== 0 ? cmp : a.entityIntId - b.entityIntId
       })
     } else {

src/utils/collation.ts

@@ -0,0 +1,37 @@
+/**
+ * @module utils/collation
+ * @description Deterministic string ordering for persisted indexes and any comparison
+ * that must agree with the native (Rust) column store / aggregation engine.
+ *
+ * Strings are compared by **UTF-8 byte order**, which equals Unicode code-point order
+ * and is byte-for-byte identical to Rust's `str::cmp` (`as_bytes().cmp()`). This is:
+ * - **deterministic** across environments (unlike `localeCompare`, whose default-locale
+ *   ordering varies by OS / Node / ICU build — unsafe for a persisted sorted index), and
+ * - **cross-language consistent** with `@soulcraft/cortex`'s native column store and
+ *   aggregation sort, so query results are identical with or without the native plugin.
+ *
+ * Use this instead of `String.localeCompare` and the `<`/`>` operators (which compare
+ * UTF-16 code units and diverge from code-point order for supplementary-plane characters)
+ * wherever ordering is persisted or must match the native engine. See cortex `docs/ADR-001`.
+ */
+
+const utf8 = new TextEncoder()
+
+/**
+ * Compare two strings by UTF-8 byte order (== Unicode code-point order == Rust `str::cmp`).
+ *
+ * @param a - First string.
+ * @param b - Second string.
+ * @returns Negative if `a < b`, positive if `a > b`, `0` if equal — by UTF-8 byte order.
+ * @example
+ * ['b', 'A', 'a'].sort(compareCodePoints) // ['A', 'a', 'b']  (byte order: A=65, a=97, b=98)
+ */
+export function compareCodePoints(a: string, b: string): number {
+  const ea = utf8.encode(a)
+  const eb = utf8.encode(b)
+  const len = ea.length < eb.length ? ea.length : eb.length
+  for (let i = 0; i < len; i++) {
+    if (ea[i] !== eb[i]) return ea[i] - eb[i]
+  }
+  return ea.length - eb.length
+}

tests/unit/utils/collation.test.ts

@@ -0,0 +1,45 @@
+import { describe, it, expect } from 'vitest'
+import { compareCodePoints } from '../../../src/utils/collation.js'
+
+/**
+ * compareCodePoints must be byte-for-byte identical to Rust `str::cmp`
+ * (UTF-8 byte order == Unicode code-point order), so the JS and native
+ * (cortex) column store / aggregation sort agree, and persisted ordering
+ * is deterministic across environments (unlike localeCompare).
+ */
+describe('compareCodePoints (UTF-8 byte / code-point order)', () => {
+  it('orders mixed-case ASCII by byte value, NOT locale', () => {
+    // 'B' = 0x42 (66), 'a' = 0x61 (97) → 'B' sorts before 'a' (localeCompare would not).
+    expect(compareCodePoints('B', 'a')).toBeLessThan(0)
+    expect(['a', 'B', 'A', 'b'].sort(compareCodePoints)).toEqual(['A', 'B', 'a', 'b'])
+  })
+
+  it('orders ASCII below multi-byte UTF-8', () => {
+    // 'z' = 0x7A, 'é' = 0xC3 0xA9 → 'z' < 'é'
+    expect(compareCodePoints('z', 'é')).toBeLessThan(0)
+    expect(compareCodePoints('é', 'z')).toBeGreaterThan(0)
+  })
+
+  it('treats a prefix as less than the longer string', () => {
+    expect(compareCodePoints('ab', 'abc')).toBeLessThan(0)
+    expect(compareCodePoints('abc', 'ab')).toBeGreaterThan(0)
+  })
+
+  it('returns 0 for equal strings', () => {
+    expect(compareCodePoints('hello', 'hello')).toBe(0)
+    expect(compareCodePoints('', '')).toBe(0)
+  })
+
+  it('sorts by code point across the BMP boundary (matches Rust byte order)', () => {
+    // '€' = U+20AC (0xE2 0x82 0xAC), '😀' = U+1F600 (0xF0 0x9F 0x98 0x80).
+    // Code-point / UTF-8 byte order: 'a' < '€' < '😀'. (UTF-16 `<` would mis-order the astral char.)
+    expect(['😀', 'a', '€'].sort(compareCodePoints)).toEqual(['a', '€', '😀'])
+  })
+
+  it('is a total order consistent with sort (deterministic)', () => {
+    const input = ['banana', 'Apple', 'apple', 'Banana', '101', 'Éclair', 'éclair']
+    const a = input.slice().sort(compareCodePoints)
+    const b = input.slice().reverse().sort(compareCodePoints)
+    expect(a).toEqual(b) // order is independent of input order
+  })
+})