From 547721ae14be09b36d39e0244b9fb2c26913e567 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Tue, 26 May 2026 17:35:18 -0700 Subject: [PATCH] fix: deterministic code-point string collation for column store + aggregation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace localeCompare (default-locale, non-deterministic across environments — unsafe for a persisted sorted index) with UTF-8 byte / code-point order via a shared compareCodePoints() helper. String ordering is now deterministic and byte-identical to the native column store / aggregation sort, so results are the same with or without the native accelerator. Covers the aggregate orderBy sort and all 5 column-store comparison sites (tail buffer, merge sort, binary search). Adds compareCodePoints unit tests. --- src/aggregation/AggregationIndex.ts | 3 +- .../columnStore/ColumnSegmentCursor.ts | 5 ++- src/indexes/columnStore/ColumnStore.ts | 5 ++- src/indexes/columnStore/ColumnTailBuffer.ts | 3 +- src/utils/collation.ts | 37 +++++++++++++++ tests/unit/utils/collation.test.ts | 45 +++++++++++++++++++ 6 files changed, 92 insertions(+), 6 deletions(-) create mode 100644 src/utils/collation.ts create mode 100644 tests/unit/utils/collation.test.ts diff --git a/src/aggregation/AggregationIndex.ts b/src/aggregation/AggregationIndex.ts index 36cc6645..c04d6374 100644 --- a/src/aggregation/AggregationIndex.ts +++ b/src/aggregation/AggregationIndex.ts @@ -26,6 +26,7 @@ import type { MetricState } from '../types/brainy.types.js' import { matchesMetadataFilter } from '../utils/metadataFilter.js' +import { compareCodePoints } from '../utils/collation.js' import { bucketTimestamp } from './timeWindows.js' import { NounType } from '../types/graphTypes.js' @@ -685,7 +686,7 @@ export class AggregationIndex { if (typeof aVal === 'number' && typeof bVal === 'number') { return (aVal - bVal) * dir } - return String(aVal).localeCompare(String(bVal)) * dir + return compareCodePoints(String(aVal), String(bVal)) * dir }) } diff --git a/src/indexes/columnStore/ColumnSegmentCursor.ts b/src/indexes/columnStore/ColumnSegmentCursor.ts index 50395dfe..2990994e 100644 --- a/src/indexes/columnStore/ColumnSegmentCursor.ts +++ b/src/indexes/columnStore/ColumnSegmentCursor.ts @@ -14,6 +14,7 @@ import type { SegmentHeader } from './types.js' import { ValueType } from './types.js' import type { RoaringBitmap32 } from '../../utils/roaring/index.js' +import { compareCodePoints } from '../../utils/collation.js' /** * Binary search result. @@ -100,7 +101,7 @@ export class ColumnSegmentCursor { while (lo < hi) { const mid = (lo + hi) >>> 1 const cmp = isString - ? String(this.values[mid]).localeCompare(String(target)) + ? compareCodePoints(String(this.values[mid]), String(target)) : (this.values[mid] as number) - (target as number) if (cmp < 0) { @@ -136,7 +137,7 @@ export class ColumnSegmentCursor { while (endLo < endHi) { const mid = (endLo + endHi) >>> 1 const cmp = isString - ? String(this.values[mid]).localeCompare(String(hi)) + ? compareCodePoints(String(this.values[mid]), String(hi)) : (this.values[mid] as number) - (hi as number) if (cmp <= 0) { endLo = mid + 1 diff --git a/src/indexes/columnStore/ColumnStore.ts b/src/indexes/columnStore/ColumnStore.ts index b8e755c6..6b264463 100644 --- a/src/indexes/columnStore/ColumnStore.ts +++ b/src/indexes/columnStore/ColumnStore.ts @@ -30,6 +30,7 @@ import { ColumnManifest } from './ColumnManifest.js' import { ColumnSegmentCursor, TailBufferCursor, type CursorEntry } from './ColumnSegmentCursor.js' import { writeSegmentToBuffer, readSegmentFromBuffer } from './ColumnSegmentFormat.js' import { RoaringBitmap32 } from '../../utils/roaring/index.js' +import { compareCodePoints } from '../../utils/collation.js' /** * Configuration for the ColumnStore. @@ -554,7 +555,7 @@ export class ColumnStore implements ColumnStoreProvider { // Sort a copy for the cursor const sorted = entries.slice().sort((a, b) => { if (valueType === ValueType.String) { - return String(a.value).localeCompare(String(b.value)) + return compareCodePoints(String(a.value), String(b.value)) } return (a.value as number) - (b.value as number) }) @@ -615,7 +616,7 @@ export class ColumnStore implements ColumnStoreProvider { const compare = (a: HeapEntry, b: HeapEntry): number => { let cmp: number if (isString) { - cmp = String(a.value).localeCompare(String(b.value)) + cmp = compareCodePoints(String(a.value), String(b.value)) } else { cmp = (a.value as number) - (b.value as number) } diff --git a/src/indexes/columnStore/ColumnTailBuffer.ts b/src/indexes/columnStore/ColumnTailBuffer.ts index d2bb7c7e..c5874ac2 100644 --- a/src/indexes/columnStore/ColumnTailBuffer.ts +++ b/src/indexes/columnStore/ColumnTailBuffer.ts @@ -12,6 +12,7 @@ */ import { ValueType, DEFAULT_FLUSH_THRESHOLD } from './types.js' +import { compareCodePoints } from '../../utils/collation.js' /** * Entry in the tail buffer: a (value, entityIntId) pair. @@ -131,7 +132,7 @@ export class ColumnTailBuffer { const sorted = this.entries.slice() if (this.valueType === ValueType.String) { sorted.sort((a, b) => { - const cmp = String(a.value).localeCompare(String(b.value)) + const cmp = compareCodePoints(String(a.value), String(b.value)) return cmp !== 0 ? cmp : a.entityIntId - b.entityIntId }) } else { diff --git a/src/utils/collation.ts b/src/utils/collation.ts new file mode 100644 index 00000000..f56b0940 --- /dev/null +++ b/src/utils/collation.ts @@ -0,0 +1,37 @@ +/** + * @module utils/collation + * @description Deterministic string ordering for persisted indexes and any comparison + * that must agree with the native (Rust) column store / aggregation engine. + * + * Strings are compared by **UTF-8 byte order**, which equals Unicode code-point order + * and is byte-for-byte identical to Rust's `str::cmp` (`as_bytes().cmp()`). This is: + * - **deterministic** across environments (unlike `localeCompare`, whose default-locale + * ordering varies by OS / Node / ICU build — unsafe for a persisted sorted index), and + * - **cross-language consistent** with `@soulcraft/cortex`'s native column store and + * aggregation sort, so query results are identical with or without the native plugin. + * + * Use this instead of `String.localeCompare` and the `<`/`>` operators (which compare + * UTF-16 code units and diverge from code-point order for supplementary-plane characters) + * wherever ordering is persisted or must match the native engine. See cortex `docs/ADR-001`. + */ + +const utf8 = new TextEncoder() + +/** + * Compare two strings by UTF-8 byte order (== Unicode code-point order == Rust `str::cmp`). + * + * @param a - First string. + * @param b - Second string. + * @returns Negative if `a < b`, positive if `a > b`, `0` if equal — by UTF-8 byte order. + * @example + * ['b', 'A', 'a'].sort(compareCodePoints) // ['A', 'a', 'b'] (byte order: A=65, a=97, b=98) + */ +export function compareCodePoints(a: string, b: string): number { + const ea = utf8.encode(a) + const eb = utf8.encode(b) + const len = ea.length < eb.length ? ea.length : eb.length + for (let i = 0; i < len; i++) { + if (ea[i] !== eb[i]) return ea[i] - eb[i] + } + return ea.length - eb.length +} diff --git a/tests/unit/utils/collation.test.ts b/tests/unit/utils/collation.test.ts new file mode 100644 index 00000000..09f4e4b7 --- /dev/null +++ b/tests/unit/utils/collation.test.ts @@ -0,0 +1,45 @@ +import { describe, it, expect } from 'vitest' +import { compareCodePoints } from '../../../src/utils/collation.js' + +/** + * compareCodePoints must be byte-for-byte identical to Rust `str::cmp` + * (UTF-8 byte order == Unicode code-point order), so the JS and native + * (cortex) column store / aggregation sort agree, and persisted ordering + * is deterministic across environments (unlike localeCompare). + */ +describe('compareCodePoints (UTF-8 byte / code-point order)', () => { + it('orders mixed-case ASCII by byte value, NOT locale', () => { + // 'B' = 0x42 (66), 'a' = 0x61 (97) → 'B' sorts before 'a' (localeCompare would not). + expect(compareCodePoints('B', 'a')).toBeLessThan(0) + expect(['a', 'B', 'A', 'b'].sort(compareCodePoints)).toEqual(['A', 'B', 'a', 'b']) + }) + + it('orders ASCII below multi-byte UTF-8', () => { + // 'z' = 0x7A, 'é' = 0xC3 0xA9 → 'z' < 'é' + expect(compareCodePoints('z', 'é')).toBeLessThan(0) + expect(compareCodePoints('é', 'z')).toBeGreaterThan(0) + }) + + it('treats a prefix as less than the longer string', () => { + expect(compareCodePoints('ab', 'abc')).toBeLessThan(0) + expect(compareCodePoints('abc', 'ab')).toBeGreaterThan(0) + }) + + it('returns 0 for equal strings', () => { + expect(compareCodePoints('hello', 'hello')).toBe(0) + expect(compareCodePoints('', '')).toBe(0) + }) + + it('sorts by code point across the BMP boundary (matches Rust byte order)', () => { + // '€' = U+20AC (0xE2 0x82 0xAC), '😀' = U+1F600 (0xF0 0x9F 0x98 0x80). + // Code-point / UTF-8 byte order: 'a' < '€' < '😀'. (UTF-16 `<` would mis-order the astral char.) + expect(['😀', 'a', '€'].sort(compareCodePoints)).toEqual(['a', '€', '😀']) + }) + + it('is a total order consistent with sort (deterministic)', () => { + const input = ['banana', 'Apple', 'apple', 'Banana', '101', 'Éclair', 'éclair'] + const a = input.slice().sort(compareCodePoints) + const b = input.slice().reverse().sort(compareCodePoints) + expect(a).toEqual(b) // order is independent of input order + }) +})