fix: deterministic code-point string collation for column store + aggregation
Replace localeCompare (default-locale, non-deterministic across environments — unsafe for a persisted sorted index) with UTF-8 byte / code-point order via a shared compareCodePoints() helper. String ordering is now deterministic and byte-identical to the native column store / aggregation sort, so results are the same with or without the native accelerator. Covers the aggregate orderBy sort and all 5 column-store comparison sites (tail buffer, merge sort, binary search). Adds compareCodePoints unit tests.
This commit is contained in:
parent
fe4f5df8c9
commit
547721ae14
6 changed files with 92 additions and 6 deletions
45
tests/unit/utils/collation.test.ts
Normal file
45
tests/unit/utils/collation.test.ts
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
import { describe, it, expect } from 'vitest'
|
||||
import { compareCodePoints } from '../../../src/utils/collation.js'
|
||||
|
||||
/**
|
||||
* compareCodePoints must be byte-for-byte identical to Rust `str::cmp`
|
||||
* (UTF-8 byte order == Unicode code-point order), so the JS and native
|
||||
* (cortex) column store / aggregation sort agree, and persisted ordering
|
||||
* is deterministic across environments (unlike localeCompare).
|
||||
*/
|
||||
describe('compareCodePoints (UTF-8 byte / code-point order)', () => {
|
||||
it('orders mixed-case ASCII by byte value, NOT locale', () => {
|
||||
// 'B' = 0x42 (66), 'a' = 0x61 (97) → 'B' sorts before 'a' (localeCompare would not).
|
||||
expect(compareCodePoints('B', 'a')).toBeLessThan(0)
|
||||
expect(['a', 'B', 'A', 'b'].sort(compareCodePoints)).toEqual(['A', 'B', 'a', 'b'])
|
||||
})
|
||||
|
||||
it('orders ASCII below multi-byte UTF-8', () => {
|
||||
// 'z' = 0x7A, 'é' = 0xC3 0xA9 → 'z' < 'é'
|
||||
expect(compareCodePoints('z', 'é')).toBeLessThan(0)
|
||||
expect(compareCodePoints('é', 'z')).toBeGreaterThan(0)
|
||||
})
|
||||
|
||||
it('treats a prefix as less than the longer string', () => {
|
||||
expect(compareCodePoints('ab', 'abc')).toBeLessThan(0)
|
||||
expect(compareCodePoints('abc', 'ab')).toBeGreaterThan(0)
|
||||
})
|
||||
|
||||
it('returns 0 for equal strings', () => {
|
||||
expect(compareCodePoints('hello', 'hello')).toBe(0)
|
||||
expect(compareCodePoints('', '')).toBe(0)
|
||||
})
|
||||
|
||||
it('sorts by code point across the BMP boundary (matches Rust byte order)', () => {
|
||||
// '€' = U+20AC (0xE2 0x82 0xAC), '😀' = U+1F600 (0xF0 0x9F 0x98 0x80).
|
||||
// Code-point / UTF-8 byte order: 'a' < '€' < '😀'. (UTF-16 `<` would mis-order the astral char.)
|
||||
expect(['😀', 'a', '€'].sort(compareCodePoints)).toEqual(['a', '€', '😀'])
|
||||
})
|
||||
|
||||
it('is a total order consistent with sort (deterministic)', () => {
|
||||
const input = ['banana', 'Apple', 'apple', 'Banana', '101', 'Éclair', 'éclair']
|
||||
const a = input.slice().sort(compareCodePoints)
|
||||
const b = input.slice().reverse().sort(compareCodePoints)
|
||||
expect(a).toEqual(b) // order is independent of input order
|
||||
})
|
||||
})
|
||||
Loading…
Add table
Add a link
Reference in a new issue