fix: deterministic code-point string collation for column store + aggregation
Replace localeCompare (default-locale, non-deterministic across environments — unsafe for a persisted sorted index) with UTF-8 byte / code-point order via a shared compareCodePoints() helper. String ordering is now deterministic and byte-identical to the native column store / aggregation sort, so results are the same with or without the native accelerator. Covers the aggregate orderBy sort and all 5 column-store comparison sites (tail buffer, merge sort, binary search). Adds compareCodePoints unit tests.
This commit is contained in:
parent
fe4f5df8c9
commit
547721ae14
6 changed files with 92 additions and 6 deletions
|
|
@ -12,6 +12,7 @@
|
|||
*/
|
||||
|
||||
import { ValueType, DEFAULT_FLUSH_THRESHOLD } from './types.js'
|
||||
import { compareCodePoints } from '../../utils/collation.js'
|
||||
|
||||
/**
|
||||
* Entry in the tail buffer: a (value, entityIntId) pair.
|
||||
|
|
@ -131,7 +132,7 @@ export class ColumnTailBuffer {
|
|||
const sorted = this.entries.slice()
|
||||
if (this.valueType === ValueType.String) {
|
||||
sorted.sort((a, b) => {
|
||||
const cmp = String(a.value).localeCompare(String(b.value))
|
||||
const cmp = compareCodePoints(String(a.value), String(b.value))
|
||||
return cmp !== 0 ? cmp : a.entityIntId - b.entityIntId
|
||||
})
|
||||
} else {
|
||||
|
|
|
|||
Reference in a new issue