soulcraft/brainy
2
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

fix: deterministic code-point string collation for column store + aggregation

Browse source 
Replace localeCompare (default-locale, non-deterministic across environments —
unsafe for a persisted sorted index) with UTF-8 byte / code-point order via a
shared compareCodePoints() helper. String ordering is now deterministic and
byte-identical to the native column store / aggregation sort, so results are the
same with or without the native accelerator. Covers the aggregate orderBy sort
and all 5 column-store comparison sites (tail buffer, merge sort, binary search).
Adds compareCodePoints unit tests.
This commit is contained in:
David Snelling 2026-05-26 17:35:18 -07:00
parent fe4f5df8c9
commit 547721ae14
6 changed files with 92 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

5
src/indexes/columnStore/ColumnSegmentCursor.ts
View file

@ -14,6 +14,7 @@
import type { SegmentHeader } from './types.js'
import { ValueType } from './types.js'
import type { RoaringBitmap32 } from '../../utils/roaring/index.js'
import { compareCodePoints } from '../../utils/collation.js'
/**
* Binary search result.
@ -100,7 +101,7 @@ export class ColumnSegmentCursor {
while (lo < hi) {
const mid = (lo + hi) >>> 1
const cmp = isString
? String(this.values[mid]).localeCompare(String(target))
? compareCodePoints(String(this.values[mid]), String(target))
: (this.values[mid] as number) - (target as number)
if (cmp < 0) {
@ -136,7 +137,7 @@ export class ColumnSegmentCursor {
while (endLo < endHi) {
const mid = (endLo + endHi) >>> 1
const cmp = isString
? String(this.values[mid]).localeCompare(String(hi))
? compareCodePoints(String(this.values[mid]), String(hi))
: (this.values[mid] as number) - (hi as number)
if (cmp <= 0) {
endLo = mid + 1