soulcraft/brainy
2
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

fix: deterministic code-point string collation for column store + aggregation

Browse source 
Replace localeCompare (default-locale, non-deterministic across environments —
unsafe for a persisted sorted index) with UTF-8 byte / code-point order via a
shared compareCodePoints() helper. String ordering is now deterministic and
byte-identical to the native column store / aggregation sort, so results are the
same with or without the native accelerator. Covers the aggregate orderBy sort
and all 5 column-store comparison sites (tail buffer, merge sort, binary search).
Adds compareCodePoints unit tests.
This commit is contained in:
David Snelling 2026-05-26 17:35:18 -07:00
parent fe4f5df8c9
commit 547721ae14
6 changed files with 92 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

5
src/indexes/columnStore/ColumnStore.ts
View file

@ -30,6 +30,7 @@ import { ColumnManifest } from './ColumnManifest.js'
import { ColumnSegmentCursor, TailBufferCursor, type CursorEntry } from './ColumnSegmentCursor.js'
import { writeSegmentToBuffer, readSegmentFromBuffer } from './ColumnSegmentFormat.js'
import { RoaringBitmap32 } from '../../utils/roaring/index.js'
import { compareCodePoints } from '../../utils/collation.js'
/**
* Configuration for the ColumnStore.
@ -554,7 +555,7 @@ export class ColumnStore implements ColumnStoreProvider {
// Sort a copy for the cursor
const sorted = entries.slice().sort((a, b) => {
if (valueType === ValueType.String) {
return String(a.value).localeCompare(String(b.value))
return compareCodePoints(String(a.value), String(b.value))
}
return (a.value as number) - (b.value as number)
})
@ -615,7 +616,7 @@ export class ColumnStore implements ColumnStoreProvider {
const compare = (a: HeapEntry, b: HeapEntry): number => {
let cmp: number
if (isString) {
cmp = String(a.value).localeCompare(String(b.value))
cmp = compareCodePoints(String(a.value), String(b.value))
} else {
cmp = (a.value as number) - (b.value as number)
}