Skip to content

Commit 5d29560

Browse files
committed
[SPARK-32485][SQL][TEST] Fix RecordBinaryComparator tests on big-endian platforms.
Comparisons performed by RecordBinaryComparator are executed in lexicographic order (byte by byte starting from the byte at index 0). This means that the underlying endianness of the platform can affect the result of a comparison between multi-byte values such as longs. This difference means that two tests fail on big-endian platforms. Also, the two tests compare 'special' long values to test edge cases that triggered old bugs in the fast path of the comparator. However since PR #26548 these 'special' values are byte-reversed before being compared on little-endian platforms, which means that the edge cases these 'special' values were designed to trigger are no longer tested. This PR fixes both these issues by byte reversing the values on little-endian systems. RecordBinaryComparatorSuite tests now pass on a big-endian machine (s390x).
1 parent 15b7333 commit 5d29560

File tree

1 file changed

+42
-6
lines changed

1 file changed

+42
-6
lines changed

sql/core/src/test/java/test/org/apache/spark/sql/execution/sort/RecordBinaryComparatorSuite.java

Lines changed: 42 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,8 @@
3737
import org.junit.Before;
3838
import org.junit.Test;
3939

40+
import java.nio.ByteOrder;
41+
4042
/**
4143
* Test the RecordBinaryComparator, which compares two UnsafeRows by their binary form.
4244
*/
@@ -261,40 +263,74 @@ public void testBinaryComparatorForNullColumns() throws Exception {
261263
public void testBinaryComparatorWhenSubtractionIsDivisibleByMaxIntValue() throws Exception {
262264
int numFields = 1;
263265

266+
// Place the following bytes (hex) into UnsafeRows for the comparison:
267+
//
268+
// index | 00 01 02 03 04 05 06 07
269+
// ------+------------------------
270+
// row1 | 00 00 00 00 00 00 00 0b
271+
// row2 | 00 00 00 00 80 00 00 0a
272+
//
273+
// The byte layout needs to be identical on all platforms regardless of
274+
// of endianness. To achieve this the bytes in each value are reversed
275+
// on little-endian platforms.
276+
long row1Data = 11L;
277+
long row2Data = 11L + Integer.MAX_VALUE;
278+
if (ByteOrder.nativeOrder().equals(ByteOrder.LITTLE_ENDIAN)) {
279+
row1Data = Long.reverseBytes(row1Data);
280+
row2Data = Long.reverseBytes(row2Data);
281+
}
282+
264283
UnsafeRow row1 = new UnsafeRow(numFields);
265284
byte[] data1 = new byte[100];
266285
row1.pointTo(data1, computeSizeInBytes(numFields * 8));
267-
row1.setLong(0, 11);
286+
row1.setLong(0, row1Data);
268287

269288
UnsafeRow row2 = new UnsafeRow(numFields);
270289
byte[] data2 = new byte[100];
271290
row2.pointTo(data2, computeSizeInBytes(numFields * 8));
272-
row2.setLong(0, 11L + Integer.MAX_VALUE);
291+
row2.setLong(0, row2Data);
273292

274293
insertRow(row1);
275294
insertRow(row2);
276295

277-
Assert.assertTrue(compare(0, 1) > 0);
296+
Assert.assertTrue(compare(0, 1) < 0);
278297
}
279298

280299
@Test
281300
public void testBinaryComparatorWhenSubtractionCanOverflowLongValue() throws Exception {
282301
int numFields = 1;
283302

303+
// Place the following bytes (hex) into UnsafeRows for the comparison:
304+
//
305+
// index | 00 01 02 03 04 05 06 07
306+
// ------+------------------------
307+
// row1 | 80 00 00 00 00 00 00 00
308+
// row2 | 00 00 00 00 00 00 00 01
309+
//
310+
// The byte layout needs to be identical on all platforms regardless of
311+
// of endianness. To achieve this the bytes in each value are reversed
312+
// on little-endian platforms.
313+
long row1Data = Long.MIN_VALUE;
314+
long row2Data = 1L;
315+
if (ByteOrder.nativeOrder().equals(ByteOrder.LITTLE_ENDIAN)) {
316+
row1Data = Long.reverseBytes(row1Data);
317+
row2Data = Long.reverseBytes(row2Data);
318+
}
319+
284320
UnsafeRow row1 = new UnsafeRow(numFields);
285321
byte[] data1 = new byte[100];
286322
row1.pointTo(data1, computeSizeInBytes(numFields * 8));
287-
row1.setLong(0, Long.MIN_VALUE);
323+
row1.setLong(0, row1Data);
288324

289325
UnsafeRow row2 = new UnsafeRow(numFields);
290326
byte[] data2 = new byte[100];
291327
row2.pointTo(data2, computeSizeInBytes(numFields * 8));
292-
row2.setLong(0, 1);
328+
row2.setLong(0, row2Data);
293329

294330
insertRow(row1);
295331
insertRow(row2);
296332

297-
Assert.assertTrue(compare(0, 1) < 0);
333+
Assert.assertTrue(compare(0, 1) > 0);
298334
}
299335

300336
@Test

0 commit comments

Comments
 (0)