diff --git a/core/benchmarks/MapStatusesSerDeserBenchmark-jdk11-results.txt b/core/benchmarks/MapStatusesSerDeserBenchmark-jdk11-results.txt index 74c8165fb4fec..cb886199f7436 100644 --- a/core/benchmarks/MapStatusesSerDeserBenchmark-jdk11-results.txt +++ b/core/benchmarks/MapStatusesSerDeserBenchmark-jdk11-results.txt @@ -1,64 +1,64 @@ -OpenJDK 64-Bit Server VM 11.0.15+10-LTS on Linux 5.13.0-1023-azure -Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz +OpenJDK 64-Bit Server VM 11.0.15+10-LTS on Linux 5.13.0-1031-azure +Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz 200000 MapOutputs, 10 blocks w/ broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------- -Serialization 151 165 8 1.3 754.7 1.0X -Deserialization 223 303 69 0.9 1112.6 0.7X +Serialization 191 202 8 1.0 954.7 1.0X +Deserialization 250 335 75 0.8 1249.4 0.8X Compressed Serialized MapStatus sizes: 410 bytes Compressed Serialized Broadcast MapStatus sizes: 2 MB -OpenJDK 64-Bit Server VM 11.0.15+10-LTS on Linux 5.13.0-1023-azure -Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz +OpenJDK 64-Bit Server VM 11.0.15+10-LTS on Linux 5.13.0-1031-azure +Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz 200000 MapOutputs, 10 blocks w/o broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative -------------------------------------------------------------------------------------------------------------------------- -Serialization 132 137 6 1.5 661.0 1.0X -Deserialization 222 287 88 0.9 1111.8 0.6X +Serialization 161 171 8 1.2 806.3 1.0X +Deserialization 248 318 83 0.8 1238.5 0.7X Compressed Serialized MapStatus sizes: 2 MB Compressed Serialized Broadcast MapStatus sizes: 0 bytes -OpenJDK 64-Bit Server VM 11.0.15+10-LTS on Linux 5.13.0-1023-azure -Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz +OpenJDK 64-Bit Server VM 11.0.15+10-LTS on Linux 5.13.0-1031-azure +Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz 200000 MapOutputs, 100 blocks w/ broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative -------------------------------------------------------------------------------------------------------------------------- -Serialization 299 314 12 0.7 1493.4 1.0X -Deserialization 248 315 77 0.8 1238.8 1.2X +Serialization 354 380 19 0.6 1767.9 1.0X +Deserialization 272 341 86 0.7 1361.9 1.3X Compressed Serialized MapStatus sizes: 429 bytes Compressed Serialized Broadcast MapStatus sizes: 13 MB -OpenJDK 64-Bit Server VM 11.0.15+10-LTS on Linux 5.13.0-1023-azure -Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz +OpenJDK 64-Bit Server VM 11.0.15+10-LTS on Linux 5.13.0-1031-azure +Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz 200000 MapOutputs, 100 blocks w/o broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative --------------------------------------------------------------------------------------------------------------------------- -Serialization 250 257 12 0.8 1251.7 1.0X -Deserialization 246 315 84 0.8 1230.8 1.0X +Serialization 307 321 14 0.7 1535.6 1.0X +Deserialization 305 457 255 0.7 1524.3 1.0X Compressed Serialized MapStatus sizes: 13 MB Compressed Serialized Broadcast MapStatus sizes: 0 bytes -OpenJDK 64-Bit Server VM 11.0.15+10-LTS on Linux 5.13.0-1023-azure -Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz +OpenJDK 64-Bit Server VM 11.0.15+10-LTS on Linux 5.13.0-1031-azure +Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz 200000 MapOutputs, 1000 blocks w/ broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative --------------------------------------------------------------------------------------------------------------------------- -Serialization 1238 1245 10 0.2 6190.5 1.0X -Deserialization 605 667 81 0.3 3027.1 2.0X +Serialization 1612 1673 86 0.1 8061.2 1.0X +Deserialization 752 806 62 0.3 3758.3 2.1X Compressed Serialized MapStatus sizes: 555 bytes Compressed Serialized Broadcast MapStatus sizes: 121 MB -OpenJDK 64-Bit Server VM 11.0.15+10-LTS on Linux 5.13.0-1023-azure -Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz +OpenJDK 64-Bit Server VM 11.0.15+10-LTS on Linux 5.13.0-1031-azure +Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz 200000 MapOutputs, 1000 blocks w/o broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ---------------------------------------------------------------------------------------------------------------------------- -Serialization 1127 1138 17 0.2 5632.6 1.0X -Deserialization 621 680 53 0.3 3102.5 1.8X +Serialization 1416 1424 11 0.1 7081.1 1.0X +Deserialization 728 734 9 0.3 3639.1 1.9X Compressed Serialized MapStatus sizes: 121 MB Compressed Serialized Broadcast MapStatus sizes: 0 bytes diff --git a/core/benchmarks/MapStatusesSerDeserBenchmark-jdk17-results.txt b/core/benchmarks/MapStatusesSerDeserBenchmark-jdk17-results.txt index b4246f97e2408..51978554ffda4 100644 --- a/core/benchmarks/MapStatusesSerDeserBenchmark-jdk17-results.txt +++ b/core/benchmarks/MapStatusesSerDeserBenchmark-jdk17-results.txt @@ -1,64 +1,64 @@ -OpenJDK 64-Bit Server VM 17.0.3+7-LTS on Linux 5.13.0-1023-azure -Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz +OpenJDK 64-Bit Server VM 17.0.3+7-LTS on Linux 5.13.0-1031-azure +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz 200000 MapOutputs, 10 blocks w/ broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------- -Serialization 148 155 6 1.3 741.4 1.0X -Deserialization 237 257 30 0.8 1184.1 0.6X +Serialization 131 139 6 1.5 657.2 1.0X +Deserialization 245 267 34 0.8 1223.6 0.5X Compressed Serialized MapStatus sizes: 410 bytes Compressed Serialized Broadcast MapStatus sizes: 2 MB -OpenJDK 64-Bit Server VM 17.0.3+7-LTS on Linux 5.13.0-1023-azure -Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz +OpenJDK 64-Bit Server VM 17.0.3+7-LTS on Linux 5.13.0-1031-azure +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz 200000 MapOutputs, 10 blocks w/o broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative -------------------------------------------------------------------------------------------------------------------------- -Serialization 132 136 3 1.5 660.7 1.0X -Deserialization 257 261 2 0.8 1287.3 0.5X +Serialization 126 129 2 1.6 628.8 1.0X +Deserialization 244 252 10 0.8 1222.0 0.5X Compressed Serialized MapStatus sizes: 2 MB Compressed Serialized Broadcast MapStatus sizes: 0 bytes -OpenJDK 64-Bit Server VM 17.0.3+7-LTS on Linux 5.13.0-1023-azure -Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz +OpenJDK 64-Bit Server VM 17.0.3+7-LTS on Linux 5.13.0-1031-azure +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz 200000 MapOutputs, 100 blocks w/ broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative -------------------------------------------------------------------------------------------------------------------------- -Serialization 275 285 18 0.7 1376.4 1.0X -Deserialization 263 283 31 0.8 1316.8 1.0X +Serialization 259 270 11 0.8 1293.9 1.0X +Deserialization 272 302 32 0.7 1359.5 1.0X Compressed Serialized MapStatus sizes: 429 bytes Compressed Serialized Broadcast MapStatus sizes: 13 MB -OpenJDK 64-Bit Server VM 17.0.3+7-LTS on Linux 5.13.0-1023-azure -Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz +OpenJDK 64-Bit Server VM 17.0.3+7-LTS on Linux 5.13.0-1031-azure +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz 200000 MapOutputs, 100 blocks w/o broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative --------------------------------------------------------------------------------------------------------------------------- -Serialization 256 259 3 0.8 1279.6 1.0X -Deserialization 259 269 16 0.8 1292.7 1.0X +Serialization 248 250 2 0.8 1238.5 1.0X +Deserialization 287 309 32 0.7 1436.1 0.9X Compressed Serialized MapStatus sizes: 13 MB Compressed Serialized Broadcast MapStatus sizes: 0 bytes -OpenJDK 64-Bit Server VM 17.0.3+7-LTS on Linux 5.13.0-1023-azure -Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz +OpenJDK 64-Bit Server VM 17.0.3+7-LTS on Linux 5.13.0-1031-azure +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz 200000 MapOutputs, 1000 blocks w/ broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative --------------------------------------------------------------------------------------------------------------------------- -Serialization 1144 1145 2 0.2 5718.2 1.0X -Deserialization 483 492 14 0.4 2416.8 2.4X +Serialization 1143 1161 25 0.2 5716.1 1.0X +Deserialization 499 551 54 0.4 2495.6 2.3X -Compressed Serialized MapStatus sizes: 556 bytes +Compressed Serialized MapStatus sizes: 554 bytes Compressed Serialized Broadcast MapStatus sizes: 121 MB -OpenJDK 64-Bit Server VM 17.0.3+7-LTS on Linux 5.13.0-1023-azure -Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz +OpenJDK 64-Bit Server VM 17.0.3+7-LTS on Linux 5.13.0-1031-azure +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz 200000 MapOutputs, 1000 blocks w/o broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ---------------------------------------------------------------------------------------------------------------------------- -Serialization 1090 1094 5 0.2 5451.7 1.0X -Deserialization 486 505 16 0.4 2431.9 2.2X +Serialization 933 967 30 0.2 4665.8 1.0X +Deserialization 493 520 31 0.4 2465.0 1.9X Compressed Serialized MapStatus sizes: 121 MB Compressed Serialized Broadcast MapStatus sizes: 0 bytes diff --git a/core/benchmarks/MapStatusesSerDeserBenchmark-results.txt b/core/benchmarks/MapStatusesSerDeserBenchmark-results.txt index 1ac4870032273..ce1151c0add5a 100644 --- a/core/benchmarks/MapStatusesSerDeserBenchmark-results.txt +++ b/core/benchmarks/MapStatusesSerDeserBenchmark-results.txt @@ -1,64 +1,64 @@ -OpenJDK 64-Bit Server VM 1.8.0_332-b09 on Linux 5.13.0-1023-azure +OpenJDK 64-Bit Server VM 1.8.0_332-b09 on Linux 5.13.0-1031-azure Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz 200000 MapOutputs, 10 blocks w/ broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------- -Serialization 116 132 55 1.7 578.6 1.0X -Deserialization 215 231 21 0.9 1076.6 0.5X +Serialization 118 122 7 1.7 588.0 1.0X +Deserialization 218 233 16 0.9 1091.0 0.5X Compressed Serialized MapStatus sizes: 410 bytes Compressed Serialized Broadcast MapStatus sizes: 2 MB -OpenJDK 64-Bit Server VM 1.8.0_332-b09 on Linux 5.13.0-1023-azure +OpenJDK 64-Bit Server VM 1.8.0_332-b09 on Linux 5.13.0-1031-azure Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz 200000 MapOutputs, 10 blocks w/o broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative -------------------------------------------------------------------------------------------------------------------------- -Serialization 107 109 1 1.9 533.3 1.0X -Deserialization 215 218 3 0.9 1073.4 0.5X +Serialization 111 112 1 1.8 555.1 1.0X +Deserialization 219 229 14 0.9 1094.7 0.5X Compressed Serialized MapStatus sizes: 2 MB Compressed Serialized Broadcast MapStatus sizes: 0 bytes -OpenJDK 64-Bit Server VM 1.8.0_332-b09 on Linux 5.13.0-1023-azure +OpenJDK 64-Bit Server VM 1.8.0_332-b09 on Linux 5.13.0-1031-azure Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz 200000 MapOutputs, 100 blocks w/ broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative -------------------------------------------------------------------------------------------------------------------------- -Serialization 238 270 84 0.8 1190.8 1.0X -Deserialization 236 254 29 0.8 1180.9 1.0X +Serialization 242 272 81 0.8 1210.8 1.0X +Deserialization 238 253 24 0.8 1189.2 1.0X Compressed Serialized MapStatus sizes: 429 bytes Compressed Serialized Broadcast MapStatus sizes: 13 MB -OpenJDK 64-Bit Server VM 1.8.0_332-b09 on Linux 5.13.0-1023-azure +OpenJDK 64-Bit Server VM 1.8.0_332-b09 on Linux 5.13.0-1031-azure Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz 200000 MapOutputs, 100 blocks w/o broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative --------------------------------------------------------------------------------------------------------------------------- -Serialization 218 220 2 0.9 1091.6 1.0X -Deserialization 239 246 10 0.8 1193.3 0.9X +Serialization 222 224 2 0.9 1108.4 1.0X +Deserialization 238 249 17 0.8 1190.3 0.9X Compressed Serialized MapStatus sizes: 13 MB Compressed Serialized Broadcast MapStatus sizes: 0 bytes -OpenJDK 64-Bit Server VM 1.8.0_332-b09 on Linux 5.13.0-1023-azure +OpenJDK 64-Bit Server VM 1.8.0_332-b09 on Linux 5.13.0-1031-azure Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz 200000 MapOutputs, 1000 blocks w/ broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative --------------------------------------------------------------------------------------------------------------------------- -Serialization 1077 1344 378 0.2 5384.0 1.0X -Deserialization 470 506 41 0.4 2348.0 2.3X +Serialization 1095 1383 407 0.2 5475.8 1.0X +Deserialization 469 533 43 0.4 2345.9 2.3X Compressed Serialized MapStatus sizes: 556 bytes Compressed Serialized Broadcast MapStatus sizes: 121 MB -OpenJDK 64-Bit Server VM 1.8.0_332-b09 on Linux 5.13.0-1023-azure +OpenJDK 64-Bit Server VM 1.8.0_332-b09 on Linux 5.13.0-1031-azure Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz 200000 MapOutputs, 1000 blocks w/o broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ---------------------------------------------------------------------------------------------------------------------------- -Serialization 922 933 10 0.2 4612.0 1.0X -Deserialization 472 493 31 0.4 2359.4 2.0X +Serialization 914 919 8 0.2 4569.5 1.0X +Deserialization 466 490 32 0.4 2328.1 2.0X Compressed Serialized MapStatus sizes: 121 MB Compressed Serialized Broadcast MapStatus sizes: 0 bytes diff --git a/dev/deps/spark-deps-hadoop-2-hive-2.3 b/dev/deps/spark-deps-hadoop-2-hive-2.3 index bba2a37655f93..85bcb52790d3d 100644 --- a/dev/deps/spark-deps-hadoop-2-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-2-hive-2.3 @@ -1,7 +1,7 @@ HikariCP/2.5.1//HikariCP-2.5.1.jar JLargeArrays/1.5//JLargeArrays-1.5.jar JTransforms/3.1//JTransforms-3.1.jar -RoaringBitmap/0.9.28//RoaringBitmap-0.9.28.jar +RoaringBitmap/0.9.30//RoaringBitmap-0.9.30.jar ST4/4.0.4//ST4-4.0.4.jar activation/1.1.1//activation-1.1.1.jar aircompressor/0.21//aircompressor-0.21.jar @@ -243,7 +243,7 @@ scala-parser-combinators_2.12/1.1.2//scala-parser-combinators_2.12-1.1.2.jar scala-reflect/2.12.16//scala-reflect-2.12.16.jar scala-xml_2.12/1.2.0//scala-xml_2.12-1.2.0.jar shapeless_2.12/2.3.9//shapeless_2.12-2.3.9.jar -shims/0.9.28//shims-0.9.28.jar +shims/0.9.30//shims-0.9.30.jar slf4j-api/1.7.32//slf4j-api-1.7.32.jar snakeyaml/1.30//snakeyaml-1.30.jar snappy-java/1.1.8.4//snappy-java-1.1.8.4.jar diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3 index 0c51a1e691b51..a91f97faced18 100644 --- a/dev/deps/spark-deps-hadoop-3-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-3-hive-2.3 @@ -1,7 +1,7 @@ HikariCP/2.5.1//HikariCP-2.5.1.jar JLargeArrays/1.5//JLargeArrays-1.5.jar JTransforms/3.1//JTransforms-3.1.jar -RoaringBitmap/0.9.28//RoaringBitmap-0.9.28.jar +RoaringBitmap/0.9.30//RoaringBitmap-0.9.30.jar ST4/4.0.4//ST4-4.0.4.jar activation/1.1.1//activation-1.1.1.jar aircompressor/0.21//aircompressor-0.21.jar @@ -232,7 +232,7 @@ scala-parser-combinators_2.12/1.1.2//scala-parser-combinators_2.12-1.1.2.jar scala-reflect/2.12.16//scala-reflect-2.12.16.jar scala-xml_2.12/1.2.0//scala-xml_2.12-1.2.0.jar shapeless_2.12/2.3.9//shapeless_2.12-2.3.9.jar -shims/0.9.28//shims-0.9.28.jar +shims/0.9.30//shims-0.9.30.jar slf4j-api/1.7.32//slf4j-api-1.7.32.jar snakeyaml/1.30//snakeyaml-1.30.jar snappy-java/1.1.8.4//snappy-java-1.1.8.4.jar diff --git a/pom.xml b/pom.xml index 464eb40a0102e..7b4522078065c 100644 --- a/pom.xml +++ b/pom.xml @@ -822,7 +822,7 @@ org.roaringbitmap RoaringBitmap - 0.9.28 + 0.9.30