diff --git a/core/benchmarks/MapStatusesSerDeserBenchmark-jdk11-results.txt b/core/benchmarks/MapStatusesSerDeserBenchmark-jdk11-results.txt index cb886199f7436..28521e358790b 100644 --- a/core/benchmarks/MapStatusesSerDeserBenchmark-jdk11-results.txt +++ b/core/benchmarks/MapStatusesSerDeserBenchmark-jdk11-results.txt @@ -1,64 +1,64 @@ -OpenJDK 64-Bit Server VM 11.0.15+10-LTS on Linux 5.13.0-1031-azure -Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz +OpenJDK 64-Bit Server VM 11.0.16+8-LTS on Linux 5.15.0-1017-azure +Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz 200000 MapOutputs, 10 blocks w/ broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------- -Serialization 191 202 8 1.0 954.7 1.0X -Deserialization 250 335 75 0.8 1249.4 0.8X +Serialization 162 175 14 1.2 811.6 1.0X +Deserialization 229 314 68 0.9 1144.8 0.7X -Compressed Serialized MapStatus sizes: 410 bytes +Compressed Serialized MapStatus sizes: 427 bytes Compressed Serialized Broadcast MapStatus sizes: 2 MB -OpenJDK 64-Bit Server VM 11.0.15+10-LTS on Linux 5.13.0-1031-azure -Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz +OpenJDK 64-Bit Server VM 11.0.16+8-LTS on Linux 5.15.0-1017-azure +Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz 200000 MapOutputs, 10 blocks w/o broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative -------------------------------------------------------------------------------------------------------------------------- -Serialization 161 171 8 1.2 806.3 1.0X -Deserialization 248 318 83 0.8 1238.5 0.7X +Serialization 134 145 11 1.5 670.1 1.0X +Deserialization 227 311 96 0.9 1134.0 0.6X Compressed Serialized MapStatus sizes: 2 MB Compressed Serialized Broadcast MapStatus sizes: 0 bytes -OpenJDK 64-Bit Server VM 11.0.15+10-LTS on Linux 5.13.0-1031-azure -Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz +OpenJDK 64-Bit Server VM 11.0.16+8-LTS on Linux 5.15.0-1017-azure +Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz 200000 MapOutputs, 100 blocks w/ broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative -------------------------------------------------------------------------------------------------------------------------- -Serialization 354 380 19 0.6 1767.9 1.0X -Deserialization 272 341 86 0.7 1361.9 1.3X +Serialization 308 323 9 0.6 1541.7 1.0X +Deserialization 250 340 82 0.8 1250.9 1.2X -Compressed Serialized MapStatus sizes: 429 bytes +Compressed Serialized MapStatus sizes: 445 bytes Compressed Serialized Broadcast MapStatus sizes: 13 MB -OpenJDK 64-Bit Server VM 11.0.15+10-LTS on Linux 5.13.0-1031-azure -Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz +OpenJDK 64-Bit Server VM 11.0.16+8-LTS on Linux 5.15.0-1017-azure +Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz 200000 MapOutputs, 100 blocks w/o broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative --------------------------------------------------------------------------------------------------------------------------- -Serialization 307 321 14 0.7 1535.6 1.0X -Deserialization 305 457 255 0.7 1524.3 1.0X +Serialization 256 266 13 0.8 1281.5 1.0X +Deserialization 255 336 105 0.8 1276.2 1.0X Compressed Serialized MapStatus sizes: 13 MB Compressed Serialized Broadcast MapStatus sizes: 0 bytes -OpenJDK 64-Bit Server VM 11.0.15+10-LTS on Linux 5.13.0-1031-azure -Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz +OpenJDK 64-Bit Server VM 11.0.16+8-LTS on Linux 5.15.0-1017-azure +Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz 200000 MapOutputs, 1000 blocks w/ broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative --------------------------------------------------------------------------------------------------------------------------- -Serialization 1612 1673 86 0.1 8061.2 1.0X -Deserialization 752 806 62 0.3 3758.3 2.1X +Serialization 1340 1352 16 0.1 6700.2 1.0X +Deserialization 684 738 86 0.3 3421.2 2.0X -Compressed Serialized MapStatus sizes: 555 bytes +Compressed Serialized MapStatus sizes: 571 bytes Compressed Serialized Broadcast MapStatus sizes: 121 MB -OpenJDK 64-Bit Server VM 11.0.15+10-LTS on Linux 5.13.0-1031-azure -Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz +OpenJDK 64-Bit Server VM 11.0.16+8-LTS on Linux 5.15.0-1017-azure +Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz 200000 MapOutputs, 1000 blocks w/o broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ---------------------------------------------------------------------------------------------------------------------------- -Serialization 1416 1424 11 0.1 7081.1 1.0X -Deserialization 728 734 9 0.3 3639.1 1.9X +Serialization 1193 1202 12 0.2 5967.0 1.0X +Deserialization 680 722 63 0.3 3398.1 1.8X Compressed Serialized MapStatus sizes: 121 MB Compressed Serialized Broadcast MapStatus sizes: 0 bytes diff --git a/core/benchmarks/MapStatusesSerDeserBenchmark-jdk17-results.txt b/core/benchmarks/MapStatusesSerDeserBenchmark-jdk17-results.txt index 51978554ffda4..072c5caf5bc2e 100644 --- a/core/benchmarks/MapStatusesSerDeserBenchmark-jdk17-results.txt +++ b/core/benchmarks/MapStatusesSerDeserBenchmark-jdk17-results.txt @@ -1,64 +1,64 @@ -OpenJDK 64-Bit Server VM 17.0.3+7-LTS on Linux 5.13.0-1031-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +OpenJDK 64-Bit Server VM 17.0.4+8-LTS on Linux 5.15.0-1017-azure +Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz 200000 MapOutputs, 10 blocks w/ broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------- -Serialization 131 139 6 1.5 657.2 1.0X -Deserialization 245 267 34 0.8 1223.6 0.5X +Serialization 148 157 7 1.4 738.4 1.0X +Deserialization 231 241 15 0.9 1152.7 0.6X -Compressed Serialized MapStatus sizes: 410 bytes +Compressed Serialized MapStatus sizes: 427 bytes Compressed Serialized Broadcast MapStatus sizes: 2 MB -OpenJDK 64-Bit Server VM 17.0.3+7-LTS on Linux 5.13.0-1031-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +OpenJDK 64-Bit Server VM 17.0.4+8-LTS on Linux 5.15.0-1017-azure +Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz 200000 MapOutputs, 10 blocks w/o broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative -------------------------------------------------------------------------------------------------------------------------- -Serialization 126 129 2 1.6 628.8 1.0X -Deserialization 244 252 10 0.8 1222.0 0.5X +Serialization 136 139 2 1.5 678.8 1.0X +Deserialization 229 237 9 0.9 1144.9 0.6X Compressed Serialized MapStatus sizes: 2 MB Compressed Serialized Broadcast MapStatus sizes: 0 bytes -OpenJDK 64-Bit Server VM 17.0.3+7-LTS on Linux 5.13.0-1031-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +OpenJDK 64-Bit Server VM 17.0.4+8-LTS on Linux 5.15.0-1017-azure +Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz 200000 MapOutputs, 100 blocks w/ broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative -------------------------------------------------------------------------------------------------------------------------- -Serialization 259 270 11 0.8 1293.9 1.0X -Deserialization 272 302 32 0.7 1359.5 1.0X +Serialization 275 278 2 0.7 1375.4 1.0X +Deserialization 258 267 17 0.8 1289.2 1.1X -Compressed Serialized MapStatus sizes: 429 bytes +Compressed Serialized MapStatus sizes: 445 bytes Compressed Serialized Broadcast MapStatus sizes: 13 MB -OpenJDK 64-Bit Server VM 17.0.3+7-LTS on Linux 5.13.0-1031-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +OpenJDK 64-Bit Server VM 17.0.4+8-LTS on Linux 5.15.0-1017-azure +Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz 200000 MapOutputs, 100 blocks w/o broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative --------------------------------------------------------------------------------------------------------------------------- -Serialization 248 250 2 0.8 1238.5 1.0X -Deserialization 287 309 32 0.7 1436.1 0.9X +Serialization 256 260 4 0.8 1277.6 1.0X +Deserialization 253 259 9 0.8 1266.5 1.0X Compressed Serialized MapStatus sizes: 13 MB Compressed Serialized Broadcast MapStatus sizes: 0 bytes -OpenJDK 64-Bit Server VM 17.0.3+7-LTS on Linux 5.13.0-1031-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +OpenJDK 64-Bit Server VM 17.0.4+8-LTS on Linux 5.15.0-1017-azure +Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz 200000 MapOutputs, 1000 blocks w/ broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative --------------------------------------------------------------------------------------------------------------------------- -Serialization 1143 1161 25 0.2 5716.1 1.0X -Deserialization 499 551 54 0.4 2495.6 2.3X +Serialization 1214 1215 1 0.2 6072.5 1.0X +Deserialization 466 488 40 0.4 2330.3 2.6X -Compressed Serialized MapStatus sizes: 554 bytes +Compressed Serialized MapStatus sizes: 571 bytes Compressed Serialized Broadcast MapStatus sizes: 121 MB -OpenJDK 64-Bit Server VM 17.0.3+7-LTS on Linux 5.13.0-1031-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +OpenJDK 64-Bit Server VM 17.0.4+8-LTS on Linux 5.15.0-1017-azure +Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz 200000 MapOutputs, 1000 blocks w/o broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ---------------------------------------------------------------------------------------------------------------------------- -Serialization 933 967 30 0.2 4665.8 1.0X -Deserialization 493 520 31 0.4 2465.0 1.9X +Serialization 1086 1087 1 0.2 5430.4 1.0X +Deserialization 468 480 22 0.4 2339.1 2.3X Compressed Serialized MapStatus sizes: 121 MB Compressed Serialized Broadcast MapStatus sizes: 0 bytes diff --git a/core/benchmarks/MapStatusesSerDeserBenchmark-results.txt b/core/benchmarks/MapStatusesSerDeserBenchmark-results.txt index ce1151c0add5a..15b71258ed2bc 100644 --- a/core/benchmarks/MapStatusesSerDeserBenchmark-results.txt +++ b/core/benchmarks/MapStatusesSerDeserBenchmark-results.txt @@ -1,64 +1,64 @@ -OpenJDK 64-Bit Server VM 1.8.0_332-b09 on Linux 5.13.0-1031-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +OpenJDK 64-Bit Server VM 1.8.0_345-b01 on Linux 5.15.0-1017-azure +Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz 200000 MapOutputs, 10 blocks w/ broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------- -Serialization 118 122 7 1.7 588.0 1.0X -Deserialization 218 233 16 0.9 1091.0 0.5X +Serialization 195 226 80 1.0 975.0 1.0X +Deserialization 307 344 45 0.7 1536.5 0.6X -Compressed Serialized MapStatus sizes: 410 bytes +Compressed Serialized MapStatus sizes: 427 bytes Compressed Serialized Broadcast MapStatus sizes: 2 MB -OpenJDK 64-Bit Server VM 1.8.0_332-b09 on Linux 5.13.0-1031-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +OpenJDK 64-Bit Server VM 1.8.0_345-b01 on Linux 5.15.0-1017-azure +Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz 200000 MapOutputs, 10 blocks w/o broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative -------------------------------------------------------------------------------------------------------------------------- -Serialization 111 112 1 1.8 555.1 1.0X -Deserialization 219 229 14 0.9 1094.7 0.5X +Serialization 190 195 4 1.1 950.8 1.0X +Deserialization 312 319 8 0.6 1558.0 0.6X Compressed Serialized MapStatus sizes: 2 MB Compressed Serialized Broadcast MapStatus sizes: 0 bytes -OpenJDK 64-Bit Server VM 1.8.0_332-b09 on Linux 5.13.0-1031-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +OpenJDK 64-Bit Server VM 1.8.0_345-b01 on Linux 5.15.0-1017-azure +Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz 200000 MapOutputs, 100 blocks w/ broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative -------------------------------------------------------------------------------------------------------------------------- -Serialization 242 272 81 0.8 1210.8 1.0X -Deserialization 238 253 24 0.8 1189.2 1.0X +Serialization 374 383 7 0.5 1868.5 1.0X +Deserialization 337 369 51 0.6 1683.4 1.1X -Compressed Serialized MapStatus sizes: 429 bytes +Compressed Serialized MapStatus sizes: 445 bytes Compressed Serialized Broadcast MapStatus sizes: 13 MB -OpenJDK 64-Bit Server VM 1.8.0_332-b09 on Linux 5.13.0-1031-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +OpenJDK 64-Bit Server VM 1.8.0_345-b01 on Linux 5.15.0-1017-azure +Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz 200000 MapOutputs, 100 blocks w/o broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative --------------------------------------------------------------------------------------------------------------------------- -Serialization 222 224 2 0.9 1108.4 1.0X -Deserialization 238 249 17 0.8 1190.3 0.9X +Serialization 344 348 3 0.6 1720.6 1.0X +Deserialization 335 346 7 0.6 1677.1 1.0X Compressed Serialized MapStatus sizes: 13 MB Compressed Serialized Broadcast MapStatus sizes: 0 bytes -OpenJDK 64-Bit Server VM 1.8.0_332-b09 on Linux 5.13.0-1031-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +OpenJDK 64-Bit Server VM 1.8.0_345-b01 on Linux 5.15.0-1017-azure +Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz 200000 MapOutputs, 1000 blocks w/ broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative --------------------------------------------------------------------------------------------------------------------------- -Serialization 1095 1383 407 0.2 5475.8 1.0X -Deserialization 469 533 43 0.4 2345.9 2.3X +Serialization 1600 2180 820 0.1 8001.6 1.0X +Deserialization 678 791 99 0.3 3390.8 2.4X -Compressed Serialized MapStatus sizes: 556 bytes +Compressed Serialized MapStatus sizes: 571 bytes Compressed Serialized Broadcast MapStatus sizes: 121 MB -OpenJDK 64-Bit Server VM 1.8.0_332-b09 on Linux 5.13.0-1031-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +OpenJDK 64-Bit Server VM 1.8.0_345-b01 on Linux 5.15.0-1017-azure +Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz 200000 MapOutputs, 1000 blocks w/o broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ---------------------------------------------------------------------------------------------------------------------------- -Serialization 914 919 8 0.2 4569.5 1.0X -Deserialization 466 490 32 0.4 2328.1 2.0X +Serialization 1530 1531 1 0.1 7651.5 1.0X +Deserialization 659 680 27 0.3 3295.3 2.3X Compressed Serialized MapStatus sizes: 121 MB Compressed Serialized Broadcast MapStatus sizes: 0 bytes diff --git a/dev/deps/spark-deps-hadoop-2-hive-2.3 b/dev/deps/spark-deps-hadoop-2-hive-2.3 index ac9981d06d0b3..aef36b9c8c87b 100644 --- a/dev/deps/spark-deps-hadoop-2-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-2-hive-2.3 @@ -1,7 +1,7 @@ HikariCP/2.5.1//HikariCP-2.5.1.jar JLargeArrays/1.5//JLargeArrays-1.5.jar JTransforms/3.1//JTransforms-3.1.jar -RoaringBitmap/0.9.30//RoaringBitmap-0.9.30.jar +RoaringBitmap/0.9.31//RoaringBitmap-0.9.31.jar ST4/4.0.4//ST4-4.0.4.jar activation/1.1.1//activation-1.1.1.jar aircompressor/0.21//aircompressor-0.21.jar @@ -241,7 +241,7 @@ scala-library/2.12.16//scala-library-2.12.16.jar scala-parser-combinators_2.12/1.1.2//scala-parser-combinators_2.12-1.1.2.jar scala-reflect/2.12.16//scala-reflect-2.12.16.jar scala-xml_2.12/1.2.0//scala-xml_2.12-1.2.0.jar -shims/0.9.30//shims-0.9.30.jar +shims/0.9.31//shims-0.9.31.jar slf4j-api/1.7.36//slf4j-api-1.7.36.jar snakeyaml/1.30//snakeyaml-1.30.jar snappy-java/1.1.8.4//snappy-java-1.1.8.4.jar diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3 index 6c5e2559bb285..08f1059e34e2a 100644 --- a/dev/deps/spark-deps-hadoop-3-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-3-hive-2.3 @@ -1,7 +1,7 @@ HikariCP/2.5.1//HikariCP-2.5.1.jar JLargeArrays/1.5//JLargeArrays-1.5.jar JTransforms/3.1//JTransforms-3.1.jar -RoaringBitmap/0.9.30//RoaringBitmap-0.9.30.jar +RoaringBitmap/0.9.31//RoaringBitmap-0.9.31.jar ST4/4.0.4//ST4-4.0.4.jar activation/1.1.1//activation-1.1.1.jar aircompressor/0.21//aircompressor-0.21.jar @@ -228,7 +228,7 @@ scala-library/2.12.16//scala-library-2.12.16.jar scala-parser-combinators_2.12/1.1.2//scala-parser-combinators_2.12-1.1.2.jar scala-reflect/2.12.16//scala-reflect-2.12.16.jar scala-xml_2.12/1.2.0//scala-xml_2.12-1.2.0.jar -shims/0.9.30//shims-0.9.30.jar +shims/0.9.31//shims-0.9.31.jar slf4j-api/1.7.36//slf4j-api-1.7.36.jar snakeyaml/1.30//snakeyaml-1.30.jar snappy-java/1.1.8.4//snappy-java-1.1.8.4.jar diff --git a/pom.xml b/pom.xml index 6bba2d5388851..e40fbdbfa824d 100644 --- a/pom.xml +++ b/pom.xml @@ -818,7 +818,7 @@ org.roaringbitmap RoaringBitmap - 0.9.30 + 0.9.31