diff --git a/core/benchmarks/MapStatusesSerDeserBenchmark-jdk11-results.txt b/core/benchmarks/MapStatusesSerDeserBenchmark-jdk11-results.txt index 1c68a129d8d81..74c8165fb4fec 100644 --- a/core/benchmarks/MapStatusesSerDeserBenchmark-jdk11-results.txt +++ b/core/benchmarks/MapStatusesSerDeserBenchmark-jdk11-results.txt @@ -1,64 +1,64 @@ -OpenJDK 64-Bit Server VM 11.0.14+9-LTS on Linux 5.13.0-1021-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +OpenJDK 64-Bit Server VM 11.0.15+10-LTS on Linux 5.13.0-1023-azure +Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz 200000 MapOutputs, 10 blocks w/ broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------- -Serialization 139 145 6 1.4 695.8 1.0X -Deserialization 225 286 60 0.9 1127.4 0.6X +Serialization 151 165 8 1.3 754.7 1.0X +Deserialization 223 303 69 0.9 1112.6 0.7X Compressed Serialized MapStatus sizes: 410 bytes Compressed Serialized Broadcast MapStatus sizes: 2 MB -OpenJDK 64-Bit Server VM 11.0.14+9-LTS on Linux 5.13.0-1021-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +OpenJDK 64-Bit Server VM 11.0.15+10-LTS on Linux 5.13.0-1023-azure +Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz 200000 MapOutputs, 10 blocks w/o broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative -------------------------------------------------------------------------------------------------------------------------- -Serialization 117 121 5 1.7 583.9 1.0X -Deserialization 223 273 69 0.9 1115.3 0.5X +Serialization 132 137 6 1.5 661.0 1.0X +Deserialization 222 287 88 0.9 1111.8 0.6X Compressed Serialized MapStatus sizes: 2 MB Compressed Serialized Broadcast MapStatus sizes: 0 bytes -OpenJDK 64-Bit Server VM 11.0.14+9-LTS on Linux 5.13.0-1021-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +OpenJDK 64-Bit Server VM 11.0.15+10-LTS on Linux 5.13.0-1023-azure +Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz 200000 MapOutputs, 100 blocks w/ broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative -------------------------------------------------------------------------------------------------------------------------- -Serialization 271 277 3 0.7 1353.0 1.0X -Deserialization 248 345 117 0.8 1241.5 1.1X +Serialization 299 314 12 0.7 1493.4 1.0X +Deserialization 248 315 77 0.8 1238.8 1.2X Compressed Serialized MapStatus sizes: 429 bytes Compressed Serialized Broadcast MapStatus sizes: 13 MB -OpenJDK 64-Bit Server VM 11.0.14+9-LTS on Linux 5.13.0-1021-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +OpenJDK 64-Bit Server VM 11.0.15+10-LTS on Linux 5.13.0-1023-azure +Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz 200000 MapOutputs, 100 blocks w/o broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative --------------------------------------------------------------------------------------------------------------------------- -Serialization 229 235 9 0.9 1146.1 1.0X -Deserialization 252 363 101 0.8 1257.7 0.9X +Serialization 250 257 12 0.8 1251.7 1.0X +Deserialization 246 315 84 0.8 1230.8 1.0X Compressed Serialized MapStatus sizes: 13 MB Compressed Serialized Broadcast MapStatus sizes: 0 bytes -OpenJDK 64-Bit Server VM 11.0.14+9-LTS on Linux 5.13.0-1021-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +OpenJDK 64-Bit Server VM 11.0.15+10-LTS on Linux 5.13.0-1023-azure +Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz 200000 MapOutputs, 1000 blocks w/ broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative --------------------------------------------------------------------------------------------------------------------------- -Serialization 1162 1174 16 0.2 5811.3 1.0X -Deserialization 627 644 14 0.3 3136.1 1.9X +Serialization 1238 1245 10 0.2 6190.5 1.0X +Deserialization 605 667 81 0.3 3027.1 2.0X -Compressed Serialized MapStatus sizes: 554 bytes +Compressed Serialized MapStatus sizes: 555 bytes Compressed Serialized Broadcast MapStatus sizes: 121 MB -OpenJDK 64-Bit Server VM 11.0.14+9-LTS on Linux 5.13.0-1021-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +OpenJDK 64-Bit Server VM 11.0.15+10-LTS on Linux 5.13.0-1023-azure +Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz 200000 MapOutputs, 1000 blocks w/o broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ---------------------------------------------------------------------------------------------------------------------------- -Serialization 1025 1033 12 0.2 5123.7 1.0X -Deserialization 625 645 20 0.3 3127.1 1.6X +Serialization 1127 1138 17 0.2 5632.6 1.0X +Deserialization 621 680 53 0.3 3102.5 1.8X Compressed Serialized MapStatus sizes: 121 MB Compressed Serialized Broadcast MapStatus sizes: 0 bytes diff --git a/core/benchmarks/MapStatusesSerDeserBenchmark-jdk17-results.txt b/core/benchmarks/MapStatusesSerDeserBenchmark-jdk17-results.txt index 9db3ae7bb09e7..b4246f97e2408 100644 --- a/core/benchmarks/MapStatusesSerDeserBenchmark-jdk17-results.txt +++ b/core/benchmarks/MapStatusesSerDeserBenchmark-jdk17-results.txt @@ -1,64 +1,64 @@ -OpenJDK 64-Bit Server VM 17.0.2+8-LTS on Linux 5.13.0-1021-azure -Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz +OpenJDK 64-Bit Server VM 17.0.3+7-LTS on Linux 5.13.0-1023-azure +Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz 200000 MapOutputs, 10 blocks w/ broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------- -Serialization 163 167 7 1.2 812.9 1.0X -Deserialization 291 310 15 0.7 1455.8 0.6X +Serialization 148 155 6 1.3 741.4 1.0X +Deserialization 237 257 30 0.8 1184.1 0.6X Compressed Serialized MapStatus sizes: 410 bytes Compressed Serialized Broadcast MapStatus sizes: 2 MB -OpenJDK 64-Bit Server VM 17.0.2+8-LTS on Linux 5.13.0-1021-azure -Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz +OpenJDK 64-Bit Server VM 17.0.3+7-LTS on Linux 5.13.0-1023-azure +Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz 200000 MapOutputs, 10 blocks w/o broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative -------------------------------------------------------------------------------------------------------------------------- -Serialization 154 156 1 1.3 770.3 1.0X -Deserialization 284 308 15 0.7 1420.0 0.5X +Serialization 132 136 3 1.5 660.7 1.0X +Deserialization 257 261 2 0.8 1287.3 0.5X Compressed Serialized MapStatus sizes: 2 MB Compressed Serialized Broadcast MapStatus sizes: 0 bytes -OpenJDK 64-Bit Server VM 17.0.2+8-LTS on Linux 5.13.0-1021-azure -Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz +OpenJDK 64-Bit Server VM 17.0.3+7-LTS on Linux 5.13.0-1023-azure +Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz 200000 MapOutputs, 100 blocks w/ broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative -------------------------------------------------------------------------------------------------------------------------- -Serialization 309 327 24 0.6 1544.0 1.0X -Deserialization 315 330 15 0.6 1574.2 1.0X +Serialization 275 285 18 0.7 1376.4 1.0X +Deserialization 263 283 31 0.8 1316.8 1.0X Compressed Serialized MapStatus sizes: 429 bytes Compressed Serialized Broadcast MapStatus sizes: 13 MB -OpenJDK 64-Bit Server VM 17.0.2+8-LTS on Linux 5.13.0-1021-azure -Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz +OpenJDK 64-Bit Server VM 17.0.3+7-LTS on Linux 5.13.0-1023-azure +Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz 200000 MapOutputs, 100 blocks w/o broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative --------------------------------------------------------------------------------------------------------------------------- -Serialization 293 295 3 0.7 1463.6 1.0X -Deserialization 310 317 8 0.6 1550.4 0.9X +Serialization 256 259 3 0.8 1279.6 1.0X +Deserialization 259 269 16 0.8 1292.7 1.0X Compressed Serialized MapStatus sizes: 13 MB Compressed Serialized Broadcast MapStatus sizes: 0 bytes -OpenJDK 64-Bit Server VM 17.0.2+8-LTS on Linux 5.13.0-1021-azure -Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz +OpenJDK 64-Bit Server VM 17.0.3+7-LTS on Linux 5.13.0-1023-azure +Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz 200000 MapOutputs, 1000 blocks w/ broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative --------------------------------------------------------------------------------------------------------------------------- -Serialization 1304 1383 111 0.2 6520.8 1.0X -Deserialization 560 636 53 0.4 2799.1 2.3X +Serialization 1144 1145 2 0.2 5718.2 1.0X +Deserialization 483 492 14 0.4 2416.8 2.4X -Compressed Serialized MapStatus sizes: 554 bytes +Compressed Serialized MapStatus sizes: 556 bytes Compressed Serialized Broadcast MapStatus sizes: 121 MB -OpenJDK 64-Bit Server VM 17.0.2+8-LTS on Linux 5.13.0-1021-azure -Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz +OpenJDK 64-Bit Server VM 17.0.3+7-LTS on Linux 5.13.0-1023-azure +Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz 200000 MapOutputs, 1000 blocks w/o broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ---------------------------------------------------------------------------------------------------------------------------- -Serialization 1240 1268 41 0.2 6197.9 1.0X -Deserialization 560 575 22 0.4 2798.9 2.2X +Serialization 1090 1094 5 0.2 5451.7 1.0X +Deserialization 486 505 16 0.4 2431.9 2.2X Compressed Serialized MapStatus sizes: 121 MB Compressed Serialized Broadcast MapStatus sizes: 0 bytes diff --git a/core/benchmarks/MapStatusesSerDeserBenchmark-results.txt b/core/benchmarks/MapStatusesSerDeserBenchmark-results.txt index ac4a0c6f55870..1ac4870032273 100644 --- a/core/benchmarks/MapStatusesSerDeserBenchmark-results.txt +++ b/core/benchmarks/MapStatusesSerDeserBenchmark-results.txt @@ -1,64 +1,64 @@ -OpenJDK 64-Bit Server VM 1.8.0_322-b06 on Linux 5.13.0-1021-azure -Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz +OpenJDK 64-Bit Server VM 1.8.0_332-b09 on Linux 5.13.0-1023-azure +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz 200000 MapOutputs, 10 blocks w/ broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------- -Serialization 135 156 69 1.5 677.4 1.0X -Deserialization 225 232 14 0.9 1123.2 0.6X +Serialization 116 132 55 1.7 578.6 1.0X +Deserialization 215 231 21 0.9 1076.6 0.5X Compressed Serialized MapStatus sizes: 410 bytes Compressed Serialized Broadcast MapStatus sizes: 2 MB -OpenJDK 64-Bit Server VM 1.8.0_322-b06 on Linux 5.13.0-1021-azure -Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz +OpenJDK 64-Bit Server VM 1.8.0_332-b09 on Linux 5.13.0-1023-azure +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz 200000 MapOutputs, 10 blocks w/o broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative -------------------------------------------------------------------------------------------------------------------------- -Serialization 126 127 1 1.6 630.1 1.0X -Deserialization 225 228 6 0.9 1123.2 0.6X +Serialization 107 109 1 1.9 533.3 1.0X +Deserialization 215 218 3 0.9 1073.4 0.5X Compressed Serialized MapStatus sizes: 2 MB Compressed Serialized Broadcast MapStatus sizes: 0 bytes -OpenJDK 64-Bit Server VM 1.8.0_322-b06 on Linux 5.13.0-1021-azure -Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz +OpenJDK 64-Bit Server VM 1.8.0_332-b09 on Linux 5.13.0-1023-azure +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz 200000 MapOutputs, 100 blocks w/ broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative -------------------------------------------------------------------------------------------------------------------------- -Serialization 260 262 1 0.8 1301.1 1.0X -Deserialization 246 259 32 0.8 1232.4 1.1X +Serialization 238 270 84 0.8 1190.8 1.0X +Deserialization 236 254 29 0.8 1180.9 1.0X Compressed Serialized MapStatus sizes: 429 bytes Compressed Serialized Broadcast MapStatus sizes: 13 MB -OpenJDK 64-Bit Server VM 1.8.0_322-b06 on Linux 5.13.0-1021-azure -Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz +OpenJDK 64-Bit Server VM 1.8.0_332-b09 on Linux 5.13.0-1023-azure +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz 200000 MapOutputs, 100 blocks w/o broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative --------------------------------------------------------------------------------------------------------------------------- -Serialization 244 245 1 0.8 1221.4 1.0X -Deserialization 245 249 8 0.8 1226.6 1.0X +Serialization 218 220 2 0.9 1091.6 1.0X +Deserialization 239 246 10 0.8 1193.3 0.9X Compressed Serialized MapStatus sizes: 13 MB Compressed Serialized Broadcast MapStatus sizes: 0 bytes -OpenJDK 64-Bit Server VM 1.8.0_322-b06 on Linux 5.13.0-1021-azure -Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz +OpenJDK 64-Bit Server VM 1.8.0_332-b09 on Linux 5.13.0-1023-azure +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz 200000 MapOutputs, 1000 blocks w/ broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative --------------------------------------------------------------------------------------------------------------------------- -Serialization 1117 1119 4 0.2 5584.7 1.0X -Deserialization 455 510 64 0.4 2273.0 2.5X +Serialization 1077 1344 378 0.2 5384.0 1.0X +Deserialization 470 506 41 0.4 2348.0 2.3X -Compressed Serialized MapStatus sizes: 554 bytes +Compressed Serialized MapStatus sizes: 556 bytes Compressed Serialized Broadcast MapStatus sizes: 121 MB -OpenJDK 64-Bit Server VM 1.8.0_322-b06 on Linux 5.13.0-1021-azure -Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz +OpenJDK 64-Bit Server VM 1.8.0_332-b09 on Linux 5.13.0-1023-azure +Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz 200000 MapOutputs, 1000 blocks w/o broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ---------------------------------------------------------------------------------------------------------------------------- -Serialization 961 966 8 0.2 4805.6 1.0X -Deserialization 453 465 15 0.4 2264.8 2.1X +Serialization 922 933 10 0.2 4612.0 1.0X +Deserialization 472 493 31 0.4 2359.4 2.0X Compressed Serialized MapStatus sizes: 121 MB Compressed Serialized Broadcast MapStatus sizes: 0 bytes diff --git a/dev/deps/spark-deps-hadoop-2-hive-2.3 b/dev/deps/spark-deps-hadoop-2-hive-2.3 index a8f21e4618703..6f9b068180f8c 100644 --- a/dev/deps/spark-deps-hadoop-2-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-2-hive-2.3 @@ -1,7 +1,7 @@ HikariCP/2.5.1//HikariCP-2.5.1.jar JLargeArrays/1.5//JLargeArrays-1.5.jar JTransforms/3.1//JTransforms-3.1.jar -RoaringBitmap/0.9.25//RoaringBitmap-0.9.25.jar +RoaringBitmap/0.9.28//RoaringBitmap-0.9.28.jar ST4/4.0.4//ST4-4.0.4.jar activation/1.1.1//activation-1.1.1.jar aircompressor/0.21//aircompressor-0.21.jar @@ -243,7 +243,7 @@ scala-parser-combinators_2.12/1.1.2//scala-parser-combinators_2.12-1.1.2.jar scala-reflect/2.12.15//scala-reflect-2.12.15.jar scala-xml_2.12/1.2.0//scala-xml_2.12-1.2.0.jar shapeless_2.12/2.3.7//shapeless_2.12-2.3.7.jar -shims/0.9.25//shims-0.9.25.jar +shims/0.9.28//shims-0.9.28.jar slf4j-api/1.7.32//slf4j-api-1.7.32.jar snakeyaml/1.30//snakeyaml-1.30.jar snappy-java/1.1.8.4//snappy-java-1.1.8.4.jar diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3 index b1db79ecf7d5b..46559b1fa27c8 100644 --- a/dev/deps/spark-deps-hadoop-3-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-3-hive-2.3 @@ -1,7 +1,7 @@ HikariCP/2.5.1//HikariCP-2.5.1.jar JLargeArrays/1.5//JLargeArrays-1.5.jar JTransforms/3.1//JTransforms-3.1.jar -RoaringBitmap/0.9.25//RoaringBitmap-0.9.25.jar +RoaringBitmap/0.9.28//RoaringBitmap-0.9.28.jar ST4/4.0.4//ST4-4.0.4.jar activation/1.1.1//activation-1.1.1.jar aircompressor/0.21//aircompressor-0.21.jar @@ -232,7 +232,7 @@ scala-parser-combinators_2.12/1.1.2//scala-parser-combinators_2.12-1.1.2.jar scala-reflect/2.12.15//scala-reflect-2.12.15.jar scala-xml_2.12/1.2.0//scala-xml_2.12-1.2.0.jar shapeless_2.12/2.3.7//shapeless_2.12-2.3.7.jar -shims/0.9.25//shims-0.9.25.jar +shims/0.9.28//shims-0.9.28.jar slf4j-api/1.7.32//slf4j-api-1.7.32.jar snakeyaml/1.30//snakeyaml-1.30.jar snappy-java/1.1.8.4//snappy-java-1.1.8.4.jar diff --git a/pom.xml b/pom.xml index 44d8fb68451fa..9e51ddccc8bef 100644 --- a/pom.xml +++ b/pom.xml @@ -819,7 +819,7 @@ org.roaringbitmap RoaringBitmap - 0.9.25 + 0.9.28