From a44b866dd2609fddad3455c710658813cda0bd86 Mon Sep 17 00:00:00 2001 From: David Christle Date: Mon, 7 Jun 2021 11:02:54 -0700 Subject: [PATCH 1/3] [SPARK-35670][BUILD] Upgrade ZSTD-JNI to 1.5.0-2 --- dev/deps/spark-deps-hadoop-2.7-hive-2.3 | 2 +- dev/deps/spark-deps-hadoop-3.2-hive-2.3 | 2 +- pom.xml | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/dev/deps/spark-deps-hadoop-2.7-hive-2.3 b/dev/deps/spark-deps-hadoop-2.7-hive-2.3 index cfe8b2dd6e70a..5ab36a9bcf580 100644 --- a/dev/deps/spark-deps-hadoop-2.7-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-2.7-hive-2.3 @@ -242,4 +242,4 @@ xz/1.8//xz-1.8.jar zjsonpatch/0.3.0//zjsonpatch-0.3.0.jar zookeeper-jute/3.6.2//zookeeper-jute-3.6.2.jar zookeeper/3.6.2//zookeeper-3.6.2.jar -zstd-jni/1.4.9-1//zstd-jni-1.4.9-1.jar +zstd-jni/1.5.0-2//zstd-jni-1.5.0-2.jar diff --git a/dev/deps/spark-deps-hadoop-3.2-hive-2.3 b/dev/deps/spark-deps-hadoop-3.2-hive-2.3 index 3482dd219024a..494072b57c9fc 100644 --- a/dev/deps/spark-deps-hadoop-3.2-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-3.2-hive-2.3 @@ -209,4 +209,4 @@ xz/1.8//xz-1.8.jar zjsonpatch/0.3.0//zjsonpatch-0.3.0.jar zookeeper-jute/3.6.2//zookeeper-jute-3.6.2.jar zookeeper/3.6.2//zookeeper-3.6.2.jar -zstd-jni/1.4.9-1//zstd-jni-1.4.9-1.jar +zstd-jni/1.5.0-2//zstd-jni-1.5.0-2.jar diff --git a/pom.xml b/pom.xml index ca038b29d75c6..f37c565745ad2 100644 --- a/pom.xml +++ b/pom.xml @@ -702,7 +702,7 @@ com.github.luben zstd-jni - 1.4.9-1 + 1.5.0-2 com.clearspring.analytics @@ -3316,9 +3316,9 @@ scala-2.12 - 2.12.14 From 2799e6a80a7c22711f8a57eabda9d2b73950b64c Mon Sep 17 00:00:00 2001 From: David Christle Date: Wed, 16 Jun 2021 11:30:06 -0700 Subject: [PATCH 2/3] Update pom.xml Undo unintended whitespace changes. --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index f37c565745ad2..b491c01c729e5 100644 --- a/pom.xml +++ b/pom.xml @@ -3316,9 +3316,9 @@ scala-2.12 - 2.12.14 From 016641fc4322b898d5ae1dca3e8dae36ef4e88a2 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Thu, 17 Jun 2021 07:41:33 -0700 Subject: [PATCH 3/3] benchmark --- .../ZStandardBenchmark-jdk11-results.txt | 32 +++++++++---------- .../benchmarks/ZStandardBenchmark-results.txt | 32 +++++++++---------- 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/core/benchmarks/ZStandardBenchmark-jdk11-results.txt b/core/benchmarks/ZStandardBenchmark-jdk11-results.txt index 3895e7bb6f27d..e93671dbf24e2 100644 --- a/core/benchmarks/ZStandardBenchmark-jdk11-results.txt +++ b/core/benchmarks/ZStandardBenchmark-jdk11-results.txt @@ -2,26 +2,26 @@ Benchmark ZStandardCompressionCodec ================================================================================================ -OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure -Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz +OpenJDK 64-Bit Server VM 11.0.11+9-LTS on Linux 5.8.0-1033-azure +Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz Benchmark ZStandardCompressionCodec: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative -------------------------------------------------------------------------------------------------------------------------------------- -Compression 10000 times at level 1 without buffer pool 606 614 6 0.0 60645.3 1.0X -Compression 10000 times at level 2 without buffer pool 686 693 7 0.0 68594.9 0.9X -Compression 10000 times at level 3 without buffer pool 906 920 14 0.0 90642.7 0.7X -Compression 10000 times at level 1 with buffer pool 389 403 20 0.0 38901.4 1.6X -Compression 10000 times at level 2 with buffer pool 450 466 13 0.0 45032.0 1.3X -Compression 10000 times at level 3 with buffer pool 680 682 2 0.0 68004.2 0.9X +Compression 10000 times at level 1 without buffer pool 805 1103 500 0.0 80501.4 1.0X +Compression 10000 times at level 2 without buffer pool 728 744 20 0.0 72819.9 1.1X +Compression 10000 times at level 3 without buffer pool 987 995 7 0.0 98719.4 0.8X +Compression 10000 times at level 1 with buffer pool 371 377 8 0.0 37092.3 2.2X +Compression 10000 times at level 2 with buffer pool 465 473 6 0.0 46509.8 1.7X +Compression 10000 times at level 3 with buffer pool 715 738 20 0.0 71500.2 1.1X -OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure -Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz +OpenJDK 64-Bit Server VM 11.0.11+9-LTS on Linux 5.8.0-1033-azure +Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz Benchmark ZStandardCompressionCodec: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------------------ -Decompression 10000 times from level 1 without buffer pool 1209 1226 25 0.0 120862.8 1.0X -Decompression 10000 times from level 2 without buffer pool 1191 1193 3 0.0 119064.9 1.0X -Decompression 10000 times from level 3 without buffer pool 1188 1193 6 0.0 118843.3 1.0X -Decompression 10000 times from level 1 with buffer pool 998 1004 9 0.0 99754.7 1.2X -Decompression 10000 times from level 2 with buffer pool 990 1001 11 0.0 99043.8 1.2X -Decompression 10000 times from level 3 with buffer pool 983 999 20 0.0 98269.5 1.2X +Decompression 10000 times from level 1 without buffer pool 776 786 11 0.0 77649.5 1.0X +Decompression 10000 times from level 2 without buffer pool 787 792 5 0.0 78686.6 1.0X +Decompression 10000 times from level 3 without buffer pool 782 790 7 0.0 78195.4 1.0X +Decompression 10000 times from level 1 with buffer pool 529 551 21 0.0 52914.0 1.5X +Decompression 10000 times from level 2 with buffer pool 523 537 11 0.0 52266.2 1.5X +Decompression 10000 times from level 3 with buffer pool 519 527 10 0.0 51932.3 1.5X diff --git a/core/benchmarks/ZStandardBenchmark-results.txt b/core/benchmarks/ZStandardBenchmark-results.txt index 6990c28690072..d1aa07a675020 100644 --- a/core/benchmarks/ZStandardBenchmark-results.txt +++ b/core/benchmarks/ZStandardBenchmark-results.txt @@ -2,26 +2,26 @@ Benchmark ZStandardCompressionCodec ================================================================================================ -OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure -Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz +OpenJDK 64-Bit Server VM 1.8.0_292-b10 on Linux 5.8.0-1033-azure +Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz Benchmark ZStandardCompressionCodec: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative -------------------------------------------------------------------------------------------------------------------------------------- -Compression 10000 times at level 1 without buffer pool 670 681 9 0.0 67011.0 1.0X -Compression 10000 times at level 2 without buffer pool 569 571 2 0.0 56932.0 1.2X -Compression 10000 times at level 3 without buffer pool 748 751 2 0.0 74813.8 0.9X -Compression 10000 times at level 1 with buffer pool 336 337 1 0.0 33630.6 2.0X -Compression 10000 times at level 2 with buffer pool 395 397 2 0.0 39472.6 1.7X -Compression 10000 times at level 3 with buffer pool 563 567 4 0.0 56272.8 1.2X +Compression 10000 times at level 1 without buffer pool 444 606 183 0.0 44440.9 1.0X +Compression 10000 times at level 2 without buffer pool 514 527 10 0.0 51421.8 0.9X +Compression 10000 times at level 3 without buffer pool 725 729 6 0.0 72531.4 0.6X +Compression 10000 times at level 1 with buffer pool 229 235 6 0.0 22886.7 1.9X +Compression 10000 times at level 2 with buffer pool 288 303 15 0.0 28802.3 1.5X +Compression 10000 times at level 3 with buffer pool 493 521 26 0.0 49339.5 0.9X -OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure -Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz +OpenJDK 64-Bit Server VM 1.8.0_292-b10 on Linux 5.8.0-1033-azure +Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz Benchmark ZStandardCompressionCodec: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------------------------ -Decompression 10000 times from level 1 without buffer pool 1029 1031 3 0.0 102887.4 1.0X -Decompression 10000 times from level 2 without buffer pool 1028 1031 4 0.0 102847.8 1.0X -Decompression 10000 times from level 3 without buffer pool 1029 1029 0 0.0 102941.0 1.0X -Decompression 10000 times from level 1 with buffer pool 798 799 0 0.0 79838.0 1.3X -Decompression 10000 times from level 2 with buffer pool 799 799 0 0.0 79852.9 1.3X -Decompression 10000 times from level 3 with buffer pool 796 798 2 0.0 79630.5 1.3X +Decompression 10000 times from level 1 without buffer pool 1188 1192 6 0.0 118770.4 1.0X +Decompression 10000 times from level 2 without buffer pool 1176 1199 33 0.0 117574.4 1.0X +Decompression 10000 times from level 3 without buffer pool 1174 1175 1 0.0 117426.0 1.0X +Decompression 10000 times from level 1 with buffer pool 1020 1046 36 0.0 102021.9 1.2X +Decompression 10000 times from level 2 with buffer pool 996 1005 14 0.0 99561.0 1.2X +Decompression 10000 times from level 3 with buffer pool 1021 1022 1 0.0 102050.9 1.2X