From 10b8a1c0976f5a2e1120f1c99368b91f44f7e3d2 Mon Sep 17 00:00:00 2001 From: Luca Canali Date: Tue, 26 Feb 2019 17:10:47 +0100 Subject: [PATCH 1/4] Workaround for issue when running Spark in Docker image with Alpine Linux 3.9.0 SPARK-26995 --- .../kubernetes/docker/src/main/dockerfiles/spark/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile index 084304032470..1d8ac3ce89c7 100644 --- a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile +++ b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile @@ -28,6 +28,7 @@ ARG spark_uid=185 RUN set -ex && \ apk upgrade --no-cache && \ + ln -s /lib /lib64 && \ apk add --no-cache bash tini libc6-compat linux-pam krb5 krb5-libs && \ mkdir -p /opt/spark && \ mkdir -p /opt/spark/examples && \ From d7d8c1fd7b525fffb88abe67f113138d93e7d212 Mon Sep 17 00:00:00 2001 From: Luca Canali Date: Fri, 1 Mar 2019 22:35:49 +0100 Subject: [PATCH 2/4] Added test. --- .../spark/deploy/k8s/integrationtest/BasicTestsSuite.scala | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala index 4e749c40563d..2833656ae47f 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala @@ -81,6 +81,12 @@ private[spark] trait BasicTestsSuite { k8sSuite: KubernetesSuite => }) } + test("Run SparkPi with spark.io.compression.codec=snappy for SPARK-26995.", k8sTestTag) { + sparkAppConf + .set("spark.io.compression.codec", "snappy") + runSparkPiAndVerifyCompletion() + } + test("Run extraJVMOptions check on driver", k8sTestTag) { sparkAppConf .set("spark.driver.extraJavaOptions", "-Dspark.test.foo=spark.test.bar") @@ -103,4 +109,5 @@ private[spark] object BasicTestsSuite { val REMOTE_PAGE_RANK_DATA_FILE = "https://storage.googleapis.com/spark-k8s-integration-tests/files/pagerank_data.txt" val REMOTE_PAGE_RANK_FILE_NAME = "pagerank_data.txt" + val SPARK_HOME_PATH: String = "/opt/spark/" } From a854a97fd434d2f49c74571fe7da9e7c3e801723 Mon Sep 17 00:00:00 2001 From: Luca Canali Date: Fri, 1 Mar 2019 22:38:30 +0100 Subject: [PATCH 3/4] Cleanup leftover noise. --- .../spark/deploy/k8s/integrationtest/BasicTestsSuite.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala index 2833656ae47f..07a154a03f44 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala @@ -109,5 +109,4 @@ private[spark] object BasicTestsSuite { val REMOTE_PAGE_RANK_DATA_FILE = "https://storage.googleapis.com/spark-k8s-integration-tests/files/pagerank_data.txt" val REMOTE_PAGE_RANK_FILE_NAME = "pagerank_data.txt" - val SPARK_HOME_PATH: String = "/opt/spark/" } From c62606740c4d1e7d6927f25d7f96a78e83cec5f4 Mon Sep 17 00:00:00 2001 From: Luca Canali Date: Sun, 3 Mar 2019 21:57:42 +0100 Subject: [PATCH 4/4] Merge test for SPARK-26995 with existing test. --- .../deploy/k8s/integrationtest/BasicTestsSuite.scala | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala index 07a154a03f44..3c1d9ea54ab2 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala @@ -52,6 +52,9 @@ private[spark] trait BasicTestsSuite { k8sSuite: KubernetesSuite => } test("Run SparkPi with an argument.", k8sTestTag) { + // This additional configuration with snappy is for SPARK-26995 + sparkAppConf + .set("spark.io.compression.codec", "snappy") runSparkPiAndVerifyCompletion(appArgs = Array("5")) } @@ -81,12 +84,6 @@ private[spark] trait BasicTestsSuite { k8sSuite: KubernetesSuite => }) } - test("Run SparkPi with spark.io.compression.codec=snappy for SPARK-26995.", k8sTestTag) { - sparkAppConf - .set("spark.io.compression.codec", "snappy") - runSparkPiAndVerifyCompletion() - } - test("Run extraJVMOptions check on driver", k8sTestTag) { sparkAppConf .set("spark.driver.extraJavaOptions", "-Dspark.test.foo=spark.test.bar")