Update spark version to 3.5.0 #15363
Merged
Conversation
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Diff for d6e110e:diff --git a/_bashbrew-cat b/_bashbrew-cat
index 6bd4235..71edde5 100644
--- a/_bashbrew-cat
+++ b/_bashbrew-cat
@@ -6,17 +6,37 @@ Architectures: amd64, arm64v8
GitCommit: 58d288546e8419d229f14b62b6a653999e0390f1
Directory: 3.4.1/scala2.12-java11-python3-r-ubuntu
-Tags: 3.4.1-scala2.12-java11-python3-ubuntu, 3.4.1-python3, python3, 3.4.1, latest
+Tags: 3.4.1-scala2.12-java11-python3-ubuntu, 3.4.1-python3, 3.4.1
Architectures: amd64, arm64v8
GitCommit: 58d288546e8419d229f14b62b6a653999e0390f1
Directory: 3.4.1/scala2.12-java11-python3-ubuntu
-Tags: 3.4.1-scala2.12-java11-r-ubuntu, 3.4.1-r, r
+Tags: 3.4.1-scala2.12-java11-r-ubuntu, 3.4.1-r
Architectures: amd64, arm64v8
GitCommit: 58d288546e8419d229f14b62b6a653999e0390f1
Directory: 3.4.1/scala2.12-java11-r-ubuntu
-Tags: 3.4.1-scala2.12-java11-ubuntu, 3.4.1-scala, scala
+Tags: 3.4.1-scala2.12-java11-ubuntu, 3.4.1-scala
Architectures: amd64, arm64v8
GitCommit: 58d288546e8419d229f14b62b6a653999e0390f1
Directory: 3.4.1/scala2.12-java11-ubuntu
+
+Tags: 3.5.0-scala2.12-java11-python3-r-ubuntu
+Architectures: amd64, arm64v8
+GitCommit: 028efd4637fb2cf791d5bd9ea70b2fca472de4b7
+Directory: 3.5.0/scala2.12-java11-python3-r-ubuntu
+
+Tags: 3.5.0-scala2.12-java11-python3-ubuntu, 3.5.0-python3, 3.5.0, python3, latest
+Architectures: amd64, arm64v8
+GitCommit: 028efd4637fb2cf791d5bd9ea70b2fca472de4b7
+Directory: 3.5.0/scala2.12-java11-python3-ubuntu
+
+Tags: 3.5.0-scala2.12-java11-r-ubuntu, 3.5.0-r, r
+Architectures: amd64, arm64v8
+GitCommit: 028efd4637fb2cf791d5bd9ea70b2fca472de4b7
+Directory: 3.5.0/scala2.12-java11-r-ubuntu
+
+Tags: 3.5.0-scala2.12-java11-ubuntu, 3.5.0-scala, scala
+Architectures: amd64, arm64v8
+GitCommit: 028efd4637fb2cf791d5bd9ea70b2fca472de4b7
+Directory: 3.5.0/scala2.12-java11-ubuntu
diff --git a/_bashbrew-list b/_bashbrew-list
index d4a584b..19035e4 100644
--- a/_bashbrew-list
+++ b/_bashbrew-list
@@ -6,6 +6,14 @@ spark:3.4.1-scala2.12-java11-python3-r-ubuntu
spark:3.4.1-scala2.12-java11-python3-ubuntu
spark:3.4.1-scala2.12-java11-r-ubuntu
spark:3.4.1-scala2.12-java11-ubuntu
+spark:3.5.0
+spark:3.5.0-python3
+spark:3.5.0-r
+spark:3.5.0-scala
+spark:3.5.0-scala2.12-java11-python3-r-ubuntu
+spark:3.5.0-scala2.12-java11-python3-ubuntu
+spark:3.5.0-scala2.12-java11-r-ubuntu
+spark:3.5.0-scala2.12-java11-ubuntu
spark:latest
spark:python3
spark:r
diff --git a/_bashbrew-list-build-order b/_bashbrew-list-build-order
index 66dee52..168fcfe 100644
--- a/_bashbrew-list-build-order
+++ b/_bashbrew-list-build-order
@@ -1,4 +1,8 @@
+spark:3.4.1-scala
spark:scala
spark:3.4.1-scala2.12-java11-python3-r-ubuntu
+spark:3.4.1
+spark:3.4.1-r
+spark:3.5.0-scala2.12-java11-python3-r-ubuntu
spark:latest
spark:r
diff --git a/spark_r/Dockerfile b/spark_3.4.1-r/Dockerfile
similarity index 100%
copy from spark_r/Dockerfile
copy to spark_3.4.1-r/Dockerfile
diff --git a/spark_scala/Dockerfile b/spark_3.4.1-scala/Dockerfile
similarity index 100%
copy from spark_scala/Dockerfile
copy to spark_3.4.1-scala/Dockerfile
diff --git a/spark_scala/entrypoint.sh b/spark_3.4.1-scala/entrypoint.sh
similarity index 100%
copy from spark_scala/entrypoint.sh
copy to spark_3.4.1-scala/entrypoint.sh
diff --git a/spark_latest/Dockerfile b/spark_3.4.1/Dockerfile
similarity index 100%
copy from spark_latest/Dockerfile
copy to spark_3.4.1/Dockerfile
diff --git a/spark_3.4.1-scala2.12-java11-python3-r-ubuntu/Dockerfile b/spark_3.5.0-scala2.12-java11-python3-r-ubuntu/Dockerfile
similarity index 96%
copy from spark_3.4.1-scala2.12-java11-python3-r-ubuntu/Dockerfile
copy to spark_3.5.0-scala2.12-java11-python3-r-ubuntu/Dockerfile
index 30e6b86..d6faaa7 100644
--- a/spark_3.4.1-scala2.12-java11-python3-r-ubuntu/Dockerfile
+++ b/spark_3.5.0-scala2.12-java11-python3-r-ubuntu/Dockerfile
@@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-FROM spark:3.4.1-scala2.12-java11-ubuntu
+FROM spark:3.5.0-scala2.12-java11-ubuntu
USER root
diff --git a/spark_latest/Dockerfile b/spark_latest/Dockerfile
index 124ef71..78db218 100644
--- a/spark_latest/Dockerfile
+++ b/spark_latest/Dockerfile
@@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-FROM spark:3.4.1-scala2.12-java11-ubuntu
+FROM spark:3.5.0-scala2.12-java11-ubuntu
USER root
diff --git a/spark_r/Dockerfile b/spark_r/Dockerfile
index 1c9fc38..b5fa221 100644
--- a/spark_r/Dockerfile
+++ b/spark_r/Dockerfile
@@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-FROM spark:3.4.1-scala2.12-java11-ubuntu
+FROM spark:3.5.0-scala2.12-java11-ubuntu
USER root
diff --git a/spark_scala/Dockerfile b/spark_scala/Dockerfile
index d8bba7e..15f4b31 100644
--- a/spark_scala/Dockerfile
+++ b/spark_scala/Dockerfile
@@ -36,9 +36,9 @@ RUN set -ex; \
# Install Apache Spark
# https://downloads.apache.org/spark/KEYS
-ENV SPARK_TGZ_URL=https://archive.apache.org/dist/spark/spark-3.4.1/spark-3.4.1-bin-hadoop3.tgz \
- SPARK_TGZ_ASC_URL=https://archive.apache.org/dist/spark/spark-3.4.1/spark-3.4.1-bin-hadoop3.tgz.asc \
- GPG_KEY=F28C9C925C188C35E345614DEDA00CE834F0FC5C
+ENV SPARK_TGZ_URL=https://archive.apache.org/dist/spark/spark-3.5.0/spark-3.5.0-bin-hadoop3.tgz \
+ SPARK_TGZ_ASC_URL=https://archive.apache.org/dist/spark/spark-3.5.0/spark-3.5.0-bin-hadoop3.tgz.asc \
+ GPG_KEY=FC3AE3A7EAA1BAC98770840E7E1ABCC53AAA2216
RUN set -ex; \
export SPARK_TMP="$(mktemp -d)"; \
diff --git a/spark_scala/entrypoint.sh b/spark_scala/entrypoint.sh
index 2e3d2a8..c576d8f 100755
--- a/spark_scala/entrypoint.sh
+++ b/spark_scala/entrypoint.sh
@@ -77,6 +77,9 @@ elif ! [ -z "${SPARK_HOME+x}" ]; then
SPARK_CLASSPATH="$SPARK_HOME/conf:$SPARK_CLASSPATH";
fi
+# SPARK-43540: add current working directory into executor classpath
+SPARK_CLASSPATH="$SPARK_CLASSPATH:$PWD"
+
# Switch to spark if no USER specified (root by default) otherwise use USER directly
switch_spark_if_root() {
if [ $(id -u) -eq 0 ]; then
@@ -90,6 +93,7 @@ case "$1" in
CMD=(
"$SPARK_HOME/bin/spark-submit"
--conf "spark.driver.bindAddress=$SPARK_DRIVER_BIND_ADDRESS"
+ --conf "spark.executorEnv.SPARK_DRIVER_POD_IP=$SPARK_DRIVER_BIND_ADDRESS"
--deploy-mode client
"$@"
)Relevant Maintainers:
|
yosifkit
approved these changes
Sep 14, 2023
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
This patch add 3.5.0 version for Apache Spark https://spark.apache.org/docs/3.5.0/ [1]
[1] apache/spark-website#476
[2] apache/spark-docker#55