From 1e9b0f6da967647f6f9f8125e1cf4b45736d346c Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Sun, 28 May 2023 11:21:52 -0700 Subject: [PATCH] [SPARK-43836][BUILD] Make Scala 2.13 as default in Spark 3.5 ### What changes were proposed in this pull request? This PR aims to make Scala 2.13 the default Scala version in Apache Spark 3.5. ### Why are the changes needed? The current releases of Scala community are `Scala 3.2.2` and `Scala 2.13.10`. - https://scala-lang.org/download/all.html Although the Apache Spark community has been using Scala 2.12 by default since Apache Spark 3.0 and Scala community will release Scala 2.12.18 for Java 21 support, we had better focus on `Scala 2.13+` more from Apache Spark 3.5 timeline to adopt Scala community's activity. Since SPARK-25075 added Scala 2.13 at Apache Spark 3.2.0, the Apache Spark community has been using it as a second Scala version. This PR aims to switch only the default Scala version from 2.12 to 2.13. Apache Spark will support both Scala 2.12 and 2.13 still. ### Does this PR introduce _any_ user-facing change? Yes, but we still have Scala 2.12. ### How was this patch tested? Pass the CIs. Closes #41344 from dongjoon-hyun/SPARK-43836. Authored-by: Dongjoon Hyun Signed-off-by: Dongjoon Hyun --- assembly/pom.xml | 4 +- common/kvstore/pom.xml | 4 +- common/network-common/pom.xml | 4 +- common/network-shuffle/pom.xml | 4 +- common/network-yarn/pom.xml | 4 +- common/sketch/pom.xml | 4 +- common/tags/pom.xml | 4 +- common/unsafe/pom.xml | 4 +- common/utils/pom.xml | 4 +- connector/avro/pom.xml | 8 +- connector/connect/client/jvm/pom.xml | 4 +- connector/connect/common/pom.xml | 4 +- connector/connect/server/pom.xml | 8 +- connector/docker-integration-tests/pom.xml | 4 +- connector/kafka-0-10-assembly/pom.xml | 4 +- connector/kafka-0-10-sql/pom.xml | 8 +- connector/kafka-0-10-token-provider/pom.xml | 4 +- connector/kafka-0-10/pom.xml | 8 +- connector/kinesis-asl-assembly/pom.xml | 4 +- connector/kinesis-asl/pom.xml | 4 +- connector/protobuf/pom.xml | 8 +- connector/spark-ganglia-lgpl/pom.xml | 4 +- core/pom.xml | 8 +- dev/deps/spark-deps-hadoop-3-hive-2.3 | 43 +++--- dev/mima | 8 +- dev/test-dependencies.sh | 2 +- docs/_plugins/copy_api_dirs.rb | 14 +- examples/pom.xml | 4 +- graphx/pom.xml | 4 +- hadoop-cloud/pom.xml | 4 +- launcher/pom.xml | 4 +- mllib-local/pom.xml | 4 +- mllib/pom.xml | 8 +- pom.xml | 125 ++++++++---------- repl/pom.xml | 4 +- resource-managers/kubernetes/core/pom.xml | 4 +- .../kubernetes/integration-tests/pom.xml | 4 +- resource-managers/mesos/pom.xml | 4 +- resource-managers/yarn/pom.xml | 4 +- sql/catalyst/pom.xml | 8 +- sql/core/pom.xml | 8 +- sql/hive-thriftserver/pom.xml | 8 +- sql/hive/pom.xml | 8 +- streaming/pom.xml | 8 +- tools/pom.xml | 4 +- 45 files changed, 197 insertions(+), 203 deletions(-) diff --git a/assembly/pom.xml b/assembly/pom.xml index 09d6bd8a33f79..d4d7a1db4a29e 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -20,12 +20,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 3.5.0-SNAPSHOT ../pom.xml - spark-assembly_2.12 + spark-assembly_2.13 Spark Project Assembly https://spark.apache.org/ pom diff --git a/common/kvstore/pom.xml b/common/kvstore/pom.xml index bef8303874b20..69f9f186e0889 100644 --- a/common/kvstore/pom.xml +++ b/common/kvstore/pom.xml @@ -21,12 +21,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 3.5.0-SNAPSHOT ../../pom.xml - spark-kvstore_2.12 + spark-kvstore_2.13 jar Spark Project Local DB https://spark.apache.org/ diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml index 8a63e999c53cd..9f90d12216e69 100644 --- a/common/network-common/pom.xml +++ b/common/network-common/pom.xml @@ -21,12 +21,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 3.5.0-SNAPSHOT ../../pom.xml - spark-network-common_2.12 + spark-network-common_2.13 jar Spark Project Networking https://spark.apache.org/ diff --git a/common/network-shuffle/pom.xml b/common/network-shuffle/pom.xml index a8bde14a259f0..864f1cc2d3715 100644 --- a/common/network-shuffle/pom.xml +++ b/common/network-shuffle/pom.xml @@ -21,12 +21,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 3.5.0-SNAPSHOT ../../pom.xml - spark-network-shuffle_2.12 + spark-network-shuffle_2.13 jar Spark Project Shuffle Streaming Service https://spark.apache.org/ diff --git a/common/network-yarn/pom.xml b/common/network-yarn/pom.xml index 671d5cb7e0178..c19ac33afa5cd 100644 --- a/common/network-yarn/pom.xml +++ b/common/network-yarn/pom.xml @@ -21,12 +21,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 3.5.0-SNAPSHOT ../../pom.xml - spark-network-yarn_2.12 + spark-network-yarn_2.13 jar Spark Project YARN Shuffle Service https://spark.apache.org/ diff --git a/common/sketch/pom.xml b/common/sketch/pom.xml index 4cc597519c3dd..6cf1a4fb83e56 100644 --- a/common/sketch/pom.xml +++ b/common/sketch/pom.xml @@ -21,12 +21,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 3.5.0-SNAPSHOT ../../pom.xml - spark-sketch_2.12 + spark-sketch_2.13 jar Spark Project Sketch https://spark.apache.org/ diff --git a/common/tags/pom.xml b/common/tags/pom.xml index 9a44c847d8a03..1eb8352e32df3 100644 --- a/common/tags/pom.xml +++ b/common/tags/pom.xml @@ -21,12 +21,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 3.5.0-SNAPSHOT ../../pom.xml - spark-tags_2.12 + spark-tags_2.13 jar Spark Project Tags https://spark.apache.org/ diff --git a/common/unsafe/pom.xml b/common/unsafe/pom.xml index a61f00084eb64..e37f1b80a5045 100644 --- a/common/unsafe/pom.xml +++ b/common/unsafe/pom.xml @@ -21,12 +21,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 3.5.0-SNAPSHOT ../../pom.xml - spark-unsafe_2.12 + spark-unsafe_2.13 jar Spark Project Unsafe https://spark.apache.org/ diff --git a/common/utils/pom.xml b/common/utils/pom.xml index 8d24d888e6db9..53df20b646de9 100644 --- a/common/utils/pom.xml +++ b/common/utils/pom.xml @@ -21,12 +21,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 3.5.0-SNAPSHOT ../../pom.xml - spark-common-utils_2.12 + spark-common-utils_2.13 jar Spark Project Common Utils https://spark.apache.org/ diff --git a/connector/avro/pom.xml b/connector/avro/pom.xml index 597e3c2235f7a..7087fdbccd04d 100644 --- a/connector/avro/pom.xml +++ b/connector/avro/pom.xml @@ -20,12 +20,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 3.5.0-SNAPSHOT ../../pom.xml - spark-avro_2.12 + spark-avro_2.13 avro @@ -70,12 +70,12 @@ org.apache.spark spark-tags_${scala.binary.version} - org.scala-lang.modules scala-parallel-collections_${scala.binary.version} - --> + org.tukaani xz diff --git a/connector/connect/client/jvm/pom.xml b/connector/connect/client/jvm/pom.xml index 4d0a437932964..8dc4c3b7a19dd 100644 --- a/connector/connect/client/jvm/pom.xml +++ b/connector/connect/client/jvm/pom.xml @@ -21,12 +21,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 3.5.0-SNAPSHOT ../../../../pom.xml - spark-connect-client-jvm_2.12 + spark-connect-client-jvm_2.13 jar Spark Project Connect Client https://spark.apache.org/ diff --git a/connector/connect/common/pom.xml b/connector/connect/common/pom.xml index 06076646df7a5..fcd72480e0ef3 100644 --- a/connector/connect/common/pom.xml +++ b/connector/connect/common/pom.xml @@ -21,12 +21,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 3.5.0-SNAPSHOT ../../../pom.xml - spark-connect-common_2.12 + spark-connect-common_2.13 jar Spark Project Connect Common https://spark.apache.org/ diff --git a/connector/connect/server/pom.xml b/connector/connect/server/pom.xml index f9dc40ce85a44..f3b43e14cdab2 100644 --- a/connector/connect/server/pom.xml +++ b/connector/connect/server/pom.xml @@ -21,12 +21,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 3.5.0-SNAPSHOT ../../../pom.xml - spark-connect_2.12 + spark-connect_2.13 jar Spark Project Connect Server https://spark.apache.org/ @@ -156,12 +156,12 @@ - org.scala-lang.modules scala-parallel-collections_${scala.binary.version} - --> + com.google.guava guava diff --git a/connector/docker-integration-tests/pom.xml b/connector/docker-integration-tests/pom.xml index cc549487a8b57..e40269838522d 100644 --- a/connector/docker-integration-tests/pom.xml +++ b/connector/docker-integration-tests/pom.xml @@ -21,12 +21,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 3.5.0-SNAPSHOT ../../pom.xml - spark-docker-integration-tests_2.12 + spark-docker-integration-tests_2.13 jar Spark Project Docker Integration Tests https://spark.apache.org/ diff --git a/connector/kafka-0-10-assembly/pom.xml b/connector/kafka-0-10-assembly/pom.xml index 340974cc789bd..f339e8c2e4fd2 100644 --- a/connector/kafka-0-10-assembly/pom.xml +++ b/connector/kafka-0-10-assembly/pom.xml @@ -20,12 +20,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 3.5.0-SNAPSHOT ../../pom.xml - spark-streaming-kafka-0-10-assembly_2.12 + spark-streaming-kafka-0-10-assembly_2.13 jar Spark Integration for Kafka 0.10 Assembly https://spark.apache.org/ diff --git a/connector/kafka-0-10-sql/pom.xml b/connector/kafka-0-10-sql/pom.xml index fdd1196cd446a..f5a12b61c2bea 100644 --- a/connector/kafka-0-10-sql/pom.xml +++ b/connector/kafka-0-10-sql/pom.xml @@ -20,13 +20,13 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 3.5.0-SNAPSHOT ../../pom.xml org.apache.spark - spark-sql-kafka-0-10_2.12 + spark-sql-kafka-0-10_2.13 sql-kafka-0-10 @@ -74,12 +74,12 @@ test-jar test - org.scala-lang.modules scala-parallel-collections_${scala.binary.version} - --> + org.apache.kafka kafka-clients diff --git a/connector/kafka-0-10-token-provider/pom.xml b/connector/kafka-0-10-token-provider/pom.xml index 3256130c50f3b..b3c0889d9475a 100644 --- a/connector/kafka-0-10-token-provider/pom.xml +++ b/connector/kafka-0-10-token-provider/pom.xml @@ -20,13 +20,13 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 3.5.0-SNAPSHOT ../../pom.xml org.apache.spark - spark-token-provider-kafka-0-10_2.12 + spark-token-provider-kafka-0-10_2.13 token-provider-kafka-0-10 diff --git a/connector/kafka-0-10/pom.xml b/connector/kafka-0-10/pom.xml index 706eb2dd2c399..f1820bb595a2d 100644 --- a/connector/kafka-0-10/pom.xml +++ b/connector/kafka-0-10/pom.xml @@ -20,12 +20,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 3.5.0-SNAPSHOT ../../pom.xml - spark-streaming-kafka-0-10_2.12 + spark-streaming-kafka-0-10_2.13 streaming-kafka-0-10 @@ -59,12 +59,12 @@ test-jar test - org.scala-lang.modules scala-parallel-collections_${scala.binary.version} - --> + org.apache.kafka kafka-clients diff --git a/connector/kinesis-asl-assembly/pom.xml b/connector/kinesis-asl-assembly/pom.xml index cd5c0393f6f84..2cba2668f049a 100644 --- a/connector/kinesis-asl-assembly/pom.xml +++ b/connector/kinesis-asl-assembly/pom.xml @@ -20,12 +20,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 3.5.0-SNAPSHOT ../../pom.xml - spark-streaming-kinesis-asl-assembly_2.12 + spark-streaming-kinesis-asl-assembly_2.13 jar Spark Project Kinesis Assembly https://spark.apache.org/ diff --git a/connector/kinesis-asl/pom.xml b/connector/kinesis-asl/pom.xml index c70a073e73407..af9cd4b7ec96e 100644 --- a/connector/kinesis-asl/pom.xml +++ b/connector/kinesis-asl/pom.xml @@ -19,13 +19,13 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 3.5.0-SNAPSHOT ../../pom.xml - spark-streaming-kinesis-asl_2.12 + spark-streaming-kinesis-asl_2.13 jar Spark Kinesis Integration diff --git a/connector/protobuf/pom.xml b/connector/protobuf/pom.xml index e85f07841dfab..c99b9a0608745 100644 --- a/connector/protobuf/pom.xml +++ b/connector/protobuf/pom.xml @@ -20,12 +20,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 3.5.0-SNAPSHOT ../../pom.xml - spark-protobuf_2.12 + spark-protobuf_2.13 protobuf @@ -70,12 +70,12 @@ org.apache.spark spark-tags_${scala.binary.version} - org.scala-lang.modules scala-parallel-collections_${scala.binary.version} - --> + com.google.protobuf protobuf-java diff --git a/connector/spark-ganglia-lgpl/pom.xml b/connector/spark-ganglia-lgpl/pom.xml index c0dcde1355849..00f4769fd60ad 100644 --- a/connector/spark-ganglia-lgpl/pom.xml +++ b/connector/spark-ganglia-lgpl/pom.xml @@ -19,13 +19,13 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 3.5.0-SNAPSHOT ../../pom.xml - spark-ganglia-lgpl_2.12 + spark-ganglia-lgpl_2.13 jar Spark Ganglia Integration diff --git a/core/pom.xml b/core/pom.xml index 09b0a2af96fac..4413ba6b6341f 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -20,12 +20,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 3.5.0-SNAPSHOT ../pom.xml - spark-core_2.12 + spark-core_2.13 jar Spark Project Core https://spark.apache.org/ @@ -35,12 +35,12 @@ - org.scala-lang.modules scala-parallel-collections_${scala.binary.version} - --> + org.apache.avro avro diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3 index d275c827ef115..bed3be0b086bc 100644 --- a/dev/deps/spark-deps-hadoop-3-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-3-hive-2.3 @@ -5,7 +5,7 @@ RoaringBitmap/0.9.44//RoaringBitmap-0.9.44.jar ST4/4.0.4//ST4-4.0.4.jar activation/1.1.1//activation-1.1.1.jar aircompressor/0.21//aircompressor-0.21.jar -algebra_2.12/2.0.1//algebra_2.12-2.0.1.jar +algebra_2.13/2.8.0//algebra_2.13-2.8.0.jar aliyun-java-sdk-core/4.5.10//aliyun-java-sdk-core-4.5.10.jar aliyun-java-sdk-kms/2.11.0//aliyun-java-sdk-kms-2.11.0.jar aliyun-java-sdk-ram/3.1.0//aliyun-java-sdk-ram-3.1.0.jar @@ -30,11 +30,11 @@ azure-keyvault-core/1.0.0//azure-keyvault-core-1.0.0.jar azure-storage/7.0.1//azure-storage-7.0.1.jar blas/3.0.3//blas-3.0.3.jar bonecp/0.8.0.RELEASE//bonecp-0.8.0.RELEASE.jar -breeze-macros_2.12/2.1.0//breeze-macros_2.12-2.1.0.jar -breeze_2.12/2.1.0//breeze_2.12-2.1.0.jar -cats-kernel_2.12/2.1.1//cats-kernel_2.12-2.1.1.jar +breeze-macros_2.13/2.1.0//breeze-macros_2.13-2.1.0.jar +breeze_2.13/2.1.0//breeze_2.13-2.1.0.jar +cats-kernel_2.13/2.8.0//cats-kernel_2.13-2.8.0.jar chill-java/0.10.0//chill-java-0.10.0.jar -chill_2.12/0.10.0//chill_2.12-0.10.0.jar +chill_2.13/0.10.0//chill_2.13-0.10.0.jar commons-cli/1.5.0//commons-cli-1.5.0.jar commons-codec/1.15//commons-codec-1.15.jar commons-collections/3.2.2//commons-collections-3.2.2.jar @@ -104,7 +104,7 @@ jackson-databind/2.15.1//jackson-databind-2.15.1.jar jackson-dataformat-cbor/2.15.1//jackson-dataformat-cbor-2.15.1.jar jackson-dataformat-yaml/2.15.1//jackson-dataformat-yaml-2.15.1.jar jackson-datatype-jsr310/2.15.1//jackson-datatype-jsr310-2.15.1.jar -jackson-module-scala_2.12/2.15.1//jackson-module-scala_2.12-2.15.1.jar +jackson-module-scala_2.13/2.15.1//jackson-module-scala_2.13-2.15.1.jar jakarta.annotation-api/1.3.5//jakarta.annotation-api-1.3.5.jar jakarta.inject/2.6.1//jakarta.inject-2.6.1.jar jakarta.servlet-api/4.0.3//jakarta.servlet-api-4.0.3.jar @@ -129,14 +129,16 @@ jettison/1.5.3//jettison-1.5.3.jar jetty-util-ajax/9.4.51.v20230217//jetty-util-ajax-9.4.51.v20230217.jar jetty-util/9.4.51.v20230217//jetty-util-9.4.51.v20230217.jar jline/2.14.6//jline-2.14.6.jar +jline/3.21.0//jline-3.21.0.jar +jna/5.9.0//jna-5.9.0.jar joda-time/2.12.5//joda-time-2.12.5.jar jodd-core/3.5.2//jodd-core-3.5.2.jar jpam/1.1//jpam-1.1.jar json/1.8//json-1.8.jar -json4s-ast_2.12/3.7.0-M11//json4s-ast_2.12-3.7.0-M11.jar -json4s-core_2.12/3.7.0-M11//json4s-core_2.12-3.7.0-M11.jar -json4s-jackson_2.12/3.7.0-M11//json4s-jackson_2.12-3.7.0-M11.jar -json4s-scalap_2.12/3.7.0-M11//json4s-scalap_2.12-3.7.0-M11.jar +json4s-ast_2.13/3.7.0-M11//json4s-ast_2.13-3.7.0-M11.jar +json4s-core_2.13/3.7.0-M11//json4s-core_2.13-3.7.0-M11.jar +json4s-jackson_2.13/3.7.0-M11//json4s-jackson_2.13-3.7.0-M11.jar +json4s-scalap_2.13/3.7.0-M11//json4s-scalap_2.13-3.7.0-M11.jar jsr305/3.0.0//jsr305-3.0.0.jar jta/1.1//jta-1.1.jar jul-to-slf4j/2.0.7//jul-to-slf4j-2.0.7.jar @@ -224,21 +226,22 @@ pickle/1.3//pickle-1.3.jar py4j/0.10.9.7//py4j-0.10.9.7.jar remotetea-oncrpc/1.1.2//remotetea-oncrpc-1.1.2.jar rocksdbjni/8.1.1.1//rocksdbjni-8.1.1.1.jar -scala-collection-compat_2.12/2.7.0//scala-collection-compat_2.12-2.7.0.jar -scala-compiler/2.12.17//scala-compiler-2.12.17.jar -scala-library/2.12.17//scala-library-2.12.17.jar -scala-parser-combinators_2.12/2.2.0//scala-parser-combinators_2.12-2.2.0.jar -scala-reflect/2.12.17//scala-reflect-2.12.17.jar -scala-xml_2.12/2.1.0//scala-xml_2.12-2.1.0.jar +scala-collection-compat_2.13/2.7.0//scala-collection-compat_2.13-2.7.0.jar +scala-compiler/2.13.8//scala-compiler-2.13.8.jar +scala-library/2.13.8//scala-library-2.13.8.jar +scala-parallel-collections_2.13/1.0.4//scala-parallel-collections_2.13-1.0.4.jar +scala-parser-combinators_2.13/2.2.0//scala-parser-combinators_2.13-2.2.0.jar +scala-reflect/2.13.8//scala-reflect-2.13.8.jar +scala-xml_2.13/2.1.0//scala-xml_2.13-2.1.0.jar shims/0.9.44//shims-0.9.44.jar slf4j-api/2.0.7//slf4j-api-2.0.7.jar snakeyaml-engine/2.6//snakeyaml-engine-2.6.jar snakeyaml/2.0//snakeyaml-2.0.jar snappy-java/1.1.10.0//snappy-java-1.1.10.0.jar -spire-macros_2.12/0.17.0//spire-macros_2.12-0.17.0.jar -spire-platform_2.12/0.17.0//spire-platform_2.12-0.17.0.jar -spire-util_2.12/0.17.0//spire-util_2.12-0.17.0.jar -spire_2.12/0.17.0//spire_2.12-0.17.0.jar +spire-macros_2.13/0.18.0//spire-macros_2.13-0.18.0.jar +spire-platform_2.13/0.18.0//spire-platform_2.13-0.18.0.jar +spire-util_2.13/0.18.0//spire-util_2.13-0.18.0.jar +spire_2.13/0.18.0//spire_2.13-0.18.0.jar stax-api/1.0.1//stax-api-1.0.1.jar stream/2.9.6//stream-2.9.6.jar super-csv/2.2.0//super-csv-2.2.0.jar diff --git a/dev/mima b/dev/mima index 4a9e343b0a78f..32c3718e4ccca 100755 --- a/dev/mima +++ b/dev/mima @@ -24,9 +24,9 @@ set -e FWDIR="$(cd "`dirname "$0"`"/..; pwd)" cd "$FWDIR" -SPARK_PROFILES=${1:-"-Pmesos -Pkubernetes -Pyarn -Pspark-ganglia-lgpl -Pkinesis-asl -Phive-thriftserver -Phive"} -TOOLS_CLASSPATH="$(build/sbt -DcopyDependencies=false "export tools/fullClasspath" | grep jar | tail -n1)" -OLD_DEPS_CLASSPATH="$(build/sbt -DcopyDependencies=false $SPARK_PROFILES "export oldDeps/fullClasspath" | grep jar | tail -n1)" +SPARK_PROFILES=${1:-"-Pscala-2.13 -Pmesos -Pkubernetes -Pyarn -Pspark-ganglia-lgpl -Pkinesis-asl -Phive-thriftserver -Phive"} +TOOLS_CLASSPATH="$(build/sbt -Pscala-2.13 -DcopyDependencies=false "export tools/fullClasspath" | grep jar | tail -n1)" +OLD_DEPS_CLASSPATH="$(build/sbt -Pscala-2.13 -DcopyDependencies=false $SPARK_PROFILES "export oldDeps/fullClasspath" | grep jar | tail -n1)" rm -f .generated-mima* @@ -42,7 +42,7 @@ $JAVA_CMD \ -cp "$TOOLS_CLASSPATH:$OLD_DEPS_CLASSPATH" \ org.apache.spark.tools.GenerateMIMAIgnore -echo -e "q\n" | build/sbt -mem 5120 -DcopyDependencies=false "$@" mimaReportBinaryIssues | grep -v -e "info.*Resolving" +echo -e "q\n" | build/sbt -Pscala-2.13 -mem 5120 -DcopyDependencies=false "$@" mimaReportBinaryIssues | grep -v -e "info.*Resolving" ret_val=$? if [ $ret_val != 0 ]; then diff --git a/dev/test-dependencies.sh b/dev/test-dependencies.sh index d7967ac3afa90..07da6497b97d2 100755 --- a/dev/test-dependencies.sh +++ b/dev/test-dependencies.sh @@ -62,7 +62,7 @@ SCALA_BINARY_VERSION=$($MVN -q \ -Dexec.args='${scala.binary.version}' \ --non-recursive \ org.codehaus.mojo:exec-maven-plugin:1.6.0:exec | grep -E '[0-9]+\.[0-9]+') -if [[ "$SCALA_BINARY_VERSION" != "2.12" ]]; then +if [[ "$SCALA_BINARY_VERSION" != "2.13" ]]; then echo "Skip dependency testing on $SCALA_BINARY_VERSION" exit 0 fi diff --git a/docs/_plugins/copy_api_dirs.rb b/docs/_plugins/copy_api_dirs.rb index 28d5e0d82c93a..9cb073ef1e00c 100644 --- a/docs/_plugins/copy_api_dirs.rb +++ b/docs/_plugins/copy_api_dirs.rb @@ -26,8 +26,8 @@ curr_dir = pwd cd("..") - puts "Running 'build/sbt -Pkinesis-asl clean compile unidoc' from " + pwd + "; this may take a few minutes..." - system("build/sbt -Pkinesis-asl clean compile unidoc") || raise("Unidoc generation failed") + puts "Running 'build/sbt -Pscala-2.13 -Pkinesis-asl clean compile unidoc' from " + pwd + "; this may take a few minutes..." + system("build/sbt -Pscala-2.13 -Pkinesis-asl clean compile unidoc") || raise("Unidoc generation failed") puts "Moving back into docs dir." cd("docs") @@ -37,7 +37,7 @@ # Copy over the unified ScalaDoc for all projects to api/scala. # This directory will be copied over to _site when `jekyll` command is run. - source = "../target/scala-2.12/unidoc" + source = "../target/scala-2.13/unidoc" dest = "api/scala" puts "Making directory " + dest @@ -119,8 +119,8 @@ puts "Moving to project root and building API docs." cd("..") - puts "Running 'build/sbt clean package -Phive' from " + pwd + "; this may take a few minutes..." - system("build/sbt clean package -Phive") || raise("PySpark doc generation failed") + puts "Running 'build/sbt -Pscala-2.13 clean package -Phive' from " + pwd + "; this may take a few minutes..." + system("build/sbt -Pscala-2.13 clean package -Phive") || raise("PySpark doc generation failed") puts "Moving back into docs dir." cd("docs") @@ -165,8 +165,8 @@ puts "Moving to project root and building API docs." cd("..") - puts "Running 'build/sbt clean package -Phive' from " + pwd + "; this may take a few minutes..." - system("build/sbt clean package -Phive") || raise("SQL doc generation failed") + puts "Running 'build/sbt -Pscala-2.13 clean package -Phive' from " + pwd + "; this may take a few minutes..." + system("build/sbt -Pscala-2.13 clean package -Phive") || raise("SQL doc generation failed") puts "Moving back into docs dir." cd("docs") diff --git a/examples/pom.xml b/examples/pom.xml index e8f22b995fded..57e41724bdca4 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -20,12 +20,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 3.5.0-SNAPSHOT ../pom.xml - spark-examples_2.12 + spark-examples_2.13 jar Spark Project Examples https://spark.apache.org/ diff --git a/graphx/pom.xml b/graphx/pom.xml index 48baeb9a87560..5d01dd06c0ecb 100644 --- a/graphx/pom.xml +++ b/graphx/pom.xml @@ -20,12 +20,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 3.5.0-SNAPSHOT ../pom.xml - spark-graphx_2.12 + spark-graphx_2.13 graphx diff --git a/hadoop-cloud/pom.xml b/hadoop-cloud/pom.xml index e213052dbc14a..7a3e601553dc0 100644 --- a/hadoop-cloud/pom.xml +++ b/hadoop-cloud/pom.xml @@ -21,12 +21,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 3.5.0-SNAPSHOT ../pom.xml - spark-hadoop-cloud_2.12 + spark-hadoop-cloud_2.13 jar Spark Project Hadoop Cloud Integration diff --git a/launcher/pom.xml b/launcher/pom.xml index aba7ee82d53cf..0bc3ae20ee183 100644 --- a/launcher/pom.xml +++ b/launcher/pom.xml @@ -21,12 +21,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 3.5.0-SNAPSHOT ../pom.xml - spark-launcher_2.12 + spark-launcher_2.13 jar Spark Project Launcher https://spark.apache.org/ diff --git a/mllib-local/pom.xml b/mllib-local/pom.xml index 00c16a8b6a544..83ca643f43bce 100644 --- a/mllib-local/pom.xml +++ b/mllib-local/pom.xml @@ -20,12 +20,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 3.5.0-SNAPSHOT ../pom.xml - spark-mllib-local_2.12 + spark-mllib-local_2.13 mllib-local diff --git a/mllib/pom.xml b/mllib/pom.xml index 73af83c758688..07290124273f2 100644 --- a/mllib/pom.xml +++ b/mllib/pom.xml @@ -20,12 +20,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 3.5.0-SNAPSHOT ../pom.xml - spark-mllib_2.12 + spark-mllib_2.13 mllib @@ -91,12 +91,12 @@ test-jar test - org.scala-lang.modules scala-parallel-collections_${scala.binary.version} - --> + org.scalanlp breeze_${scala.binary.version} diff --git a/pom.xml b/pom.xml index 2394e42921862..cc61c72ed755e 100644 --- a/pom.xml +++ b/pom.xml @@ -25,7 +25,7 @@ 18 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 3.5.0-SNAPSHOT pom Spark Project Parent POM @@ -167,8 +167,8 @@ 3.2.2 4.4 - 2.12.17 - 2.12 + 2.13.8 + 2.13 2.2.0 4.8.0 @@ -422,13 +422,13 @@ ${project.version} test-jar - org.scala-lang.modules scala-parallel-collections_${scala.binary.version} 1.0.4 - --> + com.twitter chill_${scala.binary.version} @@ -1073,7 +1073,7 @@ org.scala-lang.modules - scala-xml_2.12 + scala-xml_2.13 @@ -2854,9 +2854,43 @@ -feature -explaintypes -target:jvm-1.8 - -Xfatal-warnings - -Ywarn-unused:imports - -P:silencer:globalFilters=.*deprecated.* + -Wconf:cat=deprecation:wv,any:e + + -Wconf:cat=scaladoc:wv + -Wconf:cat=lint-multiarg-infix:wv + -Wconf:cat=other-nullary-override:wv + -Wconf:cat=other-match-analysis&site=org.apache.spark.sql.catalyst.catalog.SessionCatalog.lookupFunction.catalogFunction:wv + -Wconf:cat=other-pure-statement&site=org.apache.spark.streaming.util.FileBasedWriteAheadLog.readAll.readFile:wv + -Wconf:cat=other-pure-statement&site=org.apache.spark.scheduler.OutputCommitCoordinatorSuite.<local OutputCommitCoordinatorSuite>.futureAction:wv + + -Wconf:msg=^(?=.*?method|value|type|object|trait|inheritance)(?=.*?deprecated)(?=.*?since 2.13).+$:s + -Wconf:msg=^(?=.*?Widening conversion from)(?=.*?is deprecated because it loses precision).+$:s + -Wconf:msg=Auto-application to \`\(\)\` is deprecated:s + -Wconf:msg=method with a single empty parameter list overrides method without any parameter list:s + -Wconf:msg=method without a parameter list overrides a method with a single empty one:s + + -Wconf:cat=deprecation&msg=procedure syntax is deprecated:e + + -Wconf:cat=unchecked&msg=outer reference:s + -Wconf:cat=unchecked&msg=eliminated by erasure:s + -Wconf:msg=^(?=.*?a value of type)(?=.*?cannot also be).+$:s -Xss128m @@ -2872,13 +2906,6 @@ ${java.version} -Xlint:all,-serial,-path,-try - - - com.github.ghik - silencer-plugin_${scala.version} - 1.7.10 - - @@ -3529,6 +3556,13 @@ + + scala-2.13 + + 2.13.8 + + + scala-2.12 @@ -3538,20 +3572,6 @@ --> 2.12.17 - - - - - - - - - - scala-2.13 - - 2.13.8 - 2.13 - @@ -3565,45 +3585,16 @@ -feature -explaintypes -target:jvm-1.8 - -Wconf:cat=deprecation:wv,any:e - - -Wconf:cat=scaladoc:wv - -Wconf:cat=lint-multiarg-infix:wv - -Wconf:cat=other-nullary-override:wv - -Wconf:cat=other-match-analysis&site=org.apache.spark.sql.catalyst.catalog.SessionCatalog.lookupFunction.catalogFunction:wv - -Wconf:cat=other-pure-statement&site=org.apache.spark.streaming.util.FileBasedWriteAheadLog.readAll.readFile:wv - -Wconf:cat=other-pure-statement&site=org.apache.spark.scheduler.OutputCommitCoordinatorSuite.<local OutputCommitCoordinatorSuite>.futureAction:wv - - -Wconf:msg=^(?=.*?method|value|type|object|trait|inheritance)(?=.*?deprecated)(?=.*?since 2.13).+$:s - -Wconf:msg=^(?=.*?Widening conversion from)(?=.*?is deprecated because it loses precision).+$:s - -Wconf:msg=Auto-application to \`\(\)\` is deprecated:s - -Wconf:msg=method with a single empty parameter list overrides method without any parameter list:s - -Wconf:msg=method without a parameter list overrides a method with a single empty one:s - - -Wconf:cat=deprecation&msg=procedure syntax is deprecated:e - - -Wconf:cat=unchecked&msg=outer reference:s - -Wconf:cat=unchecked&msg=eliminated by erasure:s - -Wconf:msg=^(?=.*?a value of type)(?=.*?cannot also be).+$:s + -Xfatal-warnings + -Ywarn-unused:imports + -P:silencer:globalFilters=.*deprecated.* + + com.github.ghik + silencer-plugin_${scala.version} + 1.7.10 + diff --git a/repl/pom.xml b/repl/pom.xml index 8c0f9f989c170..74ac775100cb8 100644 --- a/repl/pom.xml +++ b/repl/pom.xml @@ -20,12 +20,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 3.5.0-SNAPSHOT ../pom.xml - spark-repl_2.12 + spark-repl_2.13 jar Spark Project REPL https://spark.apache.org/ diff --git a/resource-managers/kubernetes/core/pom.xml b/resource-managers/kubernetes/core/pom.xml index 9dab5496184e2..72c7f1f12f42d 100644 --- a/resource-managers/kubernetes/core/pom.xml +++ b/resource-managers/kubernetes/core/pom.xml @@ -19,12 +19,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 3.5.0-SNAPSHOT ../../../pom.xml - spark-kubernetes_2.12 + spark-kubernetes_2.13 jar Spark Project Kubernetes diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml index 02894f82eec9d..3e25e7053707a 100644 --- a/resource-managers/kubernetes/integration-tests/pom.xml +++ b/resource-managers/kubernetes/integration-tests/pom.xml @@ -19,12 +19,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 3.5.0-SNAPSHOT ../../../pom.xml - spark-kubernetes-integration-tests_2.12 + spark-kubernetes-integration-tests_2.13 kubernetes-integration-tests diff --git a/resource-managers/mesos/pom.xml b/resource-managers/mesos/pom.xml index 7510ecac3e7fc..267d6c7d84f21 100644 --- a/resource-managers/mesos/pom.xml +++ b/resource-managers/mesos/pom.xml @@ -19,12 +19,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 3.5.0-SNAPSHOT ../../pom.xml - spark-mesos_2.12 + spark-mesos_2.13 jar Spark Project Mesos diff --git a/resource-managers/yarn/pom.xml b/resource-managers/yarn/pom.xml index dcc7bcdd1af38..2cda552a9c47c 100644 --- a/resource-managers/yarn/pom.xml +++ b/resource-managers/yarn/pom.xml @@ -19,12 +19,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 3.5.0-SNAPSHOT ../../pom.xml - spark-yarn_2.12 + spark-yarn_2.13 jar Spark Project YARN diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml index 9057cc8a20efe..cf01c0af4a452 100644 --- a/sql/catalyst/pom.xml +++ b/sql/catalyst/pom.xml @@ -21,12 +21,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 3.5.0-SNAPSHOT ../../pom.xml - spark-catalyst_2.12 + spark-catalyst_2.13 jar Spark Project Catalyst https://spark.apache.org/ @@ -87,12 +87,12 @@ spark-sketch_${scala.binary.version} ${project.version} - org.scala-lang.modules scala-parallel-collections_${scala.binary.version} - --> + org.scalacheck scalacheck_${scala.binary.version} diff --git a/sql/core/pom.xml b/sql/core/pom.xml index 7f4c2a4cfa54d..5d4f7572d0022 100644 --- a/sql/core/pom.xml +++ b/sql/core/pom.xml @@ -21,12 +21,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 3.5.0-SNAPSHOT ../../pom.xml - spark-sql_2.12 + spark-sql_2.13 jar Spark Project SQL https://spark.apache.org/ @@ -89,12 +89,12 @@ test - org.scala-lang.modules scala-parallel-collections_${scala.binary.version} - --> + org.apache.orc orc-core diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml index ad7fc0d2ac4bd..4bbb92d1376a0 100644 --- a/sql/hive-thriftserver/pom.xml +++ b/sql/hive-thriftserver/pom.xml @@ -21,12 +21,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 3.5.0-SNAPSHOT ../../pom.xml - spark-hive-thriftserver_2.12 + spark-hive-thriftserver_2.13 jar Spark Project Hive Thrift Server https://spark.apache.org/ @@ -61,12 +61,12 @@ test-jar test - org.scala-lang.modules scala-parallel-collections_${scala.binary.version} - --> + com.google.guava guava diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml index 16d915c233ee4..b267830a3ad5f 100644 --- a/sql/hive/pom.xml +++ b/sql/hive/pom.xml @@ -21,12 +21,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 3.5.0-SNAPSHOT ../../pom.xml - spark-hive_2.12 + spark-hive_2.13 jar Spark Project Hive https://spark.apache.org/ @@ -79,12 +79,12 @@ test-jar test - org.scala-lang.modules scala-parallel-collections_${scala.binary.version} - --> + ${hive.group} hive-common diff --git a/streaming/pom.xml b/streaming/pom.xml index bebfd3abcce39..a36370a1e8b61 100644 --- a/streaming/pom.xml +++ b/streaming/pom.xml @@ -20,12 +20,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 3.5.0-SNAPSHOT ../pom.xml - spark-streaming_2.12 + spark-streaming_2.13 streaming @@ -50,12 +50,12 @@ org.apache.spark spark-tags_${scala.binary.version} - org.scala-lang.modules scala-parallel-collections_${scala.binary.version} - --> +