[SPARK-43836][BUILD] Make Scala 2.13 as default in Spark 3.5

### What changes were proposed in this pull request? This PR aims to make Scala 2.13 the default Scala version in Apache Spark 3.5. ### Why are the changes needed? The current releases of Scala community are `Scala 3.2.2` and `Scala 2.13.10`. - https://scala-lang.org/download/all.html Although the Apache Spark community has been using Scala 2.12 by default since Apache Spark 3.0 and Scala community will release Scala 2.12.18 for Java 21 support, we had better focus on `Scala 2.13+` more from Apache Spark 3.5 timeline to adopt Scala community's activity. Since SPARK-25075 added Scala 2.13 at Apache Spark 3.2.0, the Apache Spark community has been using it as a second Scala version. This PR aims to switch only the default Scala version from 2.12 to 2.13. Apache Spark will support both Scala 2.12 and 2.13 still. ### Does this PR introduce _any_ user-facing change? Yes, but we still have Scala 2.12. ### How was this patch tested? Pass the CIs. Closes #41344 from dongjoon-hyun/SPARK-43836. Authored-by: Dongjoon Hyun <dongjoon@apache.org> Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
apache · May 28, 2023 · 1e9b0f6 · 1e9b0f6
1 parent e6e242e
commit 1e9b0f6
Show file tree

Hide file tree

Showing 45 changed files with 197 additions and 203 deletions.
diff --git a/assembly/pom.xml b/assembly/pom.xml
@@ -20,12 +20,12 @@
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.spark</groupId>
-    <artifactId>spark-parent_2.12</artifactId>
+    <artifactId>spark-parent_2.13</artifactId>
     <version>3.5.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
-  <artifactId>spark-assembly_2.12</artifactId>
+  <artifactId>spark-assembly_2.13</artifactId>
   <name>Spark Project Assembly</name>
   <url>https://spark.apache.org/</url>
   <packaging>pom</packaging>

diff --git a/common/kvstore/pom.xml b/common/kvstore/pom.xml
@@ -21,12 +21,12 @@
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.spark</groupId>
-    <artifactId>spark-parent_2.12</artifactId>
+    <artifactId>spark-parent_2.13</artifactId>
     <version>3.5.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
-  <artifactId>spark-kvstore_2.12</artifactId>
+  <artifactId>spark-kvstore_2.13</artifactId>
   <packaging>jar</packaging>
   <name>Spark Project Local DB</name>
   <url>https://spark.apache.org/</url>

diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml
@@ -21,12 +21,12 @@
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.spark</groupId>
-    <artifactId>spark-parent_2.12</artifactId>
+    <artifactId>spark-parent_2.13</artifactId>
     <version>3.5.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
-  <artifactId>spark-network-common_2.12</artifactId>
+  <artifactId>spark-network-common_2.13</artifactId>
   <packaging>jar</packaging>
   <name>Spark Project Networking</name>
   <url>https://spark.apache.org/</url>

diff --git a/common/network-shuffle/pom.xml b/common/network-shuffle/pom.xml
@@ -21,12 +21,12 @@
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.spark</groupId>
-    <artifactId>spark-parent_2.12</artifactId>
+    <artifactId>spark-parent_2.13</artifactId>
     <version>3.5.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
-  <artifactId>spark-network-shuffle_2.12</artifactId>
+  <artifactId>spark-network-shuffle_2.13</artifactId>
   <packaging>jar</packaging>
   <name>Spark Project Shuffle Streaming Service</name>
   <url>https://spark.apache.org/</url>

diff --git a/common/network-yarn/pom.xml b/common/network-yarn/pom.xml
@@ -21,12 +21,12 @@
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.spark</groupId>
-    <artifactId>spark-parent_2.12</artifactId>
+    <artifactId>spark-parent_2.13</artifactId>
     <version>3.5.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
-  <artifactId>spark-network-yarn_2.12</artifactId>
+  <artifactId>spark-network-yarn_2.13</artifactId>
   <packaging>jar</packaging>
   <name>Spark Project YARN Shuffle Service</name>
   <url>https://spark.apache.org/</url>

diff --git a/common/sketch/pom.xml b/common/sketch/pom.xml
@@ -21,12 +21,12 @@
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.spark</groupId>
-    <artifactId>spark-parent_2.12</artifactId>
+    <artifactId>spark-parent_2.13</artifactId>
     <version>3.5.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
-  <artifactId>spark-sketch_2.12</artifactId>
+  <artifactId>spark-sketch_2.13</artifactId>
   <packaging>jar</packaging>
   <name>Spark Project Sketch</name>
   <url>https://spark.apache.org/</url>

diff --git a/common/tags/pom.xml b/common/tags/pom.xml
@@ -21,12 +21,12 @@
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.spark</groupId>
-    <artifactId>spark-parent_2.12</artifactId>
+    <artifactId>spark-parent_2.13</artifactId>
     <version>3.5.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
-  <artifactId>spark-tags_2.12</artifactId>
+  <artifactId>spark-tags_2.13</artifactId>
   <packaging>jar</packaging>
   <name>Spark Project Tags</name>
   <url>https://spark.apache.org/</url>

diff --git a/common/unsafe/pom.xml b/common/unsafe/pom.xml
@@ -21,12 +21,12 @@
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.spark</groupId>
-    <artifactId>spark-parent_2.12</artifactId>
+    <artifactId>spark-parent_2.13</artifactId>
     <version>3.5.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
-  <artifactId>spark-unsafe_2.12</artifactId>
+  <artifactId>spark-unsafe_2.13</artifactId>
   <packaging>jar</packaging>
   <name>Spark Project Unsafe</name>
   <url>https://spark.apache.org/</url>

diff --git a/common/utils/pom.xml b/common/utils/pom.xml
@@ -21,12 +21,12 @@
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.spark</groupId>
-    <artifactId>spark-parent_2.12</artifactId>
+    <artifactId>spark-parent_2.13</artifactId>
     <version>3.5.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
-  <artifactId>spark-common-utils_2.12</artifactId>
+  <artifactId>spark-common-utils_2.13</artifactId>
   <packaging>jar</packaging>
   <name>Spark Project Common Utils</name>
   <url>https://spark.apache.org/</url>

diff --git a/connector/avro/pom.xml b/connector/avro/pom.xml
@@ -20,12 +20,12 @@
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.spark</groupId>
-    <artifactId>spark-parent_2.12</artifactId>
+    <artifactId>spark-parent_2.13</artifactId>
     <version>3.5.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
-  <artifactId>spark-avro_2.12</artifactId>
+  <artifactId>spark-avro_2.13</artifactId>
   <properties>
     <sbt.project.name>avro</sbt.project.name>
   </properties>
@@ -70,12 +70,12 @@
       <groupId>org.apache.spark</groupId>
       <artifactId>spark-tags_${scala.binary.version}</artifactId>
     </dependency>
-    <!-- #if scala-2.13 --><!--
+    <!-- #if scala-2.13 -->
     <dependency>
       <groupId>org.scala-lang.modules</groupId>
       <artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
     </dependency>
-    --><!-- #endif scala-2.13 -->
+    <!-- #endif scala-2.13 -->
     <dependency>
       <groupId>org.tukaani</groupId>
       <artifactId>xz</artifactId>

diff --git a/connector/connect/client/jvm/pom.xml b/connector/connect/client/jvm/pom.xml
@@ -21,12 +21,12 @@
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.spark</groupId>
-    <artifactId>spark-parent_2.12</artifactId>
+    <artifactId>spark-parent_2.13</artifactId>
     <version>3.5.0-SNAPSHOT</version>
     <relativePath>../../../../pom.xml</relativePath>
   </parent>
 
-  <artifactId>spark-connect-client-jvm_2.12</artifactId>
+  <artifactId>spark-connect-client-jvm_2.13</artifactId>
   <packaging>jar</packaging>
   <name>Spark Project Connect Client</name>
   <url>https://spark.apache.org/</url>

diff --git a/connector/connect/common/pom.xml b/connector/connect/common/pom.xml
@@ -21,12 +21,12 @@
     <modelVersion>4.0.0</modelVersion>
     <parent>
         <groupId>org.apache.spark</groupId>
-        <artifactId>spark-parent_2.12</artifactId>
+        <artifactId>spark-parent_2.13</artifactId>
         <version>3.5.0-SNAPSHOT</version>
         <relativePath>../../../pom.xml</relativePath>
     </parent>
 
-    <artifactId>spark-connect-common_2.12</artifactId>
+    <artifactId>spark-connect-common_2.13</artifactId>
     <packaging>jar</packaging>
     <name>Spark Project Connect Common</name>
     <url>https://spark.apache.org/</url>

diff --git a/connector/connect/server/pom.xml b/connector/connect/server/pom.xml
@@ -21,12 +21,12 @@
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.spark</groupId>
-    <artifactId>spark-parent_2.12</artifactId>
+    <artifactId>spark-parent_2.13</artifactId>
     <version>3.5.0-SNAPSHOT</version>
     <relativePath>../../../pom.xml</relativePath>
   </parent>
 
-  <artifactId>spark-connect_2.12</artifactId>
+  <artifactId>spark-connect_2.13</artifactId>
   <packaging>jar</packaging>
   <name>Spark Project Connect Server</name>
   <url>https://spark.apache.org/</url>
@@ -156,12 +156,12 @@
         </exclusion>
       </exclusions>
     </dependency>
-    <!-- #if scala-2.13 --><!--
+    <!-- #if scala-2.13 -->
     <dependency>
       <groupId>org.scala-lang.modules</groupId>
       <artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
     </dependency>
-    --><!-- #endif scala-2.13 -->
+    <!-- #endif scala-2.13 -->
     <dependency>
       <groupId>com.google.guava</groupId>
       <artifactId>guava</artifactId>

diff --git a/connector/docker-integration-tests/pom.xml b/connector/docker-integration-tests/pom.xml
@@ -21,12 +21,12 @@
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.spark</groupId>
-    <artifactId>spark-parent_2.12</artifactId>
+    <artifactId>spark-parent_2.13</artifactId>
     <version>3.5.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
-  <artifactId>spark-docker-integration-tests_2.12</artifactId>
+  <artifactId>spark-docker-integration-tests_2.13</artifactId>
   <packaging>jar</packaging>
   <name>Spark Project Docker Integration Tests</name>
   <url>https://spark.apache.org/</url>

diff --git a/connector/kafka-0-10-assembly/pom.xml b/connector/kafka-0-10-assembly/pom.xml
@@ -20,12 +20,12 @@
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.spark</groupId>
-    <artifactId>spark-parent_2.12</artifactId>
+    <artifactId>spark-parent_2.13</artifactId>
     <version>3.5.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
-  <artifactId>spark-streaming-kafka-0-10-assembly_2.12</artifactId>
+  <artifactId>spark-streaming-kafka-0-10-assembly_2.13</artifactId>
   <packaging>jar</packaging>
   <name>Spark Integration for Kafka 0.10 Assembly</name>
   <url>https://spark.apache.org/</url>

diff --git a/connector/kafka-0-10-sql/pom.xml b/connector/kafka-0-10-sql/pom.xml
@@ -20,13 +20,13 @@
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.spark</groupId>
-    <artifactId>spark-parent_2.12</artifactId>
+    <artifactId>spark-parent_2.13</artifactId>
     <version>3.5.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
   <groupId>org.apache.spark</groupId>
-  <artifactId>spark-sql-kafka-0-10_2.12</artifactId>
+  <artifactId>spark-sql-kafka-0-10_2.13</artifactId>
   <properties>
     <sbt.project.name>sql-kafka-0-10</sbt.project.name>
   </properties>
@@ -74,12 +74,12 @@
       <type>test-jar</type>
       <scope>test</scope>
     </dependency>
-    <!-- #if scala-2.13 --><!--
+    <!-- #if scala-2.13 -->
     <dependency>
       <groupId>org.scala-lang.modules</groupId>
       <artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
     </dependency>
-    --><!-- #endif scala-2.13 -->
+    <!-- #endif scala-2.13 -->
     <dependency>
       <groupId>org.apache.kafka</groupId>
       <artifactId>kafka-clients</artifactId>

diff --git a/connector/kafka-0-10-token-provider/pom.xml b/connector/kafka-0-10-token-provider/pom.xml
@@ -20,13 +20,13 @@
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.spark</groupId>
-    <artifactId>spark-parent_2.12</artifactId>
+    <artifactId>spark-parent_2.13</artifactId>
     <version>3.5.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
   <groupId>org.apache.spark</groupId>
-  <artifactId>spark-token-provider-kafka-0-10_2.12</artifactId>
+  <artifactId>spark-token-provider-kafka-0-10_2.13</artifactId>
   <properties>
     <sbt.project.name>token-provider-kafka-0-10</sbt.project.name>
   </properties>

diff --git a/connector/kafka-0-10/pom.xml b/connector/kafka-0-10/pom.xml
@@ -20,12 +20,12 @@
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.spark</groupId>
-    <artifactId>spark-parent_2.12</artifactId>
+    <artifactId>spark-parent_2.13</artifactId>
     <version>3.5.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
-  <artifactId>spark-streaming-kafka-0-10_2.12</artifactId>
+  <artifactId>spark-streaming-kafka-0-10_2.13</artifactId>
   <properties>
     <sbt.project.name>streaming-kafka-0-10</sbt.project.name>
   </properties>
@@ -59,12 +59,12 @@
       <type>test-jar</type>
       <scope>test</scope>
     </dependency>
-    <!-- #if scala-2.13 --><!--
+    <!-- #if scala-2.13 -->
     <dependency>
       <groupId>org.scala-lang.modules</groupId>
       <artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
     </dependency>
-    --><!-- #endif scala-2.13 -->
+    <!-- #endif scala-2.13 -->
     <dependency>
       <groupId>org.apache.kafka</groupId>
       <artifactId>kafka-clients</artifactId>

diff --git a/connector/kinesis-asl-assembly/pom.xml b/connector/kinesis-asl-assembly/pom.xml
@@ -20,12 +20,12 @@
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.spark</groupId>
-    <artifactId>spark-parent_2.12</artifactId>
+    <artifactId>spark-parent_2.13</artifactId>
     <version>3.5.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
-  <artifactId>spark-streaming-kinesis-asl-assembly_2.12</artifactId>
+  <artifactId>spark-streaming-kinesis-asl-assembly_2.13</artifactId>
   <packaging>jar</packaging>
   <name>Spark Project Kinesis Assembly</name>
   <url>https://spark.apache.org/</url>

diff --git a/connector/kinesis-asl/pom.xml b/connector/kinesis-asl/pom.xml
@@ -19,13 +19,13 @@
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.spark</groupId>
-    <artifactId>spark-parent_2.12</artifactId>
+    <artifactId>spark-parent_2.13</artifactId>
     <version>3.5.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
   <!-- Kinesis integration is not included by default due to ASL-licensed code. -->
-  <artifactId>spark-streaming-kinesis-asl_2.12</artifactId>
+  <artifactId>spark-streaming-kinesis-asl_2.13</artifactId>
   <packaging>jar</packaging>
   <name>Spark Kinesis Integration</name>
 

diff --git a/connector/protobuf/pom.xml b/connector/protobuf/pom.xml
@@ -20,12 +20,12 @@
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.spark</groupId>
-    <artifactId>spark-parent_2.12</artifactId>
+    <artifactId>spark-parent_2.13</artifactId>
     <version>3.5.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
-  <artifactId>spark-protobuf_2.12</artifactId>
+  <artifactId>spark-protobuf_2.13</artifactId>
   <properties>
     <sbt.project.name>protobuf</sbt.project.name>
   </properties>
@@ -70,12 +70,12 @@
       <groupId>org.apache.spark</groupId>
       <artifactId>spark-tags_${scala.binary.version}</artifactId>
     </dependency>
-    <!-- #if scala-2.13 --><!--
+    <!-- #if scala-2.13 -->
     <dependency>
       <groupId>org.scala-lang.modules</groupId>
       <artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
     </dependency>
-    --><!-- #endif scala-2.13 -->
+    <!-- #endif scala-2.13 -->
     <dependency>
       <groupId>com.google.protobuf</groupId>
       <artifactId>protobuf-java</artifactId>