Skip to content

Commit ad853c5

Browse files
dbtsaidongjoon-hyun
dbtsai
authored and
dongjoon-hyun
committed
[SPARK-25956] Make Scala 2.12 as default Scala version in Spark 3.0
## What changes were proposed in this pull request? This PR makes Spark's default Scala version as 2.12, and Scala 2.11 will be the alternative version. This implies that Scala 2.12 will be used by our CI builds including pull request builds. We'll update the Jenkins to include a new compile-only jobs for Scala 2.11 to ensure the code can be still compiled with Scala 2.11. ## How was this patch tested? existing tests Closes apache#22967 from dbtsai/scala2.12. Authored-by: DB Tsai <d_tsai@apple.com> Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
1 parent 2977e23 commit ad853c5

File tree

45 files changed

+138
-138
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

45 files changed

+138
-138
lines changed

‎assembly/pom.xml

+2-2
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,12 @@
2020
<modelVersion>4.0.0</modelVersion>
2121
<parent>
2222
<groupId>org.apache.spark</groupId>
23-
<artifactId>spark-parent_2.11</artifactId>
23+
<artifactId>spark-parent_2.12</artifactId>
2424
<version>3.0.0-SNAPSHOT</version>
2525
<relativePath>../pom.xml</relativePath>
2626
</parent>
2727

28-
<artifactId>spark-assembly_2.11</artifactId>
28+
<artifactId>spark-assembly_2.12</artifactId>
2929
<name>Spark Project Assembly</name>
3030
<url>http://spark.apache.org/</url>
3131
<packaging>pom</packaging>

‎common/kvstore/pom.xml

+2-2
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,12 @@
2121
<modelVersion>4.0.0</modelVersion>
2222
<parent>
2323
<groupId>org.apache.spark</groupId>
24-
<artifactId>spark-parent_2.11</artifactId>
24+
<artifactId>spark-parent_2.12</artifactId>
2525
<version>3.0.0-SNAPSHOT</version>
2626
<relativePath>../../pom.xml</relativePath>
2727
</parent>
2828

29-
<artifactId>spark-kvstore_2.11</artifactId>
29+
<artifactId>spark-kvstore_2.12</artifactId>
3030
<packaging>jar</packaging>
3131
<name>Spark Project Local DB</name>
3232
<url>http://spark.apache.org/</url>

‎common/network-common/pom.xml

+2-2
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,12 @@
2121
<modelVersion>4.0.0</modelVersion>
2222
<parent>
2323
<groupId>org.apache.spark</groupId>
24-
<artifactId>spark-parent_2.11</artifactId>
24+
<artifactId>spark-parent_2.12</artifactId>
2525
<version>3.0.0-SNAPSHOT</version>
2626
<relativePath>../../pom.xml</relativePath>
2727
</parent>
2828

29-
<artifactId>spark-network-common_2.11</artifactId>
29+
<artifactId>spark-network-common_2.12</artifactId>
3030
<packaging>jar</packaging>
3131
<name>Spark Project Networking</name>
3232
<url>http://spark.apache.org/</url>

‎common/network-shuffle/pom.xml

+2-2
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,12 @@
2121
<modelVersion>4.0.0</modelVersion>
2222
<parent>
2323
<groupId>org.apache.spark</groupId>
24-
<artifactId>spark-parent_2.11</artifactId>
24+
<artifactId>spark-parent_2.12</artifactId>
2525
<version>3.0.0-SNAPSHOT</version>
2626
<relativePath>../../pom.xml</relativePath>
2727
</parent>
2828

29-
<artifactId>spark-network-shuffle_2.11</artifactId>
29+
<artifactId>spark-network-shuffle_2.12</artifactId>
3030
<packaging>jar</packaging>
3131
<name>Spark Project Shuffle Streaming Service</name>
3232
<url>http://spark.apache.org/</url>

‎common/network-yarn/pom.xml

+2-2
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,12 @@
2121
<modelVersion>4.0.0</modelVersion>
2222
<parent>
2323
<groupId>org.apache.spark</groupId>
24-
<artifactId>spark-parent_2.11</artifactId>
24+
<artifactId>spark-parent_2.12</artifactId>
2525
<version>3.0.0-SNAPSHOT</version>
2626
<relativePath>../../pom.xml</relativePath>
2727
</parent>
2828

29-
<artifactId>spark-network-yarn_2.11</artifactId>
29+
<artifactId>spark-network-yarn_2.12</artifactId>
3030
<packaging>jar</packaging>
3131
<name>Spark Project YARN Shuffle Service</name>
3232
<url>http://spark.apache.org/</url>

‎common/sketch/pom.xml

+2-2
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,12 @@
2121
<modelVersion>4.0.0</modelVersion>
2222
<parent>
2323
<groupId>org.apache.spark</groupId>
24-
<artifactId>spark-parent_2.11</artifactId>
24+
<artifactId>spark-parent_2.12</artifactId>
2525
<version>3.0.0-SNAPSHOT</version>
2626
<relativePath>../../pom.xml</relativePath>
2727
</parent>
2828

29-
<artifactId>spark-sketch_2.11</artifactId>
29+
<artifactId>spark-sketch_2.12</artifactId>
3030
<packaging>jar</packaging>
3131
<name>Spark Project Sketch</name>
3232
<url>http://spark.apache.org/</url>

‎common/tags/pom.xml

+2-2
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,12 @@
2121
<modelVersion>4.0.0</modelVersion>
2222
<parent>
2323
<groupId>org.apache.spark</groupId>
24-
<artifactId>spark-parent_2.11</artifactId>
24+
<artifactId>spark-parent_2.12</artifactId>
2525
<version>3.0.0-SNAPSHOT</version>
2626
<relativePath>../../pom.xml</relativePath>
2727
</parent>
2828

29-
<artifactId>spark-tags_2.11</artifactId>
29+
<artifactId>spark-tags_2.12</artifactId>
3030
<packaging>jar</packaging>
3131
<name>Spark Project Tags</name>
3232
<url>http://spark.apache.org/</url>

‎common/unsafe/pom.xml

+2-2
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,12 @@
2121
<modelVersion>4.0.0</modelVersion>
2222
<parent>
2323
<groupId>org.apache.spark</groupId>
24-
<artifactId>spark-parent_2.11</artifactId>
24+
<artifactId>spark-parent_2.12</artifactId>
2525
<version>3.0.0-SNAPSHOT</version>
2626
<relativePath>../../pom.xml</relativePath>
2727
</parent>
2828

29-
<artifactId>spark-unsafe_2.11</artifactId>
29+
<artifactId>spark-unsafe_2.12</artifactId>
3030
<packaging>jar</packaging>
3131
<name>Spark Project Unsafe</name>
3232
<url>http://spark.apache.org/</url>

‎core/pom.xml

+2-2
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,12 @@
2020
<modelVersion>4.0.0</modelVersion>
2121
<parent>
2222
<groupId>org.apache.spark</groupId>
23-
<artifactId>spark-parent_2.11</artifactId>
23+
<artifactId>spark-parent_2.12</artifactId>
2424
<version>3.0.0-SNAPSHOT</version>
2525
<relativePath>../pom.xml</relativePath>
2626
</parent>
2727

28-
<artifactId>spark-core_2.11</artifactId>
28+
<artifactId>spark-core_2.12</artifactId>
2929
<properties>
3030
<sbt.project.name>core</sbt.project.name>
3131
</properties>

‎dev/deps/spark-deps-hadoop-2.7

+18-18
Original file line numberDiff line numberDiff line change
@@ -22,13 +22,13 @@ avro-1.8.2.jar
2222
avro-ipc-1.8.2.jar
2323
avro-mapred-1.8.2-hadoop2.jar
2424
bonecp-0.8.0.RELEASE.jar
25-
breeze-macros_2.11-0.13.2.jar
26-
breeze_2.11-0.13.2.jar
25+
breeze-macros_2.12-0.13.2.jar
26+
breeze_2.12-0.13.2.jar
2727
calcite-avatica-1.2.0-incubating.jar
2828
calcite-core-1.2.0-incubating.jar
2929
calcite-linq4j-1.2.0-incubating.jar
3030
chill-java-0.9.3.jar
31-
chill_2.11-0.9.3.jar
31+
chill_2.12-0.9.3.jar
3232
commons-beanutils-1.7.0.jar
3333
commons-beanutils-core-1.8.0.jar
3434
commons-cli-1.2.jar
@@ -96,7 +96,7 @@ jackson-jaxrs-1.9.13.jar
9696
jackson-mapper-asl-1.9.13.jar
9797
jackson-module-jaxb-annotations-2.9.6.jar
9898
jackson-module-paranamer-2.9.6.jar
99-
jackson-module-scala_2.11-2.9.6.jar
99+
jackson-module-scala_2.12-2.9.6.jar
100100
jackson-xc-1.9.13.jar
101101
janino-3.0.10.jar
102102
javassist-3.18.1-GA.jar
@@ -122,10 +122,10 @@ jline-2.14.6.jar
122122
joda-time-2.9.3.jar
123123
jodd-core-3.5.2.jar
124124
jpam-1.1.jar
125-
json4s-ast_2.11-3.5.3.jar
126-
json4s-core_2.11-3.5.3.jar
127-
json4s-jackson_2.11-3.5.3.jar
128-
json4s-scalap_2.11-3.5.3.jar
125+
json4s-ast_2.12-3.5.3.jar
126+
json4s-core_2.12-3.5.3.jar
127+
json4s-jackson_2.12-3.5.3.jar
128+
json4s-scalap_2.12-3.5.3.jar
129129
jsp-api-2.1.jar
130130
jsr305-3.0.0.jar
131131
jta-1.1.jar
@@ -140,8 +140,8 @@ libthrift-0.9.3.jar
140140
log4j-1.2.17.jar
141141
logging-interceptor-3.9.1.jar
142142
lz4-java-1.5.0.jar
143-
machinist_2.11-0.6.1.jar
144-
macro-compat_2.11-1.1.1.jar
143+
machinist_2.12-0.6.1.jar
144+
macro-compat_2.12-1.1.1.jar
145145
mesos-1.4.0-shaded-protobuf.jar
146146
metrics-core-3.1.5.jar
147147
metrics-graphite-3.1.5.jar
@@ -170,19 +170,19 @@ parquet-jackson-1.10.0.jar
170170
protobuf-java-2.5.0.jar
171171
py4j-0.10.8.1.jar
172172
pyrolite-4.13.jar
173-
scala-compiler-2.11.12.jar
174-
scala-library-2.11.12.jar
175-
scala-parser-combinators_2.11-1.1.0.jar
176-
scala-reflect-2.11.12.jar
177-
scala-xml_2.11-1.0.5.jar
178-
shapeless_2.11-2.3.2.jar
173+
scala-compiler-2.12.7.jar
174+
scala-library-2.12.7.jar
175+
scala-parser-combinators_2.12-1.1.0.jar
176+
scala-reflect-2.12.7.jar
177+
scala-xml_2.12-1.0.5.jar
178+
shapeless_2.12-2.3.2.jar
179179
slf4j-api-1.7.16.jar
180180
slf4j-log4j12-1.7.16.jar
181181
snakeyaml-1.18.jar
182182
snappy-0.2.jar
183183
snappy-java-1.1.7.1.jar
184-
spire-macros_2.11-0.13.0.jar
185-
spire_2.11-0.13.0.jar
184+
spire-macros_2.12-0.13.0.jar
185+
spire_2.12-0.13.0.jar
186186
stax-api-1.0-2.jar
187187
stax-api-1.0.1.jar
188188
stream-2.7.0.jar

‎dev/deps/spark-deps-hadoop-3.1

+18-18
Original file line numberDiff line numberDiff line change
@@ -20,13 +20,13 @@ avro-1.8.2.jar
2020
avro-ipc-1.8.2.jar
2121
avro-mapred-1.8.2-hadoop2.jar
2222
bonecp-0.8.0.RELEASE.jar
23-
breeze-macros_2.11-0.13.2.jar
24-
breeze_2.11-0.13.2.jar
23+
breeze-macros_2.12-0.13.2.jar
24+
breeze_2.12-0.13.2.jar
2525
calcite-avatica-1.2.0-incubating.jar
2626
calcite-core-1.2.0-incubating.jar
2727
calcite-linq4j-1.2.0-incubating.jar
2828
chill-java-0.9.3.jar
29-
chill_2.11-0.9.3.jar
29+
chill_2.12-0.9.3.jar
3030
commons-beanutils-1.9.3.jar
3131
commons-cli-1.2.jar
3232
commons-codec-1.10.jar
@@ -96,7 +96,7 @@ jackson-jaxrs-json-provider-2.7.8.jar
9696
jackson-mapper-asl-1.9.13.jar
9797
jackson-module-jaxb-annotations-2.9.6.jar
9898
jackson-module-paranamer-2.9.6.jar
99-
jackson-module-scala_2.11-2.9.6.jar
99+
jackson-module-scala_2.12-2.9.6.jar
100100
janino-3.0.10.jar
101101
javassist-3.18.1-GA.jar
102102
javax.annotation-api-1.2.jar
@@ -123,10 +123,10 @@ joda-time-2.9.3.jar
123123
jodd-core-3.5.2.jar
124124
jpam-1.1.jar
125125
json-smart-2.3.jar
126-
json4s-ast_2.11-3.5.3.jar
127-
json4s-core_2.11-3.5.3.jar
128-
json4s-jackson_2.11-3.5.3.jar
129-
json4s-scalap_2.11-3.5.3.jar
126+
json4s-ast_2.12-3.5.3.jar
127+
json4s-core_2.12-3.5.3.jar
128+
json4s-jackson_2.12-3.5.3.jar
129+
json4s-scalap_2.12-3.5.3.jar
130130
jsp-api-2.1.jar
131131
jsr305-3.0.0.jar
132132
jta-1.1.jar
@@ -155,8 +155,8 @@ libthrift-0.9.3.jar
155155
log4j-1.2.17.jar
156156
logging-interceptor-3.9.1.jar
157157
lz4-java-1.5.0.jar
158-
machinist_2.11-0.6.1.jar
159-
macro-compat_2.11-1.1.1.jar
158+
machinist_2.12-0.6.1.jar
159+
macro-compat_2.12-1.1.1.jar
160160
mesos-1.4.0-shaded-protobuf.jar
161161
metrics-core-3.1.5.jar
162162
metrics-graphite-3.1.5.jar
@@ -189,19 +189,19 @@ protobuf-java-2.5.0.jar
189189
py4j-0.10.8.1.jar
190190
pyrolite-4.13.jar
191191
re2j-1.1.jar
192-
scala-compiler-2.11.12.jar
193-
scala-library-2.11.12.jar
194-
scala-parser-combinators_2.11-1.1.0.jar
195-
scala-reflect-2.11.12.jar
196-
scala-xml_2.11-1.0.5.jar
197-
shapeless_2.11-2.3.2.jar
192+
scala-compiler-2.12.7.jar
193+
scala-library-2.12.7.jar
194+
scala-parser-combinators_2.12-1.1.0.jar
195+
scala-reflect-2.12.7.jar
196+
scala-xml_2.12-1.0.5.jar
197+
shapeless_2.12-2.3.2.jar
198198
slf4j-api-1.7.16.jar
199199
slf4j-log4j12-1.7.16.jar
200200
snakeyaml-1.18.jar
201201
snappy-0.2.jar
202202
snappy-java-1.1.7.1.jar
203-
spire-macros_2.11-0.13.0.jar
204-
spire_2.11-0.13.0.jar
203+
spire-macros_2.12-0.13.0.jar
204+
spire_2.12-0.13.0.jar
205205
stax-api-1.0.1.jar
206206
stax2-api-3.1.4.jar
207207
stream-2.7.0.jar

‎docs/_config.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,8 @@ include:
1616
# of Spark, Scala, and Mesos.
1717
SPARK_VERSION: 3.0.0-SNAPSHOT
1818
SPARK_VERSION_SHORT: 3.0.0
19-
SCALA_BINARY_VERSION: "2.11"
20-
SCALA_VERSION: "2.11.12"
19+
SCALA_BINARY_VERSION: "2.12"
20+
SCALA_VERSION: "2.12.7"
2121
MESOS_VERSION: 1.0.0
2222
SPARK_ISSUE_TRACKER_URL: https://issues.apache.org/jira/browse/SPARK
2323
SPARK_GITHUB_URL: https://github.com/apache/spark

‎docs/_plugins/copy_api_dirs.rb

+1-1
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@
3737

3838
# Copy over the unified ScalaDoc for all projects to api/scala.
3939
# This directory will be copied over to _site when `jekyll` command is run.
40-
source = "../target/scala-2.11/unidoc"
40+
source = "../target/scala-2.12/unidoc"
4141
dest = "api/scala"
4242

4343
puts "Making directory " + dest

‎docs/building-spark.md

+9-9
Original file line numberDiff line numberDiff line change
@@ -96,9 +96,9 @@ It's possible to build Spark submodules using the `mvn -pl` option.
9696

9797
For instance, you can build the Spark Streaming module using:
9898

99-
./build/mvn -pl :spark-streaming_2.11 clean install
99+
./build/mvn -pl :spark-streaming_{{site.SCALA_BINARY_VERSION}} clean install
100100

101-
where `spark-streaming_2.11` is the `artifactId` as defined in `streaming/pom.xml` file.
101+
where `spark-streaming_{{site.SCALA_BINARY_VERSION}}` is the `artifactId` as defined in `streaming/pom.xml` file.
102102

103103
## Continuous Compilation
104104

@@ -230,25 +230,25 @@ Once installed, the `docker` service needs to be started, if not already running
230230
On Linux, this can be done by `sudo service docker start`.
231231

232232
./build/mvn install -DskipTests
233-
./build/mvn test -Pdocker-integration-tests -pl :spark-docker-integration-tests_2.11
233+
./build/mvn test -Pdocker-integration-tests -pl :spark-docker-integration-tests_{{site.SCALA_BINARY_VERSION}}
234234

235235
or
236236

237237
./build/sbt docker-integration-tests/test
238238

239239
## Change Scala Version
240240

241-
To build Spark using another supported Scala version, please change the major Scala version using (e.g. 2.12):
241+
To build Spark using another supported Scala version, please change the major Scala version using (e.g. 2.11):
242242

243-
./dev/change-scala-version.sh 2.12
243+
./dev/change-scala-version.sh 2.11
244244

245-
For Maven, please enable the profile (e.g. 2.12):
245+
For Maven, please enable the profile (e.g. 2.11):
246246

247-
./build/mvn -Pscala-2.12 compile
247+
./build/mvn -Pscala-2.11 compile
248248

249-
For SBT, specify a complete scala version using (e.g. 2.12.6):
249+
For SBT, specify a complete scala version using (e.g. 2.11.12):
250250

251-
./build/sbt -Dscala.version=2.12.6
251+
./build/sbt -Dscala.version=2.11.12
252252

253253
Otherwise, the sbt-pom-reader plugin will use the `scala.version` specified in the spark-parent pom.
254254

‎docs/cloud-integration.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ is set to the chosen version of Spark:
8585
...
8686
<dependency>
8787
<groupId>org.apache.spark</groupId>
88-
<artifactId>hadoop-cloud_2.11</artifactId>
88+
<artifactId>hadoop-cloud_{{site.SCALA_BINARY_VERSION}}</artifactId>
8989
<version>${spark.version}</version>
9090
</dependency>
9191
...

‎docs/sparkr.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -133,7 +133,7 @@ specifying `--packages` with `spark-submit` or `sparkR` commands, or if initiali
133133

134134
<div data-lang="r" markdown="1">
135135
{% highlight r %}
136-
sparkR.session(sparkPackages = "com.databricks:spark-avro_2.11:3.0.0")
136+
sparkR.session(sparkPackages = "org.apache.spark:spark-avro_{{site.SCALA_BINARY_VERSION}}:{{site.SPARK_VERSION}}")
137137
{% endhighlight %}
138138
</div>
139139

‎examples/pom.xml

+2-2
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,12 @@
2020
<modelVersion>4.0.0</modelVersion>
2121
<parent>
2222
<groupId>org.apache.spark</groupId>
23-
<artifactId>spark-parent_2.11</artifactId>
23+
<artifactId>spark-parent_2.12</artifactId>
2424
<version>3.0.0-SNAPSHOT</version>
2525
<relativePath>../pom.xml</relativePath>
2626
</parent>
2727

28-
<artifactId>spark-examples_2.11</artifactId>
28+
<artifactId>spark-examples_2.12</artifactId>
2929
<packaging>jar</packaging>
3030
<name>Spark Project Examples</name>
3131
<url>http://spark.apache.org/</url>

0 commit comments

Comments
 (0)