#22 Add support for Scala 2.13

.github/workflows/jacoco_check.yml

@@ -29,17 +29,17 @@ jobs:
  include:
  - scala: 2.11.12
  scalaShort: "2.11"
- spark: 2.4.7
+ spark: 2.4.8
  overall: 0.0
  changed: 80.0
- - scala: 2.12.15
+ - scala: 2.12.18
  scalaShort: "2.12"
- spark: 2.4.7
+ spark: 3.2.4
  overall: 0.0
  changed: 80.0
- - scala: 2.12.15
- scalaShort: "2.12"
- spark: 3.2.1
+ - scala: 2.13.11
+ scalaShort: "2.13"
+ spark: 3.4.1
  overall: 0.0
  changed: 80.0
  name: Check code-coverage by JaCoCo - Spark ${{matrix.spark}} on Scala ${{matrix.scala}}

.github/workflows/test.yml

@@ -27,11 +27,15 @@ jobs:
  strategy:
  fail-fast: false
  matrix:
- scala: [2.11.12, 2.12.15]
- spark: [2.4.7, 3.2.1]
+ scala: [2.11.12, 2.12.18, 2.13.11]
+ spark: [2.4.8, 3.2.4, 3.4.1]
  exclude:
  - scala: 2.11.12
- spark: 3.2.1
+ spark: 3.2.4
+ - scala: 2.11.12
+ spark: 3.4.1
+ - scala: 2.13.11
+ spark: 2.4.8
  name: Test Spark ${{matrix.spark}} on Scala ${{matrix.scala}}
  steps:
  - name: Checkout code

build.sbt

@@ -17,12 +17,13 @@
 import Dependencies._
 
 val scala211 = "2.11.12"
-val scala212 = "2.12.10"
+val scala212 = "2.12.18"
+val scala213 = "2.13.11"
 
 ThisBuild / organization := "za.co.absa"
 
-ThisBuild / scalaVersion := scala211
-ThisBuild / crossScalaVersions := Seq(scala211, scala212)
+ThisBuild / scalaVersion := scala212
+ThisBuild / crossScalaVersions := Seq(scala211, scala212, scala213)
 
 // Scala shouldn't be packaged so it is explicitly added as a provided dependency below
 ThisBuild / autoScalaLibrary := false
@@ -34,10 +35,11 @@ lazy val hofs = (project in file("."))
  name := "spark-hofs",
  printSparkVersion := {
  val log = streams.value.log
- log.info(s"Building with Spark $sparkVersion")
- sparkVersion
+ val effectiveSparkVersion = sparkVersion(scalaVersion.value)
+ log.info(s"Building with Spark $effectiveSparkVersion")
+ effectiveSparkVersion
  },
- libraryDependencies ++= SparkHofsDependencies :+ getScalaDependency(scalaVersion.value),
+ libraryDependencies ++= getSparkHofsDependencies(scalaVersion.value) :+ getScalaDependency(scalaVersion.value),
  releasePublishArtifactsAction := PgpKeys.publishSigned.value,
  Test / fork := true
  ).enablePlugins(AutomateHeaderPlugin)

project/Dependencies.scala

@@ -17,21 +17,36 @@
 import sbt._
 
 object Dependencies {
+ val defaultSparkVersionForScala211 = "2.4.8"
+ val defaultSparkVersionForScala212 = "3.3.2"
+ val defaultSparkVersionForScala213 = "3.4.1"
 
- def sparkVersion: String = sys.props.getOrElse("SPARK_VERSION", "2.4.7")
-
- private val scalatestVersion = "3.0.3"
+ private val scalatestVersion = "3.2.14"
 
  def getScalaDependency(scalaVersion: String): ModuleID = "org.scala-lang" % "scala-library" % scalaVersion % Provided
 
- val SparkHofsDependencies: Seq[ModuleID] = Seq(
+ def getSparkHofsDependencies(scalaVersion: String): Seq[ModuleID] = Seq(
  // provided
- "org.apache.spark" %% "spark-core" % sparkVersion % Provided,
- "org.apache.spark" %% "spark-sql" % sparkVersion % Provided,
- "org.apache.spark" %% "spark-catalyst" % sparkVersion % Provided,
+ "org.apache.spark" %% "spark-core" % sparkVersion(scalaVersion) % Provided,
+ "org.apache.spark" %% "spark-sql" % sparkVersion(scalaVersion) % Provided,
+ "org.apache.spark" %% "spark-catalyst" % sparkVersion(scalaVersion) % Provided,
 
  // test
  "org.scalatest" %% "scalatest" % scalatestVersion % Test
  )
 
+ def sparkVersion(scalaVersion: String): String = sys.props.getOrElse("SPARK_VERSION", sparkFallbackVersion(scalaVersion))
+
+ def sparkFallbackVersion(scalaVersion: String): String = {
+ if (scalaVersion.startsWith("2.11.")) {
+ defaultSparkVersionForScala211
+ } else if (scalaVersion.startsWith("2.12.")) {
+ defaultSparkVersionForScala212
+ } else if (scalaVersion.startsWith("2.13.")) {
+ defaultSparkVersionForScala213
+ } else {
+ throw new IllegalArgumentException(s"Scala $scalaVersion not supported.")
+ }
+ }
+
 }

project/build.properties

@@ -13,4 +13,4 @@
 # limitations under the License.
 #
 
-sbt.version=1.5.8
+sbt.version=1.9.2

project/plugins.sbt

@@ -14,9 +14,9 @@
  * limitations under the License.
  */
 
-addSbtPlugin("com.jsuereth"  % "sbt-pgp" % "2.0.0")
-addSbtPlugin("com.github.gseitz" % "sbt-release" % "1.0.12")
-addSbtPlugin("de.heikoseeberger" % "sbt-header" % "5.6.0")
+addSbtPlugin("com.github.sbt" % "sbt-pgp" % "2.2.1")
+addSbtPlugin("com.github.sbt" % "sbt-release" % "1.1.0")
+addSbtPlugin("de.heikoseeberger" % "sbt-header" % "5.7.0")
 
 // sbt-jacoco - workaround related dependencies required to download
 lazy val ow2Version = "9.5"

src/test/resources/log4j.properties

@@ -0,0 +1,19 @@
+# Copyright 2018 ABSA Group Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+log4j.rootCategory=INFO, console
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n
+log4j.appender.console.Threshold=ERROR

src/test/resources/log4j2.properties

@@ -0,0 +1,19 @@
+# Copyright 2018 ABSA Group Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+log4j.rootCategory=INFO, console
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n
+log4j.appender.console.Threshold=ERROR

src/test/scala/za/co/absa/spark/hofs/AggregateFunctionSuite.scala

@@ -18,11 +18,12 @@ package za.co.absa.spark.hofs
 
 import org.apache.spark.sql.Column
 import org.apache.spark.sql.functions._
-import org.scalatest.{FunSuite, Matchers}
 import DataFrameExtensions._
+import org.scalatest.funsuite.AnyFunSuite
+import org.scalatest.matchers.should.Matchers
 import za.co.absa.spark.hofs
 
-class AggregateFunctionSuite extends FunSuite with TestBase with Matchers {
+class AggregateFunctionSuite extends AnyFunSuite with TestBase with Matchers {
  import spark.implicits._
 
  private val df = Seq(Seq(2, 4, 5, 7)).toDF("array")

src/test/scala/za/co/absa/spark/hofs/FilterFunctionSuite.scala

@@ -17,11 +17,11 @@
 package za.co.absa.spark.hofs
 
 import org.apache.spark.sql.Column
-import org.scalatest.{FunSuite, Matchers}
-
 import DataFrameExtensions._
+import org.scalatest.funsuite.AnyFunSuite
+import org.scalatest.matchers.should.Matchers
 
-class FilterFunctionSuite extends FunSuite with TestBase with Matchers {
+class FilterFunctionSuite extends AnyFunSuite with TestBase with Matchers {
  import spark.implicits._
 
  private val predicate = (x: Column) => x % 2 === 0

src/test/scala/za/co/absa/spark/hofs/TransformFunctionSuite.scala

@@ -17,11 +17,12 @@
 package za.co.absa.spark.hofs
 
 import org.apache.spark.sql.functions._
-import org.scalatest.{FunSuite, Matchers}
 import DataFrameExtensions._
+import org.scalatest.funsuite.AnyFunSuite
+import org.scalatest.matchers.should.Matchers
 import za.co.absa.spark.hofs
 
-class TransformFunctionSuite extends FunSuite with TestBase with Matchers {
+class TransformFunctionSuite extends AnyFunSuite with TestBase with Matchers {
  import spark.implicits._
 
  test("transform function with anonymous variables") {

src/test/scala/za/co/absa/spark/hofs/ZipWithFunctionSuite.scala

@@ -17,11 +17,11 @@
 package za.co.absa.spark.hofs
 
 import org.apache.spark.sql.Column
-import org.scalatest.{FunSuite, Matchers}
-
 import DataFrameExtensions._
+import org.scalatest.funsuite.AnyFunSuite
+import org.scalatest.matchers.should.Matchers
 
-class ZipWithFunctionSuite extends FunSuite with TestBase with Matchers {
+class ZipWithFunctionSuite extends AnyFunSuite with TestBase with Matchers {
  import spark.implicits._
 
  private val df = Seq((Seq(1, 4, 5, 7), Seq(2, 6, 5, 8))).toDF("array1", "array2")

version.sbt

@@ -14,4 +14,4 @@
  * limitations under the License.
  */
 
-ThisBuild / version := "0.4.1-SNAPSHOT"
+ThisBuild / version := "0.5.0-SNAPSHOT"