[KYUUBI #298]Add Support for Apache Spark 3.1.1

.github/workflows/master.yml

@@ -6,6 +6,11 @@ jobs:
   build:
     name: Build
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        spark:
+          - 3.0
+          - 3.1
     steps:
       - uses: actions/checkout@v2
       - name: Setup JDK 1.8
@@ -38,7 +43,7 @@ jobs:
             ${{ runner.os }}-maven-io-
       - name: Build with Maven
         run: |
-          mvn clean install -Dmaven.javadoc.skip=true -B -V
+          mvn clean install -Pspark-${{ matrix.spark }} -Dmaven.javadoc.skip=true -B -V
           bash <(curl -s https://codecov.io/bash)
       - name: Collect unit tests log
         run: |

.travis.yml

@@ -54,14 +54,17 @@ before_script:
   - mvn help:evaluate -Dexpression=hadoop.version
   - mvn help:evaluate -Dexpression=hive.version
 
-#jobs:
-#  include:
-#    - stage: spark3.0.0
-#      script:
-#        - build/mvn --no-transfer-progress clean verify -pl :kyuubi-common,:kyuubi-ha,:kyuubi-main,:kyuubi-spark-sql-engine,:kyuubi-codecov,:kyuubi-download,:kyuubi-assembly -Dmaven.javadoc.skip=true -B -V
+jobs:
+  include:
+    - stage: spark3.0
+      script:
+        - mvn clean install -Dmaven.javadoc.skip=true -V
+    - stage: spark3.1
+      script:
+        - mvn clean install -Dmaven.javadoc.skip=true -V -Pspark-3.1
 
-script:
-  - mvn clean install -Dmaven.javadoc.skip=true -V
+#script:
+#  - mvn clean install -Dmaven.javadoc.skip=true -V
 
 after_script:
   - rm -r $HOME/.m2/repository/org/apache/kyuubi

externals/kyuubi-download/pom.xml

@@ -31,22 +31,13 @@
     <packaging>pom</packaging>
     <name>Kyuubi Project Download Externals</name>
 
-    <properties>
-        <spark.download.skip>false</spark.download.skip>
-        <spark.archive.name>spark-${spark.version}-bin-hadoop2.7.tgz</spark.archive.name>
-        <!-- see more at http://www.apache.org/mirrors/, e.g. https://mirror.bit.edu.cn, https://mirrors.tuna.tsinghua.edu.cn e.t.c -->
-        <!-- spark.archive.mirror>https://mirrors.bfsu.edu.cn</spark.archive.mirror -->
-        <spark.archive.mirror>https://archive.apache.org/dist/spark/spark-${spark.version}</spark.archive.mirror>
-        <!-- spark.archive.sha512>f5652835094d9f69eb3260e20ca9c2d58e8bdf85a8ed15797549a518b23c862b75a329b38d4248f8427e4310718238c60fae0f9d1afb3c70fb390d3e9cce2e49</spark.archive.sha512 -->
-    </properties>
-
     <build>
         <plugins>
             <plugin>
                 <groupId>com.googlecode.maven-download-plugin</groupId>
                 <artifactId>download-maven-plugin</artifactId>
                 <configuration>
-                    <skip>${spark.download.skip}</skip>
+                    <skip>${spark.archive.download.skip}</skip>
                 </configuration>
                 <executions>
                     <execution>
@@ -68,13 +59,4 @@
             </plugin>
         </plugins>
     </build>
-
-    <profiles>
-        <profile>
-            <id>spark-provided</id>
-            <properties>
-                <spark.download.skip>true</spark.download.skip>
-            </properties>
-        </profile>
-    </profiles>
 </project>

externals/kyuubi-spark-sql-engine/pom.xml

@@ -44,6 +44,12 @@
             <version>${project.version}</version>
         </dependency>
 
+        <dependency>
+            <groupId>org.apache.spark</groupId>
+            <artifactId>spark-core_${scala.binary.version}</artifactId>
+            <scope>provided</scope>
+        </dependency>
+
         <dependency>
             <groupId>org.apache.spark</groupId>
             <artifactId>spark-sql_${scala.binary.version}</artifactId>
@@ -79,12 +85,25 @@
             <artifactId>hive-serde</artifactId>
             <scope>test</scope>
         </dependency>
+
         <dependency>
             <groupId>org.apache.hive</groupId>
             <artifactId>hive-service</artifactId>
             <scope>test</scope>
         </dependency>
 
+        <dependency>
+            <groupId>org.apache.hive</groupId>
+            <artifactId>hive-llap-client</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.hive</groupId>
+            <artifactId>hive-llap-common</artifactId>
+            <scope>test</scope>
+        </dependency>
+
         <dependency>
             <groupId>org.apache.iceberg</groupId>
             <artifactId>${iceberg.name}</artifactId>

externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/WithSparkSQLEngine.scala

@@ -42,6 +42,8 @@ trait WithSparkSQLEngine extends KyuubiFunSuite {
       s"jdbc:derby:;databaseName=$metastorePath;create=true")
     System.setProperty("spark.sql.warehouse.dir", warehousePath.toString)
     System.setProperty("spark.sql.hive.metastore.sharedPrefixes", "org.apache.hive.jdbc")
+    System.setProperty("spark.ui.enabled", "false")
+
     conf.foreach { case (k, v) =>
       System.setProperty(k, v)
       SparkSQLEngine.kyuubiConf.set(k, v)

kyuubi-common/src/test/scala/org/apache/kyuubi/KerberizedTestHelper.scala

@@ -19,11 +19,11 @@ package org.apache.kyuubi
 
 import java.io.File
 import java.nio.charset.StandardCharsets
+import java.nio.file.Files
 
 import scala.io.Source
 import scala.util.control.NonFatal
 
-import com.google.common.io.Files
 import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.minikdc.MiniKdc
 import org.apache.hadoop.security.UserGroupInformation
@@ -90,7 +90,9 @@ trait KerberizedTestHelper extends KyuubiFunSuite {
     }
 
     kdc.getKrb5conf.delete()
-    Files.write(krb5confStr, kdc.getKrb5conf, StandardCharsets.UTF_8)
+    val writer = Files.newBufferedWriter(kdc.getKrb5conf.toPath, StandardCharsets.UTF_8)
+    writer.write(krb5confStr)
+    writer.close()
     info(s"krb5.conf file content: $krb5confStr")
   }