[KYUUBI #298]Add Support for Apache Spark 3.1.1

.github/workflows/master.yml

@@ -1,11 +1,24 @@
 name: Kyuubi
 
-on: [push, pull_request]
+on:
+  push:
+    branches:
+      - master
+      - branch-*
+  pull_request:
+    branches:
+      - master
+      - branch-*
 
 jobs:
   build:
     name: Build
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        profiles:
+          - '-Pspark-3.0 -Phadoop-2.7'
+          - '-Pspark-3.1 -Phadoop-2.7'
     steps:
       - uses: actions/checkout@v2
       - name: Setup JDK 1.8
@@ -38,7 +51,7 @@ jobs:
             ${{ runner.os }}-maven-io-
       - name: Build with Maven
         run: |
-          mvn clean install -Dmaven.javadoc.skip=true -B -V
+          mvn clean install ${{ matrix.profiles }} -Dmaven.javadoc.skip=true -B -V
           bash <(curl -s https://codecov.io/bash)
       - name: Collect unit tests log
         run: |

.travis.yml

@@ -26,26 +26,24 @@ cache:
   directories:
     - $HOME/.m2
 
-deploy:
-#  - provider: pages
-#    cleanup: false
-#    keep_history: true
-#    github_token: $GITHUB_TOKEN
-#    email: yaooqinn@hotmail.com
-#    name: Kent Yao
-#    on:
-#      branch: master
-#  - provider: releases
-#    api_key: $GITHUB_TOKEN
-#    file_glob: true
-#    file: kyuubi-*.tar.gz
-#    cleanup: false
-#    on:
-#      tags: true
+
+matrix:
+  include:
+    - name: Test Kyuubi -Pspark-3.0 -Phadoop-2.7
+      env:
+        - PROFILE="-Pspark-3.0 -Phadoop-2.7"
+        - EXCLUDE_TAGS=""
+    - name: Tes Kyuubi -Pspark-3.1 -Phadoop-2.7
+      env:
+        - PROFILE="-Pspark-3.1 -Phadoop-2.7"
+        - EXCLUDE_TAGS="org.apache.kyuubi.tags.DataLakeTest"
+#    - name: Tes Kyuubi -Pspark-3.1 -Phadoop-3.2
+#      env:
+#        - PROFILE="-Pspark-3.1 -Phadoop-3.2"
+#        - EXCLUDE_TAGS="org.apache.kyuubi.tags.DataLakeTest"
 
 install:
   - mvn --version
-  - mvn clean
 
 before_script:
   - mvn help:evaluate -Dexpression=project.version
@@ -54,20 +52,11 @@ before_script:
   - mvn help:evaluate -Dexpression=hadoop.version
   - mvn help:evaluate -Dexpression=hive.version
 
-#jobs:
-#  include:
-#    - stage: spark3.0.0
-#      script:
-#        - build/mvn --no-transfer-progress clean verify -pl :kyuubi-common,:kyuubi-ha,:kyuubi-main,:kyuubi-spark-sql-engine,:kyuubi-codecov,:kyuubi-download,:kyuubi-assembly -Dmaven.javadoc.skip=true -B -V
-
 script:
-  - mvn clean install -Dmaven.javadoc.skip=true -V
-
-after_script:
-  - rm -r $HOME/.m2/repository/org/apache/kyuubi
+  - mvn clean install $PROFILE -Dmaven.plugin.scalatest.exclude.tags=$EXCLUDE_TAGS -Dmaven.javadoc.skip=true -V
 
 after_success:
-  - bash <(curl -s https://codecov.io/bash)
+  - if [[ -z $EXCLUDE_TAGS ]]; then bash <(curl -s https://codecov.io/bash); fi
 
 after_failure:
   - for log in `find * -name "kyuubi-spark-sql-engine.log*"`; do echo "=========$log========="; cat $log; done

externals/kyuubi-download/pom.xml

@@ -31,22 +31,13 @@
     <packaging>pom</packaging>
     <name>Kyuubi Project Download Externals</name>
 
-    <properties>
-        <spark.download.skip>false</spark.download.skip>
-        <spark.archive.name>spark-${spark.version}-bin-hadoop2.7.tgz</spark.archive.name>
-        <!-- see more at http://www.apache.org/mirrors/, e.g. https://mirror.bit.edu.cn, https://mirrors.tuna.tsinghua.edu.cn e.t.c -->
-        <!-- spark.archive.mirror>https://mirrors.bfsu.edu.cn</spark.archive.mirror -->
-        <spark.archive.mirror>https://archive.apache.org/dist/spark/spark-${spark.version}</spark.archive.mirror>
-        <!-- spark.archive.sha512>f5652835094d9f69eb3260e20ca9c2d58e8bdf85a8ed15797549a518b23c862b75a329b38d4248f8427e4310718238c60fae0f9d1afb3c70fb390d3e9cce2e49</spark.archive.sha512 -->
-    </properties>
-
     <build>
         <plugins>
             <plugin>
                 <groupId>com.googlecode.maven-download-plugin</groupId>
                 <artifactId>download-maven-plugin</artifactId>
                 <configuration>
-                    <skip>${spark.download.skip}</skip>
+                    <skip>${spark.archive.download.skip}</skip>
                 </configuration>
                 <executions>
                     <execution>
@@ -68,13 +59,4 @@
             </plugin>
         </plugins>
     </build>
-
-    <profiles>
-        <profile>
-            <id>spark-provided</id>
-            <properties>
-                <spark.download.skip>true</spark.download.skip>
-            </properties>
-        </profile>
-    </profiles>
 </project>

externals/kyuubi-spark-sql-engine/pom.xml

@@ -44,6 +44,12 @@
             <version>${project.version}</version>
         </dependency>
 
+        <dependency>
+            <groupId>org.apache.spark</groupId>
+            <artifactId>spark-core_${scala.binary.version}</artifactId>
+            <scope>provided</scope>
+        </dependency>
+
         <dependency>
             <groupId>org.apache.spark</groupId>
             <artifactId>spark-sql_${scala.binary.version}</artifactId>
@@ -79,12 +85,31 @@
             <artifactId>hive-serde</artifactId>
             <scope>test</scope>
         </dependency>
+
         <dependency>
             <groupId>org.apache.hive</groupId>
             <artifactId>hive-service</artifactId>
             <scope>test</scope>
         </dependency>
 
+        <dependency>
+            <groupId>org.apache.hive</groupId>
+            <artifactId>hive-llap-client</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.hive</groupId>
+            <artifactId>hive-llap-common</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-client</artifactId>
+            <scope>test</scope>
+        </dependency>
+
         <dependency>
             <groupId>org.apache.iceberg</groupId>
             <artifactId>${iceberg.name}</artifactId>

externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/SparkSQLEngine.scala

@@ -73,6 +73,7 @@ object SparkSQLEngine extends Logging {
 
   def createSpark(): SparkSession = {
     val sparkConf = new SparkConf()
+    sparkConf.setIfMissing("spark.sql.legacy.castComplexTypesToString.enabled", "true")
     sparkConf.setIfMissing("spark.master", "local")
     sparkConf.setIfMissing("spark.ui.port", "0")
 

externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/WithSparkSQLEngine.scala

@@ -45,6 +45,7 @@ trait WithSparkSQLEngine extends KyuubiFunSuite {
       s"jdbc:derby:;databaseName=$metastorePath;create=true")
     System.setProperty("spark.sql.warehouse.dir", warehousePath.toString)
     System.setProperty("spark.sql.hive.metastore.sharedPrefixes", "org.apache.hive.jdbc")
+    System.setProperty("spark.ui.enabled", "false")
     withKyuubiConf.foreach { case (k, v) =>
       System.setProperty(k, v)
       kyuubiConf.set(k, v)
@@ -59,7 +60,6 @@ trait WithSparkSQLEngine extends KyuubiFunSuite {
 
   override def afterAll(): Unit = {
     super.afterAll()
-
     stopSparkEngine()
   }
 

externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/operation/SparkIcebergOperationSuite.scala

@@ -19,7 +19,9 @@ package org.apache.kyuubi.engine.spark.operation
 
 import org.apache.kyuubi.engine.spark.WithSparkSQLEngine
 import org.apache.kyuubi.operation.BasicIcebergJDBCTests
+import org.apache.kyuubi.tags.DataLakeTest
 
+@DataLakeTest
 class SparkIcebergOperationSuite extends WithSparkSQLEngine with BasicIcebergJDBCTests {
   override protected def jdbcUrl: String = getJdbcUrl
   override def withKyuubiConf: Map[String, String] = icebergConfigs

kyuubi-common/src/test/java/org/apache/kyuubi/tags/DataLakeTest.java

@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kyuubi.tags;
+
+import org.scalatest.TagAnnotation;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@TagAnnotation
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ElementType.METHOD, ElementType.TYPE})
+public @interface DataLakeTest {}

kyuubi-common/src/test/scala/org/apache/kyuubi/KerberizedTestHelper.scala

@@ -19,11 +19,11 @@ package org.apache.kyuubi
 
 import java.io.File
 import java.nio.charset.StandardCharsets
+import java.nio.file.Files
 
 import scala.io.Source
 import scala.util.control.NonFatal
 
-import com.google.common.io.Files
 import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.minikdc.MiniKdc
 import org.apache.hadoop.security.UserGroupInformation
@@ -90,7 +90,9 @@ trait KerberizedTestHelper extends KyuubiFunSuite {
     }
 
     kdc.getKrb5conf.delete()
-    Files.write(krb5confStr, kdc.getKrb5conf, StandardCharsets.UTF_8)
+    val writer = Files.newBufferedWriter(kdc.getKrb5conf.toPath, StandardCharsets.UTF_8)
+    writer.write(krb5confStr)
+    writer.close()
     info(s"krb5.conf file content: $krb5confStr")
   }
 

kyuubi-main/src/test/scala/org/apache/kyuubi/config/AllKyuubiConfiguration.scala

@@ -217,7 +217,7 @@ class AllKyuubiConfiguration extends KyuubiFunSuite {
     writeWith2Line("In the above case, if there are related configurations from" +
       " [JDBC Connection URL](#via-jdbc-connection-url), `kent` will run his SQL engine" +
       " application on YARN and prefer the Spark AQE to be off, while `bob` will activate" +
-      " his SQL engine application on a Spark standalone cluster w/ 8g heap memory for each" +
+      " his SQL engine application on a Spark standalone cluster with 8g heap memory for each" +
       " executor and obey the Spark AQE behavior of Kyuubi system default. On the other hand," +
       " for those users who do not have custom configurations will use system defaults.")
   }

kyuubi-main/src/test/scala/org/apache/kyuubi/operation/datalake/IcebergOperationSuite.scala

@@ -19,7 +19,9 @@ package org.apache.kyuubi.operation.datalake
 
 import org.apache.kyuubi.config.KyuubiConf
 import org.apache.kyuubi.operation.{BasicIcebergJDBCTests, WithKyuubiServer}
+import org.apache.kyuubi.tags.DataLakeTest
 
+@DataLakeTest
 class IcebergOperationSuite extends WithKyuubiServer with BasicIcebergJDBCTests {
   override protected val conf: KyuubiConf = {
     val kyuubiConf = KyuubiConf().set(KyuubiConf.ENGINE_IDLE_TIMEOUT, 20000L)