From 570dc84558ce7522210d43a0a6d1bcf4b80e5737 Mon Sep 17 00:00:00 2001 From: Yuming Wang Date: Wed, 2 Jun 2021 21:11:03 +0800 Subject: [PATCH 1/5] Upgrade Hive to 2.3.9 --- dev/deps/spark-deps-hadoop-2.7-hive-2.3 | 26 ++++++++-------- dev/deps/spark-deps-hadoop-3.2-hive-2.3 | 26 ++++++++-------- docs/building-spark.md | 4 +-- docs/sql-data-sources-hive-tables.md | 8 ++--- docs/sql-migration-guide.md | 2 +- pom.xml | 15 +++++++-- project/SparkBuild.scala | 3 ++ .../apache/spark/sql/internal/SQLConf.scala | 3 +- .../org/apache/spark/sql/SQLQuerySuite.scala | 2 +- .../org/apache/spark/sql/hive/HiveUtils.scala | 2 +- .../spark/sql/hive/client/package.scala | 2 +- .../HiveExternalCatalogVersionsSuite.scala | 2 +- .../execution/HiveSerDeReadWriteSuite.scala | 31 +++++++++++++++++++ .../apache/spark/sql/hive/test/TestHive.scala | 2 +- 14 files changed, 87 insertions(+), 41 deletions(-) diff --git a/dev/deps/spark-deps-hadoop-2.7-hive-2.3 b/dev/deps/spark-deps-hadoop-2.7-hive-2.3 index 0899fd266268..9efdf192ff35 100644 --- a/dev/deps/spark-deps-hadoop-2.7-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-2.7-hive-2.3 @@ -83,21 +83,21 @@ hadoop-yarn-client/2.7.4//hadoop-yarn-client-2.7.4.jar hadoop-yarn-common/2.7.4//hadoop-yarn-common-2.7.4.jar hadoop-yarn-server-common/2.7.4//hadoop-yarn-server-common-2.7.4.jar hadoop-yarn-server-web-proxy/2.7.4//hadoop-yarn-server-web-proxy-2.7.4.jar -hive-beeline/2.3.8//hive-beeline-2.3.8.jar -hive-cli/2.3.8//hive-cli-2.3.8.jar -hive-common/2.3.8//hive-common-2.3.8.jar -hive-exec/2.3.8/core/hive-exec-2.3.8-core.jar -hive-jdbc/2.3.8//hive-jdbc-2.3.8.jar -hive-llap-common/2.3.8//hive-llap-common-2.3.8.jar -hive-metastore/2.3.8//hive-metastore-2.3.8.jar -hive-serde/2.3.8//hive-serde-2.3.8.jar +hive-beeline/2.3.9//hive-beeline-2.3.9.jar +hive-cli/2.3.9//hive-cli-2.3.9.jar +hive-common/2.3.9//hive-common-2.3.9.jar +hive-exec/2.3.9/core/hive-exec-2.3.9-core.jar +hive-jdbc/2.3.9//hive-jdbc-2.3.9.jar +hive-llap-common/2.3.9//hive-llap-common-2.3.9.jar +hive-metastore/2.3.9//hive-metastore-2.3.9.jar +hive-serde/2.3.9//hive-serde-2.3.9.jar hive-service-rpc/3.1.2//hive-service-rpc-3.1.2.jar -hive-shims-0.23/2.3.8//hive-shims-0.23-2.3.8.jar -hive-shims-common/2.3.8//hive-shims-common-2.3.8.jar -hive-shims-scheduler/2.3.8//hive-shims-scheduler-2.3.8.jar -hive-shims/2.3.8//hive-shims-2.3.8.jar +hive-shims-0.23/2.3.9//hive-shims-0.23-2.3.9.jar +hive-shims-common/2.3.9//hive-shims-common-2.3.9.jar +hive-shims-scheduler/2.3.9//hive-shims-scheduler-2.3.9.jar +hive-shims/2.3.9//hive-shims-2.3.9.jar hive-storage-api/2.7.2//hive-storage-api-2.7.2.jar -hive-vector-code-gen/2.3.8//hive-vector-code-gen-2.3.8.jar +hive-vector-code-gen/2.3.9//hive-vector-code-gen-2.3.9.jar hk2-api/2.6.1//hk2-api-2.6.1.jar hk2-locator/2.6.1//hk2-locator-2.6.1.jar hk2-utils/2.6.1//hk2-utils-2.6.1.jar diff --git a/dev/deps/spark-deps-hadoop-3.2-hive-2.3 b/dev/deps/spark-deps-hadoop-3.2-hive-2.3 index 70bcad453604..2f3da18f0569 100644 --- a/dev/deps/spark-deps-hadoop-3.2-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-3.2-hive-2.3 @@ -61,21 +61,21 @@ guava/14.0.1//guava-14.0.1.jar hadoop-client-api/3.2.2//hadoop-client-api-3.2.2.jar hadoop-client-runtime/3.2.2//hadoop-client-runtime-3.2.2.jar hadoop-yarn-server-web-proxy/3.2.2//hadoop-yarn-server-web-proxy-3.2.2.jar -hive-beeline/2.3.8//hive-beeline-2.3.8.jar -hive-cli/2.3.8//hive-cli-2.3.8.jar -hive-common/2.3.8//hive-common-2.3.8.jar -hive-exec/2.3.8/core/hive-exec-2.3.8-core.jar -hive-jdbc/2.3.8//hive-jdbc-2.3.8.jar -hive-llap-common/2.3.8//hive-llap-common-2.3.8.jar -hive-metastore/2.3.8//hive-metastore-2.3.8.jar -hive-serde/2.3.8//hive-serde-2.3.8.jar +hive-beeline/2.3.9//hive-beeline-2.3.9.jar +hive-cli/2.3.9//hive-cli-2.3.9.jar +hive-common/2.3.9//hive-common-2.3.9.jar +hive-exec/2.3.9/core/hive-exec-2.3.9-core.jar +hive-jdbc/2.3.9//hive-jdbc-2.3.9.jar +hive-llap-common/2.3.9//hive-llap-common-2.3.9.jar +hive-metastore/2.3.9//hive-metastore-2.3.9.jar +hive-serde/2.3.9//hive-serde-2.3.9.jar hive-service-rpc/3.1.2//hive-service-rpc-3.1.2.jar -hive-shims-0.23/2.3.8//hive-shims-0.23-2.3.8.jar -hive-shims-common/2.3.8//hive-shims-common-2.3.8.jar -hive-shims-scheduler/2.3.8//hive-shims-scheduler-2.3.8.jar -hive-shims/2.3.8//hive-shims-2.3.8.jar +hive-shims-0.23/2.3.9//hive-shims-0.23-2.3.9.jar +hive-shims-common/2.3.9//hive-shims-common-2.3.9.jar +hive-shims-scheduler/2.3.9//hive-shims-scheduler-2.3.9.jar +hive-shims/2.3.9//hive-shims-2.3.9.jar hive-storage-api/2.7.2//hive-storage-api-2.7.2.jar -hive-vector-code-gen/2.3.8//hive-vector-code-gen-2.3.8.jar +hive-vector-code-gen/2.3.9//hive-vector-code-gen-2.3.9.jar hk2-api/2.6.1//hk2-api-2.6.1.jar hk2-locator/2.6.1//hk2-locator-2.6.1.jar hk2-utils/2.6.1//hk2-utils-2.6.1.jar diff --git a/docs/building-spark.md b/docs/building-spark.md index 8e1c84a37b43..1a0dee6a913b 100644 --- a/docs/building-spark.md +++ b/docs/building-spark.md @@ -83,9 +83,9 @@ Example: To enable Hive integration for Spark SQL along with its JDBC server and CLI, add the `-Phive` and `-Phive-thriftserver` profiles to your existing build options. -By default Spark will build with Hive 2.3.8. +By default Spark will build with Hive 2.3.9. - # With Hive 2.3.8 support + # With Hive 2.3.9 support ./build/mvn -Pyarn -Phive -Phive-thriftserver -DskipTests clean package ## Packaging without Hadoop Dependencies for YARN diff --git a/docs/sql-data-sources-hive-tables.md b/docs/sql-data-sources-hive-tables.md index 14dd75070994..6f480d0808d7 100644 --- a/docs/sql-data-sources-hive-tables.md +++ b/docs/sql-data-sources-hive-tables.md @@ -127,10 +127,10 @@ The following options can be used to configure the version of Hive that is used Property NameDefaultMeaningSince Version spark.sql.hive.metastore.version - 2.3.8 + 2.3.9 Version of the Hive metastore. Available - options are 0.12.0 through 2.3.8 and 3.0.0 through 3.1.2. + options are 0.12.0 through 2.3.9 and 3.0.0 through 3.1.2. 1.4.0 @@ -142,9 +142,9 @@ The following options can be used to configure the version of Hive that is used property can be one of four options:
  1. builtin
    2. - Use Hive 2.3.8, which is bundled with the Spark assembly when -Phive is + Use Hive 2.3.9, which is bundled with the Spark assembly when -Phive is enabled. When this option is chosen, spark.sql.hive.metastore.version must be - either 2.3.8 or not defined. + either 2.3.9 or not defined.
  2. maven
    3. Use Hive jars of specified version downloaded from Maven repositories. This configuration is not generally recommended for production deployments. diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md index a5b2d8be7aaa..fd36e19a4963 100644 --- a/docs/sql-migration-guide.md +++ b/docs/sql-migration-guide.md @@ -921,7 +921,7 @@ Python UDF registration is unchanged. Spark SQL is designed to be compatible with the Hive Metastore, SerDes and UDFs. Currently, Hive SerDes and UDFs are based on built-in Hive, and Spark SQL can be connected to different versions of Hive Metastore -(from 0.12.0 to 2.3.8 and 3.0.0 to 3.1.2. Also see [Interacting with Different Versions of Hive Metastore](sql-data-sources-hive-tables.html#interacting-with-different-versions-of-hive-metastore)). +(from 0.12.0 to 2.3.9 and 3.0.0 to 3.1.2. Also see [Interacting with Different Versions of Hive Metastore](sql-data-sources-hive-tables.html#interacting-with-different-versions-of-hive-metastore)). #### Deploying in Existing Hive Warehouses {:.no_toc} diff --git a/pom.xml b/pom.xml index 930030ef30b5..b9caa20eac2d 100644 --- a/pom.xml +++ b/pom.xml @@ -128,8 +128,8 @@ org.apache.hive core - 2.3.8 - 2.3.8 + 2.3.9 + 2.3.9 2.3 @@ -300,6 +300,17 @@ false + + staged + hive-staged-releases + https://repository.apache.org/content/repositories/staging/ + + true + + + true + + diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 76fdc097ce3c..e68e1678fa57 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -273,6 +273,9 @@ object SparkBuild extends PomBuild { DefaultMavenRepository, Resolver.mavenLocal, Resolver.file("ivyLocal", file(Path.userHome.absolutePath + "/.ivy2/local"))(Resolver.ivyStylePatterns) + ) ++ Seq( + "hive-staged-releases-mirror" at "https://repository.apache.org/content/repositories/staging/", + Resolver.file("local", file(Path.userHome.absolutePath + "/.ivy2/local"))(Resolver.ivyStylePatterns) ), externalResolvers := resolvers.value, otherResolvers := SbtPomKeys.mvnLocalRepository(dotM2 => Seq(Resolver.file("dotM2", dotM2))).value, diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index e368ee1d532d..17f0dc4adfbd 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -2883,7 +2883,8 @@ object SQLConf { .stringConf .createWithDefault( sys.env.getOrElse("DEFAULT_ARTIFACT_REPOSITORY", - "https://maven-central.storage-download.googleapis.com/maven2/")) + "https://maven-central.storage-download.googleapis.com/maven2/," + + "https://repository.apache.org/content/repositories/staging/")) val LEGACY_FROM_DAYTIME_STRING = buildConf("spark.sql.legacy.fromDayTimeString.enabled") diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala index ee0c7c370922..76c9f790d988 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala @@ -3642,7 +3642,7 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark test("SPARK-33084: Add jar support Ivy URI in SQL") { val sc = spark.sparkContext - val hiveVersion = "2.3.8" + val hiveVersion = "2.3.9" // transitive=false, only download specified jar sql(s"ADD JAR ivy://org.apache.hive.hcatalog:hive-hcatalog-core:$hiveVersion?transitive=false") assert(sc.listJars() diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala index 8c2f02cfa118..3b010d94939f 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala @@ -71,7 +71,7 @@ private[spark] object HiveUtils extends Logging { val HIVE_METASTORE_VERSION = buildStaticConf("spark.sql.hive.metastore.version") .doc("Version of the Hive metastore. Available options are " + - "0.12.0 through 2.3.8 and " + + "0.12.0 through 2.3.9 and " + "3.0.0 through 3.1.2.") .version("1.4.0") .stringConf diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/package.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/package.scala index bb78944fb161..d8203648b736 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/package.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/package.scala @@ -102,7 +102,7 @@ package object client { // Since HIVE-23980, calcite-core included in Hive package jar. // For spark, only VersionsSuite currently creates a hive materialized view for testing. - case object v2_3 extends HiveVersion("2.3.8", + case object v2_3 extends HiveVersion("2.3.9", exclusions = Seq("org.apache.calcite:calcite-core", "org.apache.calcite:calcite-druid", "org.apache.calcite.avatica:avatica", diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala index c9712968f9bd..250c46063d5c 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala @@ -203,7 +203,7 @@ class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils { tryDownloadSpark(version, sparkTestingDir.getCanonicalPath) } - // Extract major.minor for testing Spark 3.1.x and 3.0.x with metastore 2.3.8 and Java 11. + // Extract major.minor for testing Spark 3.1.x and 3.0.x with metastore 2.3.9 and Java 11. val hiveMetastoreVersion = """^\d+\.\d+""".r.findFirstIn(hiveVersion).get val args = Seq( "--name", "prepare testing tables", diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSerDeReadWriteSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSerDeReadWriteSuite.scala index aae49f70ca93..efec011b7157 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSerDeReadWriteSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSerDeReadWriteSuite.scala @@ -185,4 +185,35 @@ class HiveSerDeReadWriteSuite extends QueryTest with SQLTestUtils with TestHiveS checkComplexTypes(fileFormat) } } + + test("SPARK-34512: Disable validate default values when parsing Avro schemas") { + withTable("t1") { + hiveClient.runSqlHive( + """ + |CREATE TABLE t1 + | ROW FORMAT SERDE + | 'org.apache.hadoop.hive.serde2.avro.AvroSerDe' + | STORED AS INPUTFORMAT + | 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat' + | OUTPUTFORMAT + | 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat' + | TBLPROPERTIES ( + | 'avro.schema.literal'='{ + | "namespace": "org.apache.spark.sql.hive.test", + | "name": "schema_with_default_value", + | "type": "record", + | "fields": [ + | { + | "name": "ARRAY_WITH_DEFAULT", + | "type": {"type": "array", "items": "string"}, + | "default": null + | } + | ] + | }') + |""".stripMargin) + + hiveClient.runSqlHive("INSERT INTO t1 SELECT array('SPARK-34512', 'HIVE-24797')") + checkAnswer(spark.table("t1"), Seq(Row(Array("SPARK-34512", "HIVE-24797")))) + } + } } diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala index ec73d7f71887..23d20239121d 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala @@ -669,7 +669,7 @@ private[sql] class TestHiveSessionStateBuilder( } private[hive] object HiveTestJars { - private val repository = SQLConf.ADDITIONAL_REMOTE_REPOSITORIES.defaultValueString.split(",")(0) + private val repository = SQLConf.ADDITIONAL_REMOTE_REPOSITORIES.defaultValueString.split(",").last private val hiveTestJarsDir = Utils.createTempDir() def getHiveContribJar(version: String = HiveUtils.builtinHiveVersion): File = From d48df9021392bf6d4225f6d139a3d8b9c5f3cce9 Mon Sep 17 00:00:00 2001 From: Yuming Wang Date: Thu, 3 Jun 2021 07:08:52 +0800 Subject: [PATCH 2/5] ivy download --- .../apache/spark/sql/hive/execution/HiveQuerySuite.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index b08368f0fecb..54b949036d47 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -1508,7 +1508,7 @@ class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAnd // and hence cannot be found in the test environment or are non-jar (.pom) which cause // failures in tests. Use transitive=false as it should be good enough to test the Ivy // support in Hive ADD JAR - sql(s"ADD JAR ivy://org.apache.hive.hcatalog:hive-hcatalog-core:$hiveVersion" + + sql(s"ADD JAR ivy://org.apache.hive.hcatalog:hive-hcatalog-core:2.3.8" + "?transitive=false") sql( """CREATE TABLE t(a string, b string) @@ -1516,10 +1516,10 @@ class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAnd sql(s"""LOAD DATA LOCAL INPATH "$testData" INTO TABLE t""") sql("SELECT * FROM src JOIN t on src.key = t.a") assert(sql("LIST JARS").filter(_.getString(0).contains( - s"org.apache.hive.hcatalog_hive-hcatalog-core-$hiveVersion.jar")).count() > 0) + s"org.apache.hive.hcatalog_hive-hcatalog-core-2.3.8.jar")).count() > 0) assert(sql("LIST JAR"). filter(_.getString(0).contains( - s"org.apache.hive.hcatalog_hive-hcatalog-core-$hiveVersion.jar")).count() > 0) + s"org.apache.hive.hcatalog_hive-hcatalog-core-2.3.8.jar")).count() > 0) } } } From f81813f3d9268b6c029a30071e18d9955bb4e49f Mon Sep 17 00:00:00 2001 From: Yuming Wang Date: Thu, 3 Jun 2021 07:30:51 +0800 Subject: [PATCH 3/5] fix --- .../org/apache/spark/sql/hive/execution/HiveQuerySuite.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index 54b949036d47..0f1ddbb15389 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -35,7 +35,6 @@ import org.apache.spark.sql.catalyst.expressions.Cast import org.apache.spark.sql.catalyst.parser.ParseException import org.apache.spark.sql.catalyst.plans.logical.Project import org.apache.spark.sql.execution.joins.BroadcastNestedLoopJoinExec -import org.apache.spark.sql.hive.HiveUtils.{builtinHiveVersion => hiveVersion} import org.apache.spark.sql.hive.test.{HiveTestJars, TestHive} import org.apache.spark.sql.hive.test.TestHive._ import org.apache.spark.sql.internal.SQLConf From c7194cfa9ff8422149bcf9db07dfa8a9e9ad6767 Mon Sep 17 00:00:00 2001 From: Yuming Wang Date: Thu, 3 Jun 2021 10:08:33 +0800 Subject: [PATCH 4/5] fix --- .../src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala index 76c9f790d988..ee0c7c370922 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala @@ -3642,7 +3642,7 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark test("SPARK-33084: Add jar support Ivy URI in SQL") { val sc = spark.sparkContext - val hiveVersion = "2.3.9" + val hiveVersion = "2.3.8" // transitive=false, only download specified jar sql(s"ADD JAR ivy://org.apache.hive.hcatalog:hive-hcatalog-core:$hiveVersion?transitive=false") assert(sc.listJars() From ff0170e51a3cb3d866ef63f73edb0192f692acf4 Mon Sep 17 00:00:00 2001 From: Yuming Wang Date: Fri, 11 Jun 2021 07:19:38 +0800 Subject: [PATCH 5/5] Fix --- pom.xml | 11 ----------- project/SparkBuild.scala | 3 --- .../scala/org/apache/spark/sql/internal/SQLConf.scala | 3 +-- .../scala/org/apache/spark/sql/SQLQuerySuite.scala | 2 +- .../spark/sql/hive/execution/HiveQuerySuite.scala | 7 ++++--- .../org/apache/spark/sql/hive/test/TestHive.scala | 2 +- 6 files changed, 7 insertions(+), 21 deletions(-) diff --git a/pom.xml b/pom.xml index b9caa20eac2d..3ca661aee727 100644 --- a/pom.xml +++ b/pom.xml @@ -300,17 +300,6 @@ false - - staged - hive-staged-releases - https://repository.apache.org/content/repositories/staging/ - - true - - - true - - diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index e68e1678fa57..76fdc097ce3c 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -273,9 +273,6 @@ object SparkBuild extends PomBuild { DefaultMavenRepository, Resolver.mavenLocal, Resolver.file("ivyLocal", file(Path.userHome.absolutePath + "/.ivy2/local"))(Resolver.ivyStylePatterns) - ) ++ Seq( - "hive-staged-releases-mirror" at "https://repository.apache.org/content/repositories/staging/", - Resolver.file("local", file(Path.userHome.absolutePath + "/.ivy2/local"))(Resolver.ivyStylePatterns) ), externalResolvers := resolvers.value, otherResolvers := SbtPomKeys.mvnLocalRepository(dotM2 => Seq(Resolver.file("dotM2", dotM2))).value, diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index 17f0dc4adfbd..e368ee1d532d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -2883,8 +2883,7 @@ object SQLConf { .stringConf .createWithDefault( sys.env.getOrElse("DEFAULT_ARTIFACT_REPOSITORY", - "https://maven-central.storage-download.googleapis.com/maven2/," + - "https://repository.apache.org/content/repositories/staging/")) + "https://maven-central.storage-download.googleapis.com/maven2/")) val LEGACY_FROM_DAYTIME_STRING = buildConf("spark.sql.legacy.fromDayTimeString.enabled") diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala index ee0c7c370922..76c9f790d988 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala @@ -3642,7 +3642,7 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark test("SPARK-33084: Add jar support Ivy URI in SQL") { val sc = spark.sparkContext - val hiveVersion = "2.3.8" + val hiveVersion = "2.3.9" // transitive=false, only download specified jar sql(s"ADD JAR ivy://org.apache.hive.hcatalog:hive-hcatalog-core:$hiveVersion?transitive=false") assert(sc.listJars() diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index 0f1ddbb15389..b08368f0fecb 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -35,6 +35,7 @@ import org.apache.spark.sql.catalyst.expressions.Cast import org.apache.spark.sql.catalyst.parser.ParseException import org.apache.spark.sql.catalyst.plans.logical.Project import org.apache.spark.sql.execution.joins.BroadcastNestedLoopJoinExec +import org.apache.spark.sql.hive.HiveUtils.{builtinHiveVersion => hiveVersion} import org.apache.spark.sql.hive.test.{HiveTestJars, TestHive} import org.apache.spark.sql.hive.test.TestHive._ import org.apache.spark.sql.internal.SQLConf @@ -1507,7 +1508,7 @@ class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAnd // and hence cannot be found in the test environment or are non-jar (.pom) which cause // failures in tests. Use transitive=false as it should be good enough to test the Ivy // support in Hive ADD JAR - sql(s"ADD JAR ivy://org.apache.hive.hcatalog:hive-hcatalog-core:2.3.8" + + sql(s"ADD JAR ivy://org.apache.hive.hcatalog:hive-hcatalog-core:$hiveVersion" + "?transitive=false") sql( """CREATE TABLE t(a string, b string) @@ -1515,10 +1516,10 @@ class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAnd sql(s"""LOAD DATA LOCAL INPATH "$testData" INTO TABLE t""") sql("SELECT * FROM src JOIN t on src.key = t.a") assert(sql("LIST JARS").filter(_.getString(0).contains( - s"org.apache.hive.hcatalog_hive-hcatalog-core-2.3.8.jar")).count() > 0) + s"org.apache.hive.hcatalog_hive-hcatalog-core-$hiveVersion.jar")).count() > 0) assert(sql("LIST JAR"). filter(_.getString(0).contains( - s"org.apache.hive.hcatalog_hive-hcatalog-core-2.3.8.jar")).count() > 0) + s"org.apache.hive.hcatalog_hive-hcatalog-core-$hiveVersion.jar")).count() > 0) } } } diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala index 23d20239121d..ec73d7f71887 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala @@ -669,7 +669,7 @@ private[sql] class TestHiveSessionStateBuilder( } private[hive] object HiveTestJars { - private val repository = SQLConf.ADDITIONAL_REMOTE_REPOSITORIES.defaultValueString.split(",").last + private val repository = SQLConf.ADDITIONAL_REMOTE_REPOSITORIES.defaultValueString.split(",")(0) private val hiveTestJarsDir = Utils.createTempDir() def getHiveContribJar(version: String = HiveUtils.builtinHiveVersion): File =