From 3dc012475274d609e673ca5ceb71b9c3c501cb1f Mon Sep 17 00:00:00 2001 From: Yuming Wang Date: Sun, 22 Sep 2019 18:38:31 +0800 Subject: [PATCH 1/6] Reduce shuffle partitions to reduce testing time --- .../sql-tests/results/subquery/in-subquery/in-joins.sql.out | 2 +- .../scala/org/apache/spark/sql/test/SharedSparkSession.scala | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/sql/core/src/test/resources/sql-tests/results/subquery/in-subquery/in-joins.sql.out b/sql/core/src/test/resources/sql-tests/results/subquery/in-subquery/in-joins.sql.out index ab6a11a2b7efa..86ab7ae9f1dbb 100644 --- a/sql/core/src/test/resources/sql-tests/results/subquery/in-subquery/in-joins.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/subquery/in-subquery/in-joins.sql.out @@ -106,8 +106,8 @@ ORDER BY t1a DESC, t3b DESC -- !query 4 schema struct -- !query 4 output -1 10 val3b 8 NULL 1 10 val1b 8 16 +1 10 val3b 8 NULL 1 10 val3a 6 12 1 8 val3a 6 12 1 8 val3a 6 12 diff --git a/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala b/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala index ee29b4b8fb32b..448ec844cbeec 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala @@ -72,6 +72,8 @@ trait SharedSparkSessionBase // this rule may potentially block testing of other optimization rules such as // ConstantPropagation etc. .set(SQLConf.OPTIMIZER_EXCLUDED_RULES.key, ConvertToLocalRelation.ruleName) + // Reduce shuffle partitions to reduce testing time. + .set(SQLConf.SHUFFLE_PARTITIONS.key, "5") conf.set( StaticSQLConf.WAREHOUSE_PATH, conf.get(StaticSQLConf.WAREHOUSE_PATH) + "/" + getClass.getCanonicalName) From 6ec97617b5f1fd8744393c43d0fcfb5e7dd6371d Mon Sep 17 00:00:00 2001 From: Yuming Wang Date: Sun, 22 Sep 2019 22:43:10 +0800 Subject: [PATCH 2/6] Avoid test error in StreamSuite --- .../scala/org/apache/spark/sql/SQLQueryTestSuite.scala | 10 +++++----- .../org/apache/spark/sql/test/SharedSparkSession.scala | 2 -- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala index f32575794f946..21c61e9daca42 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala @@ -22,7 +22,7 @@ import java.util.{Locale, TimeZone} import scala.util.control.NonFatal -import org.apache.spark.SparkException +import org.apache.spark.{SparkConf, SparkException} import org.apache.spark.sql.catalyst.planning.PhysicalOperation import org.apache.spark.sql.catalyst.plans.logical._ import org.apache.spark.sql.catalyst.plans.logical.sql.{DescribeColumnStatement, DescribeTableStatement} @@ -133,6 +133,10 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession { private val notIncludedMsg = "[not included in comparison]" private val clsName = this.getClass.getCanonicalName + override def sparkConf: SparkConf = super.sparkConf + // Reduce shuffle partitions to reduce testing time. + .set(SQLConf.SHUFFLE_PARTITIONS, 5) + /** List of test cases to ignore, in lower cases. */ protected def blackList: Set[String] = Set( "blacklist.sql" // Do NOT remove this one. It is here to test the blacklist functionality. @@ -291,10 +295,6 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession { testCase match { case udfTestCase: UDFTest => - // In Python UDF tests, the number of shuffle partitions matters considerably in - // the testing time because it requires to fork and communicate between external - // processes. - localSparkSession.conf.set(SQLConf.SHUFFLE_PARTITIONS.key, 4) registerTestUDF(udfTestCase.udf, localSparkSession) case _ => } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala b/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala index 448ec844cbeec..ee29b4b8fb32b 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala @@ -72,8 +72,6 @@ trait SharedSparkSessionBase // this rule may potentially block testing of other optimization rules such as // ConstantPropagation etc. .set(SQLConf.OPTIMIZER_EXCLUDED_RULES.key, ConvertToLocalRelation.ruleName) - // Reduce shuffle partitions to reduce testing time. - .set(SQLConf.SHUFFLE_PARTITIONS.key, "5") conf.set( StaticSQLConf.WAREHOUSE_PATH, conf.get(StaticSQLConf.WAREHOUSE_PATH) + "/" + getClass.getCanonicalName) From 55004b9566b33c3eb886376fc2d0679ed407e284 Mon Sep 17 00:00:00 2001 From: Yuming Wang Date: Mon, 23 Sep 2019 07:29:49 +0800 Subject: [PATCH 3/6] SHUFFLE_PARTITIONS to 4 --- .../results/subquery/in-subquery/not-in-joins.sql.out | 2 +- .../test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/results/subquery/in-subquery/not-in-joins.sql.out b/sql/core/src/test/resources/sql-tests/results/subquery/in-subquery/not-in-joins.sql.out index bae5d00cc8632..0ff1806ef4a3a 100644 --- a/sql/core/src/test/resources/sql-tests/results/subquery/in-subquery/not-in-joins.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/subquery/in-subquery/not-in-joins.sql.out @@ -173,8 +173,8 @@ ORDER BY t1b DESC -- !query 6 schema struct -- !query 6 output -1 16 12 10 1 16 12 21 +1 16 12 10 1 10 NULL 12 1 6 8 10 1 NULL 16 22 diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala index 21c61e9daca42..d26c82957fb36 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala @@ -134,8 +134,8 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession { private val clsName = this.getClass.getCanonicalName override def sparkConf: SparkConf = super.sparkConf - // Reduce shuffle partitions to reduce testing time. - .set(SQLConf.SHUFFLE_PARTITIONS, 5) + // Fewer shuffle partitions to speed up testing. + .set(SQLConf.SHUFFLE_PARTITIONS, 4) /** List of test cases to ignore, in lower cases. */ protected def blackList: Set[String] = Set( From b4f2d191af303704940880f20c4585e68470f2e4 Mon Sep 17 00:00:00 2001 From: Yuming Wang Date: Mon, 23 Sep 2019 12:58:13 +0800 Subject: [PATCH 4/6] SHUFFLE_PARTITIONS to 5 --- .../sql-tests/results/subquery/in-subquery/not-in-joins.sql.out | 2 +- .../test/resources/sql-tests/results/udf/pgSQL/udf-join.sql.out | 2 +- .../src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/results/subquery/in-subquery/not-in-joins.sql.out b/sql/core/src/test/resources/sql-tests/results/subquery/in-subquery/not-in-joins.sql.out index 0ff1806ef4a3a..bae5d00cc8632 100644 --- a/sql/core/src/test/resources/sql-tests/results/subquery/in-subquery/not-in-joins.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/subquery/in-subquery/not-in-joins.sql.out @@ -173,8 +173,8 @@ ORDER BY t1b DESC -- !query 6 schema struct -- !query 6 output -1 16 12 21 1 16 12 10 +1 16 12 21 1 10 NULL 12 1 6 8 10 1 NULL 16 22 diff --git a/sql/core/src/test/resources/sql-tests/results/udf/pgSQL/udf-join.sql.out b/sql/core/src/test/resources/sql-tests/results/udf/pgSQL/udf-join.sql.out index 6fcff129d7568..184fc2eb893a9 100644 --- a/sql/core/src/test/resources/sql-tests/results/udf/pgSQL/udf-join.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/udf/pgSQL/udf-join.sql.out @@ -3023,9 +3023,9 @@ select udf(b.unique1) from -- !query 147 schema struct -- !query 147 output +0 NULL NULL -0 NULL NULL diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala index d26c82957fb36..9da4a1307d9fd 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala @@ -135,7 +135,7 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession { override def sparkConf: SparkConf = super.sparkConf // Fewer shuffle partitions to speed up testing. - .set(SQLConf.SHUFFLE_PARTITIONS, 4) + .set(SQLConf.SHUFFLE_PARTITIONS, 5) /** List of test cases to ignore, in lower cases. */ protected def blackList: Set[String] = Set( From ad6bee74edc880f81d012d534a50a9752983cd5b Mon Sep 17 00:00:00 2001 From: Yuming Wang Date: Mon, 23 Sep 2019 16:22:42 +0800 Subject: [PATCH 5/6] SHUFFLE_PARTITIONS to 4 and add sort --- .../sql-tests/inputs/subquery/in-subquery/in-joins.sql | 2 +- .../sql-tests/inputs/subquery/in-subquery/not-in-joins.sql | 2 +- .../sql-tests/results/subquery/in-subquery/in-joins.sql.out | 2 +- .../sql-tests/results/subquery/in-subquery/not-in-joins.sql.out | 2 +- .../test/resources/sql-tests/results/udf/pgSQL/udf-join.sql.out | 2 +- .../src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-joins.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-joins.sql index 22f3eafd6a02d..04e58dbf0bc31 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-joins.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-joins.sql @@ -83,7 +83,7 @@ GROUP BY t1a, t3a, t3b, t3c -ORDER BY t1a DESC, t3b DESC; +ORDER BY t1a DESC, t3b DESC, t3c DESC; -- TC 01.03 SELECT Count(DISTINCT(t1a)) diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-joins.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-joins.sql index 4f8ca8bfb27c1..bebc18a61894b 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-joins.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-joins.sql @@ -128,7 +128,7 @@ GROUP BY t1b, HAVING t1d NOT IN (SELECT t2d FROM t2 WHERE t1d = t2d) -ORDER BY t1b DESC; +ORDER BY t1b DESC, t1d ASC; -- TC 01.05 SELECT COUNT(DISTINCT(t1a)), diff --git a/sql/core/src/test/resources/sql-tests/results/subquery/in-subquery/in-joins.sql.out b/sql/core/src/test/resources/sql-tests/results/subquery/in-subquery/in-joins.sql.out index 86ab7ae9f1dbb..4fea037137760 100644 --- a/sql/core/src/test/resources/sql-tests/results/subquery/in-subquery/in-joins.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/subquery/in-subquery/in-joins.sql.out @@ -102,7 +102,7 @@ GROUP BY t1a, t3a, t3b, t3c -ORDER BY t1a DESC, t3b DESC +ORDER BY t1a DESC, t3b DESC, t3c DESC -- !query 4 schema struct -- !query 4 output diff --git a/sql/core/src/test/resources/sql-tests/results/subquery/in-subquery/not-in-joins.sql.out b/sql/core/src/test/resources/sql-tests/results/subquery/in-subquery/not-in-joins.sql.out index bae5d00cc8632..498378bfa5c37 100644 --- a/sql/core/src/test/resources/sql-tests/results/subquery/in-subquery/not-in-joins.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/subquery/in-subquery/not-in-joins.sql.out @@ -169,7 +169,7 @@ GROUP BY t1b, HAVING t1d NOT IN (SELECT t2d FROM t2 WHERE t1d = t2d) -ORDER BY t1b DESC +ORDER BY t1b DESC, t1d ASC -- !query 6 schema struct -- !query 6 output diff --git a/sql/core/src/test/resources/sql-tests/results/udf/pgSQL/udf-join.sql.out b/sql/core/src/test/resources/sql-tests/results/udf/pgSQL/udf-join.sql.out index 184fc2eb893a9..6fcff129d7568 100644 --- a/sql/core/src/test/resources/sql-tests/results/udf/pgSQL/udf-join.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/udf/pgSQL/udf-join.sql.out @@ -3023,9 +3023,9 @@ select udf(b.unique1) from -- !query 147 schema struct -- !query 147 output -0 NULL NULL +0 NULL NULL diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala index 9da4a1307d9fd..d26c82957fb36 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala @@ -135,7 +135,7 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession { override def sparkConf: SparkConf = super.sparkConf // Fewer shuffle partitions to speed up testing. - .set(SQLConf.SHUFFLE_PARTITIONS, 5) + .set(SQLConf.SHUFFLE_PARTITIONS, 4) /** List of test cases to ignore, in lower cases. */ protected def blackList: Set[String] = Set( From df51b6909e6173cdb7f3bd7e6d6002b9a910d3a5 Mon Sep 17 00:00:00 2001 From: Yuming Wang Date: Mon, 23 Sep 2019 17:10:39 +0800 Subject: [PATCH 6/6] DESC -> ASC --- .../sql-tests/inputs/subquery/in-subquery/in-joins.sql | 2 +- .../sql-tests/results/subquery/in-subquery/in-joins.sql.out | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-joins.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-joins.sql index 04e58dbf0bc31..f06be5fc6aa58 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-joins.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-joins.sql @@ -83,7 +83,7 @@ GROUP BY t1a, t3a, t3b, t3c -ORDER BY t1a DESC, t3b DESC, t3c DESC; +ORDER BY t1a DESC, t3b DESC, t3c ASC; -- TC 01.03 SELECT Count(DISTINCT(t1a)) diff --git a/sql/core/src/test/resources/sql-tests/results/subquery/in-subquery/in-joins.sql.out b/sql/core/src/test/resources/sql-tests/results/subquery/in-subquery/in-joins.sql.out index 4fea037137760..7089fbd25e28c 100644 --- a/sql/core/src/test/resources/sql-tests/results/subquery/in-subquery/in-joins.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/subquery/in-subquery/in-joins.sql.out @@ -102,12 +102,12 @@ GROUP BY t1a, t3a, t3b, t3c -ORDER BY t1a DESC, t3b DESC, t3c DESC +ORDER BY t1a DESC, t3b DESC, t3c ASC -- !query 4 schema struct -- !query 4 output -1 10 val1b 8 16 1 10 val3b 8 NULL +1 10 val1b 8 16 1 10 val3a 6 12 1 8 val3a 6 12 1 8 val3a 6 12