From 11d6a6709e1173399cd6f1222043d3f5abcb198d Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Tue, 8 Feb 2022 11:52:41 +0800 Subject: [PATCH] [SPARK-36444][SQL][3.1] Remove OptimizeSubqueries from batch of PartitionPruning --- .../spark/sql/execution/SparkOptimizer.scala | 3 +-- .../spark/sql/DynamicPartitionPruningSuite.scala | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkOptimizer.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkOptimizer.scala index 33b86a2b5340..9ab2addf5c18 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkOptimizer.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkOptimizer.scala @@ -42,8 +42,7 @@ class SparkOptimizer( override def defaultBatches: Seq[Batch] = (preOptimizationBatches ++ super.defaultBatches :+ Batch("Optimize Metadata Only Query", Once, OptimizeMetadataOnlyQuery(catalog)) :+ Batch("PartitionPruning", Once, - PartitionPruning, - OptimizeSubqueries) :+ + PartitionPruning) :+ Batch("Pushdown Filters from PartitionPruning", fixedPoint, PushDownPredicates) :+ Batch("Cleanup filters that cannot be pushed down", Once, diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala index b57ad61386d3..51bd3651b8d2 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala @@ -1382,6 +1382,20 @@ abstract class DynamicPartitionPruningSuiteBase ) } } + + test("SPARK-36444: Remove OptimizeSubqueries from batch of PartitionPruning") { + withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true") { + val df = sql( + """ + |SELECT date_id, product_id FROM fact_sk f + |JOIN (select store_id + 3 as new_store_id from dim_store where country = 'US') s + |ON f.store_id = s.new_store_id + """.stripMargin) + + checkPartitionPruningPredicate(df, false, true) + checkAnswer(df, Row(1150, 1) :: Row(1130, 4) :: Row(1140, 4) :: Nil) + } + } } class DynamicPartitionPruningSuiteAEOff extends DynamicPartitionPruningSuiteBase {