diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/PropagateEmptyRelation.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/PropagateEmptyRelation.scala index 04c23ce29a8d..6ad0793fb642 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/PropagateEmptyRelation.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/PropagateEmptyRelation.scala @@ -121,6 +121,7 @@ abstract class PropagateEmptyRelationBase extends Rule[LogicalPlan] with CastSup case Aggregate(ge, _, _) if ge.nonEmpty && !p.isStreaming => empty(p) // Generators like Hive-style UDTF may return their records within `close`. case Generate(_: Explode, _, _, _, _, _) => empty(p) + case Expand(_, _, _) => empty(p) case _ => p } } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PropagateEmptyRelationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PropagateEmptyRelationSuite.scala index b5dcb8aa6764..1aa4f4cbceae 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PropagateEmptyRelationSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PropagateEmptyRelationSuite.scala @@ -24,7 +24,7 @@ import org.apache.spark.sql.catalyst.dsl.plans._ import org.apache.spark.sql.catalyst.expressions.Literal import org.apache.spark.sql.catalyst.expressions.Literal.FalseLiteral import org.apache.spark.sql.catalyst.plans._ -import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan, Project} +import org.apache.spark.sql.catalyst.plans.logical.{Expand, LocalRelation, LogicalPlan, Project} import org.apache.spark.sql.catalyst.rules.RuleExecutor import org.apache.spark.sql.types.{IntegerType, MetadataBuilder, StructType} @@ -286,4 +286,12 @@ class PropagateEmptyRelationSuite extends PlanTest { val optimized = Optimize.execute(query) comparePlans(optimized, query) } + + test("SPARK-37689: Expand should be supported PropagateEmptyRelation") { + val query = Expand(Seq(Seq('a, 'b, "null"), Seq('a, "null", 'c)), Seq('a, 'b, 'c), + LocalRelation.fromExternalRows(Seq('a.int, 'b.int, 'c.int), Nil)).analyze + val optimized = Optimize.execute(query) + val expected = LocalRelation.fromExternalRows(Seq('a.int, 'b.int, 'c.int), Nil) + comparePlans(optimized, expected) + } }