From d48adffcdaee6068415304dbcb4b7ea41689db92 Mon Sep 17 00:00:00 2001 From: mcdull-zhang Date: Fri, 18 Nov 2022 12:13:38 +0800 Subject: [PATCH 1/2] first --- .../spark/sql/catalyst/expressions/subquery.scala | 2 +- .../org/apache/spark/sql/CachedTableSuite.scala | 14 +++++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/subquery.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/subquery.scala index 069251734db2..1cc8faccb033 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/subquery.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/subquery.scala @@ -355,7 +355,7 @@ case class ListQuery( plan.canonicalized, outerAttrs.map(_.canonicalized), ExprId(0), - childOutputs.map(_.canonicalized.asInstanceOf[Attribute]), + plan.canonicalized.output, joinCond.map(_.canonicalized)) } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala index 5548108b9150..dc3fead978f3 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala @@ -35,7 +35,7 @@ import org.apache.spark.sql.catalyst.analysis.TempTableAlreadyExistsException import org.apache.spark.sql.catalyst.expressions.SubqueryExpression import org.apache.spark.sql.catalyst.plans.logical.{BROADCAST, Join, JoinStrategyHint, SHUFFLE_HASH} import org.apache.spark.sql.catalyst.util.DateTimeConstants -import org.apache.spark.sql.execution.{ColumnarToRowExec, ExecSubqueryExpression, RDDScanExec, SparkPlan} +import org.apache.spark.sql.execution.{ColumnarToRowExec, ExecSubqueryExpression, ExternalRDDScanExec, RDDScanExec, SparkPlan} import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper import org.apache.spark.sql.execution.columnar._ import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec @@ -1671,4 +1671,16 @@ class CachedTableSuite extends QueryTest with SQLTestUtils } } } + + test("SPARK-41191: Cache Table is not working while nested caches exist") { + withCache("t1", "t2") { + sql("CACHE TABLE t1 as SELECT a FROM testData3 GROUP BY a") + sql("CACHE TABLE t2 as SELECT a,b FROM testData2 WHERE a IN " + + "(SELECT a FROM t1)") + val sparkPlan = sql("SELECT key,value,b FROM testData t3 JOIN t2 ON t3.key=t2.a") + .queryExecution.sparkPlan + assert(sparkPlan.collect { case e: InMemoryTableScanExec => e }.size === 1) + assert(sparkPlan.collect { case e: ExternalRDDScanExec[_] => e }.size === 1) + } + } } From 0ce58102ab552787d423ff613b41e1ee5f73bebd Mon Sep 17 00:00:00 2001 From: mcdull-zhang Date: Fri, 18 Nov 2022 17:02:40 +0800 Subject: [PATCH 2/2] unit test