diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala index eab4c3efe4f7..42e2b2ebfdc7 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala @@ -97,7 +97,6 @@ abstract class Optimizer(catalogManager: CatalogManager) SimplifyBinaryComparison, ReplaceNullWithFalseInPredicate, PruneFilters, - EliminateSorts, SimplifyCasts, SimplifyCaseConversionExpressions, RewriteCorrelatedScalarSubquery, @@ -174,8 +173,8 @@ abstract class Optimizer(catalogManager: CatalogManager) // idempotence enforcement on this batch. We thus make it FixedPoint(1) instead of Once. Batch("Join Reorder", FixedPoint(1), CostBasedJoinReorder) :+ - Batch("Remove Redundant Sorts", Once, - RemoveRedundantSorts) :+ + Batch("Eliminate Sorts", Once, + EliminateSorts) :+ Batch("Decimal Optimizations", fixedPoint, DecimalAggregates) :+ Batch("Object Expressions Optimization", fixedPoint, @@ -953,40 +952,62 @@ object CombineFilters extends Rule[LogicalPlan] with PredicateHelper { } /** - * Removes no-op SortOrder from Sort + * Removes Sort operation. This can happen: + * 1) if the sort order is empty or the sort order does not have any reference + * 2) if the child is already sorted + * 3) if there is another Sort operator separated by 0...n Project/Filter operators + * 4) if the Sort operator is within Join separated by 0...n Project/Filter operators only, + * and the Join conditions is deterministic + * 5) if the Sort operator is within GroupBy separated by 0...n Project/Filter operators only, + * and the aggregate function is order irrelevant */ object EliminateSorts extends Rule[LogicalPlan] { def apply(plan: LogicalPlan): LogicalPlan = plan transform { case s @ Sort(orders, _, child) if orders.isEmpty || orders.exists(_.child.foldable) => val newOrders = orders.filterNot(_.child.foldable) if (newOrders.isEmpty) child else s.copy(order = newOrders) - } -} - -/** - * Removes redundant Sort operation. This can happen: - * 1) if the child is already sorted - * 2) if there is another Sort operator separated by 0...n Project/Filter operators - */ -object RemoveRedundantSorts extends Rule[LogicalPlan] { - def apply(plan: LogicalPlan): LogicalPlan = plan transformDown { case Sort(orders, true, child) if SortOrder.orderingSatisfies(child.outputOrdering, orders) => child case s @ Sort(_, _, child) => s.copy(child = recursiveRemoveSort(child)) + case j @ Join(originLeft, originRight, _, cond, _) if cond.forall(_.deterministic) => + j.copy(left = recursiveRemoveSort(originLeft), right = recursiveRemoveSort(originRight)) + case g @ Aggregate(_, aggs, originChild) if isOrderIrrelevantAggs(aggs) => + g.copy(child = recursiveRemoveSort(originChild)) } - def recursiveRemoveSort(plan: LogicalPlan): LogicalPlan = plan match { + private def recursiveRemoveSort(plan: LogicalPlan): LogicalPlan = plan match { case Sort(_, _, child) => recursiveRemoveSort(child) case other if canEliminateSort(other) => other.withNewChildren(other.children.map(recursiveRemoveSort)) case _ => plan } - def canEliminateSort(plan: LogicalPlan): Boolean = plan match { + private def canEliminateSort(plan: LogicalPlan): Boolean = plan match { case p: Project => p.projectList.forall(_.deterministic) case f: Filter => f.condition.deterministic case _ => false } + + private def isOrderIrrelevantAggs(aggs: Seq[NamedExpression]): Boolean = { + def isOrderIrrelevantAggFunction(func: AggregateFunction): Boolean = func match { + case _: Sum => true + case _: Min => true + case _: Max => true + case _: Count => true + case _: Average => true + case _: CentralMomentAgg => true + case _ => false + } + + def checkValidAggregateExpression(expr: Expression): Boolean = expr match { + case _: AttributeReference => true + case ae: AggregateExpression => isOrderIrrelevantAggFunction(ae.aggregateFunction) + case _: UserDefinedExpression => false + case e => e.children.forall(checkValidAggregateExpression) + } + + aggs.forall(checkValidAggregateExpression) + } } /** diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/EliminateSortsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/EliminateSortsSuite.scala index e318f36d7827..d9a6fbf81de9 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/EliminateSortsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/EliminateSortsSuite.scala @@ -17,6 +17,7 @@ package org.apache.spark.sql.catalyst.optimizer +import org.apache.spark.api.python.PythonEvalType import org.apache.spark.sql.catalyst.analysis.{Analyzer, EmptyFunctionRegistry} import org.apache.spark.sql.catalyst.catalog.{InMemoryCatalog, SessionCatalog} import org.apache.spark.sql.catalyst.dsl.expressions._ @@ -27,6 +28,7 @@ import org.apache.spark.sql.catalyst.plans.logical._ import org.apache.spark.sql.catalyst.rules._ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.internal.SQLConf.{CASE_SENSITIVE, ORDER_BY_ORDINAL} +import org.apache.spark.sql.types.IntegerType class EliminateSortsSuite extends PlanTest { override val conf = new SQLConf().copy(CASE_SENSITIVE -> true, ORDER_BY_ORDINAL -> false) @@ -35,12 +37,22 @@ class EliminateSortsSuite extends PlanTest { object Optimize extends RuleExecutor[LogicalPlan] { val batches = - Batch("Eliminate Sorts", FixedPoint(10), + Batch("Default", FixedPoint(10), FoldablePropagation, - EliminateSorts) :: Nil + LimitPushDown) :: + Batch("Eliminate Sorts", Once, + EliminateSorts) :: + Batch("Collapse Project", Once, + CollapseProject) :: Nil + } + + object PushDownOptimizer extends RuleExecutor[LogicalPlan] { + val batches = + Batch("Limit PushDown", FixedPoint(10), LimitPushDown) :: Nil } val testRelation = LocalRelation('a.int, 'b.int, 'c.int) + val testRelationB = LocalRelation('d.int) test("Empty order by clause") { val x = testRelation @@ -83,4 +95,217 @@ class EliminateSortsSuite extends PlanTest { comparePlans(optimized, correctAnswer) } + + test("remove redundant order by") { + val orderedPlan = testRelation.select('a, 'b).orderBy('a.asc, 'b.desc_nullsFirst) + val unnecessaryReordered = orderedPlan.limit(2).select('a).orderBy('a.asc, 'b.desc_nullsFirst) + val optimized = Optimize.execute(unnecessaryReordered.analyze) + val correctAnswer = orderedPlan.limit(2).select('a).analyze + comparePlans(Optimize.execute(optimized), correctAnswer) + } + + test("do not remove sort if the order is different") { + val orderedPlan = testRelation.select('a, 'b).orderBy('a.asc, 'b.desc_nullsFirst) + val reorderedDifferently = orderedPlan.limit(2).select('a).orderBy('a.asc, 'b.desc) + val optimized = Optimize.execute(reorderedDifferently.analyze) + val correctAnswer = reorderedDifferently.analyze + comparePlans(optimized, correctAnswer) + } + + test("filters don't affect order") { + val orderedPlan = testRelation.select('a, 'b).orderBy('a.asc, 'b.desc) + val filteredAndReordered = orderedPlan.where('a > Literal(10)).orderBy('a.asc, 'b.desc) + val optimized = Optimize.execute(filteredAndReordered.analyze) + val correctAnswer = orderedPlan.where('a > Literal(10)).analyze + comparePlans(optimized, correctAnswer) + } + + test("limits don't affect order") { + val orderedPlan = testRelation.select('a, 'b).orderBy('a.asc, 'b.desc) + val filteredAndReordered = orderedPlan.limit(Literal(10)).orderBy('a.asc, 'b.desc) + val optimized = Optimize.execute(filteredAndReordered.analyze) + val correctAnswer = orderedPlan.limit(Literal(10)).analyze + comparePlans(optimized, correctAnswer) + } + + test("different sorts are not simplified if limit is in between") { + val orderedPlan = testRelation.select('a, 'b).orderBy('b.desc).limit(Literal(10)) + .orderBy('a.asc) + val optimized = Optimize.execute(orderedPlan.analyze) + val correctAnswer = orderedPlan.analyze + comparePlans(optimized, correctAnswer) + } + + test("range is already sorted") { + val inputPlan = Range(1L, 1000L, 1, 10) + val orderedPlan = inputPlan.orderBy('id.asc) + val optimized = Optimize.execute(orderedPlan.analyze) + val correctAnswer = inputPlan.analyze + comparePlans(optimized, correctAnswer) + + val reversedPlan = inputPlan.orderBy('id.desc) + val reversedOptimized = Optimize.execute(reversedPlan.analyze) + val reversedCorrectAnswer = reversedPlan.analyze + comparePlans(reversedOptimized, reversedCorrectAnswer) + + val negativeStepInputPlan = Range(10L, 1L, -1, 10) + val negativeStepOrderedPlan = negativeStepInputPlan.orderBy('id.desc) + val negativeStepOptimized = Optimize.execute(negativeStepOrderedPlan.analyze) + val negativeStepCorrectAnswer = negativeStepInputPlan.analyze + comparePlans(negativeStepOptimized, negativeStepCorrectAnswer) + } + + test("sort should not be removed when there is a node which doesn't guarantee any order") { + val orderedPlan = testRelation.select('a, 'b) + val groupedAndResorted = orderedPlan.groupBy('a)(sum('a)).orderBy('a.asc) + val optimized = Optimize.execute(groupedAndResorted.analyze) + val correctAnswer = groupedAndResorted.analyze + comparePlans(optimized, correctAnswer) + } + + test("remove two consecutive sorts") { + val orderedTwice = testRelation.orderBy('a.asc).orderBy('b.desc) + val optimized = Optimize.execute(orderedTwice.analyze) + val correctAnswer = testRelation.orderBy('b.desc).analyze + comparePlans(optimized, correctAnswer) + } + + test("remove sorts separated by Filter/Project operators") { + val orderedTwiceWithProject = testRelation.orderBy('a.asc).select('b).orderBy('b.desc) + val optimizedWithProject = Optimize.execute(orderedTwiceWithProject.analyze) + val correctAnswerWithProject = testRelation.select('b).orderBy('b.desc).analyze + comparePlans(optimizedWithProject, correctAnswerWithProject) + + val orderedTwiceWithFilter = + testRelation.orderBy('a.asc).where('b > Literal(0)).orderBy('b.desc) + val optimizedWithFilter = Optimize.execute(orderedTwiceWithFilter.analyze) + val correctAnswerWithFilter = testRelation.where('b > Literal(0)).orderBy('b.desc).analyze + comparePlans(optimizedWithFilter, correctAnswerWithFilter) + + val orderedTwiceWithBoth = + testRelation.orderBy('a.asc).select('b).where('b > Literal(0)).orderBy('b.desc) + val optimizedWithBoth = Optimize.execute(orderedTwiceWithBoth.analyze) + val correctAnswerWithBoth = + testRelation.select('b).where('b > Literal(0)).orderBy('b.desc).analyze + comparePlans(optimizedWithBoth, correctAnswerWithBoth) + + val orderedThrice = orderedTwiceWithBoth.select(('b + 1).as('c)).orderBy('c.asc) + val optimizedThrice = Optimize.execute(orderedThrice.analyze) + val correctAnswerThrice = testRelation.select('b).where('b > Literal(0)) + .select(('b + 1).as('c)).orderBy('c.asc).analyze + comparePlans(optimizedThrice, correctAnswerThrice) + } + + test("remove orderBy in groupBy clause with count aggs") { + val projectPlan = testRelation.select('a, 'b) + val unnecessaryOrderByPlan = projectPlan.orderBy('a.asc, 'b.desc) + val groupByPlan = unnecessaryOrderByPlan.groupBy('a)(count(1)) + val optimized = Optimize.execute(groupByPlan.analyze) + val correctAnswer = projectPlan.groupBy('a)(count(1)).analyze + comparePlans(optimized, correctAnswer) + } + + test("remove orderBy in groupBy clause with sum aggs") { + val projectPlan = testRelation.select('a, 'b) + val unnecessaryOrderByPlan = projectPlan.orderBy('a.asc, 'b.desc) + val groupByPlan = unnecessaryOrderByPlan.groupBy('a)(sum('a) + 10 as "sum") + val optimized = Optimize.execute(groupByPlan.analyze) + val correctAnswer = projectPlan.groupBy('a)(sum('a) + 10 as "sum").analyze + comparePlans(optimized, correctAnswer) + } + + test("should not remove orderBy in groupBy clause with first aggs") { + val projectPlan = testRelation.select('a, 'b) + val orderByPlan = projectPlan.orderBy('a.asc, 'b.desc) + val groupByPlan = orderByPlan.groupBy('a)(first('a)) + val optimized = Optimize.execute(groupByPlan.analyze) + val correctAnswer = groupByPlan.analyze + comparePlans(optimized, correctAnswer) + } + + test("should not remove orderBy in groupBy clause with first and count aggs") { + val projectPlan = testRelation.select('a, 'b) + val orderByPlan = projectPlan.orderBy('a.asc, 'b.desc) + val groupByPlan = orderByPlan.groupBy('a)(first('a), count(1)) + val optimized = Optimize.execute(groupByPlan.analyze) + val correctAnswer = groupByPlan.analyze + comparePlans(optimized, correctAnswer) + } + + test("should not remove orderBy in groupBy clause with PythonUDF as aggs") { + val pythonUdf = PythonUDF("pyUDF", null, + IntegerType, Seq.empty, PythonEvalType.SQL_BATCHED_UDF, true) + val projectPlan = testRelation.select('a, 'b) + val orderByPlan = projectPlan.orderBy('a.asc, 'b.desc) + val groupByPlan = orderByPlan.groupBy('a)(pythonUdf) + val optimized = Optimize.execute(groupByPlan.analyze) + val correctAnswer = groupByPlan.analyze + comparePlans(optimized, correctAnswer) + } + + test("should not remove orderBy in groupBy clause with ScalaUDF as aggs") { + val scalaUdf = ScalaUDF((s: Int) => s, IntegerType, 'a :: Nil, true :: Nil) + val projectPlan = testRelation.select('a, 'b) + val orderByPlan = projectPlan.orderBy('a.asc, 'b.desc) + val groupByPlan = orderByPlan.groupBy('a)(scalaUdf) + val optimized = Optimize.execute(groupByPlan.analyze) + val correctAnswer = groupByPlan.analyze + comparePlans(optimized, correctAnswer) + } + + test("should not remove orderBy with limit in groupBy clause") { + val projectPlan = testRelation.select('a, 'b) + val orderByPlan = projectPlan.orderBy('a.asc, 'b.desc).limit(10) + val groupByPlan = orderByPlan.groupBy('a)(count(1)) + val optimized = Optimize.execute(groupByPlan.analyze) + val correctAnswer = groupByPlan.analyze + comparePlans(optimized, correctAnswer) + } + + test("remove orderBy in join clause") { + val projectPlan = testRelation.select('a, 'b) + val unnecessaryOrderByPlan = projectPlan.orderBy('a.asc, 'b.desc) + val projectPlanB = testRelationB.select('d) + val joinPlan = unnecessaryOrderByPlan.join(projectPlanB).select('a, 'd) + val optimized = Optimize.execute(joinPlan.analyze) + val correctAnswer = projectPlan.join(projectPlanB).select('a, 'd).analyze + comparePlans(optimized, correctAnswer) + } + + test("should not remove orderBy with limit in join clause") { + val projectPlan = testRelation.select('a, 'b) + val orderByPlan = projectPlan.orderBy('a.asc, 'b.desc).limit(10) + val projectPlanB = testRelationB.select('d) + val joinPlan = orderByPlan.join(projectPlanB).select('a, 'd) + val optimized = Optimize.execute(joinPlan.analyze) + val correctAnswer = joinPlan.analyze + comparePlans(optimized, correctAnswer) + } + + test("should not remove orderBy in left join clause if there is an outer limit") { + val projectPlan = testRelation.select('a, 'b) + val orderByPlan = projectPlan.orderBy('a.asc, 'b.desc) + val projectPlanB = testRelationB.select('d) + val joinPlan = orderByPlan + .join(projectPlanB, LeftOuter) + .limit(10) + val optimized = Optimize.execute(joinPlan.analyze) + val correctAnswer = PushDownOptimizer.execute(joinPlan.analyze) + comparePlans(optimized, correctAnswer) + } + + test("remove orderBy in right join clause event if there is an outer limit") { + val projectPlan = testRelation.select('a, 'b) + val orderByPlan = projectPlan.orderBy('a.asc, 'b.desc) + val projectPlanB = testRelationB.select('d) + val joinPlan = orderByPlan + .join(projectPlanB, RightOuter) + .limit(10) + val optimized = Optimize.execute(joinPlan.analyze) + val noOrderByPlan = projectPlan + .join(projectPlanB, RightOuter) + .limit(10) + val correctAnswer = PushDownOptimizer.execute(noOrderByPlan.analyze) + comparePlans(optimized, correctAnswer) + } } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/RemoveRedundantSortsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/RemoveRedundantSortsSuite.scala deleted file mode 100644 index dae5e6f3ee3d..000000000000 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/RemoveRedundantSortsSuite.scala +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.catalyst.optimizer - -import org.apache.spark.sql.catalyst.dsl.expressions._ -import org.apache.spark.sql.catalyst.dsl.plans._ -import org.apache.spark.sql.catalyst.expressions._ -import org.apache.spark.sql.catalyst.plans._ -import org.apache.spark.sql.catalyst.plans.logical._ -import org.apache.spark.sql.catalyst.rules._ - -class RemoveRedundantSortsSuite extends PlanTest { - - object Optimize extends RuleExecutor[LogicalPlan] { - val batches = - Batch("Remove Redundant Sorts", Once, - RemoveRedundantSorts) :: - Batch("Collapse Project", Once, - CollapseProject) :: Nil - } - - val testRelation = LocalRelation('a.int, 'b.int, 'c.int) - - test("remove redundant order by") { - val orderedPlan = testRelation.select('a, 'b).orderBy('a.asc, 'b.desc_nullsFirst) - val unnecessaryReordered = orderedPlan.limit(2).select('a).orderBy('a.asc, 'b.desc_nullsFirst) - val optimized = Optimize.execute(unnecessaryReordered.analyze) - val correctAnswer = orderedPlan.limit(2).select('a).analyze - comparePlans(Optimize.execute(optimized), correctAnswer) - } - - test("do not remove sort if the order is different") { - val orderedPlan = testRelation.select('a, 'b).orderBy('a.asc, 'b.desc_nullsFirst) - val reorderedDifferently = orderedPlan.limit(2).select('a).orderBy('a.asc, 'b.desc) - val optimized = Optimize.execute(reorderedDifferently.analyze) - val correctAnswer = reorderedDifferently.analyze - comparePlans(optimized, correctAnswer) - } - - test("filters don't affect order") { - val orderedPlan = testRelation.select('a, 'b).orderBy('a.asc, 'b.desc) - val filteredAndReordered = orderedPlan.where('a > Literal(10)).orderBy('a.asc, 'b.desc) - val optimized = Optimize.execute(filteredAndReordered.analyze) - val correctAnswer = orderedPlan.where('a > Literal(10)).analyze - comparePlans(optimized, correctAnswer) - } - - test("limits don't affect order") { - val orderedPlan = testRelation.select('a, 'b).orderBy('a.asc, 'b.desc) - val filteredAndReordered = orderedPlan.limit(Literal(10)).orderBy('a.asc, 'b.desc) - val optimized = Optimize.execute(filteredAndReordered.analyze) - val correctAnswer = orderedPlan.limit(Literal(10)).analyze - comparePlans(optimized, correctAnswer) - } - - test("different sorts are not simplified if limit is in between") { - val orderedPlan = testRelation.select('a, 'b).orderBy('b.desc).limit(Literal(10)) - .orderBy('a.asc) - val optimized = Optimize.execute(orderedPlan.analyze) - val correctAnswer = orderedPlan.analyze - comparePlans(optimized, correctAnswer) - } - - test("range is already sorted") { - val inputPlan = Range(1L, 1000L, 1, 10) - val orderedPlan = inputPlan.orderBy('id.asc) - val optimized = Optimize.execute(orderedPlan.analyze) - val correctAnswer = inputPlan.analyze - comparePlans(optimized, correctAnswer) - - val reversedPlan = inputPlan.orderBy('id.desc) - val reversedOptimized = Optimize.execute(reversedPlan.analyze) - val reversedCorrectAnswer = reversedPlan.analyze - comparePlans(reversedOptimized, reversedCorrectAnswer) - - val negativeStepInputPlan = Range(10L, 1L, -1, 10) - val negativeStepOrderedPlan = negativeStepInputPlan.orderBy('id.desc) - val negativeStepOptimized = Optimize.execute(negativeStepOrderedPlan.analyze) - val negativeStepCorrectAnswer = negativeStepInputPlan.analyze - comparePlans(negativeStepOptimized, negativeStepCorrectAnswer) - } - - test("sort should not be removed when there is a node which doesn't guarantee any order") { - val orderedPlan = testRelation.select('a, 'b).orderBy('a.asc) - val groupedAndResorted = orderedPlan.groupBy('a)(sum('a)).orderBy('a.asc) - val optimized = Optimize.execute(groupedAndResorted.analyze) - val correctAnswer = groupedAndResorted.analyze - comparePlans(optimized, correctAnswer) - } - - test("remove two consecutive sorts") { - val orderedTwice = testRelation.orderBy('a.asc).orderBy('b.desc) - val optimized = Optimize.execute(orderedTwice.analyze) - val correctAnswer = testRelation.orderBy('b.desc).analyze - comparePlans(optimized, correctAnswer) - } - - test("remove sorts separated by Filter/Project operators") { - val orderedTwiceWithProject = testRelation.orderBy('a.asc).select('b).orderBy('b.desc) - val optimizedWithProject = Optimize.execute(orderedTwiceWithProject.analyze) - val correctAnswerWithProject = testRelation.select('b).orderBy('b.desc).analyze - comparePlans(optimizedWithProject, correctAnswerWithProject) - - val orderedTwiceWithFilter = - testRelation.orderBy('a.asc).where('b > Literal(0)).orderBy('b.desc) - val optimizedWithFilter = Optimize.execute(orderedTwiceWithFilter.analyze) - val correctAnswerWithFilter = testRelation.where('b > Literal(0)).orderBy('b.desc).analyze - comparePlans(optimizedWithFilter, correctAnswerWithFilter) - - val orderedTwiceWithBoth = - testRelation.orderBy('a.asc).select('b).where('b > Literal(0)).orderBy('b.desc) - val optimizedWithBoth = Optimize.execute(orderedTwiceWithBoth.analyze) - val correctAnswerWithBoth = - testRelation.select('b).where('b > Literal(0)).orderBy('b.desc).analyze - comparePlans(optimizedWithBoth, correctAnswerWithBoth) - - val orderedThrice = orderedTwiceWithBoth.select(('b + 1).as('c)).orderBy('c.asc) - val optimizedThrice = Optimize.execute(orderedThrice.analyze) - val correctAnswerThrice = testRelation.select('b).where('b > Literal(0)) - .select(('b + 1).as('c)).orderBy('c.asc).analyze - comparePlans(optimizedThrice, correctAnswerThrice) - } -} diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala index a1d7792941ed..c117ee7818c0 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala @@ -1080,9 +1080,8 @@ class SubquerySuite extends QueryTest with SharedSparkSession { | HAVING max(c2) > 0 | ORDER BY c1) """.stripMargin - // The rule to remove redundant sorts is not able to remove the inner sort under - // an Aggregate operator. We only remove the top level sort. - assert(getNumSortsInQuery(query6) == 1) + + assert(getNumSortsInQuery(query6) == 0) // Cases when sort is not removed from the plan // Limit on top of sort