Skip to content

Commit 827873f

Browse files
update as feedback
1 parent 34def69 commit 827873f

File tree

1 file changed

+16
-8
lines changed
  • sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis

1 file changed

+16
-8
lines changed

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -200,29 +200,37 @@ class Analyzer(
200200
GroupingSets(bitmasks(a), a.groupByExprs, a.child, a.aggregations)
201201
case x: GroupingSets =>
202202
val gid = AttributeReference(VirtualColumn.groupingIdName, IntegerType, false)()
203+
// We will insert another Projection if the GROUP BY keys contains the
204+
// non-attribute expressions. And the top operators can references those
205+
// expressions by its alias.
206+
// e.g. SELECT key%5 as c1 FROM src GROUP BY key%5 ==>
207+
// SELECT a as c1 FROM (SELECT key%5 AS a FROM src) GROUP BY a
203208

204-
// the complex expression (non-attribute expressions) in the GROUP BY keys
205-
val nonAttributeGroupByExpression = new ArrayBuffer[Alias]()
209+
// find all of the non-attribute expressions in the GROUP BY keys
210+
val nonAttributeGroupByExpressions = new ArrayBuffer[Alias]()
211+
212+
// The pair of (the non-attributes expression, associated attribute (alias))
206213
val groupByExprPairs = x.groupByExprs.map(_ match {
207214
case e: NamedExpression => (e, e)
208215
case other => {
209216
val alias = Alias(other, other.toString)()
210-
nonAttributeGroupByExpression += alias
211-
(other, alias.toAttribute) // (Aliased complex expression, the associated attribute)
217+
nonAttributeGroupByExpressions += alias // add the non-attributes expression alias
218+
(other, alias.toAttribute)
212219
}
213220
})
214221

215-
// substitute the complex expression for aggregations.
222+
// substitute the non-attribute expressions for aggregations.
216223
val aggregation = x.aggregations.map(expr => expr.transformDown {
217224
case e => groupByExprPairs.find(_._1.semanticEquals(e)).map(_._2).getOrElse(e)
218225
}.asInstanceOf[NamedExpression])
219226

220227
// substitute the group by expressions.
221228
val newGroupByExprs = groupByExprPairs.map(_._2)
222229

223-
// add an additional projection if contains the complex expression in the GROUP BY keys
224-
val child = if (nonAttributeGroupByExpression.length > 0) {
225-
Project(x.child.output ++ nonAttributeGroupByExpression, x.child)
230+
val child = if (nonAttributeGroupByExpressions.length > 0) {
231+
// insert additional projection if contains the
232+
// non-attribute expressions in the GROUP BY keys
233+
Project(x.child.output ++ nonAttributeGroupByExpressions, x.child)
226234
} else {
227235
x.child
228236
}

0 commit comments

Comments
 (0)