Skip to content

[SPARK-13974][SQL] sub-query names do not need to be globally unique while generate SQL #11783

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
cloud-fan wants to merge 1 commit into from
Closed

[SPARK-13974][SQL] sub-query names do not need to be globally unique while generate SQL #11783

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 9 additions & 12 deletions sql/hive/src/main/scala/org/apache/spark/sql/hive/SQLBuilder.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,9 @@ class SQLBuilder(logicalPlan: LogicalPlan, sqlContext: SQLContext) extends Loggi

def this(df: DataFrame) = this(df.queryExecution.analyzed, df.sqlContext)

private val nextSubqueryId = new AtomicLong(0)
private def newSubqueryName(): String = s"gen_subquery_${nextSubqueryId.getAndIncrement()}"

def toSQL: String = {
val canonicalizedPlan = Canonicalizer.execute(logicalPlan)
val outputNames = logicalPlan.output.map(_.name)
Expand All @@ -64,7 +67,7 @@ class SQLBuilder(logicalPlan: LogicalPlan, sqlContext: SQLContext) extends Loggi
val finalName = if (qualifiers.length == 1) {
qualifiers.head
} else {
SQLBuilder.newSubqueryName
newSubqueryName()
}

// Canonicalizer will remove all naming information, we should add it back by adding an extra
Expand Down Expand Up @@ -254,16 +257,16 @@ class SQLBuilder(logicalPlan: LogicalPlan, sqlContext: SQLContext) extends Loggi
}

private def generateToSQL(g: Generate): String = {
val columnAliases = g.generatorOutput.map(_.sql).mkString(",")
val columnAliases = g.generatorOutput.map(_.sql).mkString(", ")

val childSQL = if (g.child == OneRowRelation) {
// This only happens when we put UDTF in project list and there is no FROM clause. Because we
// always generate LATERAL VIEW for `Generate`, here we use a trick to put a dummy sub-query
// after FROM clause, so that we can generate a valid LATERAL VIEW SQL string.
// For example, if the original SQL is: "SELECT EXPLODE(ARRAY(1, 2))", we will convert in to
// LATERAL VIEW format, and generate:
// SELECT col FROM (SELECT 1) sub-q0 LATERAL VIEW EXPLODE(ARRAY(1, 2)) sub_q1 AS col
s"(SELECT 1) ${SQLBuilder.newSubqueryName}"
// SELECT col FROM (SELECT 1) sub_q0 LATERAL VIEW EXPLODE(ARRAY(1, 2)) sub_q1 AS col
s"(SELECT 1) ${newSubqueryName()}"
} else {
toSQL(g.child)
}
Expand All @@ -283,7 +286,7 @@ class SQLBuilder(logicalPlan: LogicalPlan, sqlContext: SQLContext) extends Loggi
"LATERAL VIEW",
if (g.outer) "OUTER" else "",
g.generator.sql,
SQLBuilder.newSubqueryName,
newSubqueryName(),
"AS",
columnAliases
)
Expand Down Expand Up @@ -477,7 +480,7 @@ class SQLBuilder(logicalPlan: LogicalPlan, sqlContext: SQLContext) extends Loggi
}

private def addSubquery(plan: LogicalPlan): SubqueryAlias = {
SubqueryAlias(SQLBuilder.newSubqueryName, plan)
SubqueryAlias(newSubqueryName(), plan)
}

private def addSubqueryIfNeeded(plan: LogicalPlan): LogicalPlan = plan match {
Expand Down Expand Up @@ -514,9 +517,3 @@ class SQLBuilder(logicalPlan: LogicalPlan, sqlContext: SQLContext) extends Loggi
}
}
}

object SQLBuilder {
private val nextSubqueryId = new AtomicLong(0)

private def newSubqueryName: String = s"gen_subquery_${nextSubqueryId.getAndIncrement()}"
}