diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala index c551c7c9877e8..7dbaf7faff0c0 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala @@ -414,7 +414,7 @@ class SQLContext(@transient val sparkContext: SparkContext) def simpleString: String = s"""== Physical Plan == |${stringOrError(executedPlan)} - """ + """.stripMargin.trim override def toString: String = // TODO previously will output RDD details by run (${stringOrError(toRdd.toDebugString)}) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala index 21ecf17028dbc..b64002a803a65 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala @@ -215,9 +215,19 @@ private[hive] object HiveQl { def getAst(sql: String): ASTNode = ParseUtils.findRootNonNullToken((new ParseDriver).parse(sql)) /** Returns a LogicalPlan for a given HiveQL string. */ - def parseSql(sql: String): LogicalPlan = { + def parseSql(sqlRaw: String): LogicalPlan = { + var sql = sqlRaw try { - if (sql.trim.toLowerCase.startsWith("set")) { + // FIXME workaround solution to remove the first comment from the sql string + if (sql.startsWith("--")) { + var pos = sql.indexOf(System.getProperty("line.separator")) + pos = if (pos < 0) sql.length() else pos + 1 + sql = sql.substring(pos) + } + var procSql = sql.trim.toLowerCase + if (procSql.isEmpty()) { + NoRelation + } else if (procSql.startsWith("set")) { // Split in two parts since we treat the part before the first "=" // as key, and the part after as value, which may contain other "=" signs. sql.trim.drop(3).split("=", 2).map(_.trim) match { @@ -228,20 +238,31 @@ private[hive] object HiveQl { case Array(key, value) => // "set key=value" SetCommand(Some(key), Some(value)) } - } else if (sql.trim.toLowerCase.startsWith("cache table")) { + } else if (procSql.startsWith("cache table")) { CacheCommand(sql.trim.drop(12).trim, true) - } else if (sql.trim.toLowerCase.startsWith("uncache table")) { + } else if (procSql.startsWith("uncache table")) { CacheCommand(sql.trim.drop(14).trim, false) - } else if (sql.trim.toLowerCase.startsWith("add jar")) { + } else if (procSql.startsWith("add jar")) { AddJar(sql.trim.drop(8).trim) - } else if (sql.trim.toLowerCase.startsWith("add file")) { + } else if (procSql.startsWith("add file")) { AddFile(sql.trim.drop(9)) - } else if (sql.trim.toLowerCase.startsWith("dfs")) { + } else if (procSql.startsWith("dfs")) { NativeCommand(sql) - } else if (sql.trim.startsWith("source")) { - SourceCommand(sql.split(" ").toSeq match { case Seq("source", filePath) => filePath }) + } else if (procSql.startsWith("source")) { + SourceCommand(sql.trim.drop(7).trim) } else if (sql.trim.startsWith("!")) { ShellCommand(sql.drop(1)) + } else if (procSql.startsWith("explain")) { + val remaining = sql.trim.drop(8).trim + // process the "extended" and the "formatted" + if (remaining.toLowerCase.startsWith("extended")) { + ExplainCommand(parseSql(remaining.drop(9)), true) + } else if (remaining.toLowerCase.startsWith("formatted")) { + // TODO supports the "FORMATTED"? + ExplainCommand(parseSql(remaining.drop(10))) + } else { + ExplainCommand(parseSql(remaining)) + } } else { val tree = getAst(sql) if (nativeCommands contains tree.getText) { @@ -254,10 +275,10 @@ private[hive] object HiveQl { } } } catch { - case e: Exception => throw new ParseException(sql, e) + case e: Exception => throw new ParseException(sqlRaw, e) case e: NotImplementedError => sys.error( s""" - |Unsupported language features in query: $sql + |Unsupported language features in query: $sqlRaw |${dumpTree(getAst(sql))} """.stripMargin) } @@ -407,16 +428,6 @@ private[hive] object HiveQl { val tableName = tableNameParts.map { case Token(p, Nil) => p }.mkString(".") AnalyzeTable(tableName) } - // Just fake explain for any of the native commands. - case Token("TOK_EXPLAIN", explainArgs) - if noExplainCommands.contains(explainArgs.head.getText) => - ExplainCommand(NoRelation) - case Token("TOK_EXPLAIN", explainArgs) => - // Ignore FORMATTED if present. - val Some(query) :: _ :: extended :: Nil = - getClauses(Seq("TOK_QUERY", "FORMATTED", "EXTENDED"), explainArgs) - ExplainCommand(nodeToPlan(query), extended != None) - case Token("TOK_DESCTABLE", describeArgs) => // Reference: https://cwiki.apache.org/confluence/display/Hive/LanguageManual+DDL val Some(tableType) :: formatted :: extended :: pretty :: Nil = diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveExplainSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveExplainSuite.scala index 4ed58f4be1167..dfd28d0546362 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveExplainSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveExplainSuite.scala @@ -51,4 +51,12 @@ class HiveExplainSuite extends QueryTest { "== Physical Plan ==", "Code Generation", "== RDD ==") } + + test("explain native command") { + check("explain set a=123", true, "== Physical Plan ==", "SetCommand", "a", "123") + check("explain show tables", true, "== Physical Plan ==", "NativeCommand", "show tables") + check(" Explain show tables ", true, "== Physical Plan ==", "NativeCommand", "show tables") + check(" Explain create table temp__B as select * from src", true, + "== Physical Plan ==", "HiveTableScan") + } }