Skip to content

[SPARK-29092][SQL] Report additional information about DataSourceScanExec in EXPLAIN FORMATTED #26042

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
dilipbiswal wants to merge 10 commits into from
Closed

[SPARK-29092][SQL] Report additional information about DataSourceScanExec in EXPLAIN FORMATTED #26042

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
ec16299
[SPARK-29092] Report additional information about DataSourceScanExec …
dilipbiswal Oct 7, 2019
97aaadc
ignore explain test from thriftserver
dilipbiswal Oct 8, 2019
6526889
address test failure
dilipbiswal Oct 8, 2019
91ce794
Code review
dilipbiswal Oct 8, 2019
9560005
latest
dilipbiswal Oct 11, 2019
7fc4d5e
enable thrift test
dilipbiswal Oct 11, 2019
d07e2b0
test failure
dilipbiswal Oct 12, 2019
8beed08
Code review
dilipbiswal Oct 15, 2019
192960d
code review
dilipbiswal Oct 17, 2019
490ee3b
test failure
dilipbiswal Oct 17, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 42 additions & 1 deletion sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,10 +65,26 @@ trait DataSourceScanExec extends LeafExecNode {
s"$nodeNamePrefix$nodeName${truncatedString(output, "[", ",", "]", maxFields)}$metadataStr")
}

override def verboseStringWithOperatorId(): String = {
val metadataStr = metadata.toSeq.sorted.filterNot {
case (_, value) if (value.isEmpty || value.equals("[]")) => true
case (key, _) if (key.equals("DataFilters") || key.equals("Format")) => true
case (_, _) => false
}.map {
case (key, value) => s"$key: ${redact(value)}"
}

s"""
|(${ExplainUtils.getOpId(this)}) $nodeName ${ExplainUtils.getCodegenId(this)}
|Output: ${producedAttributes.mkString("[", ", ", "]")}
|${metadataStr.mkString("\n")}
""".stripMargin
}

/**
* Shorthand for calling redactString() without specifying redacting rules
*/
private def redact(text: String): String = {
protected def redact(text: String): String = {
Utils.redact(sqlContext.sessionState.conf.stringRedactionPattern, text)
}

Expand Down Expand Up @@ -342,6 +358,31 @@ case class FileSourceScanExec(
withSelectedBucketsCount
}

override def verboseStringWithOperatorId(): String = {
val metadataStr = metadata.toSeq.sorted.filterNot {
case (_, value) if (value.isEmpty || value.equals("[]")) => true
case (key, _) if (key.equals("DataFilters") || key.equals("Format")) => true
case (_, _) => false
}.map {
case (key, _) if (key.equals("Location")) =>
val location = relation.location
val numPaths = location.rootPaths.length
val abbreviatedLoaction = if (numPaths <= 1) {
location.rootPaths.mkString("[", ", ", "]")
} else {
"[" + location.rootPaths.head + s", ... ${numPaths - 1} entries]"
}
s"$key: ${location.getClass.getSimpleName} ${redact(abbreviatedLoaction)}"
case (key, value) => s"$key: ${redact(value)}"
}

s"""
|(${ExplainUtils.getOpId(this)}) $nodeName ${ExplainUtils.getCodegenId(this)}
|Output: ${producedAttributes.mkString("[", ", ", "]")}
|${metadataStr.mkString("\n")}
""".stripMargin
}

lazy val inputRDD: RDD[InternalRow] = {
val readFile: (PartitionedFile) => Iterator[InternalRow] =
relation.fileFormat.buildReaderWithPartitionValues(
Expand Down
2 changes: 1 addition & 1 deletion sql/core/src/main/scala/org/apache/spark/sql/execution/subquery.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -193,7 +193,7 @@ case class PlanSubqueries(sparkSession: SparkSession) extends Rule[SparkPlan] {
)
}
val executedPlan = new QueryExecution(sparkSession, query).executedPlan
InSubqueryExec(expr, SubqueryExec(s"subquery${exprId.id}", executedPlan), exprId)
InSubqueryExec(expr, SubqueryExec(s"subquery#${exprId.id}", executedPlan), exprId)
}
}
}
Expand Down
28 changes: 14 additions & 14 deletions sql/core/src/test/resources/sql-tests/results/describe-part-after-analyze.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,12 +56,12 @@ hr int
Database default
Table t
Partition Values [ds=2017-08-01, hr=10]
Location [not included in comparison]sql/core/spark-warehouse/t/ds=2017-08-01/hr=10
Location [not included in comparison]/{warehouse_dir}/t/ds=2017-08-01/hr=10
Created Time [not included in comparison]
Last Access [not included in comparison]

# Storage Information
Location [not included in comparison]sql/core/spark-warehouse/t
Location [not included in comparison]/{warehouse_dir}/t


-- !query 5
Expand Down Expand Up @@ -90,13 +90,13 @@ hr int
Database default
Table t
Partition Values [ds=2017-08-01, hr=10]
Location [not included in comparison]sql/core/spark-warehouse/t/ds=2017-08-01/hr=10
Location [not included in comparison]/{warehouse_dir}/t/ds=2017-08-01/hr=10
Created Time [not included in comparison]
Last Access [not included in comparison]
Partition Statistics [not included in comparison] bytes, 3 rows

# Storage Information
Location [not included in comparison]sql/core/spark-warehouse/t
Location [not included in comparison]/{warehouse_dir}/t


-- !query 7
Expand Down Expand Up @@ -125,13 +125,13 @@ hr int
Database default
Table t
Partition Values [ds=2017-08-01, hr=10]
Location [not included in comparison]sql/core/spark-warehouse/t/ds=2017-08-01/hr=10
Location [not included in comparison]/{warehouse_dir}/t/ds=2017-08-01/hr=10
Created Time [not included in comparison]
Last Access [not included in comparison]
Partition Statistics [not included in comparison] bytes, 3 rows

# Storage Information
Location [not included in comparison]sql/core/spark-warehouse/t
Location [not included in comparison]/{warehouse_dir}/t


-- !query 9
Expand All @@ -152,13 +152,13 @@ hr int
Database default
Table t
Partition Values [ds=2017-08-01, hr=11]
Location [not included in comparison]sql/core/spark-warehouse/t/ds=2017-08-01/hr=11
Location [not included in comparison]/{warehouse_dir}/t/ds=2017-08-01/hr=11
Created Time [not included in comparison]
Last Access [not included in comparison]
Partition Statistics [not included in comparison] bytes, 4 rows

# Storage Information
Location [not included in comparison]sql/core/spark-warehouse/t
Location [not included in comparison]/{warehouse_dir}/t


-- !query 10
Expand Down Expand Up @@ -187,13 +187,13 @@ hr int
Database default
Table t
Partition Values [ds=2017-08-01, hr=10]
Location [not included in comparison]sql/core/spark-warehouse/t/ds=2017-08-01/hr=10
Location [not included in comparison]/{warehouse_dir}/t/ds=2017-08-01/hr=10
Created Time [not included in comparison]
Last Access [not included in comparison]
Partition Statistics [not included in comparison] bytes, 3 rows

# Storage Information
Location [not included in comparison]sql/core/spark-warehouse/t
Location [not included in comparison]/{warehouse_dir}/t


-- !query 12
Expand All @@ -214,13 +214,13 @@ hr int
Database default
Table t
Partition Values [ds=2017-08-01, hr=11]
Location [not included in comparison]sql/core/spark-warehouse/t/ds=2017-08-01/hr=11
Location [not included in comparison]/{warehouse_dir}/t/ds=2017-08-01/hr=11
Created Time [not included in comparison]
Last Access [not included in comparison]
Partition Statistics [not included in comparison] bytes, 4 rows

# Storage Information
Location [not included in comparison]sql/core/spark-warehouse/t
Location [not included in comparison]/{warehouse_dir}/t


-- !query 13
Expand All @@ -241,13 +241,13 @@ hr int
Database default
Table t
Partition Values [ds=2017-09-01, hr=5]
Location [not included in comparison]sql/core/spark-warehouse/t/ds=2017-09-01/hr=5
Location [not included in comparison]/{warehouse_dir}/t/ds=2017-09-01/hr=5
Created Time [not included in comparison]
Last Access [not included in comparison]
Partition Statistics [not included in comparison] bytes, 2 rows

# Storage Information
Location [not included in comparison]sql/core/spark-warehouse/t
Location [not included in comparison]/{warehouse_dir}/t


-- !query 14
Expand Down
10 changes: 5 additions & 5 deletions sql/core/src/test/resources/sql-tests/results/describe-table-after-alter-table.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ Created By [not included in comparison]
Type MANAGED
Provider parquet
Comment added
Location [not included in comparison]sql/core/spark-warehouse/table_with_comment
Location [not included in comparison]/{warehouse_dir}/table_with_comment


-- !query 2
Expand Down Expand Up @@ -60,7 +60,7 @@ Type MANAGED
Provider parquet
Comment modified comment
Table Properties [type=parquet]
Location [not included in comparison]sql/core/spark-warehouse/table_with_comment
Location [not included in comparison]/{warehouse_dir}/table_with_comment


-- !query 4
Expand Down Expand Up @@ -95,7 +95,7 @@ Last Access [not included in comparison]
Created By [not included in comparison]
Type MANAGED
Provider parquet
Location [not included in comparison]sql/core/spark-warehouse/table_comment
Location [not included in comparison]/{warehouse_dir}/table_comment


-- !query 7
Expand Down Expand Up @@ -123,7 +123,7 @@ Created By [not included in comparison]
Type MANAGED
Provider parquet
Comment added comment
Location [not included in comparison]sql/core/spark-warehouse/table_comment
Location [not included in comparison]/{warehouse_dir}/table_comment


-- !query 9
Expand All @@ -150,7 +150,7 @@ Last Access [not included in comparison]
Created By [not included in comparison]
Type MANAGED
Provider parquet
Location [not included in comparison]sql/core/spark-warehouse/table_comment
Location [not included in comparison]/{warehouse_dir}/table_comment


-- !query 11
Expand Down
16 changes: 8 additions & 8 deletions sql/core/src/test/resources/sql-tests/results/describe.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,7 @@ Bucket Columns [`a`]
Sort Columns [`b`]
Comment table_comment
Table Properties [t=test, e=3]
Location [not included in comparison]sql/core/spark-warehouse/t
Location [not included in comparison]/{warehouse_dir}/t
Storage Properties [a=1, b=2]
Partition Provider Catalog

Expand Down Expand Up @@ -163,7 +163,7 @@ Bucket Columns [`a`]
Sort Columns [`b`]
Comment table_comment
Table Properties [t=test, e=3]
Location [not included in comparison]sql/core/spark-warehouse/t
Location [not included in comparison]/{warehouse_dir}/t
Storage Properties [a=1, b=2]
Partition Provider Catalog

Expand Down Expand Up @@ -203,7 +203,7 @@ Bucket Columns [`a`]
Sort Columns [`b`]
Comment table_comment
Table Properties [t=test]
Location [not included in comparison]sql/core/spark-warehouse/t
Location [not included in comparison]/{warehouse_dir}/t
Storage Properties [a=1, b=2]
Partition Provider Catalog

Expand Down Expand Up @@ -242,7 +242,7 @@ Num Buckets 2
Bucket Columns [`a`]
Sort Columns [`b`]
Table Properties [t=test]
Location [not included in comparison]sql/core/spark-warehouse/t
Location [not included in comparison]/{warehouse_dir}/t
Storage Properties [a=1, b=2]
Partition Provider Catalog

Expand Down Expand Up @@ -280,7 +280,7 @@ d string
Database default
Table t
Partition Values [c=Us, d=1]
Location [not included in comparison]sql/core/spark-warehouse/t/c=Us/d=1
Location [not included in comparison]/{warehouse_dir}/t/c=Us/d=1
Storage Properties [a=1, b=2]
Created Time [not included in comparison]
Last Access [not included in comparison]
Expand All @@ -289,7 +289,7 @@ Last Access [not included in comparison]
Num Buckets 2
Bucket Columns [`a`]
Sort Columns [`b`]
Location [not included in comparison]sql/core/spark-warehouse/t
Location [not included in comparison]/{warehouse_dir}/t
Storage Properties [a=1, b=2]


Expand All @@ -311,7 +311,7 @@ d string
Database default
Table t
Partition Values [c=Us, d=1]
Location [not included in comparison]sql/core/spark-warehouse/t/c=Us/d=1
Location [not included in comparison]/{warehouse_dir}/t/c=Us/d=1
Storage Properties [a=1, b=2]
Created Time [not included in comparison]
Last Access [not included in comparison]
Expand All @@ -320,7 +320,7 @@ Last Access [not included in comparison]
Num Buckets 2
Bucket Columns [`a`]
Sort Columns [`b`]
Location [not included in comparison]sql/core/spark-warehouse/t
Location [not included in comparison]/{warehouse_dir}/t
Storage Properties [a=1, b=2]


Expand Down
Loading