From 3904d162b295b52733b40baef0abeb7285ec4d2f Mon Sep 17 00:00:00 2001 From: ulysses Date: Mon, 18 Nov 2019 09:02:45 +0800 Subject: [PATCH 1/3] init --- .../org/apache/spark/sql/execution/DataSourceScanExec.scala | 5 ++--- .../spark/sql/execution/datasources/FileSourceStrategy.scala | 2 ++ 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala index b0fe4b741479..fa986727f791 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala @@ -187,7 +187,7 @@ case class FileSourceScanExec( partitionSchema = relation.partitionSchema, relation.sparkSession.sessionState.conf) - val driverMetrics: HashMap[String, Long] = HashMap.empty + lazy val driverMetrics: HashMap[String, Long] = HashMap.empty /** * Send the driver-side metrics. Before calling this function, selectedPartitions has @@ -325,8 +325,7 @@ case class FileSourceScanExec( } @transient - private val pushedDownFilters = dataFilters.flatMap(DataSourceStrategy.translateFilter) - logInfo(s"Pushed Filters: ${pushedDownFilters.mkString(",")}") + private lazy val pushedDownFilters = dataFilters.flatMap(DataSourceStrategy.translateFilter) override lazy val metadata: Map[String, String] = { def seqToString(seq: Seq[Any]) = seq.mkString("[", ", ", "]") diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategy.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategy.scala index c8a42f043f15..858de3a65e3e 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategy.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategy.scala @@ -177,6 +177,8 @@ object FileSourceStrategy extends Strategy with Logging { // Partition keys are not available in the statistics of the files. val dataFilters = normalizedFiltersWithoutSubqueries.filter(_.references.intersect(partitionSet).isEmpty) + logInfo(s"Pushed Filters: " + + s"${dataFilters.flatMap(DataSourceStrategy.translateFilter).mkString(",")}") // Predicates with both partition keys and attributes need to be evaluated after the scan. val afterScanFilters = filterSet -- partitionKeyFilters.filter(_.references.nonEmpty) From 7aa9d3d07545cd54d73fbc84834dbefe798c5b50 Mon Sep 17 00:00:00 2001 From: ulysses Date: Mon, 18 Nov 2019 09:06:30 +0800 Subject: [PATCH 2/3] add private --- .../org/apache/spark/sql/execution/DataSourceScanExec.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala index fa986727f791..f20a09c9a65d 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala @@ -187,7 +187,7 @@ case class FileSourceScanExec( partitionSchema = relation.partitionSchema, relation.sparkSession.sessionState.conf) - lazy val driverMetrics: HashMap[String, Long] = HashMap.empty + private lazy val driverMetrics: HashMap[String, Long] = HashMap.empty /** * Send the driver-side metrics. Before calling this function, selectedPartitions has From 0aa4a48b45aaa6220481f196fc4c8c7983a7031f Mon Sep 17 00:00:00 2001 From: ulysses Date: Mon, 18 Nov 2019 09:07:53 +0800 Subject: [PATCH 3/3] fix style --- .../spark/sql/execution/datasources/FileSourceStrategy.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategy.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategy.scala index 858de3a65e3e..afc9bfeda84a 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategy.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategy.scala @@ -177,8 +177,8 @@ object FileSourceStrategy extends Strategy with Logging { // Partition keys are not available in the statistics of the files. val dataFilters = normalizedFiltersWithoutSubqueries.filter(_.references.intersect(partitionSet).isEmpty) - logInfo(s"Pushed Filters: " + - s"${dataFilters.flatMap(DataSourceStrategy.translateFilter).mkString(",")}") + logInfo(s"Pushed Filters: " + + s"${dataFilters.flatMap(DataSourceStrategy.translateFilter).mkString(",")}") // Predicates with both partition keys and attributes need to be evaluated after the scan. val afterScanFilters = filterSet -- partitionKeyFilters.filter(_.references.nonEmpty)