diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala b/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala index 59e90564b351..cd125bf0ca8d 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala @@ -102,11 +102,16 @@ class SparkHadoopUtil extends Logging { hadoopConf.set("fs.s3n.awsSecretAccessKey", accessKey) hadoopConf.set("fs.s3a.secret.key", accessKey) } - // Copy any "spark.hadoop.foo=bar" system properties into conf as "foo=bar" conf.getAll.foreach { case (key, value) => + // Copy any "spark.hadoop.foo=bar" system properties into conf as "foo=bar" if (key.startsWith("spark.hadoop.")) { hadoopConf.set(key.substring("spark.hadoop.".length), value) } + // fix added for SPARK-13979 + // Copy any "fs.*=bar" properties into conf it will cover almost all filesystem + else if (key.startsWith("fs.")){ + hadoopConf.set(key, value) + } } val bufferSize = conf.get("spark.buffer.size", "65536") hadoopConf.set("io.file.buffer.size", bufferSize) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategy.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategy.scala index 3741a9cb32fd..5d13f873c6b6 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategy.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategy.scala @@ -121,7 +121,9 @@ private[sql] object DataSourceStrategy extends Strategy with Logging { case PhysicalOperation(projects, filters, l @ LogicalRelation(t: HadoopFsRelation, _)) => // See buildPartitionedTableScan for the reason that we need to create a shard // broadcast HadoopConf. - val sharedHadoopConf = SparkHadoopUtil.get.conf + // fix added for SPARK-13979 + // val sharedHadoopConf = SparkHadoopUtil.get.conf + val sharedHadoopConf = t.sqlContext.sparkContext.hadoopConfiguration val confBroadcast = t.sqlContext.sparkContext.broadcast(new SerializableConfiguration(sharedHadoopConf)) pruneFilterProject( @@ -156,7 +158,9 @@ private[sql] object DataSourceStrategy extends Strategy with Logging { // Because we are creating one RDD per partition, we need to have a shared HadoopConf. // Otherwise, the cost of broadcasting HadoopConf in every RDD will be high. - val sharedHadoopConf = SparkHadoopUtil.get.conf + // fix added for SPARK-13979 + // val sharedHadoopConf = SparkHadoopUtil.get.conf + val sharedHadoopConf = relation.sqlContext.sparkContext.hadoopConfiguration val confBroadcast = relation.sqlContext.sparkContext.broadcast(new SerializableConfiguration(sharedHadoopConf)) val partitionColumnNames = partitionColumns.fieldNames.toSet