[GLUTEN-7420][VL] Fix GCS configuration

apache · Oct 15, 2024 · 4af7ea2 · 4af7ea2
1 parent 0bc6613
commit 4af7ea2
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 7 deletions.
diff --git a/cpp/velox/utils/ConfigExtractor.cc b/cpp/velox/utils/ConfigExtractor.cc
@@ -19,9 +19,6 @@
 
 #include "ConfigExtractor.h"
 #include <stdexcept>
-#ifdef ENABLE_GCS
-#include <fstream>
-#endif
 
 #include "utils/Exception.h"
 #include "velox/connectors/hive/HiveConfig.h"
@@ -166,10 +163,8 @@ std::shared_ptr<facebook::velox::config::ConfigBase> getHiveConfig(
       auto gsAuthServiceAccountJsonKeyfile =
           conf->get<std::string>("spark.hadoop.fs.gs.auth.service.account.json.keyfile");
       if (gsAuthServiceAccountJsonKeyfile.hasValue()) {
-        auto stream = std::ifstream(gsAuthServiceAccountJsonKeyfile.value());
-        stream.exceptions(std::ios::badbit);
-        std::string gsAuthServiceAccountJson = std::string(std::istreambuf_iterator<char>(stream.rdbuf()), {});
-        hiveConfMap[facebook::velox::connector::hive::HiveConfig::kGCSCredentialsPath] = gsAuthServiceAccountJson;
+        hiveConfMap[facebook::velox::connector::hive::HiveConfig::kGCSCredentialsPath] =
+            gsAuthServiceAccountJsonKeyfile.value();
       } else {
         LOG(WARNING) << "STARTUP: conf spark.hadoop.fs.gs.auth.type is set to SERVICE_ACCOUNT_JSON_KEYFILE, "
                         "however conf spark.hadoop.fs.gs.auth.service.account.json.keyfile is not set";

diff --git a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala
@@ -792,10 +792,22 @@ object GlutenConfig {
       .filter(_._1.startsWith(HADOOP_PREFIX + S3A_PREFIX))
       .foreach(entry => nativeConfMap.put(entry._1, entry._2))
 
+    // handle ABFS config
     conf
       .filter(_._1.startsWith(SPARK_ABFS_ACCOUNT_KEY))
       .foreach(entry => nativeConfMap.put(entry._1, entry._2))
 
+    // handle GCS config
+    if (conf.contains(SPARK_GCS_AUTH_TYPE)) {
+      nativeConfMap.put(SPARK_GCS_AUTH_TYPE, conf(SPARK_GCS_AUTH_TYPE))
+    }
+
+    if (conf.contains(SPARK_GCS_AUTH_SERVICE_ACCOUNT_JSON_KEYFILE)) {
+      nativeConfMap.put(
+        SPARK_GCS_AUTH_SERVICE_ACCOUNT_JSON_KEYFILE,
+        conf(SPARK_GCS_AUTH_SERVICE_ACCOUNT_JSON_KEYFILE))
+    }
+
     // return
     nativeConfMap
   }