Added Hive support for Sparklyr

LLNL · Apr 25, 2019 · f0f0c67 · f0f0c67
1 parent 2332a8f
commit f0f0c67
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 0 deletions.
diff --git a/conf/spark/spark-defaults.conf b/conf/spark/spark-defaults.conf
@@ -66,4 +66,8 @@ spark.shuffle.sasl.timeout              SPARKNETWORKTIMEOUTSECSs
 spark.rdd.compress                      SPARKRDDCOMPRESS
 spark.io.compression.codec              SPARKIOCOMPRESSIONCODEC
 
+# Sparklyr needs to know the catalogImplementation in order to read
+# the HIVE tables
+# spark.sql.catalogImplementation hive
+
 @YARN@
diff --git a/magpie/setup/magpie-setup-project-spark b/magpie/setup/magpie-setup-project-spark
@@ -625,6 +625,12 @@ then
         sed -i -e "/@YARN@/,+1d" ${post_sparkdefaultsconf}
     fi
 
+    # Sparklyr needs to know the catalogImplementation
+    if [ "${HIVE_SETUP}" == "yes" ]
+    then
+        sed -i -e "s/# spark.sql.catalogImplementation hive/spark.sql.catalogImplementation hive/g" ${post_sparkdefaultsconf}
+    fi
+
     # This is not for legitimate security as it can easily be
     # calculated.  It is for sanity "just in case we screwed up a
     # config" security.