Don't create SparkContext before SparkSession

Author: Andrew Or

python/pyspark/shell.py

@@ -35,12 +35,11 @@
 if os.environ.get("SPARK_EXECUTOR_URI"):
     SparkContext.setSystemProperty("spark.executor.uri", os.environ["SPARK_EXECUTOR_URI"])
 
-sc = SparkContext()
-atexit.register(lambda: sc.stop())
+SparkContext._ensure_initialized()
 
 try:
     # Try to access HiveConf, it will raise exception if Hive is not added
-    sc._jvm.org.apache.hadoop.hive.conf.HiveConf()
+    SparkContext._jvm.org.apache.hadoop.hive.conf.HiveConf()
     spark = SparkSession.builder\
         .enableHiveSupport()\
         .getOrCreate()
@@ -49,6 +48,9 @@
 except TypeError:
     spark = SparkSession(sc)
 
+sc = spark.sparkContext
+atexit.register(lambda: sc.stop())
+
 # for compatibility
 sqlContext = spark._wrapped
 sqlCtx = sqlContext

python/pyspark/sql/session.py

@@ -191,6 +191,12 @@ def newSession(self):
         """
         return self.__class__(self._sc, self._jsparkSession.newSession())
 
+    @property
+    @since(2.0)
+    def sparkContext(self):
+        """Returns the underlying :class:`SparkContext`."""
+        return self._sc
+
     @property
     @since(2.0)
     def conf(self):