From 4412f469f1c9142d4ef48faa4f5d6e14e2d98ba7 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Sat, 1 Jul 2017 17:17:45 -0700 Subject: [PATCH 1/6] [SPARK-20256][SQL] SessionState should be created lazily --- .../scala/org/apache/spark/sql/SparkSession.scala | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala index 2c38f7d7c88d..2dd6c96b1481 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala @@ -117,6 +117,12 @@ class SparkSession private( existingSharedState.getOrElse(new SharedState(sparkContext)) } + /** + * Initial options for session. This options are applied once when sessionState is created. + */ + @transient + lazy val initialSessionOptions = new scala.collection.mutable.HashMap[String, String] + /** * State isolated across sessions, including SQL configurations, temporary tables, registered * functions, and everything else that accepts a [[org.apache.spark.sql.internal.SQLConf]]. @@ -132,9 +138,11 @@ class SparkSession private( parentSessionState .map(_.clone(this)) .getOrElse { - SparkSession.instantiateSessionState( + val state = SparkSession.instantiateSessionState( SparkSession.sessionStateClassName(sparkContext.conf), self) + initialSessionOptions.foreach { case (k, v) => state.conf.setConfString(k, v) } + state } } @@ -940,7 +948,7 @@ object SparkSession { } session = new SparkSession(sparkContext, None, None, extensions) - options.foreach { case (k, v) => session.sessionState.conf.setConfString(k, v) } + options.foreach { case (k, v) => session.initialSessionOptions.put(k, v) } defaultSession.set(session) // Register a successfully instantiated context to the singleton. This should be at the From eb4bcdbd2cec8d0213bd601806c19cc570181ab4 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Sun, 2 Jul 2017 10:54:29 -0700 Subject: [PATCH 2/6] Revert "[SPARK-20256][SQL] SessionState should be created lazily" This reverts commit 8517fe89068e5fe695a3647c15e539c9a55f0d7e. --- .../scala/org/apache/spark/sql/SparkSession.scala | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala index 2dd6c96b1481..2c38f7d7c88d 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala @@ -117,12 +117,6 @@ class SparkSession private( existingSharedState.getOrElse(new SharedState(sparkContext)) } - /** - * Initial options for session. This options are applied once when sessionState is created. - */ - @transient - lazy val initialSessionOptions = new scala.collection.mutable.HashMap[String, String] - /** * State isolated across sessions, including SQL configurations, temporary tables, registered * functions, and everything else that accepts a [[org.apache.spark.sql.internal.SQLConf]]. @@ -138,11 +132,9 @@ class SparkSession private( parentSessionState .map(_.clone(this)) .getOrElse { - val state = SparkSession.instantiateSessionState( + SparkSession.instantiateSessionState( SparkSession.sessionStateClassName(sparkContext.conf), self) - initialSessionOptions.foreach { case (k, v) => state.conf.setConfString(k, v) } - state } } @@ -948,7 +940,7 @@ object SparkSession { } session = new SparkSession(sparkContext, None, None, extensions) - options.foreach { case (k, v) => session.initialSessionOptions.put(k, v) } + options.foreach { case (k, v) => session.sessionState.conf.setConfString(k, v) } defaultSession.set(session) // Register a successfully instantiated context to the singleton. This should be at the From 6853a564ead0f14bf77be7b6451aa43fec5cf08d Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Sun, 2 Jul 2017 10:58:12 -0700 Subject: [PATCH 3/6] Address the comments. --- sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala index 2c38f7d7c88d..d07496b840f7 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala @@ -940,7 +940,6 @@ object SparkSession { } session = new SparkSession(sparkContext, None, None, extensions) - options.foreach { case (k, v) => session.sessionState.conf.setConfString(k, v) } defaultSession.set(session) // Register a successfully instantiated context to the singleton. This should be at the From 73df93dacf63f6d9410d91d0e074ee72d6f02d90 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Sun, 2 Jul 2017 21:32:07 -0700 Subject: [PATCH 4/6] Revert "Address the comments." This reverts commit b22369b289ddcf16118c30c62e212b3ad9736a48. --- sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala index d07496b840f7..2c38f7d7c88d 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala @@ -940,6 +940,7 @@ object SparkSession { } session = new SparkSession(sparkContext, None, None, extensions) + options.foreach { case (k, v) => session.sessionState.conf.setConfString(k, v) } defaultSession.set(session) // Register a successfully instantiated context to the singleton. This should be at the From 137f252c79f3f044507a453320a66ac6d0cb6334 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Sun, 2 Jul 2017 21:32:16 -0700 Subject: [PATCH 5/6] Revert "Revert "[SPARK-20256][SQL] SessionState should be created lazily"" This reverts commit 473562b1774cbcb5064c3b269baa6487b10d9686. --- .../scala/org/apache/spark/sql/SparkSession.scala | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala index 2c38f7d7c88d..2dd6c96b1481 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala @@ -117,6 +117,12 @@ class SparkSession private( existingSharedState.getOrElse(new SharedState(sparkContext)) } + /** + * Initial options for session. This options are applied once when sessionState is created. + */ + @transient + lazy val initialSessionOptions = new scala.collection.mutable.HashMap[String, String] + /** * State isolated across sessions, including SQL configurations, temporary tables, registered * functions, and everything else that accepts a [[org.apache.spark.sql.internal.SQLConf]]. @@ -132,9 +138,11 @@ class SparkSession private( parentSessionState .map(_.clone(this)) .getOrElse { - SparkSession.instantiateSessionState( + val state = SparkSession.instantiateSessionState( SparkSession.sessionStateClassName(sparkContext.conf), self) + initialSessionOptions.foreach { case (k, v) => state.conf.setConfString(k, v) } + state } } @@ -940,7 +948,7 @@ object SparkSession { } session = new SparkSession(sparkContext, None, None, extensions) - options.foreach { case (k, v) => session.sessionState.conf.setConfString(k, v) } + options.foreach { case (k, v) => session.initialSessionOptions.put(k, v) } defaultSession.set(session) // Register a successfully instantiated context to the singleton. This should be at the From c9686f7ea46a2e5152413a8eaeb477c645eda04a Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Tue, 4 Jul 2017 07:31:53 -0700 Subject: [PATCH 6/6] Address comments. --- sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala index 2dd6c96b1481..0ddcd2111aa5 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala @@ -121,7 +121,7 @@ class SparkSession private( * Initial options for session. This options are applied once when sessionState is created. */ @transient - lazy val initialSessionOptions = new scala.collection.mutable.HashMap[String, String] + private[sql] val initialSessionOptions = new scala.collection.mutable.HashMap[String, String] /** * State isolated across sessions, including SQL configurations, temporary tables, registered