From 87f81e3c9b6e00c43f160f8903273c44bb51d5b6 Mon Sep 17 00:00:00 2001 From: wforget <643348094@qq.com> Date: Wed, 27 Apr 2022 11:50:04 +0800 Subject: [PATCH] [KYUUBI #2484] Add conf to SessionEvent and display it in EngineSessionPage ### _Why are the changes needed?_ close #2484 ### _How was this patch tested?_ - [X] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [X] Add screenshots for manual tests if appropriate ![image](https://user-images.githubusercontent.com/17894939/165425007-10a8a98e-4498-4982-b0e3-2124b976aa64.png) - [ ] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request Closes #2485 from wForget/KYUUBI-2484. Closes #2484 b527caee [wforget] sorted 88a02820 [wforget] add test 5c3bb304 [wforget] [KYUUBI-2484] Add conf to SessionEvent and display it in EngineSessionPage Authored-by: wforget <643348094@qq.com> Signed-off-by: Fei Wang (cherry picked from commit 06da8cf8966d0a4d8b29e1884e98db2eec483bce) Signed-off-by: Fei Wang --- .../engine/spark/events/SessionEvent.scala | 2 ++ .../apache/spark/ui/EngineSessionPage.scala | 34 +++++++++++++++++++ .../org/apache/spark/ui/EngineTabSuite.scala | 27 +++++++++++++++ 3 files changed, 63 insertions(+) diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/events/SessionEvent.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/events/SessionEvent.scala index b67a6762b62..8d87b1cd940 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/events/SessionEvent.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/events/SessionEvent.scala @@ -43,6 +43,7 @@ case class SessionEvent( username: String, ip: String, serverIp: String, + conf: Map[String, String], startTime: Long, var endTime: Long = -1L, var totalOperations: Int = 0) extends KyuubiEvent with SparkListenerEvent { @@ -70,6 +71,7 @@ object SessionEvent { session.user, session.ipAddress, session.serverIpAddress, + session.conf, session.createTime) } } diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EngineSessionPage.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EngineSessionPage.scala index 18c12e3d0e3..9fe8e5ac899 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EngineSessionPage.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EngineSessionPage.scala @@ -23,6 +23,7 @@ import javax.servlet.http.HttpServletRequest import scala.xml.Node import org.apache.spark.internal.Logging +import org.apache.spark.internal.config.SECRET_REDACTION_PATTERN import org.apache.spark.ui.UIUtils._ import org.apache.spark.util.Utils @@ -31,6 +32,10 @@ case class EngineSessionPage(parent: EngineTab) extends WebUIPage("session") with Logging { val store = parent.store + private def propertyHeader = Seq("Name", "Value") + private def headerClasses = Seq("sorttable_alpha", "sorttable_alpha") + private def propertyRow(kv: (String, String)) = {kv._1}{kv._2} + /** Render the page */ def render(request: HttpServletRequest): Seq[Node] = { val parameterId = request.getParameter("id") @@ -40,6 +45,34 @@ case class EngineSessionPage(parent: EngineTab) val sessionStat = store.getSession(parameterId).getOrElse(null) require(sessionStat != null, "Invalid sessionID[" + parameterId + "]") + val redactionPattern = parent.sparkUI match { + case Some(ui) => Some(ui.conf.get(SECRET_REDACTION_PATTERN)) + case None => SECRET_REDACTION_PATTERN.defaultValue + } + + val sessionPropertiesTable = + if (sessionStat.conf != null && !sessionStat.conf.isEmpty) { + val table = UIUtils.listingTable( + propertyHeader, + propertyRow, + Utils.redact(redactionPattern, sessionStat.conf.toSeq.sorted), + fixedWidth = true, + headerClasses = headerClasses) + +

+ + Session Properties +

+ +
+ {table} +
+ } else { + Seq() + } + generateBasicStats() ++
++

@@ -49,6 +82,7 @@ case class EngineSessionPage(parent: EngineTab) Session created at {formatDate(sessionStat.startTime)}, Total run {sessionStat.totalOperations} SQL

++ + sessionPropertiesTable ++ generateSQLStatsTable(request, sessionStat.sessionId) } UIUtils.headerSparkPage(request, parent.name + " Session", content, parent) diff --git a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/spark/ui/EngineTabSuite.scala b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/spark/ui/EngineTabSuite.scala index cb3ef240312..3d27935de98 100644 --- a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/spark/ui/EngineTabSuite.scala +++ b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/spark/ui/EngineTabSuite.scala @@ -23,6 +23,7 @@ import org.apache.http.util.EntityUtils import org.apache.kyuubi.engine.spark.WithSparkSQLEngine import org.apache.kyuubi.operation.HiveJDBCTestHelper +import org.apache.kyuubi.session.SessionHandle class EngineTabSuite extends WithSparkSQLEngine with HiveJDBCTestHelper { override def withKyuubiConf: Map[String, String] = Map( @@ -164,5 +165,31 @@ class EngineTabSuite extends WithSparkSQLEngine with HiveJDBCTestHelper { } } + test("session properties for engine tab") { + assert(spark.sparkContext.ui.nonEmpty) + val redactKey = "kyuubi.test.password" + val redactValue = "testPassword" + val testKey = "kyuubi.test.key" + val testValue = "testValue" + withSessionConf(Map( + redactKey -> redactValue, + testKey -> testValue))(Map.empty)(Map.empty) { + withSessionHandle { (_, handle) => + val kyuubiHandle = SessionHandle(handle) + val httpClient = HttpClients.createDefault() + val req = new HttpGet(spark.sparkContext.uiWebUrl.get + + s"/kyuubi/session/?id=${kyuubiHandle.identifier}") + val response = httpClient.execute(req) + assert(response.getStatusLine.getStatusCode === 200) + val resp = EntityUtils.toString(response.getEntity) + assert(resp.contains("Session Properties")) + assert(resp.contains(redactKey)) + assert(!resp.contains(redactValue)) + assert(resp.contains(testKey)) + assert(resp.contains(testValue)) + } + } + } + override protected def jdbcUrl: String = getJdbcUrl }