From a6732ce46edd66a999de72799b27fa1fca578460 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Wed, 14 Sep 2022 21:05:22 +0800 Subject: [PATCH 1/2] support test rocksdb --- .../history/ChromeUIHistoryServerSuite.scala | 18 +++++++++++++++--- .../RealBrowserUIHistoryServerSuite.scala | 5 ++++- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/core/src/test/scala/org/apache/spark/deploy/history/ChromeUIHistoryServerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/ChromeUIHistoryServerSuite.scala index 1fa2d0ab882c9..9fb71f2a0a341 100644 --- a/core/src/test/scala/org/apache/spark/deploy/history/ChromeUIHistoryServerSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/history/ChromeUIHistoryServerSuite.scala @@ -20,13 +20,14 @@ package org.apache.spark.deploy.history import org.openqa.selenium.WebDriver import org.openqa.selenium.chrome.{ChromeDriver, ChromeOptions} -import org.apache.spark.tags.ChromeUITest +import org.apache.spark.internal.config.History.HybridStoreDiskBackend +import org.apache.spark.tags.{ChromeUITest, ExtendedLevelDBTest} + /** * Tests for HistoryServer with Chrome. */ -@ChromeUITest -class ChromeUIHistoryServerSuite +abstract class ChromeUIHistoryServerSuite extends RealBrowserUIHistoryServerSuite("webdriver.chrome.driver") { override var webDriver: WebDriver = _ @@ -48,3 +49,14 @@ class ChromeUIHistoryServerSuite } } } + +@ChromeUITest +@ExtendedLevelDBTest +class LevelDBBackendChromeUIHistoryServerSuite extends ChromeUIHistoryServerSuite { + override protected def diskBackend: HybridStoreDiskBackend.Value = HybridStoreDiskBackend.LEVELDB +} + +@ChromeUITest +class RocksBackendChromeUIHistoryServerSuite extends ChromeUIHistoryServerSuite { + override protected def diskBackend: HybridStoreDiskBackend.Value = HybridStoreDiskBackend.ROCKSDB +} diff --git a/core/src/test/scala/org/apache/spark/deploy/history/RealBrowserUIHistoryServerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/RealBrowserUIHistoryServerSuite.scala index 7d2d31b4adab4..ea3a5ef5ba10c 100644 --- a/core/src/test/scala/org/apache/spark/deploy/history/RealBrowserUIHistoryServerSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/history/RealBrowserUIHistoryServerSuite.scala @@ -28,7 +28,7 @@ import org.scalatestplus.selenium.WebBrowser import org.apache.spark._ import org.apache.spark.internal.config.{EVENT_LOG_STAGE_EXECUTOR_METRICS, EXECUTOR_PROCESS_TREE_METRICS_ENABLED} -import org.apache.spark.internal.config.History.{HISTORY_LOG_DIR, LOCAL_STORE_DIR, UPDATE_INTERVAL_S} +import org.apache.spark.internal.config.History.{HISTORY_LOG_DIR, HYBRID_STORE_DISK_BACKEND, HybridStoreDiskBackend, LOCAL_STORE_DIR, UPDATE_INTERVAL_S} import org.apache.spark.internal.config.Tests.IS_TESTING import org.apache.spark.util.{ResetSystemProperties, Utils} @@ -48,6 +48,8 @@ abstract class RealBrowserUIHistoryServerSuite(val driverProp: String) private var server: HistoryServer = null private var port: Int = -1 + protected def diskBackend: HybridStoreDiskBackend.Value + override def beforeAll(): Unit = { super.beforeAll() assume( @@ -79,6 +81,7 @@ abstract class RealBrowserUIHistoryServerSuite(val driverProp: String) .set(LOCAL_STORE_DIR, storeDir.getAbsolutePath()) .set(EVENT_LOG_STAGE_EXECUTOR_METRICS, true) .set(EXECUTOR_PROCESS_TREE_METRICS_ENABLED, true) + .set(HYBRID_STORE_DISK_BACKEND, diskBackend.toString) conf.setAll(extraConf) provider = new FsHistoryProvider(conf) provider.checkForLogs() From c457b1d09c852c2142d09f1711073ac79191e7d7 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Sun, 18 Sep 2022 18:25:35 -0700 Subject: [PATCH 2/2] Rename to RocksDBBackendChromeUIHistoryServerSuite --- .../spark/deploy/history/ChromeUIHistoryServerSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/test/scala/org/apache/spark/deploy/history/ChromeUIHistoryServerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/ChromeUIHistoryServerSuite.scala index 9fb71f2a0a341..ec910e9bf3436 100644 --- a/core/src/test/scala/org/apache/spark/deploy/history/ChromeUIHistoryServerSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/history/ChromeUIHistoryServerSuite.scala @@ -57,6 +57,6 @@ class LevelDBBackendChromeUIHistoryServerSuite extends ChromeUIHistoryServerSuit } @ChromeUITest -class RocksBackendChromeUIHistoryServerSuite extends ChromeUIHistoryServerSuite { +class RocksDBBackendChromeUIHistoryServerSuite extends ChromeUIHistoryServerSuite { override protected def diskBackend: HybridStoreDiskBackend.Value = HybridStoreDiskBackend.ROCKSDB }