From b1745f8f94649cb9f46fc3ea09357119accadf84 Mon Sep 17 00:00:00 2001 From: xuwq1993 Date: Fri, 25 Jun 2021 13:36:35 +0800 Subject: [PATCH 1/9] fix: update Guava version --- build.sbt | 1 + .../com/microsoft/ml/spark/automl/TuneHyperparameters.scala | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index d01fd15d6e..c76dfea307 100644 --- a/build.sbt +++ b/build.sbt @@ -35,6 +35,7 @@ val extraDependencies = Seq( "org.apache.httpcomponents" % "httpclient" % "4.5.6", "org.apache.httpcomponents" % "httpmime" % "4.5.6", "com.linkedin.isolation-forest" %% "isolation-forest_3.0.0" % "1.0.1", + "com.google.guava" % "guava" % "28.0-jre", ).map(d => d excludeAll (excludes: _*)) val dependencies = coreDependencies ++ extraDependencies diff --git a/core/src/main/scala/com/microsoft/ml/spark/automl/TuneHyperparameters.scala b/core/src/main/scala/com/microsoft/ml/spark/automl/TuneHyperparameters.scala index a0c4ffdd28..d5cd45b9ab 100644 --- a/core/src/main/scala/com/microsoft/ml/spark/automl/TuneHyperparameters.scala +++ b/core/src/main/scala/com/microsoft/ml/spark/automl/TuneHyperparameters.scala @@ -95,7 +95,7 @@ class TuneHyperparameters(override val uid: String) extends Estimator[TuneHyperp private def getExecutionContext: ExecutionContext = { getParallelism match { case 1 => - ExecutionContext.fromExecutorService(MoreExecutors.sameThreadExecutor()) + ExecutionContext.fromExecutor(MoreExecutors.directExecutor()) case _ => val keepAliveSeconds = 60L val prefix = s"${this.getClass.getSimpleName}-thread-pool" From e09729e5d49cd048f4502c7030b7fb11483f569f Mon Sep 17 00:00:00 2001 From: xuwq1993 Date: Tue, 6 Jul 2021 12:58:52 +0800 Subject: [PATCH 2/9] update: remove guava dependency --- build.sbt | 1 - 1 file changed, 1 deletion(-) diff --git a/build.sbt b/build.sbt index c76dfea307..d01fd15d6e 100644 --- a/build.sbt +++ b/build.sbt @@ -35,7 +35,6 @@ val extraDependencies = Seq( "org.apache.httpcomponents" % "httpclient" % "4.5.6", "org.apache.httpcomponents" % "httpmime" % "4.5.6", "com.linkedin.isolation-forest" %% "isolation-forest_3.0.0" % "1.0.1", - "com.google.guava" % "guava" % "28.0-jre", ).map(d => d excludeAll (excludes: _*)) val dependencies = coreDependencies ++ extraDependencies From ceafcdff4bc11f322dd142d0980064d67bd51d31 Mon Sep 17 00:00:00 2001 From: xuwq1993 Date: Tue, 6 Jul 2021 19:53:43 +0800 Subject: [PATCH 3/9] update: use newDirectExecutorService --- .../com/microsoft/ml/spark/automl/TuneHyperparameters.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/com/microsoft/ml/spark/automl/TuneHyperparameters.scala b/core/src/main/scala/com/microsoft/ml/spark/automl/TuneHyperparameters.scala index d5cd45b9ab..233ef7ab68 100644 --- a/core/src/main/scala/com/microsoft/ml/spark/automl/TuneHyperparameters.scala +++ b/core/src/main/scala/com/microsoft/ml/spark/automl/TuneHyperparameters.scala @@ -24,6 +24,7 @@ import org.apache.spark.sql.types.StructType import scala.collection.mutable.ListBuffer import scala.concurrent.{Awaitable, ExecutionContext, Future} import scala.concurrent.duration.Duration +import scala.reflect.internal.util.ScalaClassLoader import scala.util.control.NonFatal /** Tunes model hyperparameters @@ -95,7 +96,8 @@ class TuneHyperparameters(override val uid: String) extends Estimator[TuneHyperp private def getExecutionContext: ExecutionContext = { getParallelism match { case 1 => - ExecutionContext.fromExecutor(MoreExecutors.directExecutor()) + val executorService = MoreExecutors.newDirectExecutorService() + ExecutionContext.fromExecutorService(executorService) case _ => val keepAliveSeconds = 60L val prefix = s"${this.getClass.getSimpleName}-thread-pool" From 259ff9746684ca411becfef0ec6454c6a7063574 Mon Sep 17 00:00:00 2001 From: xuwq1993 Date: Wed, 7 Jul 2021 14:55:54 +0800 Subject: [PATCH 4/9] update: update MoreExecutor --- .../com/microsoft/ml/spark/automl/TuneHyperparameters.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/com/microsoft/ml/spark/automl/TuneHyperparameters.scala b/core/src/main/scala/com/microsoft/ml/spark/automl/TuneHyperparameters.scala index 233ef7ab68..cc8f9643c3 100644 --- a/core/src/main/scala/com/microsoft/ml/spark/automl/TuneHyperparameters.scala +++ b/core/src/main/scala/com/microsoft/ml/spark/automl/TuneHyperparameters.scala @@ -96,7 +96,7 @@ class TuneHyperparameters(override val uid: String) extends Estimator[TuneHyperp private def getExecutionContext: ExecutionContext = { getParallelism match { case 1 => - val executorService = MoreExecutors.newDirectExecutorService() + val executorService = MoreExecutors.sameThreadExecutor() ExecutionContext.fromExecutorService(executorService) case _ => val keepAliveSeconds = 60L From fbeef4e75b4ffea9616e10b95c2ad329202c842f Mon Sep 17 00:00:00 2001 From: xuwq1993 Date: Thu, 8 Jul 2021 14:07:15 +0800 Subject: [PATCH 5/9] update: use reflection --- .../microsoft/ml/spark/automl/TuneHyperparameters.scala | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/com/microsoft/ml/spark/automl/TuneHyperparameters.scala b/core/src/main/scala/com/microsoft/ml/spark/automl/TuneHyperparameters.scala index cc8f9643c3..8f7bf120df 100644 --- a/core/src/main/scala/com/microsoft/ml/spark/automl/TuneHyperparameters.scala +++ b/core/src/main/scala/com/microsoft/ml/spark/automl/TuneHyperparameters.scala @@ -96,7 +96,12 @@ class TuneHyperparameters(override val uid: String) extends Estimator[TuneHyperp private def getExecutionContext: ExecutionContext = { getParallelism match { case 1 => - val executorService = MoreExecutors.sameThreadExecutor() + // val executorService = MoreExecutors.sameThreadExecutor() + val classPath = "com.google.common.util.concurrent.MoreExecutors" + val funcName = "sameThreadExecutor" + val c = ScalaClassLoader(getClass.getClassLoader).tryToLoadClass(classPath) + val method = c.getClass.getMethod(funcName) + val executorService: ExecutorService = method.invoke(c).asInstanceOf[ExecutorService] ExecutionContext.fromExecutorService(executorService) case _ => val keepAliveSeconds = 60L From da5bdafedd68beecb0e45049b85ec2251a2ec7e9 Mon Sep 17 00:00:00 2001 From: xuwq1993 Date: Thu, 8 Jul 2021 14:58:45 +0800 Subject: [PATCH 6/9] update: use reflection --- .../com/microsoft/ml/spark/automl/TuneHyperparameters.scala | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/core/src/main/scala/com/microsoft/ml/spark/automl/TuneHyperparameters.scala b/core/src/main/scala/com/microsoft/ml/spark/automl/TuneHyperparameters.scala index 8f7bf120df..1156176550 100644 --- a/core/src/main/scala/com/microsoft/ml/spark/automl/TuneHyperparameters.scala +++ b/core/src/main/scala/com/microsoft/ml/spark/automl/TuneHyperparameters.scala @@ -96,12 +96,11 @@ class TuneHyperparameters(override val uid: String) extends Estimator[TuneHyperp private def getExecutionContext: ExecutionContext = { getParallelism match { case 1 => - // val executorService = MoreExecutors.sameThreadExecutor() val classPath = "com.google.common.util.concurrent.MoreExecutors" val funcName = "sameThreadExecutor" val c = ScalaClassLoader(getClass.getClassLoader).tryToLoadClass(classPath) - val method = c.getClass.getMethod(funcName) - val executorService: ExecutorService = method.invoke(c).asInstanceOf[ExecutorService] + val method = c.get.getMethod(funcName) + val executorService = method.invoke(c.get).asInstanceOf[ExecutorService] ExecutionContext.fromExecutorService(executorService) case _ => val keepAliveSeconds = 60L From cf2aa39f80c667e0b6409e8a0bb7d96be266705c Mon Sep 17 00:00:00 2001 From: xuwq1993 Date: Thu, 8 Jul 2021 15:37:08 +0800 Subject: [PATCH 7/9] update: use reflection --- .../ml/spark/automl/TuneHyperparameters.scala | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/com/microsoft/ml/spark/automl/TuneHyperparameters.scala b/core/src/main/scala/com/microsoft/ml/spark/automl/TuneHyperparameters.scala index 1156176550..6efd2e77f8 100644 --- a/core/src/main/scala/com/microsoft/ml/spark/automl/TuneHyperparameters.scala +++ b/core/src/main/scala/com/microsoft/ml/spark/automl/TuneHyperparameters.scala @@ -21,6 +21,7 @@ import org.apache.spark.mllib.util.MLUtils import org.apache.spark.sql._ import org.apache.spark.sql.types.StructType +import java.lang.reflect.Method import scala.collection.mutable.ListBuffer import scala.concurrent.{Awaitable, ExecutionContext, Future} import scala.concurrent.duration.Duration @@ -97,9 +98,17 @@ class TuneHyperparameters(override val uid: String) extends Estimator[TuneHyperp getParallelism match { case 1 => val classPath = "com.google.common.util.concurrent.MoreExecutors" - val funcName = "sameThreadExecutor" + val funcNameOld = "sameThreadExecutor" + val funcNameNew = "newDirectExecutorService" val c = ScalaClassLoader(getClass.getClassLoader).tryToLoadClass(classPath) - val method = c.get.getMethod(funcName) + val method: Method = { + try { + c.get.getMethod(funcNameNew) + } + catch { + case ex: NoSuchMethodException => c.get.getMethod(funcNameOld) + } + } val executorService = method.invoke(c.get).asInstanceOf[ExecutorService] ExecutionContext.fromExecutorService(executorService) case _ => From 8ebebe78cefe34ed3185069af62df7c42e638465 Mon Sep 17 00:00:00 2001 From: xuwq1993 Date: Thu, 8 Jul 2021 15:38:21 +0800 Subject: [PATCH 8/9] update: use reflection --- .../com/microsoft/ml/spark/automl/TuneHyperparameters.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/com/microsoft/ml/spark/automl/TuneHyperparameters.scala b/core/src/main/scala/com/microsoft/ml/spark/automl/TuneHyperparameters.scala index 6efd2e77f8..0660ef74d4 100644 --- a/core/src/main/scala/com/microsoft/ml/spark/automl/TuneHyperparameters.scala +++ b/core/src/main/scala/com/microsoft/ml/spark/automl/TuneHyperparameters.scala @@ -106,7 +106,7 @@ class TuneHyperparameters(override val uid: String) extends Estimator[TuneHyperp c.get.getMethod(funcNameNew) } catch { - case ex: NoSuchMethodException => c.get.getMethod(funcNameOld) + case ex: NoSuchMethodError => c.get.getMethod(funcNameOld) } } val executorService = method.invoke(c.get).asInstanceOf[ExecutorService] From d6781f06045ebc44496ab59360cdf208a08713b1 Mon Sep 17 00:00:00 2001 From: xuwq1993 Date: Thu, 8 Jul 2021 15:45:49 +0800 Subject: [PATCH 9/9] update: use reflection --- .../com/microsoft/ml/spark/automl/TuneHyperparameters.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/com/microsoft/ml/spark/automl/TuneHyperparameters.scala b/core/src/main/scala/com/microsoft/ml/spark/automl/TuneHyperparameters.scala index 0660ef74d4..1d74928780 100644 --- a/core/src/main/scala/com/microsoft/ml/spark/automl/TuneHyperparameters.scala +++ b/core/src/main/scala/com/microsoft/ml/spark/automl/TuneHyperparameters.scala @@ -106,7 +106,8 @@ class TuneHyperparameters(override val uid: String) extends Estimator[TuneHyperp c.get.getMethod(funcNameNew) } catch { - case ex: NoSuchMethodError => c.get.getMethod(funcNameOld) + case _: NoSuchMethodError => c.get.getMethod(funcNameOld) + case _: NoSuchMethodException => c.get.getMethod(funcNameOld) } } val executorService = method.invoke(c.get).asInstanceOf[ExecutorService]