From 1aed1ea82dac8694dabaf9ad12c7ee4e856bcabb Mon Sep 17 00:00:00 2001 From: Angerszhuuuu Date: Tue, 2 Feb 2021 20:18:21 +0800 Subject: [PATCH 1/6] Update ApplicationMaster.scala --- .../org/apache/spark/deploy/yarn/ApplicationMaster.scala | 6 ------ 1 file changed, 6 deletions(-) diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala index eb927a3c296c..850a9eb4d1dd 100644 --- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala +++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala @@ -35,7 +35,6 @@ import org.apache.hadoop.util.StringUtils import org.apache.hadoop.yarn.api._ import org.apache.hadoop.yarn.api.records._ import org.apache.hadoop.yarn.conf.YarnConfiguration -import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException import org.apache.hadoop.yarn.util.{ConverterUtils, Records} import org.apache.spark._ @@ -578,11 +577,6 @@ private[spark] class ApplicationMaster( failureCount = 0 } catch { case i: InterruptedException => // do nothing - case e: ApplicationAttemptNotFoundException => - failureCount += 1 - logError("Exception from Reporter thread.", e) - finish(FinalApplicationStatus.FAILED, ApplicationMaster.EXIT_REPORTER_FAILURE, - e.getMessage) case e: Throwable => failureCount += 1 if (!NonFatal(e)) { From b36cf618f0d4222443d8a91bf11112e961adb94d Mon Sep 17 00:00:00 2001 From: Angerszhuuuu Date: Thu, 4 Feb 2021 23:21:15 +0800 Subject: [PATCH 2/6] Update ApplicationMaster.scala --- .../org/apache/spark/deploy/yarn/ApplicationMaster.scala | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala index 850a9eb4d1dd..aa360c8a023b 100644 --- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala +++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala @@ -21,7 +21,7 @@ import java.io.{File, IOException} import java.lang.reflect.{InvocationTargetException, Modifier} import java.net.{URI, URL, URLEncoder} import java.security.PrivilegedExceptionAction -import java.util.concurrent.{TimeoutException, TimeUnit} +import java.util.concurrent.{TimeUnit, TimeoutException} import scala.collection.mutable.HashMap import scala.concurrent.Promise @@ -35,8 +35,8 @@ import org.apache.hadoop.util.StringUtils import org.apache.hadoop.yarn.api._ import org.apache.hadoop.yarn.api.records._ import org.apache.hadoop.yarn.conf.YarnConfiguration +import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException import org.apache.hadoop.yarn.util.{ConverterUtils, Records} - import org.apache.spark._ import org.apache.spark.deploy.SparkHadoopUtil import org.apache.spark.deploy.history.HistoryServer @@ -577,6 +577,11 @@ private[spark] class ApplicationMaster( failureCount = 0 } catch { case i: InterruptedException => // do nothing + case e: ApplicationAttemptNotFoundException if isClusterMode => + failureCount += 1 + logError("Exception from Reporter thread.", e) + finish(FinalApplicationStatus.FAILED, ApplicationMaster.EXIT_REPORTER_FAILURE, + e.getMessage) case e: Throwable => failureCount += 1 if (!NonFatal(e)) { From e238b9a9555dfc74a16dd9845d8250d07e9ecad1 Mon Sep 17 00:00:00 2001 From: Angerszhuuuu Date: Fri, 5 Feb 2021 10:12:31 +0800 Subject: [PATCH 3/6] Update ApplicationMaster.scala --- .../scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala index aa360c8a023b..8bedadf05466 100644 --- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala +++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala @@ -21,7 +21,7 @@ import java.io.{File, IOException} import java.lang.reflect.{InvocationTargetException, Modifier} import java.net.{URI, URL, URLEncoder} import java.security.PrivilegedExceptionAction -import java.util.concurrent.{TimeUnit, TimeoutException} +import java.util.concurrent.{TimeoutException, TimeUnit} import scala.collection.mutable.HashMap import scala.concurrent.Promise From db37f5e791315263d4c83f7ee93d2d4a1b356b16 Mon Sep 17 00:00:00 2001 From: Angerszhuuuu Date: Fri, 5 Feb 2021 10:14:21 +0800 Subject: [PATCH 4/6] Update ApplicationMaster.scala --- .../scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala index 8bedadf05466..176987936410 100644 --- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala +++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala @@ -37,6 +37,7 @@ import org.apache.hadoop.yarn.api.records._ import org.apache.hadoop.yarn.conf.YarnConfiguration import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException import org.apache.hadoop.yarn.util.{ConverterUtils, Records} + import org.apache.spark._ import org.apache.spark.deploy.SparkHadoopUtil import org.apache.spark.deploy.history.HistoryServer From cde1b246c1e3bb232ecf660a2171e414d40ebd8d Mon Sep 17 00:00:00 2001 From: Angerszhuuuu Date: Wed, 21 Apr 2021 13:16:46 +0800 Subject: [PATCH 5/6] Update ApplicationMaster.scala --- .../org/apache/spark/deploy/yarn/ApplicationMaster.scala | 5 ----- 1 file changed, 5 deletions(-) diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala index 9798fdb97de6..05a1afe1694b 100644 --- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala +++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala @@ -577,11 +577,6 @@ private[spark] class ApplicationMaster( failureCount = 0 } catch { case i: InterruptedException => // do nothing - case e: ApplicationAttemptNotFoundException if isClusterMode => - failureCount += 1 - logError("Exception from Reporter thread.", e) - finish(FinalApplicationStatus.FAILED, ApplicationMaster.EXIT_REPORTER_FAILURE, - e.getMessage) case e: Throwable => failureCount += 1 if (!NonFatal(e)) { From 69abde655c00ff7dc4502bb601d6f695791e5ec0 Mon Sep 17 00:00:00 2001 From: Angerszhuuuu Date: Wed, 21 Apr 2021 13:17:02 +0800 Subject: [PATCH 6/6] Update ApplicationMaster.scala --- .../scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala index 05a1afe1694b..2caceb5674a9 100644 --- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala +++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala @@ -35,7 +35,6 @@ import org.apache.hadoop.util.StringUtils import org.apache.hadoop.yarn.api._ import org.apache.hadoop.yarn.api.records._ import org.apache.hadoop.yarn.conf.YarnConfiguration -import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException import org.apache.hadoop.yarn.util.{ConverterUtils, Records} import org.apache.spark._