From aa1e7fa48eb705be7746f86f02031a9548ec7130 Mon Sep 17 00:00:00 2001 From: Ashutosh Gupta Date: Fri, 29 Apr 2022 02:11:46 +0100 Subject: [PATCH 1/6] MAPREDUCE-7369. Fixed MapReduce tasks timing out when spends more time on MultipleOutputs#close --- .../mapred/TaskAttemptListenerImpl.java | 15 ++++-- .../mapred/TestTaskAttemptListenerImpl.java | 46 ++++++++++++------- .../apache/hadoop/mapreduce/MRJobConfig.java | 6 +++ .../src/main/resources/mapred-default.xml | 7 +++ 4 files changed, 52 insertions(+), 22 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java index 7d151adea7d6c..5dffd735fdafd 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java @@ -28,6 +28,10 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicReference; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.ipc.ProtocolSignature; @@ -50,8 +54,8 @@ import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptFailEvent; -import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent.TaskAttemptStatus; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent; +import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent.TaskAttemptStatus; import org.apache.hadoop.mapreduce.v2.app.rm.RMHeartbeatHandler; import org.apache.hadoop.mapreduce.v2.app.rm.preemption.AMPreemptionPolicy; import org.apache.hadoop.mapreduce.v2.app.security.authorize.MRAMPolicyProvider; @@ -61,10 +65,6 @@ import org.apache.hadoop.util.StringInterner; import org.apache.hadoop.util.Time; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import org.apache.hadoop.classification.VisibleForTesting; /** * This class is responsible for talking to the task umblical. @@ -409,6 +409,11 @@ public AMFeedback statusUpdate(TaskAttemptID taskAttemptID, if (LOG.isDebugEnabled()) { LOG.debug("Ping from " + taskAttemptID.toString()); } + // Consider ping from the tasks for liveliness check + if (getConfig().getBoolean(MRJobConfig.MR_TASK_ENABLE_PING_FOR_LIVELINESS_CHECK, + MRJobConfig.DEFAULT_MR_TASK_ENABLE_PING_FOR_LIVELINESS_CHECK)) { + taskHeartbeatHandler.progressing(yarnAttemptID); + } return feedback; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestTaskAttemptListenerImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestTaskAttemptListenerImpl.java index f8b8c6ccdf1de..d3f5889e0ab34 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestTaskAttemptListenerImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestTaskAttemptListenerImpl.java @@ -17,23 +17,30 @@ */ package org.apache.hadoop.mapred; -import java.util.function.Supplier; -import org.apache.hadoop.mapred.Counters.Counter; -import org.apache.hadoop.mapreduce.checkpoint.EnumCounter; - import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Supplier; + +import org.junit.After; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.mapred.Counters.Counter; import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.TaskType; import org.apache.hadoop.mapreduce.TypeConverter; import org.apache.hadoop.mapreduce.checkpoint.CheckpointID; +import org.apache.hadoop.mapreduce.checkpoint.EnumCounter; import org.apache.hadoop.mapreduce.checkpoint.FSCheckpointID; import org.apache.hadoop.mapreduce.checkpoint.TaskCheckpointID; import org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager; @@ -48,9 +55,9 @@ import org.apache.hadoop.mapreduce.v2.app.job.Job; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent.TaskAttemptStatus; +import org.apache.hadoop.mapreduce.v2.app.rm.RMHeartbeatHandler; import org.apache.hadoop.mapreduce.v2.app.rm.preemption.AMPreemptionPolicy; import org.apache.hadoop.mapreduce.v2.app.rm.preemption.CheckpointAMPreemptionPolicy; -import org.apache.hadoop.mapreduce.v2.app.rm.RMHeartbeatHandler; import org.apache.hadoop.mapreduce.v2.util.MRBuilderUtils; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.yarn.event.Dispatcher; @@ -60,17 +67,22 @@ import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.util.ControlledClock; import org.apache.hadoop.yarn.util.SystemClock; -import org.junit.After; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.ArgumentCaptor; -import org.mockito.Captor; -import org.mockito.Mock; -import org.mockito.junit.MockitoJUnitRunner; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.*; -import static org.mockito.Mockito.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; /** * Tests the behavior of TaskAttemptListenerImpl. @@ -403,10 +415,10 @@ public void testStatusUpdateProgress() startListener(true); verify(hbHandler).register(attemptId); - // make sure a ping doesn't report progress + // make sure a ping does report progress AMFeedback feedback = listener.statusUpdate(attemptID, null); assertTrue(feedback.getTaskFound()); - verify(hbHandler, never()).progressing(eq(attemptId)); + verify(hbHandler, times(1)).progressing(eq(attemptId)); // make sure a status update does report progress MapTaskStatus mockStatus = new MapTaskStatus(attemptID, 0.0f, 1, @@ -414,7 +426,7 @@ public void testStatusUpdateProgress() new Counters()); feedback = listener.statusUpdate(attemptID, mockStatus); assertTrue(feedback.getTaskFound()); - verify(hbHandler).progressing(eq(attemptId)); + verify(hbHandler, times(2)).progressing(eq(attemptId)); } @Test diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java index a90c58dd28b4c..d58cacd7b7229 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java @@ -919,6 +919,12 @@ public interface MRJobConfig { MR_AM_PREFIX + "scheduler.heartbeat.interval-ms"; public static final int DEFAULT_MR_AM_TO_RM_HEARTBEAT_INTERVAL_MS = 1000; + /** Whether to consider ping from tasks in liveliness check. */ + public static final String MR_TASK_ENABLE_PING_FOR_LIVELINESS_CHECK = + "mapreduce.task.enable.ping-for-liveliness-check"; + public static final boolean DEFAULT_MR_TASK_ENABLE_PING_FOR_LIVELINESS_CHECK + = false; + /** * If contact with RM is lost, the AM will wait MR_AM_TO_RM_WAIT_INTERVAL_MS * milliseconds before aborting. During this interval, AM will still try diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index d315a00ba4a6c..3787523dde108 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -286,6 +286,13 @@ + + mapreduce.task.enable.ping-for-liveliness-check + true + Whether to consider ping from tasks in liveliness check. + + + mapreduce.map.memory.mb -1 From c168a398e83cc9ad5b435dcd80d6375f427faeb6 Mon Sep 17 00:00:00 2001 From: Ashutosh Gupta Date: Fri, 29 Apr 2022 13:09:38 +0100 Subject: [PATCH 2/6] Removed redundant public static final --- .../main/java/org/apache/hadoop/mapreduce/MRJobConfig.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java index d58cacd7b7229..08da67a008ff0 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java @@ -920,9 +920,9 @@ public interface MRJobConfig { public static final int DEFAULT_MR_AM_TO_RM_HEARTBEAT_INTERVAL_MS = 1000; /** Whether to consider ping from tasks in liveliness check. */ - public static final String MR_TASK_ENABLE_PING_FOR_LIVELINESS_CHECK = + String MR_TASK_ENABLE_PING_FOR_LIVELINESS_CHECK = "mapreduce.task.enable.ping-for-liveliness-check"; - public static final boolean DEFAULT_MR_TASK_ENABLE_PING_FOR_LIVELINESS_CHECK + boolean DEFAULT_MR_TASK_ENABLE_PING_FOR_LIVELINESS_CHECK = false; /** From 9c969c3c2c255de23715d41cae2ddddda041b33f Mon Sep 17 00:00:00 2001 From: Ashutosh Gupta Date: Tue, 3 May 2022 12:30:14 +0100 Subject: [PATCH 3/6] Updating DEFAULT_MR_TASK_ENABLE_PING_FOR_LIVELINESS_CHECK to true --- .../src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java index 08da67a008ff0..425d324c7b336 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java @@ -923,7 +923,7 @@ public interface MRJobConfig { String MR_TASK_ENABLE_PING_FOR_LIVELINESS_CHECK = "mapreduce.task.enable.ping-for-liveliness-check"; boolean DEFAULT_MR_TASK_ENABLE_PING_FOR_LIVELINESS_CHECK - = false; + = true; /** * If contact with RM is lost, the AM will wait MR_AM_TO_RM_WAIT_INTERVAL_MS From f242db07ae2e99f30a68057c5a60fb6ed6afb840 Mon Sep 17 00:00:00 2001 From: Ashutosh Gupta Date: Fri, 6 May 2022 11:28:11 +0100 Subject: [PATCH 4/6] Removing config for liveness check and making it default --- .../org/apache/hadoop/mapred/TaskAttemptListenerImpl.java | 5 +---- .../main/java/org/apache/hadoop/mapreduce/MRJobConfig.java | 6 ------ .../src/main/resources/mapred-default.xml | 7 ------- 3 files changed, 1 insertion(+), 17 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java index 5dffd735fdafd..7866f2d5799df 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java @@ -410,10 +410,7 @@ public AMFeedback statusUpdate(TaskAttemptID taskAttemptID, LOG.debug("Ping from " + taskAttemptID.toString()); } // Consider ping from the tasks for liveliness check - if (getConfig().getBoolean(MRJobConfig.MR_TASK_ENABLE_PING_FOR_LIVELINESS_CHECK, - MRJobConfig.DEFAULT_MR_TASK_ENABLE_PING_FOR_LIVELINESS_CHECK)) { - taskHeartbeatHandler.progressing(yarnAttemptID); - } + taskHeartbeatHandler.progressing(yarnAttemptID); return feedback; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java index 425d324c7b336..a90c58dd28b4c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java @@ -919,12 +919,6 @@ public interface MRJobConfig { MR_AM_PREFIX + "scheduler.heartbeat.interval-ms"; public static final int DEFAULT_MR_AM_TO_RM_HEARTBEAT_INTERVAL_MS = 1000; - /** Whether to consider ping from tasks in liveliness check. */ - String MR_TASK_ENABLE_PING_FOR_LIVELINESS_CHECK = - "mapreduce.task.enable.ping-for-liveliness-check"; - boolean DEFAULT_MR_TASK_ENABLE_PING_FOR_LIVELINESS_CHECK - = true; - /** * If contact with RM is lost, the AM will wait MR_AM_TO_RM_WAIT_INTERVAL_MS * milliseconds before aborting. During this interval, AM will still try diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index 3787523dde108..d315a00ba4a6c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -286,13 +286,6 @@ - - mapreduce.task.enable.ping-for-liveliness-check - true - Whether to consider ping from tasks in liveliness check. - - - mapreduce.map.memory.mb -1 From 39b2acef7fdb24db1c084f3086dfc71563803e11 Mon Sep 17 00:00:00 2001 From: Ashutosh Gupta Date: Sun, 19 Jun 2022 17:55:29 +0100 Subject: [PATCH 5/6] Disabling the ping check feature by default and added a test case for ping check feature --- .../mapred/TaskAttemptListenerImpl.java | 5 +++- .../mapred/TestTaskAttemptListenerImpl.java | 23 ++++++++++++++++--- .../apache/hadoop/mapreduce/MRJobConfig.java | 7 ++++++ .../src/main/resources/mapred-default.xml | 7 ++++++ 4 files changed, 38 insertions(+), 4 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java index 7866f2d5799df..5dffd735fdafd 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java @@ -410,7 +410,10 @@ public AMFeedback statusUpdate(TaskAttemptID taskAttemptID, LOG.debug("Ping from " + taskAttemptID.toString()); } // Consider ping from the tasks for liveliness check - taskHeartbeatHandler.progressing(yarnAttemptID); + if (getConfig().getBoolean(MRJobConfig.MR_TASK_ENABLE_PING_FOR_LIVELINESS_CHECK, + MRJobConfig.DEFAULT_MR_TASK_ENABLE_PING_FOR_LIVELINESS_CHECK)) { + taskHeartbeatHandler.progressing(yarnAttemptID); + } return feedback; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestTaskAttemptListenerImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestTaskAttemptListenerImpl.java index d3f5889e0ab34..b5a7694e4cc6b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestTaskAttemptListenerImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestTaskAttemptListenerImpl.java @@ -415,10 +415,10 @@ public void testStatusUpdateProgress() startListener(true); verify(hbHandler).register(attemptId); - // make sure a ping does report progress + // make sure a ping doesn't report progress AMFeedback feedback = listener.statusUpdate(attemptID, null); assertTrue(feedback.getTaskFound()); - verify(hbHandler, times(1)).progressing(eq(attemptId)); + verify(hbHandler, never()).progressing(eq(attemptId)); // make sure a status update does report progress MapTaskStatus mockStatus = new MapTaskStatus(attemptID, 0.0f, 1, @@ -426,7 +426,24 @@ public void testStatusUpdateProgress() new Counters()); feedback = listener.statusUpdate(attemptID, mockStatus); assertTrue(feedback.getTaskFound()); - verify(hbHandler, times(2)).progressing(eq(attemptId)); + verify(hbHandler).progressing(eq(attemptId)); + } + + @Test + public void testPingUpdateProgress() throws IOException, InterruptedException { + configureMocks(); + Configuration conf = new Configuration(); + conf.setBoolean(MRJobConfig.MR_TASK_ENABLE_PING_FOR_LIVELINESS_CHECK, true); + listener.init(conf); + listener.start(); + listener.registerPendingTask(task, wid); + listener.registerLaunchedTask(attemptId, wid); + verify(hbHandler).register(attemptId); + + // make sure a ping does report progress + AMFeedback feedback = listener.statusUpdate(attemptID, null); + assertTrue(feedback.getTaskFound()); + verify(hbHandler, times(1)).progressing(eq(attemptId)); } @Test diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java index a90c58dd28b4c..fff4c47385c28 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java @@ -919,6 +919,13 @@ public interface MRJobConfig { MR_AM_PREFIX + "scheduler.heartbeat.interval-ms"; public static final int DEFAULT_MR_AM_TO_RM_HEARTBEAT_INTERVAL_MS = 1000; + /** Whether to consider ping from tasks in liveliness check. */ + String MR_TASK_ENABLE_PING_FOR_LIVELINESS_CHECK = + "mapreduce.task.enable.ping-for-liveliness-check"; + boolean DEFAULT_MR_TASK_ENABLE_PING_FOR_LIVELINESS_CHECK + = false; + + /** * If contact with RM is lost, the AM will wait MR_AM_TO_RM_WAIT_INTERVAL_MS * milliseconds before aborting. During this interval, AM will still try diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index d315a00ba4a6c..a0219ab4b2717 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -286,6 +286,13 @@ + + mapreduce.task.enable.ping-for-liveliness-check + false + Whether to consider ping from tasks in liveliness check. + + + mapreduce.map.memory.mb -1 From 592596b4dd3a610dc1a851fd59843ca4775876b2 Mon Sep 17 00:00:00 2001 From: Ashutosh Gupta Date: Sun, 19 Jun 2022 19:42:25 +0100 Subject: [PATCH 6/6] update mapreduce.task.enable.ping-for-liveliness-check to mapreduce.task.ping-for-liveliness-check.enabled --- .../src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java | 2 +- .../src/main/resources/mapred-default.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java index fff4c47385c28..15d57a6746b13 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java @@ -921,7 +921,7 @@ public interface MRJobConfig { /** Whether to consider ping from tasks in liveliness check. */ String MR_TASK_ENABLE_PING_FOR_LIVELINESS_CHECK = - "mapreduce.task.enable.ping-for-liveliness-check"; + "mapreduce.task.ping-for-liveliness-check.enabled"; boolean DEFAULT_MR_TASK_ENABLE_PING_FOR_LIVELINESS_CHECK = false; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index a0219ab4b2717..ac7948f92a461 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -287,7 +287,7 @@ - mapreduce.task.enable.ping-for-liveliness-check + mapreduce.task.ping-for-liveliness-check.enabled false Whether to consider ping from tasks in liveliness check.