YARN-5025. Container move (relocation) between nodes

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java

@@ -31,6 +31,8 @@
 import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.hadoop.yarn.api.protocolrecords.CommitResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerLaunchContextRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerLaunchContextResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.ReInitializeContainerRequest;
@@ -481,6 +483,13 @@ public ResourceLocalizationResponse localize(
         ResourceLocalizationRequest request) throws YarnException, IOException {
       return null;
     }
+
+    @Override
+    public GetContainerLaunchContextResponse getContainerLaunchContext(
+        GetContainerLaunchContextRequest request) throws YarnException, IOException {
+      throw new UnsupportedOperationException("getting the container launch context is not " +
+          "supported for this implementation of ContainerManagementProtocol");
+    }
 
     @Override
     public ReInitializeContainerResponse reInitializeContainer(

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java

@@ -47,10 +47,12 @@
 import org.apache.hadoop.mapreduce.v2.util.MRBuilderUtils;
 import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
 import org.apache.hadoop.yarn.api.protocolrecords.CommitResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerLaunchContextRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerLaunchContextResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.ReInitializeContainerRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.ReInitializeContainerResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationRequest;
@@ -70,7 +72,8 @@
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.Token;
-import org.apache.hadoop.yarn.client.api.impl.ContainerManagementProtocolProxy.ContainerManagementProtocolProxyData;
+import org.apache.hadoop.yarn.client.api.impl.ContainerManagementProtocolProxy
+    .ContainerManagementProtocolProxyData;
 import org.apache.hadoop.yarn.event.Event;
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -486,6 +489,13 @@ public ResourceLocalizationResponse localize(
         ResourceLocalizationRequest request) throws YarnException, IOException {
       return null;
     }
+
+    @Override
+    public GetContainerLaunchContextResponse getContainerLaunchContext(
+        GetContainerLaunchContextRequest request) throws YarnException, IOException {
+      throw new UnsupportedOperationException("getting the container launch context is not " +
+          "supported for this implementation of ContainerManagementProtocol");
+    }
 
     @Override
     public ReInitializeContainerResponse reInitializeContainer(

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ContainerManagementProtocol.java

@@ -24,6 +24,8 @@
 import org.apache.hadoop.classification.InterfaceStability.Stable;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.yarn.api.protocolrecords.CommitResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerLaunchContextRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerLaunchContextResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
@@ -51,7 +53,7 @@
 /**
  * <p>The protocol between an <code>ApplicationMaster</code> and a 
  * <code>NodeManager</code> to start/stop and increase resource of containers
- * and to get status of running containers.</p>
+ * and to get status and launch context of running containers.</p>
  *
  * <p>If security is enabled the <code>NodeManager</code> verifies that the
  * <code>ApplicationMaster</code> has truly been allocated the container
@@ -220,6 +222,22 @@ SignalContainerResponse signalToContainer(SignalContainerRequest request)
   @Unstable
   ResourceLocalizationResponse localize(ResourceLocalizationRequest request)
     throws YarnException, IOException;
+
+  /**
+   * Gets container launch context for a container specified in
+   * {@link GetContainerLaunchContextRequest}.
+   * This protocol is only used by the container relocation logic, between node managers, to
+   * transfer the launch context of the container to be relocated to the target node.
+   *
+   * @param request specifies the id of the container for which the launch context is requested
+   * @return Response that contains the requested container launch context
+   * @throws YarnException
+   * @throws IOException
+   */
+  @Public
+  @Stable
+  GetContainerLaunchContextResponse getContainerLaunchContext(
+      GetContainerLaunchContextRequest request) throws YarnException, IOException;
 
   /**
    * ReInitialize the Container with a new Launch Context.

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java

@@ -26,6 +26,7 @@
 import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerMoveRequest;
 import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
@@ -52,6 +53,11 @@
  *     the <code>ResourceManager</code> about the change in
  *     requirements of running containers.
  *   </li>
+ *   <li>
+ *     A list of {@link ContainerMoveRequest} to inform
+ *     the <code>ResourceManager</code> about the application's
+ *     container relocation needs.
+ *   </li>
  * </ul>
  * 
  * @see ApplicationMasterProtocol#allocate(AllocateRequest)
@@ -77,13 +83,27 @@ public static AllocateRequest newInstance(int responseID, float appProgress,
       List<ContainerId> containersToBeReleased,
       ResourceBlacklistRequest resourceBlacklistRequest,
       List<UpdateContainerRequest> updateRequests) {
+    return newInstance(responseID, appProgress, resourceAsk,
+        containersToBeReleased, resourceBlacklistRequest, updateRequests,
+        null);
+  }
+
+  @Public
+  @Stable
+  public static AllocateRequest newInstance(int responseID, float appProgress,
+      List<ResourceRequest> resourceAsk,
+      List<ContainerId> containersToBeReleased,
+      ResourceBlacklistRequest resourceBlacklistRequest,
+      List<UpdateContainerRequest> updateRequests,
+      List<ContainerMoveRequest> moveAsk) {
     AllocateRequest allocateRequest = Records.newRecord(AllocateRequest.class);
     allocateRequest.setResponseId(responseID);
     allocateRequest.setProgress(appProgress);
     allocateRequest.setAskList(resourceAsk);
     allocateRequest.setReleaseList(containersToBeReleased);
     allocateRequest.setResourceBlacklistRequest(resourceBlacklistRequest);
     allocateRequest.setUpdateRequests(updateRequests);
+    allocateRequest.setMoveAskList(moveAsk);
     return allocateRequest;
   }
 
@@ -211,4 +231,27 @@ public abstract void setResourceBlacklistRequest(
   @Unstable
   public abstract void setUpdateRequests(
       List<UpdateContainerRequest> updateRequests);
+
+  /**
+   * Get the list of container move requests being sent by the
+   * <code>ApplicationMaster</code>.
+   *
+   * @return list of <code>ContainerMoveRequest</code>
+   *         being sent by the <code>ApplicationMaster</code>.
+   */
+  @Public
+  @Unstable
+  public abstract List<ContainerMoveRequest> getMoveAskList();
+
+  /**
+   * Set the list of container move requests to inform the
+   * <code>ResourceManager</code> about the containers that need to be
+   * relocated.
+   *
+   * @param containerMoveRequests list of <code>ContainerMoveRequest</code>
+   *                              for containers that need to be relocated.
+   */
+  @Public
+  @Unstable
+  public abstract void setMoveAskList(List<ContainerMoveRequest> containerMoveRequests);
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetContainerLaunchContextRequest.java

@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.protocolrecords;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Stable;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * <p>The request sent by one <code>NodeManager</code> to another
+ * <code>NodeManager</code> in order to get the launch context of the container with id
+ * <em>containerId</em>.</p>
+ *
+ * <p>This request is used only for container relocation, where the launch context of
+ * the origin container is necessary for launching the relocated container on the target node.</p>
+ *
+ * @see ContainerManagementProtocol#getContainerLaunchContext(GetContainerLaunchContextRequest)
+ */
+public abstract class GetContainerLaunchContextRequest {
+
+  @Public
+  @Stable
+  public static GetContainerLaunchContextRequest newInstance(ContainerId containerId) {
+    GetContainerLaunchContextRequest request =
+        Records.newRecord(GetContainerLaunchContextRequest.class);
+    request.setContainerId(containerId);
+    return request;
+  }
+
+  /**
+   * Gets the container id for which the launch context is requested.
+   * @return the container id for which the launch context is requested
+   */
+  @Public
+  @Stable
+  public abstract ContainerId getContainerId();
+
+  /**
+   * Sets the container id for which the launch context is requested.
+   * @param containerId the container id for which the launch context is requested
+   */
+  @Public
+  @Stable
+  public abstract void setContainerId(ContainerId containerId);
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetContainerLaunchContextResponse.java

@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.protocolrecords;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Stable;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * <p>The response for a container launch context request sent between
+ * <code>NodeManager</code>s. It contains the requested container launch context.</p>
+ *
+ * <p>This request is used only for container relocation, where a launch context
+ * of the origin container is necessary for launching the relocated container on
+ * the target node.</p>
+ */
+public abstract class GetContainerLaunchContextResponse {
+
+  @Public
+  @Stable
+  public static GetContainerLaunchContextResponse newInstance(ContainerLaunchContext
+      containerLaunchContext) {
+    GetContainerLaunchContextResponse request =
+        Records.newRecord(GetContainerLaunchContextResponse.class);
+    request.setContainerLaunchContext(containerLaunchContext);
+    return request;
+  }
+
+  /**
+   * Gets the launch context of the requested container.
+   * @return the launch context of the requested container
+   */
+  @Public
+  @Stable
+  public abstract ContainerLaunchContext getContainerLaunchContext();
+
+  /**
+   * Sets the launch context of the requested container.
+   * @param containerLaunchContext the launch context of the requested container
+   */
+  @Public
+  @Stable
+  public abstract void setContainerLaunchContext(ContainerLaunchContext containerLaunchContext);
+}