YARN-1964 Removed arbitrary docker options

This removes the option to provide arbitrary options to the docker run command for DockerContainerExecutor.
Now the yarn administrator will set up the docker executor, and the user will provide the docker image.
The docker image must container all resources and environment variables setup to run the users job.

Author: ashahab-altiscale

hadoop-project/src/site/site.xml

@@ -124,6 +124,7 @@
       <item name="YARN Commands" href="hadoop-yarn/hadoop-yarn-site/YarnCommands.html"/>
       <item name="Scheduler Load Simulator" href="hadoop-sls/SchedulerLoadSimulator.html"/>
       <item name="NodeManager Restart" href="hadoop-yarn/hadoop-yarn-site/NodeManagerRestart.html"/>
+      <item name="DockerContainerExecutor" href="hadoop-yarn/hadoop-yarn-site/DockerContainerExecutor.html"/>
     </menu>
 
     <menu name="YARN REST APIs" inherit="top">

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java

@@ -896,10 +896,6 @@ private static void addDeprecatedKeys() {
   public static final String NM_DOCKER_CONTAINER_EXECUTOR_IMAGE_NAME =
     NM_PREFIX + "docker-container-executor.image-name";
 
-  /** Args passed to docker run(For DockerContainerExecutor).*/
-  public static final String NM_DOCKER_CONTAINER_EXECUTOR_RUN_ARGS =
-    NM_PREFIX + "docker-container-executor.run-args";
-
   /** The name of the docker executor (For DockerContainerExecutor).*/
   public static final String NM_DOCKER_CONTAINER_EXECUTOR_EXEC_NAME =
     NM_PREFIX + "docker-container-executor.exec-name";

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml

@@ -1147,22 +1147,6 @@
   </property>
 
   <!--Docker configuration-->
-  <property>
-    <name>yarn.nodemanager.docker-container-executor.image-name</name>
-    <value></value>
-    <description>
-      This image is used by all nodemanagers to launch containers.
-      This maybe modified by the users.
-    </description>
-  </property>
-
-  <property>
-    <name>yarn.nodemanager.docker-container-executor.run-args</name>
-    <value>--rm --net=host</value>
-    <description>
-      This arguments to pass to the 'docker run' invocation.
-    </description>
-  </property>
 
   <property>
     <name>yarn.nodemanager.docker-container-executor.exec-name</name>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java

@@ -20,10 +20,13 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintStream;
 import java.net.InetSocketAddress;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -211,6 +214,34 @@ public int reacquireContainer(String user, ContainerId containerId)
     }
   }
 
+  public void writeLaunchEnv(OutputStream out, Map<String, String> environment, Map<Path, List<String>> resources, List<String> command) throws IOException{
+    ContainerLaunch.ShellScriptBuilder sb = ContainerLaunch.ShellScriptBuilder.create();
+    if (environment != null) {
+      for (Map.Entry<String,String> env : environment.entrySet()) {
+        sb.env(env.getKey().toString(), env.getValue().toString());
+      }
+    }
+    if (resources != null) {
+      for (Map.Entry<Path,List<String>> entry : resources.entrySet()) {
+        for (String linkName : entry.getValue()) {
+          sb.symlink(entry.getKey(), new Path(linkName));
+        }
+      }
+    }
+
+    sb.command(command);
+
+    PrintStream pout = null;
+    try {
+      pout = new PrintStream(out);
+      sb.write(pout);
+    } finally {
+      if (out != null) {
+        out.close();
+      }
+    }
+  }
+
   public enum ExitCode {
     FORCE_KILLED(137),
     TERMINATED(143),