Skip to content

Commit

Permalink
YARN-5025. Added and updated container management protocols for conta…
Browse files Browse the repository at this point in the history
…iner relocation
  • Loading branch information
szape committed Oct 20, 2016
1 parent 0ece56e commit fa6a914
Show file tree
Hide file tree
Showing 24 changed files with 1,280 additions and 32 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@
import java.util.concurrent.atomic.AtomicInteger;

import org.apache.hadoop.yarn.api.protocolrecords.CommitResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerLaunchContextRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerLaunchContextResponse;
import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest;
import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse;
import org.apache.hadoop.yarn.api.protocolrecords.ReInitializeContainerRequest;
Expand Down Expand Up @@ -481,6 +483,13 @@ public ResourceLocalizationResponse localize(
ResourceLocalizationRequest request) throws YarnException, IOException {
return null;
}

@Override
public GetContainerLaunchContextResponse getContainerLaunchContext(
GetContainerLaunchContextRequest request) throws YarnException, IOException {
throw new UnsupportedOperationException("getting the container launch context is not " +
"supported for this implementation of ContainerManagementProtocol");
}

@Override
public ReInitializeContainerResponse reInitializeContainer(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,10 +47,12 @@
import org.apache.hadoop.mapreduce.v2.util.MRBuilderUtils;
import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.CommitResponse;
import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest;
import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerLaunchContextRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerLaunchContextResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse;
import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest;
import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse;
import org.apache.hadoop.yarn.api.protocolrecords.ReInitializeContainerRequest;
import org.apache.hadoop.yarn.api.protocolrecords.ReInitializeContainerResponse;
import org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationRequest;
Expand All @@ -70,7 +72,8 @@
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.client.api.impl.ContainerManagementProtocolProxy.ContainerManagementProtocolProxyData;
import org.apache.hadoop.yarn.client.api.impl.ContainerManagementProtocolProxy
.ContainerManagementProtocolProxyData;
import org.apache.hadoop.yarn.event.Event;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.exceptions.YarnException;
Expand Down Expand Up @@ -486,6 +489,13 @@ public ResourceLocalizationResponse localize(
ResourceLocalizationRequest request) throws YarnException, IOException {
return null;
}

@Override
public GetContainerLaunchContextResponse getContainerLaunchContext(
GetContainerLaunchContextRequest request) throws YarnException, IOException {
throw new UnsupportedOperationException("getting the container launch context is not " +
"supported for this implementation of ContainerManagementProtocol");
}

@Override
public ReInitializeContainerResponse reInitializeContainer(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@
import org.apache.hadoop.classification.InterfaceStability.Stable;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.protocolrecords.CommitResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerLaunchContextRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerLaunchContextResponse;
import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest;
import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
Expand Down Expand Up @@ -51,7 +53,7 @@
/**
* <p>The protocol between an <code>ApplicationMaster</code> and a
* <code>NodeManager</code> to start/stop and increase resource of containers
* and to get status of running containers.</p>
* and to get status and launch context of running containers.</p>
*
* <p>If security is enabled the <code>NodeManager</code> verifies that the
* <code>ApplicationMaster</code> has truly been allocated the container
Expand Down Expand Up @@ -220,6 +222,22 @@ SignalContainerResponse signalToContainer(SignalContainerRequest request)
@Unstable
ResourceLocalizationResponse localize(ResourceLocalizationRequest request)
throws YarnException, IOException;

/**
* Gets container launch context for a container specified in
* {@link GetContainerLaunchContextRequest}.
* This protocol is only used by the container relocation logic, between node managers, to
* transfer the launch context of the container to be relocated to the target node.
*
* @param request specifies the id of the container for which the launch context is requested
* @return Response that contains the requested container launch context
* @throws YarnException
* @throws IOException
*/
@Public
@Stable
GetContainerLaunchContextResponse getContainerLaunchContext(
GetContainerLaunchContextRequest request) throws YarnException, IOException;

/**
* ReInitialize the Container with a new Launch Context.
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.hadoop.yarn.api.protocolrecords;

import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Stable;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
import org.apache.hadoop.yarn.util.Records;

/**
* <p>The request sent by one <code>NodeManager</code> to another
* <code>NodeManager</code> in order to get the launch context of the container with id
* <em>containerId</em>.</p>
*
* <p>This request is used only for container relocation, where the launch context of
* the origin container is necessary for launching the relocated container on the target node.</p>
*
* @see ContainerManagementProtocol#getContainerLaunchContext(GetContainerLaunchContextRequest)
*/
public abstract class GetContainerLaunchContextRequest {

@Public
@Stable
public static GetContainerLaunchContextRequest newInstance(ContainerId containerId) {
GetContainerLaunchContextRequest request =
Records.newRecord(GetContainerLaunchContextRequest.class);
request.setContainerId(containerId);
return request;
}

/**
* Gets the container id for which the launch context is requested.
* @return the container id for which the launch context is requested
*/
@Public
@Stable
public abstract ContainerId getContainerId();

/**
* Sets the container id for which the launch context is requested.
* @param containerId the container id for which the launch context is requested
*/
@Public
@Stable
public abstract void setContainerId(ContainerId containerId);
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.hadoop.yarn.api.protocolrecords;

import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Stable;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.util.Records;

/**
* <p>The response for a container launch context request sent between
* <code>NodeManager</code>s. It contains the requested container launch context.</p>
*
* <p>This request is used only for container relocation, where a launch context
* of the origin container is necessary for launching the relocated container on
* the target node.</p>
*/
public abstract class GetContainerLaunchContextResponse {

@Public
@Stable
public static GetContainerLaunchContextResponse newInstance(ContainerLaunchContext
containerLaunchContext) {
GetContainerLaunchContextResponse request =
Records.newRecord(GetContainerLaunchContextResponse.class);
request.setContainerLaunchContext(containerLaunchContext);
return request;
}

/**
* Gets the launch context of the requested container.
* @return the launch context of the requested container
*/
@Public
@Stable
public abstract ContainerLaunchContext getContainerLaunchContext();

/**
* Sets the launch context of the requested container.
* @param containerLaunchContext the launch context of the requested container
*/
@Public
@Stable
public abstract void setContainerLaunchContext(ContainerLaunchContext containerLaunchContext);
}
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,12 @@

import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Stable;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.NMToken;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.util.Records;

Expand All @@ -36,6 +39,15 @@
* necessary binaries/jar/shared-objects etc. via the
* {@link ContainerLaunchContext}.</p>
*
* <p>The <em>isMove</em> flag tells whether this request corresponds to a container
* relocation. If true, then no container launch context is needed as it will be
* transferred directly from the origin node manager to the target node manager.
* The container to be relocated is identified by the <em>originNodeId</em> and
* <em>originContainerId</em>.
* The originNMToken is sent along so that the origin container can be shut down
* by the target node manager.
* </p>
*
* @see ContainerManagementProtocol#startContainers(StartContainersRequest)
*/
@Public
Expand All @@ -49,6 +61,21 @@ public static StartContainerRequest newInstance(
Records.newRecord(StartContainerRequest.class);
request.setContainerLaunchContext(context);
request.setContainerToken(container);
request.setIsMove(false);
return request;
}

@Public
@Stable
public static StartContainerRequest newInstance(
Token container, ContainerId originContainerId, NodeId originNodeId, Token originNMToken) {
StartContainerRequest request =
Records.newRecord(StartContainerRequest.class);
request.setContainerToken(container);
request.setIsMove(true);
request.setOriginContainerId(originContainerId);
request.setOriginNodeId(originNodeId);
request.setOriginNMToken(originNMToken);
return request;
}

Expand Down Expand Up @@ -91,4 +118,91 @@ public static StartContainerRequest newInstance(
@Public
@Stable
public abstract void setContainerToken(Token container);

/**
* Gets whether this start container request corresponds to a container relocation.
* @return whether this start container request corresponds to a container relocation
*/
@Public
@Unstable
public abstract boolean getIsMove();

/**
* Sets whether this start container request corresponds to a container relocation.
* @param isMove whether this start container request corresponds to a container
* relocation
*/
@Public
@Unstable
public abstract void setIsMove(boolean isMove);

/**
* Gets the origin container id for this start container request.
* The origin container id is set if and only if this start container request
* corresponds to a container relocation. It identifies the container that should
* be relocated.
*
* @return the origin container id for this start container request
*/
@Public
@Unstable
public abstract ContainerId getOriginContainerId();

/**
* Sets the origin container id for this start container request.
* The origin container id should be set if and only if this start container request
* corresponds to a container relocation. It identifies the container that should
* be relocated.
*
* @param originContainerId the origin container id for this start container request
*/
@Public
@Unstable
public abstract void setOriginContainerId(ContainerId originContainerId);

/**
* Gets the origin node id for this start container request.
* The origin node id is set if and only if this resource start container corresponds
* to a container relocation. It identifies the node of the container that should
* be relocated.
*
* @return the origin node id for this start container request
*/
@Public
@Unstable
public abstract NodeId getOriginNodeId();

/**
* Sets the origin node id for this start container request.
* The origin node id should be set if and only if this start container request
* corresponds to a container relocation. It identifies the node of the container
* that should be relocated.
*
* @param originNodeId the origin node id for this start container request
*/
@Public
@Unstable
public abstract void setOriginNodeId(NodeId originNodeId);

/**
* Gets the security token for the origin node.
* The origin NM token is set if and only if this start container request corresponds
* to a container relocation. It is used for shutting down the origin container.
*
* @return the security token for the origin node
*/
@Public
@Unstable
public abstract Token getOriginNMToken();

/**
* Sets the security token for the origin node.
* The origin NM token should be set if and only if this start container request corresponds
* to a container relocation. It is used for shutting down the origin container.
*
* @param originNMToken the security token for the origin node
*/
@Public
@Unstable
public abstract void setOriginNMToken(Token originNMToken);
}
Loading

0 comments on commit fa6a914

Please sign in to comment.