Skip to content

Commit 0c10bab

Browse files
slfan1989slfan1989
authored
YARN-11483. [Federation] Router AdminCLI Supports Clean Finish Apps. (#6251) Contributed by Shilun Fan.
Reviewed-by: Inigo Goiri <inigoiri@apache.org> Signed-off-by: Shilun Fan <slfan1989@apache.org>
1 parent 7a55442 commit 0c10bab

File tree

19 files changed

+581
-1
lines changed

19 files changed

+581
-1
lines changed

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,8 @@
6464
import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesResponse;
6565
import org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesRequest;
6666
import org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesResponse;
67+
import org.apache.hadoop.yarn.server.api.protocolrecords.DeleteFederationApplicationRequest;
68+
import org.apache.hadoop.yarn.server.api.protocolrecords.DeleteFederationApplicationResponse;
6769

6870
@Private
6971
public interface ResourceManagerAdministrationProtocol extends GetUserMappingsProtocol {
@@ -218,4 +220,17 @@ BatchSaveFederationQueuePoliciesResponse batchSaveFederationQueuePolicies(
218220
@Idempotent
219221
QueryFederationQueuePoliciesResponse listFederationQueuePolicies(
220222
QueryFederationQueuePoliciesRequest request) throws YarnException, IOException;
223+
224+
/**
225+
* In YARN-Federation mode, this method provides a way to delete federation application.
226+
*
227+
* @param request DeleteFederationApplicationRequest Request.
228+
* @return Response from deleteFederationApplication.
229+
* @throws YarnException exceptions from yarn servers.
230+
* @throws IOException if an IO error occurred.
231+
*/
232+
@Private
233+
@Idempotent
234+
DeleteFederationApplicationResponse deleteFederationApplication(
235+
DeleteFederationApplicationRequest request) throws YarnException, IOException;
221236
}

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/DeleteFederationApplicationRequest.java‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
/**
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
package org.apache.hadoop.yarn.server.api.protocolrecords;
19+
20+
import org.apache.hadoop.classification.InterfaceAudience.Private;
21+
import org.apache.hadoop.classification.InterfaceAudience.Public;
22+
import org.apache.hadoop.classification.InterfaceStability.Unstable;
23+
import org.apache.hadoop.yarn.util.Records;
24+
25+
/**
26+
* This class is used for cleaning up an application that exists in the FederationStateStore.
27+
* This is a user-specified operation;
28+
* we typically use this command to clean up an expired application.
29+
* However, it can also be used to clean up non-expired application, although it is not recommended.
30+
*/
31+
@Private
32+
@Unstable
33+
public abstract class DeleteFederationApplicationRequest {
34+
35+
@Private
36+
@Unstable
37+
public static DeleteFederationApplicationRequest newInstance(String application) {
38+
DeleteFederationApplicationRequest request =
39+
Records.newRecord(DeleteFederationApplicationRequest.class);
40+
request.setApplication(application);
41+
return request;
42+
}
43+
44+
@Public
45+
@Unstable
46+
public abstract String getApplication();
47+
48+
@Public
49+
@Unstable
50+
public abstract void setApplication(String application);
51+
}

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/DeleteFederationApplicationResponse.java‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
/**
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
package org.apache.hadoop.yarn.server.api.protocolrecords;
19+
20+
import org.apache.hadoop.classification.InterfaceAudience.Public;
21+
import org.apache.hadoop.classification.InterfaceAudience.Private;
22+
import org.apache.hadoop.classification.InterfaceStability.Unstable;
23+
import org.apache.hadoop.yarn.util.Records;
24+
25+
@Private
26+
@Unstable
27+
public abstract class DeleteFederationApplicationResponse {
28+
29+
public static DeleteFederationApplicationResponse newInstance() {
30+
return Records.newRecord(DeleteFederationApplicationResponse.class);
31+
}
32+
33+
public static DeleteFederationApplicationResponse newInstance(String msg) {
34+
DeleteFederationApplicationResponse response =
35+
Records.newRecord(DeleteFederationApplicationResponse.class);
36+
response.setMessage(msg);
37+
return response;
38+
}
39+
40+
@Public
41+
@Unstable
42+
public abstract String getMessage();
43+
44+
@Public
45+
@Unstable
46+
public abstract void setMessage(String msg);
47+
}

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/resourcemanager_administration_protocol.proto‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,4 +51,5 @@ service ResourceManagerAdministrationProtocolService {
5151
rpc saveFederationQueuePolicy(SaveFederationQueuePolicyRequestProto) returns (SaveFederationQueuePolicyResponseProto);
5252
rpc batchSaveFederationQueuePolicies(BatchSaveFederationQueuePoliciesRequestProto) returns (BatchSaveFederationQueuePoliciesResponseProto);
5353
rpc listFederationQueuePolicies(QueryFederationQueuePoliciesRequestProto) returns (QueryFederationQueuePoliciesResponseProto);
54+
rpc deleteFederationApplication(DeleteFederationApplicationRequestProto) returns (DeleteFederationApplicationResponseProto);
5455
}

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -203,6 +203,14 @@ message QueryFederationQueuePoliciesResponseProto {
203203
repeated FederationQueueWeightProto federationQueueWeights = 5;
204204
}
205205

206+
message DeleteFederationApplicationRequestProto {
207+
optional string application = 1;
208+
}
209+
210+
message DeleteFederationApplicationResponseProto {
211+
optional string message = 1;
212+
}
213+
206214
//////////////////////////////////////////////////////////////////
207215
///////////// RM Failover related records ////////////////////////
208216
//////////////////////////////////////////////////////////////////

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
package org.apache.hadoop.yarn.client.cli;
1919

2020
import org.apache.commons.cli.CommandLine;
21+
import org.apache.commons.cli.DefaultParser;
2122
import org.apache.commons.cli.GnuParser;
2223
import org.apache.commons.cli.MissingArgumentException;
2324
import org.apache.commons.cli.Option;
@@ -42,6 +43,8 @@
4243
import org.apache.hadoop.yarn.server.api.protocolrecords.DeregisterSubClusterRequest;
4344
import org.apache.hadoop.yarn.server.api.protocolrecords.DeregisterSubClusterResponse;
4445
import org.apache.hadoop.yarn.server.api.protocolrecords.DeregisterSubClusters;
46+
import org.apache.hadoop.yarn.server.api.protocolrecords.DeleteFederationApplicationRequest;
47+
import org.apache.hadoop.yarn.server.api.protocolrecords.DeleteFederationApplicationResponse;
4548
import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyRequest;
4649
import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyResponse;
4750
import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesRequest;
@@ -227,12 +230,37 @@ public class RouterCLI extends Configured implements Tool {
227230
.addExample(POLICY_LIST_USAGE.args, POLICY_LIST_USAGE_EXAMPLE_1)
228231
.addExample(POLICY_LIST_USAGE.args, POLICY_LIST_USAGE_EXAMPLE_2);
229232

233+
// Command3: application
234+
private static final String CMD_APPLICATION = "-application";
235+
236+
// Application Delete
237+
protected final static UsageInfo APPLICATION_DELETE_USAGE = new UsageInfo(
238+
"--delete <application_id>",
239+
"This command is used to delete the specified application.");
240+
241+
protected final static String APPLICATION_DELETE_USAGE_EXAMPLE_DESC =
242+
"If we want to delete application_1440536969523_0001.";
243+
244+
protected final static String APPLICATION_DELETE_USAGE_EXAMPLE_1 =
245+
"yarn routeradmin -application --delete application_1440536969523_0001";
246+
247+
protected final static RouterCmdUsageInfos APPLICATION_USAGEINFOS = new RouterCmdUsageInfos()
248+
// application delete
249+
.addUsageInfo(APPLICATION_DELETE_USAGE)
250+
.addExampleDescs(APPLICATION_DELETE_USAGE.args, APPLICATION_DELETE_USAGE_EXAMPLE_DESC)
251+
.addExample(APPLICATION_DELETE_USAGE.args, APPLICATION_DELETE_USAGE_EXAMPLE_1);
252+
253+
// delete application
254+
private static final String OPTION_DELETE_APP = "delete";
255+
230256
protected final static Map<String, RouterCmdUsageInfos> ADMIN_USAGE =
231257
ImmutableMap.<String, RouterCmdUsageInfos>builder()
232258
// Command1: deregisterSubCluster
233259
.put(CMD_DEREGISTERSUBCLUSTER, DEREGISTER_SUBCLUSTER_USAGEINFOS)
234260
// Command2: policy
235261
.put(CMD_POLICY, POLICY_USAGEINFOS)
262+
// Command3: application
263+
.put(CMD_APPLICATION, APPLICATION_USAGEINFOS)
236264
.build();
237265

238266
public RouterCLI() {
@@ -814,6 +842,50 @@ protected int handListPolicies(int pageSize, int currentPage, String queue, List
814842
}
815843
}
816844

845+
private int handleDeleteApplication(String application) {
846+
LOG.info("Delete Application = {}.", application);
847+
try {
848+
DeleteFederationApplicationRequest request =
849+
DeleteFederationApplicationRequest.newInstance(application);
850+
ResourceManagerAdministrationProtocol adminProtocol = createAdminProtocol();
851+
DeleteFederationApplicationResponse response =
852+
adminProtocol.deleteFederationApplication(request);
853+
System.out.println(response.getMessage());
854+
return EXIT_SUCCESS;
855+
} catch (Exception e) {
856+
LOG.error("handleSavePolicy error.", e);
857+
return EXIT_ERROR;
858+
}
859+
}
860+
861+
private int handleApplication(String[] args)
862+
throws IOException, YarnException, ParseException {
863+
// Prepare Options.
864+
Options opts = new Options();
865+
opts.addOption("application", false,
866+
"We provide a set of commands to query and clean applications.");
867+
Option deleteOpt = new Option(null, OPTION_DELETE_APP, true,
868+
"We will clean up the provided application.");
869+
opts.addOption(deleteOpt);
870+
871+
// Parse command line arguments.
872+
CommandLine cliParser;
873+
try {
874+
cliParser = new DefaultParser().parse(opts, args);
875+
} catch (MissingArgumentException ex) {
876+
System.out.println("Missing argument for options");
877+
printUsage(args[0]);
878+
return EXIT_ERROR;
879+
}
880+
881+
if (cliParser.hasOption(OPTION_DELETE_APP)) {
882+
String application = cliParser.getOptionValue(OPTION_DELETE_APP);
883+
return handleDeleteApplication(application);
884+
}
885+
886+
return 0;
887+
}
888+
817889
@Override
818890
public int run(String[] args) throws Exception {
819891
YarnConfiguration yarnConf = getConf() == null ?
@@ -839,6 +911,8 @@ public int run(String[] args) throws Exception {
839911
return handleDeregisterSubCluster(args);
840912
} else if (CMD_POLICY.equals(cmd)) {
841913
return handlePolicy(args);
914+
} else if (CMD_APPLICATION.equals(cmd)) {
915+
return handleApplication(args);
842916
} else {
843917
System.out.println("No related commands found.");
844918
printHelp();

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRouterCLI.java‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,7 @@ public void setup() throws Exception {
9494
return QueryFederationQueuePoliciesResponse.newInstance(1, 1, 1, 10, weights);
9595
});
9696

97+
9798
Configuration config = new Configuration();
9899
config.setBoolean(YarnConfiguration.FEDERATION_ENABLED, true);
99100

@@ -271,7 +272,7 @@ public void testListPolicies() throws Exception {
271272
@Test
272273
public void testBuildHelpMsg() throws Exception {
273274
Map<String, RouterCLI.RouterCmdUsageInfos> adminUsage = rmAdminCLI.getAdminUsage();
274-
assertEquals(2, adminUsage.size());
275+
assertEquals(3, adminUsage.size());
275276

276277
RouterCLI.RouterCmdUsageInfos deregisterSubClusterUsageInfos =
277278
adminUsage.get("-deregisterSubCluster");
@@ -291,5 +292,11 @@ public void testBuildHelpMsg() throws Exception {
291292
policyExamplesMap.forEach((cmd, cmdExamples) -> {
292293
assertEquals(2, cmdExamples.size());
293294
});
295+
296+
RouterCLI.RouterCmdUsageInfos applicationUsageInfos = adminUsage.get("-application");
297+
assertNotNull(applicationUsageInfos);
298+
Map<String, List<String>> applicationExamplesMap = applicationUsageInfos.getExamples();
299+
assertNotNull(applicationExamplesMap);
300+
assertEquals(1, applicationExamplesMap.size());
294301
}
295302
}

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/ResourceManagerAdministrationProtocolPBClientImpl.java‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@
4848
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.SaveFederationQueuePolicyRequestProto;
4949
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.BatchSaveFederationQueuePoliciesRequestProto;
5050
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.QueryFederationQueuePoliciesRequestProto;
51+
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.DeleteFederationApplicationRequestProto;
5152
import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocol;
5253
import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocolPB;
5354
import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsRequest;
@@ -86,6 +87,8 @@
8687
import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesResponse;
8788
import org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesRequest;
8889
import org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesResponse;
90+
import org.apache.hadoop.yarn.server.api.protocolrecords.DeleteFederationApplicationRequest;
91+
import org.apache.hadoop.yarn.server.api.protocolrecords.DeleteFederationApplicationResponse;
8992
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.AddToClusterNodeLabelsRequestPBImpl;
9093
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.AddToClusterNodeLabelsResponsePBImpl;
9194
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.CheckForDecommissioningNodesRequestPBImpl;
@@ -122,6 +125,8 @@
122125
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.BatchSaveFederationQueuePoliciesResponsePBImpl;
123126
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.QueryFederationQueuePoliciesRequestPBImpl;
124127
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.QueryFederationQueuePoliciesResponsePBImpl;
128+
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.DeleteFederationApplicationRequestPBImpl;
129+
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.DeleteFederationApplicationResponsePBImpl;
125130

126131
import org.apache.hadoop.thirdparty.protobuf.ServiceException;
127132

@@ -420,4 +425,18 @@ public QueryFederationQueuePoliciesResponse listFederationQueuePolicies(
420425
}
421426
return null;
422427
}
428+
429+
@Override
430+
public DeleteFederationApplicationResponse deleteFederationApplication(
431+
DeleteFederationApplicationRequest request) throws YarnException, IOException {
432+
DeleteFederationApplicationRequestProto requestProto =
433+
((DeleteFederationApplicationRequestPBImpl) request).getProto();
434+
try {
435+
return new DeleteFederationApplicationResponsePBImpl(
436+
proxy.deleteFederationApplication(null, requestProto));
437+
} catch (ServiceException e) {
438+
RPCUtil.unwrapAndThrowException(e);
439+
}
440+
return null;
441+
}
423442
}

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/ResourceManagerAdministrationProtocolPBServiceImpl.java‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,8 @@
6060
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.DeregisterSubClusterResponseProto;
6161
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.SaveFederationQueuePolicyRequestProto;
6262
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.SaveFederationQueuePolicyResponseProto;
63+
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.DeleteFederationApplicationResponseProto;
64+
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.DeleteFederationApplicationRequestProto;
6365
import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocol;
6466
import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocolPB;
6567
import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsResponse;
@@ -87,6 +89,8 @@
8789
import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesResponse;
8890
import org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesRequest;
8991
import org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesResponse;
92+
import org.apache.hadoop.yarn.server.api.protocolrecords.DeleteFederationApplicationRequest;
93+
import org.apache.hadoop.yarn.server.api.protocolrecords.DeleteFederationApplicationResponse;
9094
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.AddToClusterNodeLabelsRequestPBImpl;
9195
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.AddToClusterNodeLabelsResponsePBImpl;
9296
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.CheckForDecommissioningNodesRequestPBImpl;
@@ -123,6 +127,8 @@
123127
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.BatchSaveFederationQueuePoliciesResponsePBImpl;
124128
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.QueryFederationQueuePoliciesRequestPBImpl;
125129
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.QueryFederationQueuePoliciesResponsePBImpl;
130+
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.DeleteFederationApplicationRequestPBImpl;
131+
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.DeleteFederationApplicationResponsePBImpl;
126132

127133
import org.apache.hadoop.thirdparty.protobuf.RpcController;
128134
import org.apache.hadoop.thirdparty.protobuf.ServiceException;
@@ -445,4 +451,20 @@ public QueryFederationQueuePoliciesResponseProto listFederationQueuePolicies(
445451
throw new ServiceException(e);
446452
}
447453
}
454+
455+
@Override
456+
public DeleteFederationApplicationResponseProto deleteFederationApplication(
457+
RpcController controller, DeleteFederationApplicationRequestProto proto)
458+
throws ServiceException {
459+
DeleteFederationApplicationRequest request =
460+
new DeleteFederationApplicationRequestPBImpl(proto);
461+
try {
462+
DeleteFederationApplicationResponse response = real.deleteFederationApplication(request);
463+
return ((DeleteFederationApplicationResponsePBImpl) response).getProto();
464+
} catch (YarnException e) {
465+
throw new ServiceException(e);
466+
} catch (IOException e) {
467+
throw new ServiceException(e);
468+
}
469+
}
448470
}

0 commit comments

Comments
 (0)