Skip to content

Commit 8745857

Browse files
authored
YARN-11561. [Federation] GPG Supports Format PolicyStateStore. (#6300) Contributed by Shilun Fan.
Reviewed-by: Inigo Goiri <inigoiri@apache.org> Signed-off-by: Shilun Fan <slfan1989@apache.org>
1 parent 2323ad2 commit 8745857

File tree

13 files changed

+338
-2
lines changed

13 files changed

+338
-2
lines changed

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationPolicyStore.java

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@
2727
import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationResponse;
2828
import org.apache.hadoop.yarn.server.federation.store.records.SetSubClusterPolicyConfigurationRequest;
2929
import org.apache.hadoop.yarn.server.federation.store.records.SetSubClusterPolicyConfigurationResponse;
30+
import org.apache.hadoop.yarn.server.federation.store.records.DeletePoliciesConfigurationsRequest;
31+
import org.apache.hadoop.yarn.server.federation.store.records.DeletePoliciesConfigurationsResponse;
3032

3133
/**
3234
* The FederationPolicyStore provides a key-value interface to access the
@@ -74,4 +76,13 @@ SetSubClusterPolicyConfigurationResponse setPolicyConfiguration(
7476
GetSubClusterPoliciesConfigurationsResponse getPoliciesConfigurations(
7577
GetSubClusterPoliciesConfigurationsRequest request) throws YarnException;
7678

79+
/**
80+
* Delete all queue-to-policy configurations.
81+
*
82+
* @param request delete request.
83+
* @return If the response is empty, the queue-to-policy configurations are deleted successfully.
84+
* @throws Exception if the request is invalid/fails
85+
*/
86+
DeletePoliciesConfigurationsResponse deleteAllPoliciesConfigurations(
87+
DeletePoliciesConfigurationsRequest request) throws Exception;
7788
}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,8 @@
8787
import org.apache.hadoop.yarn.server.federation.store.records.UpdateReservationHomeSubClusterResponse;
8888
import org.apache.hadoop.yarn.server.federation.store.records.DeleteReservationHomeSubClusterRequest;
8989
import org.apache.hadoop.yarn.server.federation.store.records.DeleteReservationHomeSubClusterResponse;
90+
import org.apache.hadoop.yarn.server.federation.store.records.DeletePoliciesConfigurationsRequest;
91+
import org.apache.hadoop.yarn.server.federation.store.records.DeletePoliciesConfigurationsResponse;
9092
import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKey;
9193
import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyRequest;
9294
import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyResponse;
@@ -400,6 +402,13 @@ public GetSubClusterPoliciesConfigurationsResponse getPoliciesConfigurations(
400402
return GetSubClusterPoliciesConfigurationsResponse.newInstance(result);
401403
}
402404

405+
@Override
406+
public DeletePoliciesConfigurationsResponse deleteAllPoliciesConfigurations(
407+
DeletePoliciesConfigurationsRequest request) throws Exception {
408+
policies.clear();
409+
return DeletePoliciesConfigurationsResponse.newInstance();
410+
}
411+
403412
@Override
404413
public Version getCurrentVersion() {
405414
return CURRENT_VERSION_INFO;

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,8 @@
8989
import org.apache.hadoop.yarn.server.federation.store.records.GetReservationsHomeSubClusterRequest;
9090
import org.apache.hadoop.yarn.server.federation.store.records.DeleteReservationHomeSubClusterRequest;
9191
import org.apache.hadoop.yarn.server.federation.store.records.DeleteReservationHomeSubClusterResponse;
92+
import org.apache.hadoop.yarn.server.federation.store.records.DeletePoliciesConfigurationsRequest;
93+
import org.apache.hadoop.yarn.server.federation.store.records.DeletePoliciesConfigurationsResponse;
9294
import org.apache.hadoop.yarn.server.federation.store.records.UpdateReservationHomeSubClusterRequest;
9395
import org.apache.hadoop.yarn.server.federation.store.records.UpdateReservationHomeSubClusterResponse;
9496
import org.apache.hadoop.yarn.server.federation.store.records.ReservationHomeSubCluster;
@@ -1071,6 +1073,29 @@ public GetSubClusterPoliciesConfigurationsResponse getPoliciesConfigurations(
10711073
return GetSubClusterPoliciesConfigurationsResponse.newInstance(policyConfigurations);
10721074
}
10731075

1076+
@Override
1077+
public DeletePoliciesConfigurationsResponse deleteAllPoliciesConfigurations(
1078+
DeletePoliciesConfigurationsRequest request) throws Exception {
1079+
Connection connection = null;
1080+
try {
1081+
connection = getConnection(false);
1082+
FederationQueryRunner runner = new FederationQueryRunner();
1083+
LOG.info("delete table = policies start.");
1084+
runner.truncateTable(connection, "policies");
1085+
LOG.info("delete table = policies finished.");
1086+
} catch (Exception e) {
1087+
throw new RuntimeException("Could not delete table (policies)!", e);
1088+
} finally {
1089+
// Return to the pool the CallableStatement
1090+
try {
1091+
FederationStateStoreUtils.returnToPool(LOG, null, connection);
1092+
} catch (YarnException e) {
1093+
LOG.error("close connection error.", e);
1094+
}
1095+
}
1096+
return DeletePoliciesConfigurationsResponse.newInstance();
1097+
}
1098+
10741099
@Override
10751100
public Version getCurrentVersion() {
10761101
return CURRENT_VERSION_INFO;

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,8 @@
9191
import org.apache.hadoop.yarn.server.federation.store.records.DeleteReservationHomeSubClusterResponse;
9292
import org.apache.hadoop.yarn.server.federation.store.records.UpdateReservationHomeSubClusterRequest;
9393
import org.apache.hadoop.yarn.server.federation.store.records.UpdateReservationHomeSubClusterResponse;
94+
import org.apache.hadoop.yarn.server.federation.store.records.DeletePoliciesConfigurationsRequest;
95+
import org.apache.hadoop.yarn.server.federation.store.records.DeletePoliciesConfigurationsResponse;
9496
import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyResponse;
9597
import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyRequest;
9698
import org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenResponse;
@@ -786,6 +788,23 @@ public GetSubClusterPoliciesConfigurationsResponse getPoliciesConfigurations(
786788
return GetSubClusterPoliciesConfigurationsResponse.newInstance(result);
787789
}
788790

791+
@Override
792+
public DeletePoliciesConfigurationsResponse deleteAllPoliciesConfigurations(
793+
DeletePoliciesConfigurationsRequest request) throws Exception {
794+
795+
zkManager.delete(policiesZNode);
796+
797+
try {
798+
List<ACL> zkAcl = ZKCuratorManager.getZKAcls(configuration);
799+
zkManager.createRootDirRecursively(policiesZNode, zkAcl);
800+
} catch (Exception e) {
801+
String errMsg = "Cannot create base directories: " + e.getMessage();
802+
FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
803+
}
804+
805+
return DeletePoliciesConfigurationsResponse.newInstance();
806+
}
807+
789808
@Override
790809
public Version getCurrentVersion() {
791810
return CURRENT_VERSION_INFO;
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
/**
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with this
4+
* work for additional information regarding copyright ownership. The ASF
5+
* licenses this file to you under the Apache License, Version 2.0 (the
6+
* "License"); you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
* <p>
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
* <p>
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14+
* License for the specific language governing permissions and limitations under
15+
* the License.
16+
*/
17+
package org.apache.hadoop.yarn.server.federation.store.records;
18+
19+
import org.apache.hadoop.classification.InterfaceAudience.Private;
20+
import org.apache.hadoop.classification.InterfaceStability.Unstable;
21+
import org.apache.hadoop.yarn.util.Records;
22+
23+
/**
24+
* This class is used for handling queue policy deletion requests.
25+
* We will delete all PoliciesConfigurations.
26+
*/
27+
public abstract class DeletePoliciesConfigurationsRequest {
28+
29+
@Private
30+
@Unstable
31+
public static DeletePoliciesConfigurationsRequest newInstance() {
32+
return Records.newRecord(DeletePoliciesConfigurationsRequest.class);
33+
}
34+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
/**
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with this
4+
* work for additional information regarding copyright ownership. The ASF
5+
* licenses this file to you under the Apache License, Version 2.0 (the
6+
* "License"); you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
* <p>
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
* <p>
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14+
* License for the specific language governing permissions and limitations under
15+
* the License.
16+
*/
17+
package org.apache.hadoop.yarn.server.federation.store.records;
18+
19+
import org.apache.hadoop.classification.InterfaceAudience.Private;
20+
import org.apache.hadoop.classification.InterfaceStability.Unstable;
21+
import org.apache.hadoop.yarn.util.Records;
22+
23+
/**
24+
* This class is used to respond to requests to delete PoliciesConfigurations.
25+
*/
26+
public abstract class DeletePoliciesConfigurationsResponse {
27+
28+
@Private
29+
@Unstable
30+
public static DeletePoliciesConfigurationsResponse newInstance() {
31+
return Records.newRecord(DeletePoliciesConfigurationsResponse.class);
32+
}
33+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
/**
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with this
4+
* work for additional information regarding copyright ownership. The ASF
5+
* licenses this file to you under the Apache License, Version 2.0 (the
6+
* "License"); you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
* <p>
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
* <p>
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14+
* License for the specific language governing permissions and limitations under
15+
* the License.
16+
*/
17+
package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
18+
19+
import org.apache.hadoop.classification.InterfaceAudience.Private;
20+
import org.apache.hadoop.classification.InterfaceStability.Unstable;
21+
import org.apache.hadoop.thirdparty.protobuf.TextFormat;
22+
import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.DeletePoliciesConfigurationsRequestProto;
23+
import org.apache.hadoop.yarn.server.federation.store.records.DeletePoliciesConfigurationsRequest;
24+
25+
@Private
26+
@Unstable
27+
public class DeletePoliciesConfigurationsRequestPBImpl
28+
extends DeletePoliciesConfigurationsRequest {
29+
30+
private DeletePoliciesConfigurationsRequestProto proto =
31+
DeletePoliciesConfigurationsRequestProto.getDefaultInstance();
32+
33+
private DeletePoliciesConfigurationsRequestProto.Builder builder = null;
34+
35+
private boolean viaProto = false;
36+
37+
public DeletePoliciesConfigurationsRequestPBImpl() {
38+
builder = DeletePoliciesConfigurationsRequestProto.newBuilder();
39+
}
40+
41+
public DeletePoliciesConfigurationsRequestPBImpl(
42+
DeletePoliciesConfigurationsRequestProto proto) {
43+
this.proto = proto;
44+
viaProto = true;
45+
}
46+
47+
public DeletePoliciesConfigurationsRequestProto getProto() {
48+
proto = viaProto ? proto : builder.build();
49+
viaProto = true;
50+
return proto;
51+
}
52+
53+
@Override
54+
public int hashCode() {
55+
return getProto().hashCode();
56+
}
57+
58+
@Override
59+
public boolean equals(Object other) {
60+
if (other == null) {
61+
return false;
62+
}
63+
if (other.getClass().isAssignableFrom(this.getClass())) {
64+
return this.getProto().equals(this.getClass().cast(other).getProto());
65+
}
66+
return false;
67+
}
68+
69+
@Override
70+
public String toString() {
71+
return TextFormat.shortDebugString(getProto());
72+
}
73+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
/**
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with this
4+
* work for additional information regarding copyright ownership. The ASF
5+
* licenses this file to you under the Apache License, Version 2.0 (the
6+
* "License"); you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
* <p>
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
* <p>
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14+
* License for the specific language governing permissions and limitations under
15+
* the License.
16+
*/
17+
package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
18+
19+
import org.apache.hadoop.classification.InterfaceAudience.Private;
20+
import org.apache.hadoop.classification.InterfaceStability.Unstable;
21+
import org.apache.hadoop.thirdparty.protobuf.TextFormat;
22+
import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.DeletePoliciesConfigurationsResponseProto;
23+
import org.apache.hadoop.yarn.server.federation.store.records.DeletePoliciesConfigurationsResponse;
24+
25+
26+
@Private
27+
@Unstable
28+
public class DeletePoliciesConfigurationsResponsePBImpl
29+
extends DeletePoliciesConfigurationsResponse {
30+
31+
private DeletePoliciesConfigurationsResponseProto proto =
32+
DeletePoliciesConfigurationsResponseProto.getDefaultInstance();
33+
34+
private DeletePoliciesConfigurationsResponseProto.Builder builder = null;
35+
36+
private boolean viaProto = false;
37+
38+
public DeletePoliciesConfigurationsResponsePBImpl() {
39+
builder = DeletePoliciesConfigurationsResponseProto.newBuilder();
40+
}
41+
42+
public DeletePoliciesConfigurationsResponsePBImpl(
43+
DeletePoliciesConfigurationsResponseProto proto) {
44+
this.proto = proto;
45+
viaProto = true;
46+
}
47+
48+
public DeletePoliciesConfigurationsResponseProto getProto() {
49+
proto = viaProto ? proto : builder.build();
50+
viaProto = true;
51+
return proto;
52+
}
53+
54+
@Override
55+
public int hashCode() {
56+
return getProto().hashCode();
57+
}
58+
59+
@Override
60+
public boolean equals(Object other) {
61+
if (other == null) {
62+
return false;
63+
}
64+
if (other.getClass().isAssignableFrom(this.getClass())) {
65+
return this.getProto().equals(this.getClass().cast(other).getProto());
66+
}
67+
return false;
68+
}
69+
70+
@Override
71+
public String toString() {
72+
return TextFormat.shortDebugString(getProto());
73+
}
74+
}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@
8686
import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationsHomeSubClusterRequest;
8787
import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationsHomeSubClusterResponse;
8888
import org.apache.hadoop.yarn.server.federation.store.records.DeleteApplicationHomeSubClusterRequest;
89+
import org.apache.hadoop.yarn.server.federation.store.records.DeletePoliciesConfigurationsRequest;
8990
import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
9091
import org.apache.hadoop.yarn.webapp.NotFoundException;
9192
import org.slf4j.Logger;
@@ -1113,6 +1114,11 @@ public ApplicationSubmissionContext getApplicationSubmissionContext(ApplicationI
11131114
}
11141115
}
11151116

1117+
public void deleteAllPoliciesConfigurations() throws Exception {
1118+
DeletePoliciesConfigurationsRequest request =
1119+
DeletePoliciesConfigurationsRequest.newInstance();
1120+
stateStore.deleteAllPoliciesConfigurations(request);
1121+
}
11161122

11171123
@VisibleForTesting
11181124
public FederationCache getFederationCache() {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -196,6 +196,11 @@ message DeleteReservationHomeSubClusterRequestProto {
196196
message DeleteReservationHomeSubClusterResponseProto {
197197
}
198198

199+
message DeletePoliciesConfigurationsRequestProto {
200+
}
201+
202+
message DeletePoliciesConfigurationsResponseProto {
203+
}
199204

200205
//----- configurations ---
201206

0 commit comments

Comments
 (0)