Skip to content

Commit 6234c11

Browse files
K0K0V0KK0K0V0K
committed
YARN-11874. FIX YARN REST IF
1 parent 4e1296c commit 6234c11

File tree

91 files changed

+8792
-16520
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

91 files changed

+8792
-16520
lines changed

‎hadoop-client-modules/hadoop-client-minicluster/pom.xml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -725,6 +725,13 @@
725725
<exclude>**/*.java</exclude>
726726
</excludes>
727727
</filter>
728+
<!-- Some of our dependencies include html, so remove it. -->
729+
<filter>
730+
<artifact>*:*</artifact>
731+
<excludes>
732+
<exclude>**/*.html</exclude>
733+
</excludes>
734+
</filter>
728735
<!-- We pull in several test jars; keep out the actual test classes -->
729736
<filter>
730737
<artifact>*:*</artifact>

‎hadoop-project/pom.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2146,6 +2146,11 @@
21462146
<artifactId>jersey-media-json-jettison</artifactId>
21472147
<version>${jersey2.version}</version>
21482148
</dependency>
2149+
<dependency>
2150+
<groupId>org.glassfish.jersey.media</groupId>
2151+
<artifactId>jersey-media-moxy</artifactId>
2152+
<version>${jersey2.version}</version>
2153+
</dependency>
21492154
<dependency>
21502155
<groupId>org.glassfish.jaxb</groupId>
21512156
<artifactId>jaxb-runtime</artifactId>

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -265,6 +265,10 @@
265265
<groupId>org.glassfish.jersey.media</groupId>
266266
<artifactId>jersey-media-json-jettison</artifactId>
267267
</dependency>
268+
<dependency>
269+
<groupId>org.glassfish.jersey.media</groupId>
270+
<artifactId>jersey-media-moxy</artifactId>
271+
</dependency>
268272
<dependency>
269273
<groupId>org.mockito</groupId>
270274
<artifactId>mockito-junit-jupiter</artifactId>

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java‎

Lines changed: 27 additions & 117 deletions
Original file line numberDiff line numberDiff line change
@@ -18,15 +18,18 @@
1818

1919
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
2020

21+
import org.eclipse.persistence.jaxb.JAXBContextFactory;
22+
import org.eclipse.persistence.jaxb.MarshallerProperties;
23+
import org.glassfish.jersey.jettison.JettisonConfig;
2124
import org.glassfish.jersey.jettison.JettisonJaxbContext;
2225
import org.slf4j.Logger;
2326
import org.slf4j.LoggerFactory;
2427

25-
import java.util.Arrays;
26-
import java.util.ArrayList;
2728
import java.util.Collections;
29+
import java.util.List;
2830
import java.util.Map;
2931
import java.util.HashMap;
32+
import java.util.Set;
3033

3134
import javax.inject.Inject;
3235
import javax.inject.Singleton;
@@ -35,138 +38,45 @@
3538
import javax.xml.bind.JAXBContext;
3639

3740
import org.apache.hadoop.conf.Configuration;
38-
import org.apache.hadoop.yarn.conf.YarnConfiguration;
39-
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.UserInfo;
40-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo;
41-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppsInfo;
42-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptInfo;
43-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptsInfo;
44-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterInfo;
45-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerQueueInfo;
46-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FifoSchedulerInfo;
47-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo;
48-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo;
49-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.UserMetricsInfo;
50-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerInfo;
51-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
52-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerInfo;
53-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo;
54-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerQueueInfoList;
55-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo;
56-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.UsersInfo;
57-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationStatisticsInfo;
58-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.StatisticsItemInfo;
59-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerHealthInfo;
60-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FairSchedulerQueueInfoList;
61-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppTimeoutsInfo;
62-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppTimeoutInfo;
63-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInformationsInfo;
64-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ActivitiesInfo;
65-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppActivitiesInfo;
66-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueAclsInfo;
67-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueAclInfo;
68-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.BulkActivitiesInfo;
69-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NewApplication;
70-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationSubmissionContextInfo;
71-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ContainerLaunchContextInfo;
72-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LocalResourceInfo;
73-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.DelegationToken;
74-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppQueue;
75-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppPriority;
76-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceOptionInfo;
77-
import org.apache.hadoop.yarn.webapp.RemoteExceptionData;
41+
import org.apache.hadoop.yarn.server.resourcemanager.webapp.jsonprovider.ClassSerialisationConfig;
7842

7943
@Singleton
8044
@Provider
8145
public class JAXBContextResolver implements ContextResolver<JAXBContext> {
8246

8347
private static final Logger LOG = LoggerFactory.getLogger(JAXBContextResolver.class.getName());
8448

85-
private final Map<Class, JAXBContext> typesContextMap;
49+
private final Map<Class, JAXBContext> typesContextMap = new HashMap<>();
8650

8751
public JAXBContextResolver() throws Exception {
8852
this(new Configuration());
8953
}
9054

9155
@Inject
9256
public JAXBContextResolver(@javax.inject.Named("conf") Configuration conf) throws Exception {
93-
94-
JAXBContext context;
95-
JAXBContext unWrappedRootContext;
96-
97-
// you have to specify all the dao classes here
98-
final Class[] cTypes =
99-
{ AppInfo.class, AppAttemptInfo.class, AppAttemptsInfo.class,
100-
ClusterInfo.class, CapacitySchedulerQueueInfo.class,
101-
FifoSchedulerInfo.class, SchedulerTypeInfo.class, NodeInfo.class,
102-
UserMetricsInfo.class, CapacitySchedulerInfo.class,
103-
ClusterMetricsInfo.class, SchedulerInfo.class, AppsInfo.class,
104-
NodesInfo.class, RemoteExceptionData.class,
105-
CapacitySchedulerQueueInfoList.class, ResourceInfo.class,
106-
UsersInfo.class, UserInfo.class, ApplicationStatisticsInfo.class,
107-
StatisticsItemInfo.class, CapacitySchedulerHealthInfo.class,
108-
FairSchedulerQueueInfoList.class, AppTimeoutsInfo.class,
109-
AppTimeoutInfo.class, ResourceInformationsInfo.class,
110-
ActivitiesInfo.class, AppActivitiesInfo.class,
111-
QueueAclsInfo.class, QueueAclInfo.class,
112-
BulkActivitiesInfo.class};
113-
114-
// these dao classes need root unwrapping
115-
final Class[] rootUnwrappedTypes =
116-
{ NewApplication.class, ApplicationSubmissionContextInfo.class,
117-
ContainerLaunchContextInfo.class, LocalResourceInfo.class,
118-
DelegationToken.class, AppQueue.class, AppPriority.class,
119-
ResourceOptionInfo.class };
120-
121-
ArrayList<Class> finalcTypesList = new ArrayList<>();
122-
ArrayList<Class> finalRootUnwrappedTypesList = new ArrayList<>();
123-
124-
Collections.addAll(finalcTypesList, cTypes);
125-
Collections.addAll(finalRootUnwrappedTypesList, rootUnwrappedTypes);
126-
127-
// Add Custom DAO Classes
128-
Class[] daoClasses = null;
129-
Class[] unwrappedDaoClasses = null;
130-
boolean loadCustom = true;
131-
try {
132-
daoClasses = conf
133-
.getClasses(YarnConfiguration.YARN_HTTP_WEBAPP_CUSTOM_DAO_CLASSES);
134-
unwrappedDaoClasses = conf.getClasses(
135-
YarnConfiguration.YARN_HTTP_WEBAPP_CUSTOM_UNWRAPPED_DAO_CLASSES);
136-
} catch (Exception e) {
137-
LOG.warn("Failed to load custom dao class: ", e);
138-
loadCustom = false;
139-
}
140-
141-
if (loadCustom) {
142-
if (daoClasses != null) {
143-
Collections.addAll(finalcTypesList, daoClasses);
144-
LOG.debug("Added custom dao classes: {}.", Arrays.toString(daoClasses));
145-
}
146-
if (unwrappedDaoClasses != null) {
147-
Collections.addAll(finalRootUnwrappedTypesList, unwrappedDaoClasses);
148-
LOG.debug("Added custom Unwrapped dao classes: {}", Arrays.toString(unwrappedDaoClasses));
149-
}
150-
}
151-
152-
final Class[] finalcTypes = finalcTypesList
153-
.toArray(new Class[finalcTypesList.size()]);
154-
final Class[] finalRootUnwrappedTypes = finalRootUnwrappedTypesList
155-
.toArray(new Class[finalRootUnwrappedTypesList.size()]);
156-
157-
this.typesContextMap = new HashMap<>();
158-
context = new JettisonJaxbContext(finalcTypes);
159-
unWrappedRootContext = new JettisonJaxbContext(finalRootUnwrappedTypes);
160-
for (Class type : finalcTypes) {
161-
typesContextMap.put(type, context);
162-
}
163-
for (Class type : finalRootUnwrappedTypes) {
164-
typesContextMap.put(type, unWrappedRootContext);
165-
}
57+
ClassSerialisationConfig classSerialisationConfig = new ClassSerialisationConfig(conf);
58+
Set<Class<?>> wrappedClasses = classSerialisationConfig.getWrappedClasses();
59+
Set<Class<?>> unWrappedClasses = classSerialisationConfig.getUnWrappedClasses();
60+
61+
//WARNING: AFAIK these properties not respected by MOXyJsonProvider
62+
//For details check MOXyJsonProvider#readFrom method
63+
JAXBContext wrappedContext = JAXBContextFactory.createContext(
64+
wrappedClasses.toArray(new Class[0]),
65+
Collections.singletonMap(MarshallerProperties.JSON_INCLUDE_ROOT, true)
66+
);
67+
JAXBContext unWrappedContext = JAXBContextFactory.createContext(
68+
unWrappedClasses.toArray(new Class[0]),
69+
Collections.singletonMap(MarshallerProperties.JSON_INCLUDE_ROOT, false)
70+
);
71+
72+
wrappedClasses.forEach(type -> typesContextMap.put(type, wrappedContext));
73+
unWrappedClasses.forEach(type -> typesContextMap.put(type, unWrappedContext));
16674
}
16775

16876
@Override
16977
public JAXBContext getContext(Class<?> objectType) {
170-
return typesContextMap.get(objectType);
78+
JAXBContext jaxbContext = typesContextMap.get(objectType);
79+
LOG.trace("Context for {} is {}", objectType, jaxbContext);
80+
return jaxbContext;
17181
}
17282
}

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,13 @@
2323
import java.net.InetSocketAddress;
2424

2525
import org.apache.hadoop.conf.Configuration;
26+
27+
import org.glassfish.jersey.moxy.json.MoxyJsonConfig;
2628
import org.slf4j.Logger;
2729
import org.slf4j.LoggerFactory;
2830
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
2931
import org.apache.hadoop.yarn.conf.YarnConfiguration;
32+
import org.apache.hadoop.yarn.server.resourcemanager.webapp.jsonprovider.JsonProviderFeature;
3033
import org.apache.hadoop.yarn.util.RMHAUtils;
3134
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
3235
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
@@ -60,7 +63,8 @@ public ResourceConfig resourceConfig() {
6063
config.register(new JerseyBinder());
6164
config.register(RMWebServices.class);
6265
config.register(GenericExceptionHandler.class);
63-
config.register(new JettisonFeature()).register(JAXBContextResolver.class);
66+
config.register(JsonProviderFeature.class);
67+
config.register(JAXBContextResolver.class);
6468
return config;
6569
}
6670

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeLabelsInfo.java‎

Lines changed: 18 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -32,59 +32,63 @@
3232
public class NodeLabelsInfo {
3333

3434
@XmlElement(name = "nodeLabelInfo")
35-
private ArrayList<NodeLabelInfo> nodeLabelsInfo = new ArrayList<>();
35+
private ArrayList<NodeLabelInfo> nodeLabelInfo = new ArrayList<>();
3636

3737
public NodeLabelsInfo() {
3838
// JAXB needs this
3939
}
4040

4141
public NodeLabelsInfo(ArrayList<NodeLabelInfo> nodeLabels) {
42-
this.nodeLabelsInfo = nodeLabels;
42+
this.nodeLabelInfo = nodeLabels;
4343
}
4444

4545
public NodeLabelsInfo(List<NodeLabel> nodeLabels) {
46-
this.nodeLabelsInfo = new ArrayList<>();
46+
this.nodeLabelInfo = new ArrayList<>();
4747
for (NodeLabel label : nodeLabels) {
48-
this.nodeLabelsInfo.add(new NodeLabelInfo(label));
48+
this.nodeLabelInfo.add(new NodeLabelInfo(label));
4949
}
5050
}
51-
51+
5252
public NodeLabelsInfo(Set<String> nodeLabelsName) {
53-
this.nodeLabelsInfo = new ArrayList<>();
53+
this.nodeLabelInfo = new ArrayList<>();
5454
for (String labelName : nodeLabelsName) {
55-
this.nodeLabelsInfo.add(new NodeLabelInfo(labelName));
55+
this.nodeLabelInfo.add(new NodeLabelInfo(labelName));
5656
}
5757
}
5858

5959
public NodeLabelsInfo(Collection<NodeLabel> nodeLabels) {
60-
this.nodeLabelsInfo = new ArrayList<>();
60+
this.nodeLabelInfo = new ArrayList<>();
6161
nodeLabels.stream().forEach(nodeLabel -> {
62-
this.nodeLabelsInfo.add(new NodeLabelInfo(nodeLabel));
62+
this.nodeLabelInfo.add(new NodeLabelInfo(nodeLabel));
6363
});
6464
}
6565

6666
public ArrayList<NodeLabelInfo> getNodeLabelsInfo() {
67-
return nodeLabelsInfo;
67+
return nodeLabelInfo;
6868
}
6969

7070
public Set<NodeLabel> getNodeLabels() {
7171
Set<NodeLabel> nodeLabels = new HashSet<>();
72-
for (NodeLabelInfo label : nodeLabelsInfo) {
72+
for (NodeLabelInfo label : nodeLabelInfo) {
7373
nodeLabels.add(NodeLabel.newInstance(label.getName(),
7474
label.getExclusivity()));
7575
}
7676
return nodeLabels;
7777
}
78-
78+
7979
public List<String> getNodeLabelsName() {
8080
ArrayList<String> nodeLabelsName = new ArrayList<>();
81-
for (NodeLabelInfo label : nodeLabelsInfo) {
81+
for (NodeLabelInfo label : nodeLabelInfo) {
8282
nodeLabelsName.add(label.getName());
8383
}
8484
return nodeLabelsName;
8585
}
8686

8787
public void setNodeLabelsInfo(ArrayList<NodeLabelInfo> nodeLabelInfo) {
88-
this.nodeLabelsInfo = nodeLabelInfo;
88+
this.nodeLabelInfo = nodeLabelInfo;
89+
}
90+
91+
public void setNodeLabelInfo(ArrayList<NodeLabelInfo> nodeLabelsInfo) {
92+
this.nodeLabelInfo = nodeLabelsInfo;
8993
}
9094
}

0 commit comments

Comments
 (0)