Skip to content

Commit 4baa167

Browse files
slfan1989cnaurothzhtttylz
authored
YARN-11786. Upgrade hadoop-yarn-server-timelineservice-hbase-tests to Support Trunk Compilation and Remove compatible hadoop version. (#7453)
* Upgrade hadoop-yarn-server-timelineservice-hbase-tests to Support Trunk Compilation and Remove compatible hadoop version. Co-authored-by: Chris Nauroth <cnauroth@apache.org> Co-authored-by: Hualong Zhang <hualong.z@hotmail.com> Reviewed-by: Chris Nauroth <cnauroth@apache.org> Reviewed-by: Hualong Zhang <hualong.z@hotmail.com> Signed-off-by: Shilun Fan <slfan1989@apache.org>
1 parent 081d51a commit 4baa167

File tree

17 files changed

+991
-296
lines changed

17 files changed

+991
-296
lines changed

‎BUILDING.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -163,7 +163,7 @@ Maven build goals:
163163
YARN Application Timeline Service V2 build options:
164164

165165
YARN Timeline Service v.2 chooses Apache HBase as the primary backing storage. The supported
166-
version of Apache HBase is 2.5.8.
166+
version of Apache HBase is 2.6.1.
167167

168168
Snappy build options:
169169

‎hadoop-project/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -222,7 +222,7 @@
222222
<swagger-annotations-version>1.5.4</swagger-annotations-version>
223223
<snakeyaml.version>2.0</snakeyaml.version>
224224
<sshd.version>2.11.0</sshd.version>
225-
<hbase.version>2.5.8-hadoop3</hbase.version>
225+
<hbase.version>2.6.1-hadoop3</hbase.version>
226226
<junit.version>4.13.2</junit.version>
227227
<junit.jupiter.version>5.8.2</junit.jupiter.version>
228228
<junit.vintage.version>5.8.2</junit.vintage.version>

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/FlowRunEntity.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
import javax.xml.bind.annotation.XmlElement;
2121

22+
import com.fasterxml.jackson.annotation.JsonInclude;
2223
import org.apache.hadoop.classification.InterfaceAudience;
2324
import org.apache.hadoop.classification.InterfaceStability;
2425

@@ -27,6 +28,7 @@
2728
*/
2829
@InterfaceAudience.Public
2930
@InterfaceStability.Unstable
31+
@JsonInclude(JsonInclude.Include.NON_NULL)
3032
public class FlowRunEntity extends HierarchicalTimelineEntity {
3133
public static final String USER_INFO_KEY =
3234
TimelineEntity.SYSTEM_INFO_KEY_PREFIX + "USER";
@@ -107,7 +109,7 @@ public void setRunId(long runId) {
107109
addInfo(FLOW_RUN_ID_INFO_KEY, runId);
108110
}
109111

110-
public long getStartTime() {
112+
public Long getStartTime() {
111113
return getCreatedTime();
112114
}
113115

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/writer/TimelineEntitySetWriter.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,11 +43,13 @@
4343
public class TimelineEntitySetWriter implements MessageBodyWriter<Set<TimelineEntity>> {
4444

4545
private ObjectMapper objectMapper = new ObjectMapper();
46+
private String timelineEntityType =
47+
"java.util.Set<org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity>";
4648

4749
@Override
4850
public boolean isWriteable(Class<?> type, Type genericType,
4951
Annotation[] annotations, MediaType mediaType) {
50-
return true;
52+
return timelineEntityType.equals(genericType.getTypeName());
5153
}
5254

5355
@Override

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml‎

Lines changed: 16 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,6 @@
8989
<dependency>
9090
<groupId>org.apache.hadoop</groupId>
9191
<artifactId>hadoop-common</artifactId>
92-
<version>${hbase-compatible-hadoop.version}</version>
9392
<scope>test</scope>
9493
<exclusions>
9594
<exclusion>
@@ -104,10 +103,6 @@
104103
<groupId>org.apache.hadoop</groupId>
105104
<artifactId>hadoop-hdfs-client</artifactId>
106105
</exclusion>
107-
<exclusion>
108-
<groupId>com.sun.jersey</groupId>
109-
<artifactId>jersey-json</artifactId>
110-
</exclusion>
111106
</exclusions>
112107
</dependency>
113108

@@ -131,13 +126,22 @@
131126
<artifactId>junit-platform-launcher</artifactId>
132127
<scope>test</scope>
133128
</dependency>
129+
<dependency>
130+
<groupId>org.junit.vintage</groupId>
131+
<artifactId>junit-vintage-engine</artifactId>
132+
<scope>test</scope>
133+
</dependency>
134+
<dependency>
135+
<groupId>org.glassfish.jersey.core</groupId>
136+
<artifactId>jersey-server</artifactId>
137+
<scope>test</scope>
138+
</dependency>
134139

135140
<!-- 'mvn dependency:analyze' fails to detect use of this direct
136141
dependency -->
137142
<dependency>
138143
<groupId>org.apache.hadoop</groupId>
139144
<artifactId>hadoop-auth</artifactId>
140-
<version>${hbase-compatible-hadoop.version}</version>
141145
<scope>test</scope>
142146
<exclusions>
143147
<exclusion>
@@ -242,13 +246,6 @@
242246
<scope>runtime</scope>
243247
</dependency>
244248

245-
<dependency>
246-
<groupId>com.sun.jersey</groupId>
247-
<artifactId>jersey-client</artifactId>
248-
<version>1.19.4</version>
249-
<scope>test</scope>
250-
</dependency>
251-
252249
<dependency>
253250
<groupId>javax.ws.rs</groupId>
254251
<artifactId>jsr311-api</artifactId>
@@ -369,18 +366,13 @@
369366
<dependency>
370367
<groupId>org.apache.hadoop</groupId>
371368
<artifactId>hadoop-common</artifactId>
372-
<version>${hbase-compatible-hadoop.version}</version>
373369
<type>test-jar</type>
374370
<scope>test</scope>
375371
<exclusions>
376372
<exclusion>
377373
<groupId>org.apache.hadoop</groupId>
378374
<artifactId>hadoop-auth</artifactId>
379375
</exclusion>
380-
<exclusion>
381-
<groupId>com.sun.jersey</groupId>
382-
<artifactId>jersey-json</artifactId>
383-
</exclusion>
384376
<exclusion>
385377
<groupId>commons-logging</groupId>
386378
<artifactId>commons-logging</artifactId>
@@ -393,7 +385,6 @@
393385
<dependency>
394386
<groupId>org.apache.hadoop</groupId>
395387
<artifactId>hadoop-hdfs</artifactId>
396-
<version>${hbase-compatible-hadoop.version}</version>
397388
<scope>test</scope>
398389
<exclusions>
399390
<exclusion>
@@ -412,7 +403,6 @@
412403
<dependency>
413404
<groupId>org.apache.hadoop</groupId>
414405
<artifactId>hadoop-hdfs</artifactId>
415-
<version>${hbase-compatible-hadoop.version}</version>
416406
<type>test-jar</type>
417407
<scope>test</scope>
418408
<exclusions>
@@ -430,7 +420,6 @@
430420
<dependency>
431421
<groupId>org.apache.hadoop</groupId>
432422
<artifactId>hadoop-hdfs-client</artifactId>
433-
<version>${hbase-compatible-hadoop.version}</version>
434423
<scope>test</scope>
435424
</dependency>
436425

@@ -507,6 +496,12 @@
507496
<artifactId>mockito-inline</artifactId>
508497
<scope>test</scope>
509498
</dependency>
499+
500+
<dependency>
501+
<groupId>org.glassfish.jersey.media</groupId>
502+
<artifactId>jersey-media-json-jettison</artifactId>
503+
<scope>test</scope>
504+
</dependency>
510505
</dependencies>
511506

512507
<build>

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/AbstractTimelineReaderHBaseTestBase.java‎

Lines changed: 41 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -22,32 +22,29 @@
2222
import static org.junit.Assert.assertNotNull;
2323
import static org.junit.Assert.assertTrue;
2424

25+
import javax.ws.rs.client.Client;
26+
import javax.ws.rs.client.ClientBuilder;
27+
2528
import java.io.IOException;
26-
import java.lang.reflect.UndeclaredThrowableException;
2729
import java.net.HttpURLConnection;
2830
import java.net.URI;
29-
import java.net.URL;
3031
import java.util.List;
3132

33+
import javax.ws.rs.core.GenericType;
3234
import javax.ws.rs.core.MediaType;
35+
import javax.ws.rs.core.Response;
3336

37+
import org.apache.hadoop.classification.VisibleForTesting;
3438
import org.apache.hadoop.conf.Configuration;
3539
import org.apache.hadoop.hbase.HBaseTestingUtility;
3640
import org.apache.hadoop.yarn.api.records.timelineservice.FlowActivityEntity;
41+
import org.apache.hadoop.yarn.api.records.timelineservice.reader.TimelineEntityReader;
3742
import org.apache.hadoop.yarn.conf.YarnConfiguration;
3843
import org.apache.hadoop.yarn.server.timelineservice.storage.DataGeneratorForTest;
39-
import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
44+
import org.glassfish.jersey.client.ClientConfig;
45+
import org.glassfish.jersey.client.HttpUrlConnectorProvider;
4046
import org.junit.Assert;
4147

42-
import com.sun.jersey.api.client.Client;
43-
import com.sun.jersey.api.client.ClientResponse;
44-
import com.sun.jersey.api.client.ClientResponse.Status;
45-
import com.sun.jersey.api.client.GenericType;
46-
import com.sun.jersey.api.client.config.ClientConfig;
47-
import com.sun.jersey.api.client.config.DefaultClientConfig;
48-
import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
49-
import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
50-
5148
/**
5249
* Test Base for TimelineReaderServer HBase tests.
5350
*/
@@ -109,19 +106,25 @@ protected void addFilters(Configuration conf) {
109106
}
110107

111108
protected Client createClient() {
112-
ClientConfig cfg = new DefaultClientConfig();
113-
cfg.getClasses().add(YarnJacksonJaxbJsonProvider.class);
114-
return new Client(
115-
new URLConnectionClientHandler(new DummyURLConnectionFactory()), cfg);
109+
final ClientConfig cc = new ClientConfig();
110+
cc.connectorProvider(getHttpURLConnectionFactory());
111+
return ClientBuilder.newClient(cc)
112+
.register(TimelineEntityReader.class)
113+
.register(TimelineEntitySetReader.class)
114+
.register(TimelineEntityListReader.class)
115+
.register(FlowActivityEntityReader.class)
116+
.register(FlowRunEntityReader.class)
117+
.register(FlowActivityEntitySetReader.class)
118+
.register(FlowActivityEntityListReader.class)
119+
.register(FlowRunEntitySetReader.class);
116120
}
117121

118-
protected ClientResponse getResponse(Client client, URI uri)
122+
protected Response getResponse(Client client, URI uri)
119123
throws Exception {
120-
ClientResponse resp =
121-
client.resource(uri).accept(MediaType.APPLICATION_JSON)
122-
.type(MediaType.APPLICATION_JSON).get(ClientResponse.class);
124+
Response resp =
125+
client.target(uri).request(MediaType.APPLICATION_JSON).get();
123126
if (resp == null || resp.getStatusInfo()
124-
.getStatusCode() != ClientResponse.Status.OK.getStatusCode()) {
127+
.getStatusCode() != HttpURLConnection.HTTP_OK) {
125128
String msg = "";
126129
if (resp != null) {
127130
msg = String.valueOf(resp.getStatusInfo().getStatusCode());
@@ -132,39 +135,37 @@ protected ClientResponse getResponse(Client client, URI uri)
132135
return resp;
133136
}
134137

135-
protected void verifyHttpResponse(Client client, URI uri, Status status) {
136-
ClientResponse resp =
137-
client.resource(uri).accept(MediaType.APPLICATION_JSON)
138-
.type(MediaType.APPLICATION_JSON).get(ClientResponse.class);
138+
protected void verifyHttpResponse(Client client, URI uri, Response.Status status) {
139+
Response resp = client.target(uri).request(MediaType.APPLICATION_JSON).get();
139140
assertNotNull(resp);
140141
assertTrue("Response from server should have been " + status,
141142
resp.getStatusInfo().getStatusCode() == status.getStatusCode());
142-
System.out.println("Response is: " + resp.getEntity(String.class));
143+
System.out.println("Response is: " + resp.readEntity(String.class));
143144
}
144145

145146
protected List<FlowActivityEntity> verifyFlowEntites(Client client, URI uri,
146147
int noOfEntities) throws Exception {
147-
ClientResponse resp = getResponse(client, uri);
148+
Response resp = getResponse(client, uri);
148149
List<FlowActivityEntity> entities =
149-
resp.getEntity(new GenericType<List<FlowActivityEntity>>() {
150+
resp.readEntity(new GenericType<List<FlowActivityEntity>>() {
150151
});
151152
assertNotNull(entities);
152153
assertEquals(noOfEntities, entities.size());
153154
return entities;
154155
}
155156

156-
protected static class DummyURLConnectionFactory
157-
implements HttpURLConnectionFactory {
158-
159-
@Override
160-
public HttpURLConnection getHttpURLConnection(final URL url)
161-
throws IOException {
162-
try {
163-
return (HttpURLConnection) url.openConnection();
164-
} catch (UndeclaredThrowableException e) {
165-
throw new IOException(e.getCause());
166-
}
167-
}
157+
@VisibleForTesting
158+
protected HttpUrlConnectorProvider getHttpURLConnectionFactory() {
159+
return new HttpUrlConnectorProvider().connectionFactory(
160+
url -> {
161+
HttpURLConnection conn;
162+
try {
163+
conn = (HttpURLConnection) url.openConnection();
164+
} catch (Exception e) {
165+
throw new IOException(e);
166+
}
167+
return conn;
168+
});
168169
}
169170

170171
protected static HBaseTestingUtility getHBaseTestingUtility() {

0 commit comments

Comments
 (0)