Skip to content

Commit

Permalink
MAPREDUCE-7418. Fix CheckStyle & Junit Test.
Browse files Browse the repository at this point in the history
  • Loading branch information
fanshilun committed Feb 5, 2025
1 parent 405920d commit 37e6bc2
Show file tree
Hide file tree
Showing 12 changed files with 92 additions and 81 deletions.
4 changes: 2 additions & 2 deletions ...preduce-client-app/src/test/java/org/apache/hadoop/mapred/TestLocalContainerLauncher.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -201,8 +201,8 @@ public void testRenameMapOutputForReduce() throws Exception {
final Path mapOut = mrOutputFiles.getOutputFileForWrite(1);
conf.set(MRConfig.LOCAL_DIR, localDirs[1].toString());
final Path mapOutIdx = mrOutputFiles.getOutputIndexFileForWrite(1);
assertNotEquals(
mapOut.getParent(), mapOutIdx.getParent(), "Paths must be different!");
assertNotEquals(mapOut.getParent(), mapOutIdx.getParent(),
"Paths must be different!");

// make both dirs part of LOCAL_DIR
conf.setStrings(MRConfig.LOCAL_DIR, localDirs);
Expand Down
24 changes: 11 additions & 13 deletions ...-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -427,16 +427,15 @@ public void testPropertyRedactionForJHS() throws Exception {

// load the job_conf.xml in JHS directory and verify property redaction.
Path jhsJobConfFile = getJobConfInIntermediateDoneDir(conf, params.jobId);
assertTrue(
FileContext.getFileContext(conf).util().exists(jhsJobConfFile), "The job_conf.xml file is not in the JHS directory");
assertTrue(FileContext.getFileContext(conf).util().exists(jhsJobConfFile),
"The job_conf.xml file is not in the JHS directory");
Configuration jhsJobConf = new Configuration();

try (InputStream input = FileSystem.get(conf).open(jhsJobConfFile)) {
jhsJobConf.addResource(input);
assertEquals(

MRJobConfUtil.REDACTION_REPLACEMENT_VAL
, jhsJobConf.get(sensitivePropertyName), sensitivePropertyName + " is not redacted in HDFS.");
assertEquals(MRJobConfUtil.REDACTION_REPLACEMENT_VAL,
jhsJobConf.get(sensitivePropertyName),
sensitivePropertyName + " is not redacted in HDFS.");
}
} finally {
jheh.stop();
Expand Down Expand Up @@ -497,11 +496,11 @@ public void testDefaultFsIsUsedForHistory() throws Exception {
// If we got here then event handler worked but we don't know with which
// file system. Now we check that history stuff was written to minicluster
FileSystem dfsFileSystem = dfsCluster.getFileSystem();
assertTrue(
dfsFileSystem.globStatus(new Path(t.dfsWorkDir + "/*")).length != 0, "Minicluster contains some history files");
assertTrue(dfsFileSystem.globStatus(new Path(t.dfsWorkDir + "/*")).length != 0,
"Minicluster contains some history files");
FileSystem localFileSystem = LocalFileSystem.get(conf);
assertFalse(
localFileSystem.exists(new Path(t.dfsWorkDir)), "No history directory on non-default file system");
assertFalse(localFileSystem.exists(new Path(t.dfsWorkDir)),
"No history directory on non-default file system");
} finally {
jheh.stop();
purgeHdfsHistoryIntermediateDoneDirectory(conf);
Expand Down Expand Up @@ -947,9 +946,8 @@ public void testSigTermedFunctionality() throws IOException {
//Make sure events were handled, 4 + 1 finish event
assertTrue(jheh.eventsHandled == 5, "handleEvent should've been called only 5 times but was "
+ jheh.eventsHandled);
assertTrue(
jheh.lastEventHandled.getHistoryEvent()
instanceof JobUnsuccessfulCompletionEvent, "Last event handled wasn't JobUnsuccessfulCompletionEvent");
assertTrue(jheh.lastEventHandled.getHistoryEvent() instanceof JobUnsuccessfulCompletionEvent,
"Last event handled wasn't JobUnsuccessfulCompletionEvent");
}

@Test
Expand Down
2 changes: 1 addition & 1 deletion ...t/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -421,7 +421,7 @@ public void verifyCompleted() {
LOG.info("Job finish time :{}", jobReport.getFinishTime());
assertTrue(jobReport.getStartTime() <= jobReport.getFinishTime(),
"Job start time is not less than finish time");
assertTrue(jobReport.getFinishTime() <= System.currentTimeMillis(),
assertTrue(jobReport.getFinishTime() <= System.currentTimeMillis(),
"Job finish time is in future");
for (Task task : job.getTasks().values()) {
TaskReport taskReport = task.getReport();
Expand Down
34 changes: 17 additions & 17 deletions ...duce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebApp.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,6 @@
import org.junit.jupiter.api.Test;

import com.google.inject.Injector;
import org.junit.jupiter.api.extension.RegisterExtension;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
Expand All @@ -85,9 +84,6 @@ public class TestAMWebApp {
System.getProperty("java.io.tmpdir")),
TestAMWebApp.class.getName());

@RegisterExtension
public final EnvironmentVariablesExtension extension = new EnvironmentVariablesExtension();

@AfterEach
public void tearDown() {
TEST_DIR.delete();
Expand Down Expand Up @@ -130,7 +126,7 @@ public void tearDown() {

public static Map<String, String> getJobParams(AppContext appContext) {
JobId jobId = appContext.getAllJobs().entrySet().iterator().next().getKey();
Map<String, String> params = new HashMap<>();
Map<String, String> params = new HashMap<String, String>();
params.put(AMParams.JOB_ID, MRApps.toString(jobId));
return params;
}
Expand All @@ -139,7 +135,7 @@ public static Map<String, String> getTaskParams(AppContext appContext) {
JobId jobId = appContext.getAllJobs().entrySet().iterator().next().getKey();
Entry<TaskId, Task> e = appContext.getJob(jobId).getTasks().entrySet().iterator().next();
e.getValue().getType();
Map<String, String> params = new HashMap<>();
Map<String, String> params = new HashMap<String, String>();
params.put(AMParams.JOB_ID, MRApps.toString(jobId));
params.put(AMParams.TASK_ID, MRApps.toString(e.getKey()));
params.put(AMParams.TASK_TYPE, MRApps.taskSymbol(e.getValue().getType()));
Expand Down Expand Up @@ -256,9 +252,8 @@ protected ClientService createClientService(AppContext context) {
keystoreFile.getParentFile().mkdirs();
KeyStoreTestUtil.createKeyStore(keystoreFile.getAbsolutePath(), "password",
"server", keyPair.getPrivate(), cert);
extension.getEnvironmentVariables().set("KEYSTORE_FILE_LOCATION",
keystoreFile.getAbsolutePath());
extension.getEnvironmentVariables().set("KEYSTORE_PASSWORD", "password");
System.setProperty("KEYSTORE_FILE_LOCATION", keystoreFile.getAbsolutePath());
System.setProperty("KEYSTORE_PASSWORD", "password");

Job job = app.submit(conf);

Expand Down Expand Up @@ -291,6 +286,8 @@ protected ClientService createClientService(AppContext context) {
app.verifyCompleted();

keystoreFile.delete();
System.clearProperty("KEYSTORE_FILE_LOCATION");
System.clearProperty("KEYSTORE_PASSWORD");
}

@Test
Expand All @@ -312,9 +309,8 @@ protected ClientService createClientService(AppContext context) {
keystoreFile.getParentFile().mkdirs();
KeyStoreTestUtil.createKeyStore(keystoreFile.getAbsolutePath(), "password",
"server", keyPair.getPrivate(), cert);
extension.getEnvironmentVariables().set("KEYSTORE_FILE_LOCATION",
keystoreFile.getAbsolutePath());
extension.getEnvironmentVariables().set("KEYSTORE_PASSWORD", "password");
System.setProperty("KEYSTORE_FILE_LOCATION", keystoreFile.getAbsolutePath());
System.setProperty("KEYSTORE_PASSWORD", "password");

KeyPair clientKeyPair = KeyStoreTestUtil.generateKeyPair("RSA");
X509Certificate clientCert = KeyStoreTestUtil.generateCertificate(
Expand All @@ -323,9 +319,8 @@ protected ClientService createClientService(AppContext context) {
truststoreFile.getParentFile().mkdirs();
KeyStoreTestUtil.createTrustStore(truststoreFile.getAbsolutePath(),
"password", "client", clientCert);
extension.getEnvironmentVariables().set("TRUSTSTORE_FILE_LOCATION",
truststoreFile.getAbsolutePath());
extension.getEnvironmentVariables().set("TRUSTSTORE_PASSWORD", "password");
System.setProperty("TRUSTSTORE_FILE_LOCATION", truststoreFile.getAbsolutePath());
System.setProperty("TRUSTSTORE_PASSWORD", "password");

Job job = app.submit(conf);

Expand Down Expand Up @@ -363,6 +358,11 @@ protected ClientService createClientService(AppContext context) {

keystoreFile.delete();
truststoreFile.delete();

System.clearProperty("KEYSTORE_FILE_LOCATION");
System.clearProperty("KEYSTORE_PASSWORD");
System.clearProperty("TRUSTSTORE_FILE_LOCATION");
System.clearProperty("TRUSTSTORE_PASSWORD");
}

static String webProxyBase = null;
Expand Down Expand Up @@ -417,11 +417,11 @@ protected ClientService createClientService(AppContext context) {
}
}

/*public static void main(String[] args) {
public static void main(String[] args) {
AppContext context = new MockAppContext(0, 8, 88, 4);
WebApps.$for("yarn", AppContext.class, context).withResourceConfig(configure(context)).
at(58888).inDevMode().start(new AMWebApp(context)).joinThread();
}*/
}

protected static ResourceConfig configure(AppContext context) {
ResourceConfig config = new ResourceConfig();
Expand Down
4 changes: 2 additions & 2 deletions ...client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServices.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -337,8 +337,8 @@ public void verifyBlacklistedNodesInfoXML(String xml, AppContext ctx)
assertEquals(1, infonodes.getLength(), "incorrect number of elements");
NodeList nodes = dom.getElementsByTagName("blacklistedNodes");
Set<String> blacklistedNodes = ctx.getBlacklistedNodes();
assertEquals(blacklistedNodes.size()
, nodes.getLength(), "incorrect number of elements");
assertEquals(blacklistedNodes.size(),
nodes.getLength(), "incorrect number of elements");
for (int i = 0; i < nodes.getLength(); i++) {
Element element = (Element) nodes.item(i);
assertTrue(
Expand Down
5 changes: 0 additions & 5 deletions ...app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesAttempt.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,11 +128,6 @@ protected Properties getConfiguration(String configPrefix,
}
}

@Override
public void setUp() throws Exception {
super.setUp();
}

public TestAMWebServicesAttempt() {
}

Expand Down
9 changes: 5 additions & 4 deletions ...app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobConf.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
Expand All @@ -69,7 +70,7 @@
import org.glassfish.jersey.server.ResourceConfig;
/**
* Test the app master web service Rest API for getting the job conf. This
* requires to be created a temporary configuration file.
* requires created a temporary configuration file.
*
* /ws/v1/mapreduce/job/{jobid}/conf
*/
Expand Down Expand Up @@ -130,9 +131,9 @@ protected void configure() {
}
}

@Override
public void setUp() throws Exception {
super.setUp();

@BeforeAll
public static void setTestConfDir() throws Exception {
testConfDir.mkdir();
}

Expand Down
33 changes: 14 additions & 19 deletions ...nt-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobs.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -470,8 +470,7 @@ public void verifyAMJob(JSONObject info, Job job) throws JSONException {
} else {
fail("should have acls in the web service info");
}
assertTrue(
found, "acl: " + expectName + " not found in webservice output");
assertTrue(found, "acl: " + expectName + " not found in webservice output");
}
}

Expand Down Expand Up @@ -545,8 +544,7 @@ public void verifyAMJobXML(NodeList nodes, AppContext appContext) {
} else {
fail("should have acls in the web service info");
}
assertTrue(
found, "acl: " + expectName + " not found in webservice output");
assertTrue(found, "acl: " + expectName + " not found in webservice output");
}
}
}
Expand All @@ -568,17 +566,15 @@ public void verifyAMJobGeneric(Job job, String id, String user, String name,

assertEquals(report.getStartTime(), startTime, "startTime incorrect");
assertEquals(report.getFinishTime(), finishTime, "finishTime incorrect");
assertEquals(
Times.elapsed(report.getStartTime(), report.getFinishTime())
, elapsedTime, "elapsedTime incorrect");
assertEquals(Times.elapsed(report.getStartTime(), report.getFinishTime()),
elapsedTime, "elapsedTime incorrect");
assertEquals(job.getTotalMaps(), mapsTotal, "mapsTotal incorrect");
assertEquals(job.getCompletedMaps()
, mapsCompleted, "mapsCompleted incorrect");
assertEquals(job.getCompletedMaps(),
mapsCompleted, "mapsCompleted incorrect");
assertEquals(job.getTotalReduces(), reducesTotal, "reducesTotal incorrect");
assertEquals(job.getCompletedReduces()
, reducesCompleted, "reducesCompleted incorrect");
assertEquals(report.getMapProgress() * 100,
mapProgress, 0, "mapProgress incorrect");
assertEquals(job.getCompletedReduces(),
reducesCompleted, "reducesCompleted incorrect");
assertEquals(report.getMapProgress() * 100, mapProgress, 0, "mapProgress incorrect");
assertEquals(report.getReduceProgress() * 100,
reduceProgress, 0, "reduceProgress incorrect");
}
Expand Down Expand Up @@ -617,8 +613,7 @@ public void verifyAMJobGenericSecure(Job job, int mapsPending,
assertTrue(runningReduceAttempts >= 0, "runningReduceAttempts not >= 0");
assertTrue(failedReduceAttempts >= 0, "failedReduceAttempts not >= 0");
assertTrue(killedReduceAttempts >= 0, "killedReduceAttempts not >= 0");
assertTrue(
successfulReduceAttempts >= 0, "successfulReduceAttempts not >= 0");
assertTrue(successfulReduceAttempts >= 0, "successfulReduceAttempts not >= 0");

assertTrue(newMapAttempts >= 0, "newMapAttempts not >= 0");
assertTrue(runningMapAttempts >= 0, "runningMapAttempts not >= 0");
Expand Down Expand Up @@ -724,8 +719,8 @@ public void verifyAMJobCounters(JSONObject info, Job job)
for (int j = 0; j < counters.length(); j++) {
JSONObject counter = counters.getJSONObject(j);
String counterName = counter.getString("name");
assertTrue(
(counterName != null && !counterName.isEmpty()), "counter name not set");
assertTrue((counterName != null && !counterName.isEmpty()),
"counter name not set");

long mapValue = counter.getLong("mapCounterValue");
assertTrue(mapValue >= 0, "mapCounterValue >= 0");
Expand Down Expand Up @@ -764,8 +759,8 @@ public void verifyAMJobCountersXML(NodeList nodes, Job job) {
Element counter = (Element) counterArr.item(z);
String counterName = WebServicesTestUtils.getXmlString(counter,
"name");
assertTrue(
(counterName != null && !counterName.isEmpty()), "counter name not set");
assertTrue((counterName != null && !counterName.isEmpty()),
"counter name not set");

long mapValue = WebServicesTestUtils.getXmlLong(counter,
"mapCounterValue");
Expand Down
6 changes: 2 additions & 4 deletions ...t-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesTasks.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -738,8 +738,7 @@ public void verifyAMJobTaskCounters(JSONObject info, Task task)
for (int j = 0; j < counters.length(); j++) {
JSONObject counter = counters.getJSONObject(j);
String counterName = counter.getString("name");
assertTrue(
(counterName != null && !counterName.isEmpty()), "name not set");
assertTrue((counterName != null && !counterName.isEmpty()), "name not set");
long value = counter.getLong("value");
assertTrue(value >= 0, "value >= 0");
}
Expand Down Expand Up @@ -769,8 +768,7 @@ public void verifyAMTaskCountersXML(NodeList nodes, Task task) {
Element counter = (Element) counterArr.item(z);
String counterName = WebServicesTestUtils.getXmlString(counter,
"name");
assertTrue(
(counterName != null && !counterName.isEmpty()), "counter name not set");
assertTrue((counterName != null && !counterName.isEmpty()), "counter name not set");

long value = WebServicesTestUtils.getXmlLong(counter, "value");
assertTrue(value >= 0, "value not >= 0");
Expand Down
11 changes: 8 additions & 3 deletions ...client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAppController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,13 @@
*/
package org.apache.hadoop.mapreduce.v2.app.webapp;

import static org.mockito.Mockito.*;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;

import java.io.IOException;
import java.util.Iterator;
Expand All @@ -39,7 +45,6 @@
import org.apache.hadoop.yarn.webapp.ResponseInfo;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.*;

public class TestAppController {

Expand Down Expand Up @@ -275,7 +280,7 @@ public void testConfiguration() {
public void testDownloadConfiguration() {
appController.downloadConf();
String jobConfXml = appController.getData();
assertFalse(jobConfXml.contains("Error"), "Error downloading the job configuration file.");
assertTrue(!jobConfXml.contains("Error"), "Error downloading the job configuration file.");
}

/**
Expand Down
Loading

0 comments on commit 37e6bc2

Please sign in to comment.