HADOOP-19610. S3A: ITests to run under JUnit5: many more tests

A lot more.

Nothing which brings up a YARN cluster works; not worrying about that now.

Author: steveloughran

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestCustomSigner.java

@@ -28,7 +28,9 @@
 import java.util.concurrent.atomic.AtomicInteger;
 
 import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedClass;
 import org.junit.jupiter.params.provider.MethodSource;
 import software.amazon.awssdk.auth.signer.Aws4Signer;
 import software.amazon.awssdk.auth.signer.AwsS3V4Signer;
@@ -65,6 +67,8 @@
  * Because the v2 sdk has had some problems with bulk delete
  * and custom signing, this suite is parameterized.
  */
+@ParameterizedClass(name="bulk-delete={0}")
+@MethodSource("params")
 public class ITestCustomSigner extends AbstractS3ATestBase {
 
   private static final Logger LOG = LoggerFactory
@@ -83,7 +87,7 @@ public static Collection<Object[]> params() {
     });
   }
 
-  private boolean bulkDelete;
+  private final boolean bulkDelete;
 
   private final UserGroupInformation ugi1 = UserGroupInformation.createRemoteUser("user1");
 
@@ -93,14 +97,14 @@ public static Collection<Object[]> params() {
 
   private String endpoint;
 
-  public void initITestCustomSigner(
+  public ITestCustomSigner(
       final String ignored,
-      final boolean pBulkDelete) throws Exception {
-    this.bulkDelete = pBulkDelete;
-    setup();
+      final boolean bulkDelete) {
+    this.bulkDelete = bulkDelete;
   }
 
   @Override
+  @BeforeEach
   public void setup() throws Exception {
     super.setup();
     final S3AFileSystem fs = getFileSystem();
@@ -127,11 +131,10 @@ public void teardown() throws Exception {
     FileSystem.closeAllForUGI(ugi2);
   }
 
-  @MethodSource("params")
-  @ParameterizedTest(name = "{0}")
-  public void testCustomSignerAndInitializer(final String ignored,
-      final boolean pBulkDelete) throws Exception {
-    initITestCustomSigner(ignored, pBulkDelete);
+  @Test
+  public void testCustomSignerAndInitializer()
+      throws IOException, InterruptedException {
+
     final Path basePath = path(getMethodName());
     FileSystem fs1 = runStoreOperationsAndVerify(ugi1,
         new Path(basePath, "customsignerpath1"), "id1");

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestDelegatedMRJob.java

@@ -23,7 +23,9 @@
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedClass;
 import org.junit.jupiter.params.provider.MethodSource;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -86,6 +88,8 @@
  * This is needed to verify that job resources have their tokens extracted
  * too.
  */
+@ParameterizedClass(name="token={0}")
+@MethodSource("params")
 public class ITestDelegatedMRJob extends AbstractDelegationIT {
 
   private static final Logger LOG =
@@ -97,11 +101,11 @@ public class ITestDelegatedMRJob extends AbstractDelegationIT {
   @SuppressWarnings("StaticNonFinalField")
   private static MiniKerberizedHadoopCluster cluster;
 
-  private String name;
+  private final String name;
 
-  private String tokenBinding;
+  private final String tokenBinding;
 
-  private Text tokenKind;
+  private final Text tokenKind;
 
   /**
    * Created in test setup.
@@ -134,12 +138,10 @@ public static Collection<Object[]> params() {
     });
   }
 
-  public void initITestDelegatedMRJob(String pName, String pTokenBinding, Text pTokenKind)
-      throws Exception {
-    this.name = pName;
-    this.tokenBinding = pTokenBinding;
-    this.tokenKind = pTokenKind;
-    setup();
+  public ITestDelegatedMRJob(String name, String tokenBinding, Text tokenKind) {
+    this.name = name;
+    this.tokenBinding = tokenBinding;
+    this.tokenKind = tokenKind;
   }
 
   /***
@@ -191,6 +193,7 @@ protected YarnConfiguration getConfiguration() {
   }
 
   @Override
+  @BeforeEach
   public void setup() throws Exception {
     cluster.loginPrincipal();
     super.setup();
@@ -242,23 +245,17 @@ protected int getTestTimeoutMillis() {
     return getTestTimeoutSeconds() * 1000;
   }
 
-  @MethodSource("params")
-  @ParameterizedTest
-  public void testCommonCrawlLookup(String pName, String pTokenBinding,
-      Text pTokenKind) throws Throwable {
-    initITestDelegatedMRJob(pName, pTokenBinding, pTokenKind);
+  @Test
+  public void testCommonCrawlLookup() throws Throwable {
     FileSystem resourceFS = extraJobResourcePath.getFileSystem(
         getConfiguration());
     FileStatus status = resourceFS.getFileStatus(extraJobResourcePath);
     LOG.info("Extra job resource is {}", status);
     assertTrue(status.isEncrypted(), "Not encrypted: " + status);
   }
 
-  @MethodSource("params")
-  @ParameterizedTest
-  public void testJobSubmissionCollectsTokens(String pName, String pTokenBinding,
-      Text pTokenKind) throws Exception {
-    initITestDelegatedMRJob(pName, pTokenBinding, pTokenKind);
+  @Test
+  public void testJobSubmissionCollectsTokens() throws Exception {
     describe("Mock Job test");
     JobConf conf = new JobConf(getConfiguration());
     if (isUsingDefaultExternalDataFile(conf)) {

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/ITestS3ACommitterFactory.java

@@ -22,7 +22,9 @@
 import java.util.Arrays;
 import java.util.Collection;
 
-import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedClass;
 import org.junit.jupiter.params.provider.MethodSource;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -53,9 +55,13 @@
 /**
  * Tests for the committer factory creation/override process.
  */
+@ParameterizedClass(name="committer={3}")
+@MethodSource("params")
 public final class ITestS3ACommitterFactory extends AbstractCommitITest {
+
   private static final Logger LOG = LoggerFactory.getLogger(
       ITestS3ACommitterFactory.class);
+
   /**
    * Name for invalid committer: {@value}.
    */
@@ -114,18 +120,18 @@ public static Collection<Object[]> params() {
   /**
    * Name of committer to set in filesystem config. If "" do not set one.
    */
-  private String fsCommitterName;
+  private final String fsCommitterName;
 
   /**
    * Name of committer to set in job config.
    */
-  private String jobCommitterName;
+  private final String jobCommitterName;
 
   /**
    * Expected committer class.
    * If null: an exception is expected
    */
-  private Class<? extends AbstractS3ACommitter> committerClass;
+  private final Class<? extends AbstractS3ACommitter> committerClass;
 
   /**
    * Description from parameters, simply for thread names to be more informative.
@@ -134,21 +140,20 @@ public static Collection<Object[]> params() {
 
   /**
    * Create a parameterized instance.
-   * @param pFsCommitterName committer to set in filesystem config
-   * @param pJobCommitterName committer to set in job config
-   * @param pCommitterClass expected committer class
-   * @param pDescription debug text for thread names.
+   * @param fsCommitterName committer to set in filesystem config
+   * @param jobCommitterName committer to set in job config
+   * @param committerClass expected committer class
+   * @param description debug text for thread names.
    */
-  public void initITestS3ACommitterFactory(
-      final String pFsCommitterName,
-      final String pJobCommitterName,
-      final Class<? extends AbstractS3ACommitter> pCommitterClass,
-      final String pDescription) throws Exception {
-    this.fsCommitterName = pFsCommitterName;
-    this.jobCommitterName = pJobCommitterName;
-    this.committerClass = pCommitterClass;
-    this.description = pDescription;
-    setup();
+  public ITestS3ACommitterFactory(
+      final String fsCommitterName,
+      final String jobCommitterName,
+      final Class<? extends AbstractS3ACommitter> committerClass,
+      final String description) {
+    this.fsCommitterName = fsCommitterName;
+    this.jobCommitterName = jobCommitterName;
+    this.committerClass = committerClass;
+    this.description = description;
   }
 
   @Override
@@ -176,6 +181,7 @@ private static void maybeSetCommitterName(final Configuration conf, final String
   }
 
   @Override
+  @BeforeEach
   public void setup() throws Exception {
     // destroy all filesystems from previous runs.
     FileSystem.closeAllForUGI(UserGroupInformation.getCurrentUser());
@@ -210,14 +216,8 @@ protected void deleteTestDirInTeardown() {
    * Verify that if all config options are unset, the FileOutputCommitter
    * is returned.
    */
-  @MethodSource("params")
-  @ParameterizedTest(name = "{3}-fs=[{0}]-task=[{1}]-[{2}]")
-  public void testBinding(String pFsCommitterName,
-      String pJobCommitterName,
-      Class<? extends AbstractS3ACommitter> pCommitterClass,
-      String pDescription) throws Throwable {
-    initITestS3ACommitterFactory(pFsCommitterName, pJobCommitterName, pCommitterClass,
-        pDescription);
+  @Test
+  public void testBinding() throws Throwable {
     assertFactoryCreatesExpectedCommitter(committerClass);
   }
 

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/ITestUploadRecovery.java

@@ -29,7 +29,8 @@
 import org.assertj.core.api.Assumptions;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedClass;
 import org.junit.jupiter.params.provider.MethodSource;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -68,14 +69,16 @@
  * Test upload recovery by injecting failures into the response chain.
  * The tests are parameterized on upload buffering.
  * <p>
- * The test case {@link #testCommitOperations(String, boolean)} is independent of this option;
+ * The test case {@link #testCommitOperations()} is independent of this option;
  * the test parameterization only runs this once.
  * A bit inelegant but as the fault injection code is shared and the problem "adjacent"
  * this isolates all forms of upload recovery into the same test class without
  * wasting time with duplicate uploads.
  * <p>
  * Fault injection is implemented in {@link SdkFaultInjector}.
  */
+@ParameterizedClass(name="buffer={0}-commit-test={1}")
+@MethodSource("params")
 public class ITestUploadRecovery extends AbstractS3ACostTest {
 
   private static final Logger LOG =
@@ -102,22 +105,21 @@ public static Collection<Object[]> params() {
   /**
    * should the commit test be included?
    */
-  private boolean includeCommitTest;
+  private final boolean includeCommitTest;
 
   /**
    * Buffer type for this test run.
    */
-  private String buffer;
+  private final String buffer;
 
   /**
    * Parameterized test suite.
-   * @param pBuffer buffer type
-   * @param pIncludeCommitTest should the commit upload test be included?
+   * @param buffer buffer type
+   * @param includeCommitTest should the commit upload test be included?
    */
-  public void initITestUploadRecovery(final String pBuffer,
-      final boolean pIncludeCommitTest) {
-    this.includeCommitTest = pIncludeCommitTest;
-    this.buffer = pBuffer;
+  public ITestUploadRecovery(final String buffer, final boolean includeCommitTest) {
+    this.includeCommitTest = includeCommitTest;
+    this.buffer = buffer;
   }
 
   @Override
@@ -172,11 +174,8 @@ public void teardown() throws Exception {
   /**
    * Verify that failures of simple PUT requests can be recovered from.
    */
-  @MethodSource("params")
-  @ParameterizedTest(name = "{0}-commit-{1}")
-  public void testPutRecovery(String pBuffer,
-      boolean pIncludeCommitTest) throws Throwable {
-    initITestUploadRecovery(pBuffer, pIncludeCommitTest);
+  @Test
+  public void testPutRecovery() throws Throwable {
     describe("test put recovery");
     final S3AFileSystem fs = getFileSystem();
     final Path path = methodPath();
@@ -192,11 +191,8 @@ public void testPutRecovery(String pBuffer,
   /**
    * Validate recovery of multipart uploads within a magic write sequence.
    */
-  @MethodSource("params")
-  @ParameterizedTest(name = "{0}-commit-{1}")
-  public void testMagicWriteRecovery(String pBuffer,
-      boolean pIncludeCommitTest) throws Throwable {
-    initITestUploadRecovery(pBuffer, pIncludeCommitTest);
+  @Test
+  public void testMagicWriteRecovery() throws Throwable {
     describe("test magic write recovery with multipart uploads");
     final S3AFileSystem fs = getFileSystem();
 
@@ -235,11 +231,8 @@ public void testMagicWriteRecovery(String pBuffer,
   /**
    * Test the commit operations iff {@link #includeCommitTest} is true.
    */
-  @MethodSource("params")
-  @ParameterizedTest(name = "{0}-commit-{1}")
-  public void testCommitOperations(String pBuffer,
-      boolean pIncludeCommitTest) throws Throwable {
-    initITestUploadRecovery(pBuffer, pIncludeCommitTest);
+  @Test
+  public void testCommitOperations() throws Throwable {
     skipIfClientSideEncryption();
     Assumptions.assumeThat(includeCommitTest)
         .describedAs("commit test excluded")