HADOOP-19610. S3A: ITests to run under JUnit5 (#7814)

hadoop-aws tests which need to be parameterized on a class level
are configured to do so through the @ParameterizedClass tag.
Filesystem contract test suites in hadoop-common have
also been parameterized as appropriate.

There are custom JUnit tags declared in org.apache.hadoop.test.tags,
which add tag strings to test suites/cases declaring them.
They can be used on the command line and in IDEs to control
which tests are/are not executed.

@FlakyTest "flaky"
@loadtest "load"
@RootFilesystemTest "rootfilesystem"
@scaletest "scale"

For anyone migrating tests to JUnit 5
* Methods which subclass an existing test case MUST declare the @test
  tag again -it is no longer inherited.
* All overridden setup/teardown methods MUST be located and
  @BeforeEach/@AfterEach attribute added respectively
* Subclasses of a parameterized test suite MUST redeclare themselves
  as a @ParameterizedClass, and the binding mechanism again.
* Parameterized test suites SHOULD declare a pattern to generate an
  informative parameter value string for logs, IDEs and stack traces, e.g.
  @ParameterizedClass(name="performance-{0}")
* Test suites SHOULD add a org.apache.hadoop.test.tags tag to
  declare what kind of test it is. These tags are inherited, so it
  may be that only shared superclasses of test suites need to be tagged.
  The abstract filesystem contract tests are NOT declared as integration
  tests -implementations MUST do so if they are integration tests.

Contributed by Steve Loughran

Author: steveloughran

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMultipartUploaderTest.java

@@ -30,6 +30,7 @@
 import java.util.concurrent.CompletableFuture;
 
 import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.slf4j.Logger;
@@ -99,6 +100,7 @@ public void setup() throws Exception {
   }
 
   @Override
+  @AfterEach
   public void teardown() throws Exception {
     MultipartUploader uploader = getUploader(1);
     if (uploader != null) {

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractRootDirectoryTest.java

@@ -36,6 +36,7 @@
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.RemoteIterator;
 import org.apache.hadoop.test.LambdaTestUtils;
+import org.apache.hadoop.test.tags.RootFilesystemTest;
 
 import static org.apache.commons.lang3.StringUtils.join;
 import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile;
@@ -52,6 +53,7 @@
  * Only subclass this for tests against transient filesystems where
  * you don't care about the data.
  */
+@RootFilesystemTest
 public abstract class AbstractContractRootDirectoryTest extends AbstractFSContractTestBase {
   private static final Logger LOG =
       LoggerFactory.getLogger(AbstractContractRootDirectoryTest.class);

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractUnbufferTest.java

@@ -26,13 +26,19 @@
 
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.test.tags.FlakyTest;
 
 import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile;
 import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
 
 /**
  * Contract tests for {@link org.apache.hadoop.fs.CanUnbuffer#unbuffer}.
+ * Some of these test cases can fail if the FS read() call returns less
+ * than requested, which is a valid (possibly correct) implementation
+ * of {@code InputStream.read(buffer[])} which may return only those bytes
+ * which can be returned without blocking for more data.
  */
+@FlakyTest("buffer underflow")
 public abstract class AbstractContractUnbufferTest extends AbstractFSContractTestBase {
 
   private Path file;
@@ -105,6 +111,7 @@ public void testMultipleUnbuffers() throws IOException {
     }
   }
 
+
   @Test
   public void testUnbufferMultipleReads() throws IOException {
     describe("unbuffer a file multiple times");

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractVectoredReadTest.java

@@ -34,8 +34,10 @@
 import java.util.function.IntFunction;
 
 import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedClass;
 import org.junit.jupiter.params.provider.MethodSource;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -68,6 +70,8 @@
  * Both the original readVectored(allocator) and the readVectored(allocator, release)
  * operations are tested.
  */
+@ParameterizedClass(name="buffer-{0}")
+@MethodSource("params")
 public abstract class AbstractContractVectoredReadTest extends AbstractFSContractTestBase {
 
   private static final Logger LOG =
@@ -80,15 +84,15 @@ public abstract class AbstractContractVectoredReadTest extends AbstractFSContrac
   /**
    * Buffer allocator for vector IO.
    */
-  protected IntFunction<ByteBuffer> allocate;
+  private final IntFunction<ByteBuffer> allocate;
 
   /**
    * Buffer pool for vector IO.
    */
-  protected final ElasticByteBufferPool pool =
+  private final ElasticByteBufferPool pool =
           new WeakReferencedElasticByteBufferPool();
 
-  protected String bufferType;
+  private final String bufferType;
 
   /**
    * Path to the vector file.
@@ -106,8 +110,8 @@ public static List<String> params() {
     return Arrays.asList("direct", "array");
   }
 
-  public void initAbstractContractVectoredReadTest(String pBufferType) {
-    this.bufferType = pBufferType;
+  protected AbstractContractVectoredReadTest(String bufferType) {
+    this.bufferType = bufferType;
     final boolean isDirect = !"array".equals(bufferType);
     this.allocate = size -> pool.getBuffer(isDirect, size);
   }
@@ -147,6 +151,7 @@ public void setup() throws Exception {
     createFile(fs, vectorPath, true, DATASET);
   }
 
+  @AfterEach
   @Override
   public void teardown() throws Exception {
     pool.release();
@@ -177,10 +182,8 @@ protected FSDataInputStream openVectorFile(final FileSystem fs) throws IOExcepti
             .build());
   }
 
-  @MethodSource("params")
-  @ParameterizedTest(name = "Buffer type : {0}")
-  public void testVectoredReadMultipleRanges(String pBufferType) throws Exception {
-    initAbstractContractVectoredReadTest(pBufferType);
+  @Test
+  public void testVectoredReadMultipleRanges() throws Exception {
     List<FileRange> fileRanges = new ArrayList<>();
     for (int i = 0; i < 10; i++) {
       FileRange fileRange = FileRange.createFileRange(i * 100, 100);
@@ -201,10 +204,8 @@ public void testVectoredReadMultipleRanges(String pBufferType) throws Exception
     }
   }
 
-  @MethodSource("params")
-  @ParameterizedTest(name = "Buffer type : {0}")
-  public void testVectoredReadAndReadFully(String pBufferType)  throws Exception {
-    initAbstractContractVectoredReadTest(pBufferType);
+  @Test
+  public void testVectoredReadAndReadFully()  throws Exception {
     List<FileRange> fileRanges = new ArrayList<>();
     range(fileRanges, 100, 100);
     try (FSDataInputStream in = openVectorFile()) {
@@ -219,10 +220,8 @@ public void testVectoredReadAndReadFully(String pBufferType)  throws Exception {
     }
   }
 
-  @MethodSource("params")
-  @ParameterizedTest(name = "Buffer type : {0}")
-  public void testVectoredReadWholeFile(String pBufferType)  throws Exception {
-    initAbstractContractVectoredReadTest(pBufferType);
+  @Test
+  public void testVectoredReadWholeFile()  throws Exception {
     describe("Read the whole file in one single vectored read");
     List<FileRange> fileRanges = new ArrayList<>();
     range(fileRanges, 0, DATASET_LEN);
@@ -240,10 +239,8 @@ public void testVectoredReadWholeFile(String pBufferType)  throws Exception {
    * As the minimum seek value is 4*1024,none of the below ranges
    * will get merged.
    */
-  @MethodSource("params")
-  @ParameterizedTest(name = "Buffer type : {0}")
-  public void testDisjointRanges(String pBufferType) throws Exception {
-    initAbstractContractVectoredReadTest(pBufferType);
+  @Test
+  public void testDisjointRanges() throws Exception {
     List<FileRange> fileRanges = new ArrayList<>();
     range(fileRanges, 0, 100);
     range(fileRanges, 4_000 + 101, 100);
@@ -259,10 +256,8 @@ public void testDisjointRanges(String pBufferType) throws Exception {
    * As the minimum seek value is 4*1024, all the below ranges
    * will get merged into one.
    */
-  @MethodSource("params")
-  @ParameterizedTest(name = "Buffer type : {0}")
-  public void testAllRangesMergedIntoOne(String pBufferType) throws Exception {
-    initAbstractContractVectoredReadTest(pBufferType);
+  @Test
+  public void testAllRangesMergedIntoOne() throws Exception {
     List<FileRange> fileRanges = new ArrayList<>();
     final int length = 100;
     range(fileRanges, 0, length);
@@ -279,10 +274,8 @@ public void testAllRangesMergedIntoOne(String pBufferType) throws Exception {
    * As the minimum seek value is 4*1024, the first three ranges will be
    * merged into and other two will remain as it is.
    */
-  @MethodSource("params")
-  @ParameterizedTest(name = "Buffer type : {0}")
-  public void testSomeRangesMergedSomeUnmerged(String pBufferType) throws Exception {
-    initAbstractContractVectoredReadTest(pBufferType);
+  @Test
+  public void testSomeRangesMergedSomeUnmerged() throws Exception {
     FileSystem fs = getFileSystem();
     List<FileRange> fileRanges = new ArrayList<>();
     range(fileRanges, 8 * 1024, 100);
@@ -306,10 +299,8 @@ public void testSomeRangesMergedSomeUnmerged(String pBufferType) throws Exceptio
    * Most file systems won't support overlapping ranges.
    * Currently, only Raw Local supports it.
    */
-  @MethodSource("params")
-  @ParameterizedTest(name = "Buffer type : {0}")
-  public void testOverlappingRanges(String pBufferType) throws Exception {
-    initAbstractContractVectoredReadTest(pBufferType);
+  @Test
+  public void testOverlappingRanges() throws Exception {
     if (!isSupported(VECTOR_IO_OVERLAPPING_RANGES)) {
       verifyExceptionalVectoredRead(
               getSampleOverlappingRanges(),
@@ -327,10 +318,8 @@ public void testOverlappingRanges(String pBufferType) throws Exception {
   /**
    * Same ranges are special case of overlapping.
    */
-  @MethodSource("params")
-  @ParameterizedTest(name = "Buffer type : {0}")
-  public void testSameRanges(String pBufferType) throws Exception {
-    initAbstractContractVectoredReadTest(pBufferType);
+  @Test
+  public void testSameRanges() throws Exception {
     if (!isSupported(VECTOR_IO_OVERLAPPING_RANGES)) {
       verifyExceptionalVectoredRead(
               getSampleSameRanges(),
@@ -348,10 +337,8 @@ public void testSameRanges(String pBufferType) throws Exception {
   /**
    * A null range is not permitted.
    */
-  @MethodSource("params")
-  @ParameterizedTest(name = "Buffer type : {0}")
-  public void testNullRange(String pBufferType) throws Exception {
-    initAbstractContractVectoredReadTest(pBufferType);
+  @Test
+  public void testNullRange() throws Exception {
     List<FileRange> fileRanges = new ArrayList<>();
     range(fileRanges, 500, 100);
     fileRanges.add(null);
@@ -362,19 +349,15 @@ public void testNullRange(String pBufferType) throws Exception {
   /**
    * A null range is not permitted.
    */
-  @MethodSource("params")
-  @ParameterizedTest(name = "Buffer type : {0}")
-  public void testNullRangeList(String pBufferType) throws Exception {
-    initAbstractContractVectoredReadTest(pBufferType);
+  @Test
+  public void testNullRangeList() throws Exception {
     verifyExceptionalVectoredRead(
         null,
         NullPointerException.class);
   }
 
-  @MethodSource("params")
-  @ParameterizedTest(name = "Buffer type : {0}")
-  public void testSomeRandomNonOverlappingRanges(String pBufferType) throws Exception {
-    initAbstractContractVectoredReadTest(pBufferType);
+  @Test
+  public void testSomeRandomNonOverlappingRanges() throws Exception {
     List<FileRange> fileRanges = new ArrayList<>();
     range(fileRanges, 500, 100);
     range(fileRanges, 1000, 200);
@@ -387,10 +370,8 @@ public void testSomeRandomNonOverlappingRanges(String pBufferType) throws Except
     }
   }
 
-  @MethodSource("params")
-  @ParameterizedTest(name = "Buffer type : {0}")
-  public void testConsecutiveRanges(String pBufferType) throws Exception {
-    initAbstractContractVectoredReadTest(pBufferType);
+  @Test
+  public void testConsecutiveRanges() throws Exception {
     List<FileRange> fileRanges = new ArrayList<>();
     final int offset = 500;
     final int length = 2011;
@@ -403,10 +384,8 @@ public void testConsecutiveRanges(String pBufferType) throws Exception {
     }
   }
 
-  @MethodSource("params")
-  @ParameterizedTest(name = "Buffer type : {0}")
-  public void testEmptyRanges(String pBufferType) throws Exception {
-    initAbstractContractVectoredReadTest(pBufferType);
+  @Test
+  public void testEmptyRanges() throws Exception {
     List<FileRange> fileRanges = new ArrayList<>();
     try (FSDataInputStream in = openVectorFile()) {
       in.readVectored(fileRanges, allocate);
@@ -425,10 +404,8 @@ public void testEmptyRanges(String pBufferType) throws Exception {
    * The contract option {@link ContractOptions#VECTOR_IO_EARLY_EOF_CHECK} is used
    * to determine which check to perform.
    */
-  @MethodSource("params")
-  @ParameterizedTest(name = "Buffer type : {0}")
-  public void testEOFRanges(String pBufferType)  throws Exception {
-    initAbstractContractVectoredReadTest(pBufferType);
+  @Test
+  public void testEOFRanges()  throws Exception {
     describe("Testing reading with an offset past the end of the file");
     List<FileRange> fileRanges = range(DATASET_LEN + 1, 100);
 
@@ -441,10 +418,8 @@ public void testEOFRanges(String pBufferType)  throws Exception {
   }
 
 
-  @MethodSource("params")
-  @ParameterizedTest(name = "Buffer type : {0}")
-  public void testVectoredReadWholeFilePlusOne(String pBufferType)  throws Exception {
-    initAbstractContractVectoredReadTest(pBufferType);
+  @Test
+  public void testVectoredReadWholeFilePlusOne()  throws Exception {
     describe("Try to read whole file plus 1 byte");
     List<FileRange> fileRanges = range(0, DATASET_LEN + 1);
 
@@ -471,35 +446,29 @@ private void expectEOFinRead(final List<FileRange> fileRanges) throws Exception
     }
   }
 
-  @MethodSource("params")
-  @ParameterizedTest(name = "Buffer type : {0}")
-  public void testNegativeLengthRange(String pBufferType)  throws Exception {
-    initAbstractContractVectoredReadTest(pBufferType);
+  @Test
+  public void testNegativeLengthRange()  throws Exception {
+
     verifyExceptionalVectoredRead(range(0, -50), IllegalArgumentException.class);
   }
 
-  @MethodSource("params")
-  @ParameterizedTest(name = "Buffer type : {0}")
-  public void testNegativeOffsetRange(String pBufferType)  throws Exception {
-    initAbstractContractVectoredReadTest(pBufferType);
+  @Test
+  public void testNegativeOffsetRange()  throws Exception {
     verifyExceptionalVectoredRead(range(-1, 50), EOFException.class);
   }
 
-  @MethodSource("params")
-  @ParameterizedTest(name = "Buffer type : {0}")
-  public void testNullReleaseOperation(String pBufferType)  throws Exception {
-    initAbstractContractVectoredReadTest(pBufferType);
+  @Test
+  public void testNullReleaseOperation()  throws Exception {
+
     final List<FileRange> range = range(0, 10);
     try (FSDataInputStream in = openVectorFile()) {
-        intercept(NullPointerException.class, () ->
-            in.readVectored(range, allocate, null));
+      intercept(NullPointerException.class, () ->
+          in.readVectored(range, allocate, null));
     }
   }
 
-  @MethodSource("params")
-  @ParameterizedTest(name = "Buffer type : {0}")
-  public void testNormalReadAfterVectoredRead(String pBufferType) throws Exception {
-    initAbstractContractVectoredReadTest(pBufferType);
+  @Test
+  public void testNormalReadAfterVectoredRead() throws Exception {
     List<FileRange> fileRanges = createSampleNonOverlappingRanges();
     try (FSDataInputStream in = openVectorFile()) {
       in.readVectored(fileRanges, allocate);
@@ -514,10 +483,8 @@ public void testNormalReadAfterVectoredRead(String pBufferType) throws Exception
     }
   }
 
-  @MethodSource("params")
-  @ParameterizedTest(name = "Buffer type : {0}")
-  public void testVectoredReadAfterNormalRead(String pBufferType) throws Exception {
-    initAbstractContractVectoredReadTest(pBufferType);
+  @Test
+  public void testVectoredReadAfterNormalRead() throws Exception {
     List<FileRange> fileRanges = createSampleNonOverlappingRanges();
     try (FSDataInputStream in = openVectorFile()) {
       // read starting 200 bytes
@@ -532,10 +499,8 @@ public void testVectoredReadAfterNormalRead(String pBufferType) throws Exception
     }
   }
 
-  @MethodSource("params")
-  @ParameterizedTest(name = "Buffer type : {0}")
-  public void testMultipleVectoredReads(String pBufferType) throws Exception {
-    initAbstractContractVectoredReadTest(pBufferType);
+  @Test
+  public void testMultipleVectoredReads() throws Exception {
     List<FileRange> fileRanges1 = createSampleNonOverlappingRanges();
     List<FileRange> fileRanges2 = createSampleNonOverlappingRanges();
     try (FSDataInputStream in = openVectorFile()) {
@@ -553,10 +518,8 @@ public void testMultipleVectoredReads(String pBufferType) throws Exception {
    * operation and then uses a separate thread pool to process the
    * results asynchronously.
    */
-  @MethodSource("params")
-  @ParameterizedTest(name = "Buffer type : {0}")
-  public void testVectoredIOEndToEnd(String pBufferType) throws Exception {
-    initAbstractContractVectoredReadTest(pBufferType);
+  @Test
+  public void testVectoredIOEndToEnd() throws Exception {
     List<FileRange> fileRanges = new ArrayList<>();
     range(fileRanges, 8 * 1024, 100);
     range(fileRanges, 14 * 1024, 100);