HADOOP-19610 huge and contract test failures

If you override a test method from a superclass you need to declare it
as @test again; the tag is no longer @inherited as it was in Junit4

* Huge tests visibily failed as they were incomplete
* Contract tests were failing to run some tests cases

Author: steveloughran

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractBulkDelete.java

@@ -149,6 +149,7 @@ public void testPageSizeWhenMultiObjectsDisabled() throws Exception {
   }
 
   @Override
+  @Test
   public void testDeletePathsDirectory() throws Exception {
     List<Path> paths = new ArrayList<>();
     Path dirPath = new Path(basePath, "dir");
@@ -180,6 +181,7 @@ public void testBulkDeleteParentDirectoryWithDirectories() throws Exception {
     assertIsDirectory(subDir);
   }
 
+  @Test
   public void testBulkDeleteParentDirectoryWithFiles() throws Exception {
     List<Path> paths = new ArrayList<>();
     Path dirPath = new Path(basePath, "dir");

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractDistCp.java

@@ -18,6 +18,8 @@
 
 package org.apache.hadoop.fs.contract.s3a;
 
+import org.junit.jupiter.api.Test;
+
 import static org.apache.hadoop.fs.s3a.Constants.*;
 import static org.apache.hadoop.fs.s3a.S3ATestConstants.SCALE_TEST_TIMEOUT_MILLIS;
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.skipIfAnalyticsAcceleratorEnabled;
@@ -62,6 +64,7 @@ protected S3AContract createContract(Configuration conf) {
     return new S3AContract(conf);
   }
 
+  @Test
   @Override
   public void testDistCpWithIterator() throws Exception {
     final long renames = getRenameOperationCount();
@@ -70,6 +73,7 @@ public void testDistCpWithIterator() throws Exception {
         renames, "Expected no renames for a direct write distcp");
   }
 
+  @Test
   @Override
   public void testNonDirectWrite() throws Exception {
     final long renames = getRenameOperationCount();
@@ -78,6 +82,7 @@ public void testNonDirectWrite() throws Exception {
         "Expected 2 renames for a non-direct write distcp");
   }
 
+  @Test
   @Override
   public void testDistCpUpdateCheckFileSkip() throws Exception {
     // Currently analytics accelerator does not support reading of files that have been overwritten.

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMultipartUploader.java

@@ -25,6 +25,7 @@
 import org.apache.hadoop.test.tags.ScaleTest;
 
 import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
 
 import static org.apache.hadoop.fs.contract.ContractTestUtils.skip;
 import static org.apache.hadoop.fs.s3a.S3ATestConstants.DEFAULT_SCALE_TESTS_ENABLED;
@@ -115,21 +116,26 @@ public void setup() throws Exception {
   /**
    * S3 has no concept of directories, so this test does not apply.
    */
+  @Test
+  @Override
   public void testDirectoryInTheWay() throws Exception {
     skip("Unsupported");
   }
 
+  @Test
   @Override
   public void testMultipartUploadReverseOrder() throws Exception {
     skip("skipped for speed");
   }
 
+  @Test
   @Override
   public void testMultipartUploadReverseOrderNonContiguousPartNumbers() throws Exception {
     assumeNotS3ExpressFileSystem(getFileSystem());
     super.testMultipartUploadReverseOrderNonContiguousPartNumbers();
   }
 
+  @Test
   @Override
   public void testConcurrentUploads() throws Throwable {
     assumeNotS3ExpressFileSystem(getFileSystem());

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractRename.java

@@ -55,6 +55,7 @@ protected AbstractFSContract createContract(Configuration conf) {
     return new S3AContract(conf);
   }
 
+  @Test
   @Override
   public void testRenameDirIntoExistingDir() throws Throwable {
     describe("S3A rename into an existing directory returns false");

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestCustomSigner.java

@@ -55,6 +55,7 @@
 import org.apache.hadoop.fs.s3a.auth.delegation.DelegationTokenProvider;
 import org.apache.hadoop.security.UserGroupInformation;
 
+import static org.apache.hadoop.fs.s3a.Constants.CHECKSUM_ALGORITHM;
 import static org.apache.hadoop.fs.s3a.Constants.CUSTOM_SIGNERS;
 import static org.apache.hadoop.fs.s3a.Constants.ENABLE_MULTI_DELETE;
 import static org.apache.hadoop.fs.s3a.Constants.SIGNING_ALGORITHM_S3;
@@ -103,6 +104,14 @@ public ITestCustomSigner(
     this.bulkDelete = bulkDelete;
   }
 
+  @Override
+  protected Configuration createConfiguration() {
+    final Configuration conf = super.createConfiguration();
+    // customer signer doesn't work
+    removeBaseAndBucketOverrides(conf, CHECKSUM_ALGORITHM);
+    return conf;
+  }
+
   @Override
   @BeforeEach
   public void setup() throws Exception {
@@ -224,7 +233,7 @@ private Configuration createTestConfig(String identifier) {
     // Having the checksum algorithm in this test causes
     // x-amz-sdk-checksum-algorithm specified, but no corresponding
     // x-amz-checksum-* or x-amz-trailer headers were found
-    conf.unset(Constants.CHECKSUM_ALGORITHM);
+    conf.unset(CHECKSUM_ALGORITHM);
 
     // make absolutely sure there is no caching.
     disableFilesystemCaching(conf);

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/magic/ITestS3AHugeMagicCommits.java

@@ -24,7 +24,9 @@
 
 import org.assertj.core.api.Assertions;
 import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.MethodOrderer;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -42,6 +44,7 @@
 import org.apache.hadoop.fs.s3a.commit.impl.CommitOperations;
 import org.apache.hadoop.fs.s3a.scale.AbstractSTestS3AHugeFiles;
 import org.apache.hadoop.fs.store.audit.AuditSpan;
+import org.apache.hadoop.test.tags.ScaleTest;
 
 import static org.apache.hadoop.fs.s3a.MultipartTestUtils.listMultipartUploads;
 import static org.apache.hadoop.fs.s3a.Statistic.MULTIPART_UPLOAD_ABORT_UNDER_PATH_INVOKED;
@@ -57,6 +60,8 @@
  *
  * This is a scale test.
  */
+@ScaleTest
+@TestMethodOrder(MethodOrderer.Alphanumeric.class)
 public class ITestS3AHugeMagicCommits extends AbstractSTestS3AHugeFiles {
   private static final Logger LOG = LoggerFactory.getLogger(
       ITestS3AHugeMagicCommits.class);
@@ -136,6 +141,7 @@ public void test_000_CleanupPendingUploads() throws IOException {
     LOG.info("Aborted {} uploads under {}", count, key);
   }
 
+  @Test
   @Override
   public void test_030_postCreationAssertions() throws Throwable {
     describe("Committing file");
@@ -191,21 +197,25 @@ private void skipQuietly(String text) {
     describe("Skipping: %s", text);
   }
 
+  @Test
   @Override
   public void test_040_PositionedReadHugeFile() {
     skipQuietly("test_040_PositionedReadHugeFile");
   }
 
+  @Test
   @Override
   public void test_050_readHugeFile() {
     skipQuietly("readHugeFile");
   }
 
+  @Test
   @Override
   public void test_100_renameHugeFile() {
     skipQuietly("renameHugeFile");
   }
 
+  @Test
   @Override
   public void test_800_DeleteHugeFiles() throws IOException {
     if (getFileSystem() != null) {

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesNoMultipart.java

@@ -19,6 +19,7 @@
 package org.apache.hadoop.fs.s3a.scale;
 
 import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.Test;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.s3a.Constants;
@@ -100,6 +101,7 @@ protected Configuration createScaleConfiguration() {
   /**
    * Verify multipart copy is disabled.
    */
+  @Test
   @Override
   public void test_030_postCreationAssertions() throws Throwable {
     super.test_030_postCreationAssertions();

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesStorageClass.java

@@ -20,6 +20,9 @@
 
 import java.io.IOException;
 
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -44,6 +47,7 @@
  * for creating and renaming huge files with multipart upload requests.
  */
 @ScaleTest
+@TestMethodOrder(MethodOrderer.Alphanumeric.class)
 public class ITestS3AHugeFilesStorageClass extends AbstractSTestS3AHugeFiles {
 
   private static final Logger LOG = LoggerFactory.getLogger(ITestS3AHugeFilesStorageClass.class);
@@ -64,33 +68,39 @@ protected String getBlockOutputBufferName() {
     return Constants.FAST_UPLOAD_BUFFER_ARRAY;
   }
 
+  @Test
   @Override
   public void test_010_CreateHugeFile() throws IOException {
     super.test_010_CreateHugeFile();
     assertStorageClass(getPathOfFileToCreate());
   }
 
+  @Test
   @Override
   public void test_030_postCreationAssertions() throws Throwable {
     super.test_030_postCreationAssertions();
     assertStorageClass(getPathOfFileToCreate());
   }
 
   @Override
+  @Test
   public void test_040_PositionedReadHugeFile() throws Throwable {
     skipQuietly("PositionedReadHugeFile");
   }
 
+  @Test
   @Override
   public void test_050_readHugeFile() throws Throwable {
     skipQuietly("readHugeFile");
   }
 
+  @Test
   @Override
   public void test_090_verifyRenameSourceEncryption() throws IOException {
     skipQuietly("verifyRenameSourceEncryption");
   }
 
+  @Test
   @Override
   public void test_100_renameHugeFile() throws Throwable {
     Path hugefile = getHugefile();
@@ -112,6 +122,7 @@ public void test_100_renameHugeFile() throws Throwable {
     assertStorageClass(hugefileRenamed);
   }
 
+  @Test
   @Override
   public void test_110_verifyRenameDestEncryption() throws IOException {
     skipQuietly("verifyRenameDestEncryption");