YARN-11687. Update CGroupsResourceCalculator to track usages using cgroupv2

LICENSE-binary

@@ -216,7 +216,6 @@ com.aliyun:aliyun-java-sdk-kms:2.11.0
 com.aliyun:aliyun-java-sdk-ram:3.1.0
 com.aliyun:aliyun-java-sdk-sts:3.0.0
 com.aliyun.oss:aliyun-sdk-oss:3.13.2
-com.amazonaws:aws-java-sdk-bundle:1.12.599
 com.cedarsoftware:java-util:1.9.0
 com.cedarsoftware:json-io:2.5.1
 com.fasterxml.jackson.core:jackson-annotations:2.12.7

dev-support/bin/yetus-wrapper

@@ -77,7 +77,7 @@ WANTED="$1"
 shift
 ARGV=("$@")
 
-HADOOP_YETUS_VERSION=${HADOOP_YETUS_VERSION:-0.14.0}
+HADOOP_YETUS_VERSION=${HADOOP_YETUS_VERSION:-0.14.1}
 BIN=$(yetus_abs "${BASH_SOURCE-$0}")
 BINDIR=$(dirname "${BIN}")
 
@@ -123,7 +123,7 @@ fi
 ## need to DL, etc
 ##
 
-BASEURL="https://archive.apache.org/dist/yetus/${HADOOP_YETUS_VERSION}/"
+BASEURL="https://downloads.apache.org/yetus/${HADOOP_YETUS_VERSION}/"
 TARBALL="${YETUS_PREFIX}-${HADOOP_YETUS_VERSION}-bin.tar"
 
 GPGBIN=$(command -v gpg)

dev-support/docker/Dockerfile_windows_10

@@ -61,8 +61,8 @@ RUN powershell Invoke-WebRequest -URI https://cdn.azul.com/zulu/bin/zulu8.62.0.1
 RUN powershell Expand-Archive -Path $Env:TEMP\zulu8.62.0.19-ca-jdk8.0.332-win_x64.zip -DestinationPath "C:\Java"
 
 # Install Apache Maven.
-RUN powershell Invoke-WebRequest -URI https://archive.apache.org/dist/maven/maven-3/3.8.6/binaries/apache-maven-3.8.6-bin.zip -OutFile $Env:TEMP\apache-maven-3.8.6-bin.zip
-RUN powershell Expand-Archive -Path $Env:TEMP\apache-maven-3.8.6-bin.zip -DestinationPath "C:\Maven"
+RUN powershell Invoke-WebRequest -URI https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.zip -OutFile $Env:TEMP\apache-maven-3.8.8-bin.zip
+RUN powershell Expand-Archive -Path $Env:TEMP\apache-maven-3.8.8-bin.zip -DestinationPath "C:\Maven"
 
 # Install CMake 3.19.0.
 RUN powershell Invoke-WebRequest -URI https://cmake.org/files/v3.19/cmake-3.19.0-win64-x64.zip -OutFile $Env:TEMP\cmake-3.19.0-win64-x64.zip
@@ -135,7 +135,7 @@ ENV MAVEN_OPTS '-Xmx2048M -Xss128M'
 ENV IS_WINDOWS 1
 RUN setx PATH "%PATH%;%ALLUSERSPROFILE%\chocolatey\bin"
 RUN setx PATH "%PATH%;%JAVA_HOME%\bin"
-RUN setx PATH "%PATH%;C:\Maven\apache-maven-3.8.6\bin"
+RUN setx PATH "%PATH%;C:\Maven\apache-maven-3.8.8\bin"
 RUN setx PATH "%PATH%;C:\CMake\cmake-3.19.0-win64-x64\bin"
 RUN setx PATH "%PATH%;C:\ZStd"
 RUN setx PATH "%PATH%;C:\Program Files\Git\usr\bin"

hadoop-common-project/hadoop-auth/pom.xml

@@ -136,7 +136,11 @@
     </dependency>
     <dependency>
       <groupId>org.apache.kerby</groupId>
-      <artifactId>kerb-simplekdc</artifactId>
+      <artifactId>kerb-core</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.kerby</groupId>
+      <artifactId>kerb-util</artifactId>
     </dependency>
     <dependency>
       <groupId>org.apache.directory.server</groupId>

hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh

@@ -1911,7 +1911,7 @@ function hadoop_start_secure_daemon
   if [[ ! -f "${jsvc}" ]]; then
     hadoop_error "JSVC_HOME is not set or set incorrectly. jsvc is required to run secure"
     hadoop_error "or privileged daemons. Please download and install jsvc from "
-    hadoop_error "http://archive.apache.org/dist/commons/daemon/binaries/ "
+    hadoop_error "https://downloads.apache.org/commons/daemon/binaries/ "
     hadoop_error "and set JSVC_HOME to the directory containing the jsvc binary."
     exit 1
   fi

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoUtils.java

@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto;
+
+import java.security.Provider;
+import java.security.Security;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.store.LogExactlyOnce;
+
+/** Utility methods for the crypto related features. */
+@InterfaceAudience.Private
+public final class CryptoUtils {
+  static final Logger LOG = LoggerFactory.getLogger(CryptoUtils.class);
+  private static final LogExactlyOnce LOG_FAILED_TO_LOAD_CLASS = new LogExactlyOnce(LOG);
+  private static final LogExactlyOnce LOG_FAILED_TO_ADD_PROVIDER = new LogExactlyOnce(LOG);
+
+  private static final String BOUNCY_CASTLE_PROVIDER_CLASS
+      = "org.bouncycastle.jce.provider.BouncyCastleProvider";
+  static final String BOUNCY_CASTLE_PROVIDER_NAME = "BC";
+
+  /**
+   * Get the security provider value specified in
+   * {@link CommonConfigurationKeysPublic#HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY}
+   * from the given conf.
+   *
+   * @param conf the configuration
+   * @return the configured provider, if there is any; otherwise, return an empty string.
+   */
+  public static String getJceProvider(Configuration conf) {
+    final String provider = conf.getTrimmed(
+        CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY, "");
+    final boolean autoAdd = conf.getBoolean(
+        CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_AUTO_ADD_KEY,
+        CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_AUTO_ADD_DEFAULT);
+
+    // For backward compatible, auto-add BOUNCY_CASTLE_PROVIDER_CLASS when the provider is "BC".
+    if (autoAdd && BOUNCY_CASTLE_PROVIDER_NAME.equals(provider)) {
+      try {
+        // Use reflection in order to avoid statically loading the class.
+        final Class<?> clazz = Class.forName(BOUNCY_CASTLE_PROVIDER_CLASS);
+        Security.addProvider((Provider) clazz.getConstructor().newInstance());
+        LOG.debug("Successfully added security provider {}", provider);
+        if (LOG.isTraceEnabled()) {
+          LOG.trace("Trace", new Throwable());
+        }
+      } catch (ClassNotFoundException e) {
+        LOG_FAILED_TO_LOAD_CLASS.warn("Failed to load " + BOUNCY_CASTLE_PROVIDER_CLASS, e);
+      } catch (Exception e) {
+        LOG_FAILED_TO_ADD_PROVIDER.warn("Failed to add security provider for {}", provider, e);
+      }
+    }
+    return provider;
+  }
+
+  private CryptoUtils() { }
+}

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/JceCtrCryptoCodec.java

@@ -17,7 +17,6 @@
  */
 package org.apache.hadoop.crypto;
 
-import org.bouncycastle.jce.provider.BouncyCastleProvider;
 import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -27,13 +26,11 @@
 import java.nio.ByteBuffer;
 import java.security.GeneralSecurityException;
 import java.security.SecureRandom;
-import java.security.Security;
 import javax.crypto.Cipher;
 import javax.crypto.spec.IvParameterSpec;
 import javax.crypto.spec.SecretKeySpec;
 import org.slf4j.Logger;
 
-import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY;
 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_JAVA_SECURE_RANDOM_ALGORITHM_KEY;
 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_JAVA_SECURE_RANDOM_ALGORITHM_DEFAULT;
 
@@ -48,10 +45,6 @@ public String getProvider() {
     return provider;
   }
 
-  public void setProvider(String provider) {
-    this.provider = provider;
-  }
-
   public void calculateIV(byte[] initIV, long counter,
                             byte[] iv, int blockSize) {
     Preconditions.checkArgument(initIV.length == blockSize);
@@ -82,17 +75,15 @@ public Configuration getConf() {
 
   public void setConf(Configuration conf) {
     this.conf = conf;
-    setProvider(conf.get(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY));
-    if (BouncyCastleProvider.PROVIDER_NAME.equals(provider)) {
-      Security.addProvider(new BouncyCastleProvider());
-    }
+    this.provider = CryptoUtils.getJceProvider(conf);
+
     final String secureRandomAlg =
           conf.get(
               HADOOP_SECURITY_JAVA_SECURE_RANDOM_ALGORITHM_KEY,
               HADOOP_SECURITY_JAVA_SECURE_RANDOM_ALGORITHM_DEFAULT);
 
     try {
-      random = (provider != null)
+      random = (provider != null && !provider.isEmpty())
             ? SecureRandom.getInstance(secureRandomAlg, provider)
             : SecureRandom.getInstance(secureRandomAlg);
     } catch(GeneralSecurityException e) {

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java

@@ -26,7 +26,6 @@
 import java.io.OutputStreamWriter;
 import java.nio.charset.StandardCharsets;
 import java.security.NoSuchAlgorithmException;
-import java.security.Security;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.Date;
@@ -35,17 +34,16 @@
 import java.util.Map;
 import java.util.Objects;
 
-import org.bouncycastle.jce.provider.BouncyCastleProvider;
 import com.google.gson.stream.JsonReader;
 import com.google.gson.stream.JsonWriter;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.CryptoUtils;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 
 import javax.crypto.KeyGenerator;
 
-import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY;
 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_JCEKS_KEY_SERIALFILTER;
 
 /**
@@ -410,10 +408,7 @@ public KeyProvider(Configuration conf) {
                       JCEKS_KEY_SERIALFILTER_DEFAULT);
       System.setProperty(JCEKS_KEY_SERIAL_FILTER, serialFilter);
     }
-    String jceProvider = conf.get(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY);
-    if (BouncyCastleProvider.PROVIDER_NAME.equals(jceProvider)) {
-      Security.addProvider(new BouncyCastleProvider());
-    }
+    CryptoUtils.getJceProvider(conf);
   }
 
   /**

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/package-info.java

@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/** Crypto related classes. */
+package org.apache.hadoop.crypto;

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java

@@ -773,6 +773,9 @@ public class CommonConfigurationKeysPublic {
    */
   public static final String HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY =
     "hadoop.security.crypto.jce.provider";
+  public static final String HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_AUTO_ADD_KEY =
+      "hadoop.security.crypto.jce.provider.auto-add";
+  public static final boolean HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_AUTO_ADD_DEFAULT = true;
   /**
    * @see
    * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CodecPool.java

@@ -25,6 +25,7 @@
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configurable;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.util.ReflectionUtils;
 
@@ -152,6 +153,9 @@ public static Compressor getCompressor(CompressionCodec codec, Configuration con
       compressor = codec.createCompressor();
       LOG.info("Got brand-new compressor ["+codec.getDefaultExtension()+"]");
     } else {
+      if (conf == null && codec instanceof Configurable) {
+        conf = ((Configurable)codec).getConf();
+      }
       compressor.reinit(conf);
       if(LOG.isDebugEnabled()) {
         LOG.debug("Got recycled compressor");

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslConstants.java

@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.security;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * SASL related constants.
+ */
+@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
+@InterfaceStability.Evolving
+public class SaslConstants {
+  public static final Logger LOG = LoggerFactory.getLogger(SaslConstants.class);
+
+  private static final String SASL_MECHANISM_ENV = "HADOOP_SASL_MECHANISM";
+  public static final String SASL_MECHANISM;
+  private static final String SASL_MECHANISM_DEFAULT = "DIGEST-MD5";
+
+  static {
+    final String mechanism = System.getenv(SASL_MECHANISM_ENV);
+    LOG.debug("{} = {} (env)", SASL_MECHANISM_ENV, mechanism);
+    SASL_MECHANISM = mechanism != null? mechanism : SASL_MECHANISM_DEFAULT;
+    LOG.debug("{} = {} (effective)", SASL_MECHANISM_ENV, SASL_MECHANISM);
+  }
+
+  private SaslConstants() {}
+}

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcServer.java

@@ -223,8 +223,8 @@ public enum AuthMethod {
     SIMPLE((byte) 80, ""),
     KERBEROS((byte) 81, "GSSAPI"),
     @Deprecated
-    DIGEST((byte) 82, "DIGEST-MD5"),
-    TOKEN((byte) 82, "DIGEST-MD5"),
+    DIGEST((byte) 82, SaslConstants.SASL_MECHANISM),
+    TOKEN((byte) 82, SaslConstants.SASL_MECHANISM),
     PLAIN((byte) 83, "PLAIN");
 
     /** The code for this method. */
@@ -273,7 +273,7 @@ public void write(DataOutput out) throws IOException {
     }
   };
 
-  /** CallbackHandler for SASL DIGEST-MD5 mechanism */
+  /** CallbackHandler for SASL mechanism. */
   @InterfaceStability.Evolving
   public static class SaslDigestCallbackHandler implements CallbackHandler {
     private SecretManager<TokenIdentifier> secretManager;
@@ -309,7 +309,7 @@ public void handle(Callback[] callbacks) throws InvalidToken,
           continue; // realm is ignored
         } else {
           throw new UnsupportedCallbackException(callback,
-              "Unrecognized SASL DIGEST-MD5 Callback");
+              "Unrecognized SASL Callback");
         }
       }
       if (pc != null) {
@@ -319,11 +319,8 @@ public void handle(Callback[] callbacks) throws InvalidToken,
         UserGroupInformation user = null;
         user = tokenIdentifier.getUser(); // may throw exception
         connection.attemptingUser = user;
-
-        if (LOG.isDebugEnabled()) {
-          LOG.debug("SASL server DIGEST-MD5 callback: setting password "
-              + "for client: " + tokenIdentifier.getUser());
-        }
+
+        LOG.debug("SASL server callback: setting password for client: {}", user);
         pc.setPassword(password);
       }
       if (ac != null) {
@@ -339,8 +336,7 @@ public void handle(Callback[] callbacks) throws InvalidToken,
             UserGroupInformation logUser =
               getIdentifier(authzid, secretManager).getUser();
             String username = logUser == null ? null : logUser.getUserName();
-            LOG.debug("SASL server DIGEST-MD5 callback: setting "
-                + "canonicalized client ID: " + username);
+            LOG.debug("SASL server callback: setting authorizedID: {}", username);
           }
           ac.setAuthorizedID(authzid);
         }