Merge branch 'apache:trunk' into YARN-11484

Author: slfan1989

.gitignore

@@ -1,4 +1,4 @@
-.DS_Store
+*.DS_Store
 *.iml
 *.ipr
 *.iws

dev-support/bin/hadoop.sh

@@ -21,18 +21,20 @@
 personality_plugins "all,-ant,-gradle,-scalac,-scaladoc"
 
 # These flags are needed to run Yetus against Hadoop on Windows.
-WINDOWS_FLAGS="-Pnative-win
-  -Dhttps.protocols=TLSv1.2
-  -Drequire.openssl
-  -Drequire.test.libhadoop
-  -Dshell-executable=${BASH_EXECUTABLE}
-  -Dopenssl.prefix=${VCPKG_INSTALLED_PACKAGES}
-  -Dcmake.prefix.path=${VCPKG_INSTALLED_PACKAGES}
-  -Dwindows.cmake.toolchain.file=${CMAKE_TOOLCHAIN_FILE}
-  -Dwindows.cmake.build.type=RelWithDebInfo
-  -Dwindows.build.hdfspp.dll=off
-  -Dwindows.no.sasl=on
-  -Duse.platformToolsetVersion=v142"
+WINDOWS_FLAGS=(
+  "-Pnative-win"
+  "-Dhttps.protocols=TLSv1.2"
+  "-Drequire.openssl"
+  "-Drequire.test.libhadoop"
+  "-Dshell-executable=${BASH_EXECUTABLE}"
+  "-Dopenssl.prefix=${VCPKG_INSTALLED_PACKAGES}"
+  "-Dcmake.prefix.path=${VCPKG_INSTALLED_PACKAGES}"
+  "-Dwindows.cmake.toolchain.file=${CMAKE_TOOLCHAIN_FILE}"
+  "-Dwindows.cmake.build.type=RelWithDebInfo"
+  "-Dwindows.build.hdfspp.dll=off"
+  "-Dwindows.no.sasl=on"
+  "-Duse.platformToolsetVersion=v142"
+)
 
 ## @description  Globals specific to this personality
 ## @audience     private
@@ -292,7 +294,7 @@ function hadoop_native_flags
         -Drequire.snappy \
         -Pdist \
         -Dtar \
-        "${WINDOWS_FLAGS}"
+        "${WINDOWS_FLAGS[@]}"
     ;;
     *)
       echo \
@@ -436,7 +438,7 @@ function personality_modules
   fi
 
   if [[ "$IS_WINDOWS" && "$IS_WINDOWS" == 1 ]]; then
-    extra="-Ptest-patch -Pdist -Dtar ${WINDOWS_FLAGS} ${extra}"
+    extra="-Ptest-patch -Pdist -Dtar ${WINDOWS_FLAGS[*]} ${extra}"
   fi
 
   for module in $(hadoop_order ${ordering}); do
@@ -557,14 +559,6 @@ function shadedclient_rebuild
   declare module
   declare -a modules=()
 
-  if [[ ${OSTYPE} = Windows_NT ||
-        ${OSTYPE} =~ ^CYGWIN.* ||
-        ${OSTYPE} =~ ^MINGW32.* ||
-        ${OSTYPE} =~ ^MSYS.* ]]; then
-    echo "hadoop personality: building on windows, skipping check of client artifacts."
-    return 0
-  fi
-
   yetus_debug "hadoop personality: seeing if we need the test of client artifacts."
   for module in hadoop-client-modules/hadoop-client-check-invariants \
                 hadoop-client-modules/hadoop-client-check-test-invariants \
@@ -581,28 +575,47 @@ function shadedclient_rebuild
 
   big_console_header "Checking client artifacts on ${repostatus} with shaded clients"
 
-  extra="-Dtest=NoUnitTests -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dspotbugs.skip=true"
+  extra=(
+    "-Dtest=NoUnitTests"
+    "-Dmaven.javadoc.skip=true"
+    "-Dcheckstyle.skip=true"
+    "-Dspotbugs.skip=true"
+  )
 
   if [[ "$IS_WINDOWS" && "$IS_WINDOWS" == 1 ]]; then
+    # shellcheck disable=SC2206
+    extra+=(${WINDOWS_FLAGS[*]})
+
+    # The shaded client integration tests require the Hadoop jars that were just built to be
+    # installed in the local maven repository.
+    # shellcheck disable=SC2086
+    echo_and_redirect "${logfile}" \
+      "${MAVEN}" "${MAVEN_ARGS[@]}" install -fae --batch-mode \
+        -DskipTests -DskipDocs -Pdist -Dtar ${extra[*]}
+
+    # The shaded client integration tests spawn a MiniDFS and MiniYARN cluster for testing. Both of
+    # them require winutils.exe to be found in the PATH and HADOOP_HOME to be set.
     if load_hadoop_version; then
       export HADOOP_HOME="${SOURCEDIR}/hadoop-dist/target/hadoop-${HADOOP_VERSION}-SNAPSHOT"
+      WIN_HADOOP_HOME=$(cygpath -w -a "${HADOOP_HOME}")
+      export PATH="${PATH};${WIN_HADOOP_HOME}\bin"
     else
       yetus_error "[WARNING] Unable to extract the Hadoop version and thus HADOOP_HOME is not set. Some tests may fail."
     fi
-
-    extra="${WINDOWS_FLAGS} ${extra}"
   fi
 
+  # shellcheck disable=SC2086
   echo_and_redirect "${logfile}" \
-    "${MAVEN}" "${MAVEN_ARGS[@]}" verify -fae --batch-mode -am "${modules[@]}" "${extra}"
+    "${MAVEN}" "${MAVEN_ARGS[@]}" verify -fae --batch-mode -am "${modules[@]}" ${extra[*]}
 
   big_console_header "Checking client artifacts on ${repostatus} with non-shaded clients"
 
+  # shellcheck disable=SC2086
   echo_and_redirect "${logfile}" \
     "${MAVEN}" "${MAVEN_ARGS[@]}" verify -fae --batch-mode -am \
       "${modules[@]}" \
       -DskipShade -Dtest=NoUnitTests -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true \
-      -Dspotbugs.skip=true "${extra}"
+      -Dspotbugs.skip=true ${extra[*]}
 
   count=$("${GREP}" -c '\[ERROR\]' "${logfile}")
   if [[ ${count} -gt 0 ]]; then

dev-support/docker/Dockerfile_windows_10

@@ -38,8 +38,8 @@ RUN curl -SL --output vs_buildtools.exe https://aka.ms/vs/16/release/vs_buildtoo
     && del /q vs_buildtools.exe
 
 # Install Chocolatey.
+ENV chocolateyVersion=1.4.0
 RUN powershell -NoProfile -ExecutionPolicy Bypass -Command "iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))"
-RUN setx PATH "%PATH%;%ALLUSERSPROFILE%\chocolatey\bin"
 
 # Install git.
 RUN choco install git.install -y
@@ -55,24 +55,18 @@ RUN powershell .\vcpkg\vcpkg.exe install boost:x64-windows
 RUN powershell .\vcpkg\vcpkg.exe install protobuf:x64-windows
 RUN powershell .\vcpkg\vcpkg.exe install openssl:x64-windows
 RUN powershell .\vcpkg\vcpkg.exe install zlib:x64-windows
-ENV PROTOBUF_HOME "C:\vcpkg\installed\x64-windows"
 
 # Install Azul Java 8 JDK.
 RUN powershell Invoke-WebRequest -URI https://cdn.azul.com/zulu/bin/zulu8.62.0.19-ca-jdk8.0.332-win_x64.zip -OutFile $Env:TEMP\zulu8.62.0.19-ca-jdk8.0.332-win_x64.zip
 RUN powershell Expand-Archive -Path $Env:TEMP\zulu8.62.0.19-ca-jdk8.0.332-win_x64.zip -DestinationPath "C:\Java"
-ENV JAVA_HOME "C:\Java\zulu8.62.0.19-ca-jdk8.0.332-win_x64"
-RUN setx PATH "%PATH%;%JAVA_HOME%\bin"
 
 # Install Apache Maven.
 RUN powershell Invoke-WebRequest -URI https://archive.apache.org/dist/maven/maven-3/3.8.6/binaries/apache-maven-3.8.6-bin.zip -OutFile $Env:TEMP\apache-maven-3.8.6-bin.zip
 RUN powershell Expand-Archive -Path $Env:TEMP\apache-maven-3.8.6-bin.zip -DestinationPath "C:\Maven"
-RUN setx PATH "%PATH%;C:\Maven\apache-maven-3.8.6\bin"
-ENV MAVEN_OPTS '-Xmx2048M -Xss128M'
 
 # Install CMake 3.19.0.
 RUN powershell Invoke-WebRequest -URI https://cmake.org/files/v3.19/cmake-3.19.0-win64-x64.zip -OutFile $Env:TEMP\cmake-3.19.0-win64-x64.zip
 RUN powershell Expand-Archive -Path $Env:TEMP\cmake-3.19.0-win64-x64.zip -DestinationPath "C:\CMake"
-RUN setx PATH "%PATH%;C:\CMake\cmake-3.19.0-win64-x64\bin"
 
 # Install zstd 1.5.4.
 RUN powershell Invoke-WebRequest -Uri https://github.com/facebook/zstd/releases/download/v1.5.4/zstd-v1.5.4-win64.zip -OutFile $Env:TEMP\zstd-v1.5.4-win64.zip
@@ -112,13 +106,35 @@ RUN powershell Copy-Item -Path "C:\RSync\usr\bin\*" -Destination "C:\Program` Fi
 RUN powershell Invoke-WebRequest -Uri https://www.python.org/ftp/python/3.10.11/python-3.10.11-embed-amd64.zip -OutFile $Env:TEMP\python-3.10.11-embed-amd64.zip
 RUN powershell Expand-Archive -Path $Env:TEMP\python-3.10.11-embed-amd64.zip -DestinationPath "C:\Python3"
 RUN powershell New-Item -ItemType HardLink -Value "C:\Python3\python.exe" -Path "C:\Python3\python3.exe"
+
+# Create a user HadoopBuilder with basic privileges and use it for building Hadoop on Windows.
+RUN powershell New-LocalUser -Name 'HadoopBuilder' -Description 'User account for building Apache Hadoop' -Password ([securestring]::new()) -AccountNeverExpires -PasswordNeverExpires
+
+# Grant the privilege to create symbolic links to HadoopBuilder.
+RUN powershell secedit /export /cfg "C:\secpol.cfg"
+RUN powershell "(Get-Content C:\secpol.cfg).Replace('SeCreateSymbolicLinkPrivilege = ', 'SeCreateSymbolicLinkPrivilege = HadoopBuilder,') | Out-File C:\secpol.cfg"
+RUN powershell secedit /configure /db "C:\windows\security\local.sdb" /cfg "C:\secpol.cfg"
+RUN powershell Remove-Item -Force "C:\secpol.cfg" -Confirm:$false
+
+# Login as HadoopBuilder and set the necessary environment and PATH variables.
+USER HadoopBuilder
+ENV PROTOBUF_HOME "C:\vcpkg\installed\x64-windows"
+ENV JAVA_HOME "C:\Java\zulu8.62.0.19-ca-jdk8.0.332-win_x64"
+ENV MAVEN_OPTS '-Xmx2048M -Xss128M'
+RUN setx PATH "%PATH%;%ALLUSERSPROFILE%\chocolatey\bin"
+RUN setx PATH "%PATH%;%JAVA_HOME%\bin"
+RUN setx PATH "%PATH%;C:\Maven\apache-maven-3.8.6\bin"
+RUN setx PATH "%PATH%;C:\CMake\cmake-3.19.0-win64-x64\bin"
+RUN setx PATH "%PATH%;C:\ZStd"
 RUN setx path "%PATH%;C:\Python3"
+RUN setx PATH "%PATH%;C:\Program Files\Git\usr\bin"
 
 # We get strange Javadoc errors without this.
 RUN setx classpath ""
 
+# Setting Git configurations.
+RUN git config --global core.autocrlf true
 RUN git config --global core.longpaths true
-RUN setx PATH "%PATH%;C:\Program Files\Git\usr\bin"
 
 # Define the entry point for the docker container.
 ENTRYPOINT ["C:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\BuildTools\\VC\\Auxiliary\\Build\\vcvars64.bat", "&&", "cmd.exe"]

hadoop-common-project/hadoop-common/src/main/resources/core-default.xml

@@ -1724,14 +1724,14 @@
   <name>fs.s3a.encryption.algorithm</name>
   <description>Specify a server-side encryption or client-side
     encryption algorithm for s3a: file system. Unset by default. It supports the
-    following values: 'AES256' (for SSE-S3), 'SSE-KMS', 'SSE-C', and 'CSE-KMS'
+    following values: 'AES256' (for SSE-S3), 'SSE-KMS', 'DSSE-KMS', 'SSE-C', and 'CSE-KMS'
   </description>
 </property>
 
 <property>
   <name>fs.s3a.encryption.key</name>
   <description>Specific encryption key to use if fs.s3a.encryption.algorithm
-    has been set to 'SSE-KMS', 'SSE-C' or 'CSE-KMS'. In the case of SSE-C
+    has been set to 'SSE-KMS', 'DSSE-KMS', 'SSE-C' or 'CSE-KMS'. In the case of SSE-C
     , the value of this property should be the Base64 encoded key. If you are
     using SSE-KMS and leave this property empty, you'll be using your default's
     S3 KMS key, otherwise you should set this property to the specific KMS key

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java

@@ -63,6 +63,7 @@
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.util.ChunkedArrayList;
 import org.apache.hadoop.util.Daemon;
+import org.apache.hadoop.util.Shell;
 import org.apache.hadoop.util.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -661,7 +662,8 @@ public static boolean isValidName(String src) {
     for (int i = 0; i < components.length; i++) {
       String element = components[i];
       if (element.equals(".")  ||
-          (element.contains(":"))  ||
+          // For Windows, we must allow the : in the drive letter.
+          (!Shell.WINDOWS && i == 1 && element.contains(":"))  ||
           (element.contains("/"))) {
         return false;
       }

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java

@@ -955,7 +955,10 @@ public static class MapOutputBuffer<K extends Object, V extends Object>
       new ArrayList<SpillRecord>();
     private int totalIndexCacheMemory;
     private int indexCacheMemoryLimit;
+    private int spillFilesCountLimit;
     private static final int INDEX_CACHE_MEMORY_LIMIT_DEFAULT = 1024 * 1024;
+    private static final int SPILL_FILES_COUNT_LIMIT_DEFAULT = -1;
+    private static final int SPILL_FILES_COUNT_UNBOUNDED_LIMIT_VALUE = -1;
 
     private MapTask mapTask;
     private MapOutputFile mapOutputFile;
@@ -984,10 +987,17 @@ public void init(MapOutputCollector.Context context
           MRJobConfig.DEFAULT_IO_SORT_MB);
       indexCacheMemoryLimit = job.getInt(JobContext.INDEX_CACHE_MEMORY_LIMIT,
                                          INDEX_CACHE_MEMORY_LIMIT_DEFAULT);
+      spillFilesCountLimit = job.getInt(JobContext.SPILL_FILES_COUNT_LIMIT,
+          SPILL_FILES_COUNT_LIMIT_DEFAULT);
       if (spillper > (float)1.0 || spillper <= (float)0.0) {
         throw new IOException("Invalid \"" + JobContext.MAP_SORT_SPILL_PERCENT +
             "\": " + spillper);
       }
+      if(spillFilesCountLimit != SPILL_FILES_COUNT_UNBOUNDED_LIMIT_VALUE
+          && spillFilesCountLimit < 0) {
+        throw new IOException("Invalid value for \"" + JobContext.SPILL_FILES_COUNT_LIMIT + "\", " +
+            "current value: " + spillFilesCountLimit);
+      }
       if ((sortmb & 0x7FF) != sortmb) {
         throw new IOException(
             "Invalid \"" + JobContext.IO_SORT_MB + "\": " + sortmb);
@@ -1698,7 +1708,7 @@ private void sortAndSpill() throws IOException, ClassNotFoundException,
             spillRec.size() * MAP_OUTPUT_INDEX_RECORD_LENGTH;
         }
         LOG.info("Finished spill " + numSpills);
-        ++numSpills;
+        incrementNumSpills();
       } finally {
         if (out != null) out.close();
         if (partitionOut != null) {
@@ -1774,7 +1784,7 @@ private void spillSingleRecord(final K key, final V value,
           totalIndexCacheMemory +=
             spillRec.size() * MAP_OUTPUT_INDEX_RECORD_LENGTH;
         }
-        ++numSpills;
+        incrementNumSpills();
       } finally {
         if (out != null) out.close();
         if (partitionOut != null) {
@@ -2022,14 +2032,29 @@ private void sameVolRename(Path srcPath,
       if (!dst.getParentFile().exists()) {
         if (!dst.getParentFile().mkdirs()) {
           throw new IOException("Unable to rename " + src + " to "
-              + dst + ": couldn't create parent directory"); 
+              + dst + ": couldn't create parent directory");
         }
       }
 
       if (!src.renameTo(dst)) {
         throw new IOException("Unable to rename " + src + " to " + dst);
       }
     }
+
+    /**
+     * Increments numSpills local counter by taking into consideration
+     * the max limit on spill files being generated by the job.
+     * If limit is reached, this function throws an IOException
+     */
+    private void incrementNumSpills() throws IOException {
+      ++numSpills;
+      if(spillFilesCountLimit != SPILL_FILES_COUNT_UNBOUNDED_LIMIT_VALUE
+          && numSpills > spillFilesCountLimit) {
+        throw new IOException("Too many spill files got created, control it with " +
+            "mapreduce.task.spill.files.count.limit, current value: " + spillFilesCountLimit +
+            ", current spill count: " + numSpills);
+      }
+    }
   } // MapOutputBuffer
 
   /**

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java

@@ -323,6 +323,7 @@ public interface MRJobConfig {
   public static final int DEFAULT_IO_SORT_MB = 100;
 
   public static final String INDEX_CACHE_MEMORY_LIMIT = "mapreduce.task.index.cache.limit.bytes";
+  String SPILL_FILES_COUNT_LIMIT = "mapreduce.task.spill.files.count.limit";
 
   public static final String PRESERVE_FAILED_TASK_FILES = "mapreduce.task.files.preserve.failedtasks";
 

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml

@@ -62,6 +62,15 @@
   set to less than .5</description>
 </property>
 
+<property>
+  <name>mapreduce.task.spill.files.count.limit</name>
+  <value>-1</value>
+  <description>Number of spill files that can be created by a MapTask.
+  After breaching this, task will fail. Default value for this config is -1
+  which indicates that there is no limit on number of spill files being
+  created</description>
+</property>
+
 <property>
   <name>mapreduce.job.local-fs.single-disk-limit.bytes</name>
   <value>-1</value>