Merge branch 'HDFS-17577' of github.com:liangyu-1/hadoop into HDFS-17577

merge

Author: liangyu-1

.github/workflows/website.yml

@@ -56,4 +56,5 @@ jobs:
           publish_dir: ./staging/hadoop-project
           user_name: 'github-actions[bot]'
           user_email: 'github-actions[bot]@users.noreply.github.com'
+          force_orphan: true
 

BUILDING.txt

@@ -163,14 +163,7 @@ Maven build goals:
  YARN Application Timeline Service V2 build options:
 
    YARN Timeline Service v.2 chooses Apache HBase as the primary backing storage. The supported
-   versions of Apache HBase are 1.7.1 (default) and 2.2.4.
-
-  * HBase 1.7.1 is used by default to build Hadoop. The official releases are ready to use if you
-    plan on running Timeline Service v2 with HBase 1.7.1.
-
-  * Use -Dhbase.profile=2.0 to build Hadoop with HBase 2.2.4. Provide this option if you plan
-    on running Timeline Service v2 with HBase 2.x.
-
+   version of Apache HBase is 2.5.8.
 
  Snappy build options:
 
@@ -653,6 +646,20 @@ container.
     -Dwindows.cmake.toolchain.file=C:\vcpkg\scripts\buildsystems\vcpkg.cmake -Dwindows.cmake.build.type=RelWithDebInfo^
     -Dwindows.build.hdfspp.dll=off -Dwindows.no.sasl=on -Duse.platformToolsetVersion=v142
 
+Building the release tarball:
+Assuming that we're still running in the Docker container hadoop-windows-10-builder, run the
+following command to create the Apache Hadoop release tarball -
+
+> set IS_WINDOWS=1
+> set MVN_ARGS="-Dshell-executable=C:\Git\bin\bash.exe -Dhttps.protocols=TLSv1.2 -Pnative-win -Drequire.openssl -Dopenssl.prefix=C:\vcpkg\installed\x64-windows -Dcmake.prefix.path=C:\vcpkg\installed\x64-windows -Dwindows.cmake.toolchain.file=C:\vcpkg\scripts\buildsystems\vcpkg.cmake -Dwindows.cmake.build.type=RelWithDebInfo -Dwindows.build.hdfspp.dll=off -Duse.platformToolsetVersion=v142 -Dwindows.no.sasl=on -DskipTests -DskipDocs -Drequire.test.libhadoop"
+> C:\Git\bin\bash.exe C:\hadoop\dev-support\bin\create-release --mvnargs=%MVN_ARGS%
+
+Note:
+If the building fails due to an issue with long paths, rename the Hadoop root directory to just a
+letter (like 'h') and rebuild -
+
+> C:\Git\bin\bash.exe C:\h\dev-support\bin\create-release --mvnargs=%MVN_ARGS%
+
 ----------------------------------------------------------------------------------
 Building distributions:
 

LICENSE-binary

@@ -216,7 +216,6 @@ com.aliyun:aliyun-java-sdk-kms:2.11.0
 com.aliyun:aliyun-java-sdk-ram:3.1.0
 com.aliyun:aliyun-java-sdk-sts:3.0.0
 com.aliyun.oss:aliyun-sdk-oss:3.13.2
-com.amazonaws:aws-java-sdk-bundle:1.12.599
 com.cedarsoftware:java-util:1.9.0
 com.cedarsoftware:json-io:2.5.1
 com.fasterxml.jackson.core:jackson-annotations:2.12.7
@@ -227,10 +226,11 @@ com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.12.7
 com.fasterxml.jackson.module:jackson-module-jaxb-annotations:2.12.7
 com.fasterxml.uuid:java-uuid-generator:3.1.4
 com.fasterxml.woodstox:woodstox-core:5.4.0
+com.github.ben-manes.caffeine:caffeine:2.9.3
 com.github.davidmoten:rxjava-extras:0.8.0.17
 com.github.stephenc.jcip:jcip-annotations:1.0-1
-com.google:guice:4.0
-com.google:guice-servlet:4.0
+com.google:guice:5.1.0
+com.google:guice-servlet:5.1.0
 com.google.api.grpc:proto-google-common-protos:1.0.0
 com.google.code.gson:2.9.0
 com.google.errorprone:error_prone_annotations:2.2.0
@@ -241,7 +241,7 @@ com.google.guava:guava:20.0
 com.google.guava:guava:27.0-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.microsoft.azure:azure-storage:7.0.0
-com.nimbusds:nimbus-jose-jwt:9.31
+com.nimbusds:nimbus-jose-jwt:9.37.2
 com.zaxxer:HikariCP:4.0.3
 commons-beanutils:commons-beanutils:1.9.4
 commons-cli:commons-cli:1.5.0
@@ -299,8 +299,8 @@ net.java.dev.jna:jna:5.2.0
 net.minidev:accessors-smart:1.2
 org.apache.avro:avro:1.9.2
 org.apache.commons:commons-collections4:4.2
-org.apache.commons:commons-compress:1.24.0
-org.apache.commons:commons-configuration2:2.8.0
+org.apache.commons:commons-compress:1.26.1
+org.apache.commons:commons-configuration2:2.10.1
 org.apache.commons:commons-csv:1.9.0
 org.apache.commons:commons-digester:1.8.1
 org.apache.commons:commons-lang3:3.12.0
@@ -310,16 +310,15 @@ org.apache.commons:commons-validator:1.6
 org.apache.curator:curator-client:5.2.0
 org.apache.curator:curator-framework:5.2.0
 org.apache.curator:curator-recipes:5.2.0
-org.apache.geronimo.specs:geronimo-jcache_1.0_spec:1.0-alpha-1
-org.apache.hbase:hbase-annotations:1.7.1
-org.apache.hbase:hbase-client:1.7.1
-org.apache.hbase:hbase-common:1.7.1
-org.apache.hbase:hbase-protocol:1.7.1
+org.apache.hbase:hbase-annotations:2.5.8
+org.apache.hbase:hbase-client:2.5.8
+org.apache.hbase:hbase-common:2.5.8
+org.apache.hbase:hbase-protocol:2.5.8
 org.apache.htrace:htrace-core:3.1.0-incubating
 org.apache.htrace:htrace-core4:4.1.0-incubating
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.13
-org.apache.kafka:kafka-clients:2.8.2
+org.apache.kafka:kafka-clients:3.4.0
 org.apache.kerby:kerb-admin:2.0.3
 org.apache.kerby:kerb-client:2.0.3
 org.apache.kerby:kerb-common:2.0.3
@@ -340,7 +339,7 @@ org.apache.sshd:sshd-core:2.11.0
 org.apache.sshd:sshd-sftp:2.11.0
 org.apache.solr:solr-solrj:8.11.2
 org.apache.yetus:audience-annotations:0.5.0
-org.apache.zookeeper:zookeeper:3.8.3
+org.apache.zookeeper:zookeeper:3.8.4
 org.codehaus.jettison:jettison:1.5.4
 org.eclipse.jetty:jetty-annotations:9.4.53.v20231009
 org.eclipse.jetty:jetty-http:9.4.53.v20231009
@@ -356,14 +355,14 @@ org.eclipse.jetty:jetty-webapp:9.4.53.v20231009
 org.eclipse.jetty:jetty-xml:9.4.53.v20231009
 org.eclipse.jetty.websocket:javax-websocket-client-impl:9.4.53.v20231009
 org.eclipse.jetty.websocket:javax-websocket-server-impl:9.4.53.v20231009
-org.ehcache:ehcache:3.3.1
+org.ehcache:ehcache:3.8.2
 org.ini4j:ini4j:0.5.4
 org.lz4:lz4-java:1.7.1
 org.objenesis:objenesis:2.6
 org.xerial.snappy:snappy-java:1.1.10.4
 org.yaml:snakeyaml:2.0
 org.wildfly.openssl:wildfly-openssl:1.1.3.Final
-software.amazon.awssdk:bundle:jar:2.24.6
+software.amazon.awssdk:bundle:jar:2.25.53
 
 
 --------------------------------------------------------------------------------
@@ -379,7 +378,7 @@ hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/com
 hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/util/tree.h
 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/compat/{fstatat|openat|unlinkat}.h
 
-com.github.luben:zstd-jni:1.4.9-1
+com.github.luben:zstd-jni:1.5.2-1
 dnsjava:dnsjava:2.1.7
 org.codehaus.woodstox:stax2-api:4.2.1
 
@@ -483,9 +482,9 @@ com.microsoft.azure:azure-cosmosdb-gateway:2.4.5
 com.microsoft.azure:azure-data-lake-store-sdk:2.3.3
 com.microsoft.azure:azure-keyvault-core:1.0.0
 com.microsoft.sqlserver:mssql-jdbc:6.2.1.jre7
-org.bouncycastle:bcpkix-jdk15on:1.70
-org.bouncycastle:bcprov-jdk15on:1.70
-org.bouncycastle:bcutil-jdk15on:1.70
+org.bouncycastle:bcpkix-jdk18on:1.78.1
+org.bouncycastle:bcprov-jdk18on:1.78.1
+org.bouncycastle:bcutil-jdk18on:1.78.1
 org.checkerframework:checker-qual:2.5.2
 org.codehaus.mojo:animal-sniffer-annotations:1.21
 org.jruby.jcodings:jcodings:1.0.13
@@ -507,6 +506,7 @@ com.sun.jersey:jersey-server:1.19.4
 com.sun.jersey:jersey-servlet:1.19.4
 com.sun.xml.bind:jaxb-impl:2.2.3-1
 javax.annotation:javax.annotation-api:1.3.2
+javax.cache:cache-api:1.1.1
 javax.servlet:javax.servlet-api:3.1.0
 javax.servlet.jsp:jsp-api:2.1
 javax.websocket:javax.websocket-api:1.0

dev-support/bin/create-release

@@ -418,7 +418,8 @@ function option_parse
     fi
   fi
   if [ -n "$MVNEXTRAARGS" ]; then
-    MVN_ARGS+=("$MVNEXTRAARGS")
+    # shellcheck disable=SC2206
+    MVN_ARGS+=(${MVNEXTRAARGS[*]})
   fi
 
   if [[ "${SECURITYRELEASE}" = true ]]; then
@@ -552,18 +553,21 @@ function makearelease
   mkdir -p "${LOGDIR}"
 
   # Install the Hadoop maven plugins first
-  run_and_redirect "${LOGDIR}/mvn_install_maven_plugins.log" "${MVN}" "${MVN_ARGS[@]}" -pl hadoop-maven-plugins -am clean install
+  # shellcheck disable=SC2086
+  run_and_redirect "${LOGDIR}/mvn_install_maven_plugins.log" "${MVN}" ${MVN_ARGS[*]} -pl hadoop-maven-plugins -am clean install
 
   # mvn clean for sanity
-  run_and_redirect "${LOGDIR}/mvn_clean.log" "${MVN}" "${MVN_ARGS[@]}" clean
+  # shellcheck disable=SC2086
+  run_and_redirect "${LOGDIR}/mvn_clean.log" "${MVN}" ${MVN_ARGS[*]} clean
 
   # Create staging dir for release artifacts
   run mkdir -p "${ARTIFACTS_DIR}"
 
   big_console_header "Apache RAT Check"
 
   # Create RAT report
-  run_and_redirect "${LOGDIR}/mvn_apache_rat.log" "${MVN}" "${MVN_ARGS[@]}" apache-rat:check
+  # shellcheck disable=SC2086
+  run_and_redirect "${LOGDIR}/mvn_apache_rat.log" "${MVN}" ${MVN_ARGS[*]} apache-rat:check
 
   big_console_header "Maven Build and Install"
 
@@ -577,9 +581,9 @@ function makearelease
   fi
 
   # Create SRC and BIN tarballs for release,
-  # shellcheck disable=SC2046
+  # shellcheck disable=SC2046,SC2086
   run_and_redirect "${LOGDIR}/mvn_${target}.log" \
-    "${MVN}" "${MVN_ARGS[@]}" ${target} \
+    "${MVN}" ${MVN_ARGS[*]} ${target} \
       -Pdist,src,yarn-ui \
       "${signflags[@]}" \
       -DskipTests -Dtar $(hadoop_native_flags)
@@ -608,8 +612,9 @@ function makearelease
   # we need to do install again so that jdiff and
   # a few other things get registered in the maven
   # universe correctly
+  # shellcheck disable=SC2206,SC2086
   run_and_redirect "${LOGDIR}/mvn_site.log" \
-    "${MVN}" "${MVN_ARGS[@]}" install \
+    "${MVN}" ${MVN_ARGS[*]} install \
       site site:stage \
       -DskipTests \
       -DskipShade \

dev-support/bin/hadoop.sh

@@ -387,6 +387,11 @@ function personality_modules
       fi
     ;;
     unit)
+      if [[ "$IS_WINDOWS" && "$IS_WINDOWS" == 1 && (-z "$IS_NIGHTLY_BUILD" || "$IS_NIGHTLY_BUILD" == 0) ]]; then
+        echo "Won't run unit tests for Windows in pre-commit CI"
+        return
+      fi
+
       extra="-Dsurefire.rerunFailingTestsCount=2"
       if [[ "${BUILDMODE}" = full ]]; then
         ordering=mvnsrc

dev-support/bin/yetus-wrapper

@@ -77,7 +77,7 @@ WANTED="$1"
 shift
 ARGV=("$@")
 
-HADOOP_YETUS_VERSION=${HADOOP_YETUS_VERSION:-0.14.0}
+HADOOP_YETUS_VERSION=${HADOOP_YETUS_VERSION:-0.14.1}
 BIN=$(yetus_abs "${BASH_SOURCE-$0}")
 BINDIR=$(dirname "${BIN}")
 
@@ -123,7 +123,7 @@ fi
 ## need to DL, etc
 ##
 
-BASEURL="https://archive.apache.org/dist/yetus/${HADOOP_YETUS_VERSION}/"
+BASEURL="https://downloads.apache.org/yetus/${HADOOP_YETUS_VERSION}/"
 TARBALL="${YETUS_PREFIX}-${HADOOP_YETUS_VERSION}-bin.tar"
 
 GPGBIN=$(command -v gpg)

dev-support/docker/Dockerfile_windows_10

@@ -61,8 +61,8 @@ RUN powershell Invoke-WebRequest -URI https://cdn.azul.com/zulu/bin/zulu8.62.0.1
 RUN powershell Expand-Archive -Path $Env:TEMP\zulu8.62.0.19-ca-jdk8.0.332-win_x64.zip -DestinationPath "C:\Java"
 
 # Install Apache Maven.
-RUN powershell Invoke-WebRequest -URI https://archive.apache.org/dist/maven/maven-3/3.8.6/binaries/apache-maven-3.8.6-bin.zip -OutFile $Env:TEMP\apache-maven-3.8.6-bin.zip
-RUN powershell Expand-Archive -Path $Env:TEMP\apache-maven-3.8.6-bin.zip -DestinationPath "C:\Maven"
+RUN powershell Invoke-WebRequest -URI https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.zip -OutFile $Env:TEMP\apache-maven-3.8.8-bin.zip
+RUN powershell Expand-Archive -Path $Env:TEMP\apache-maven-3.8.8-bin.zip -DestinationPath "C:\Maven"
 
 # Install CMake 3.19.0.
 RUN powershell Invoke-WebRequest -URI https://cmake.org/files/v3.19/cmake-3.19.0-win64-x64.zip -OutFile $Env:TEMP\cmake-3.19.0-win64-x64.zip
@@ -102,10 +102,21 @@ RUN powershell Copy-Item -Path "C:\LibXXHash\usr\bin\*.dll" -Destination "C:\Pro
 RUN powershell Copy-Item -Path "C:\LibZStd\usr\bin\*.dll" -Destination "C:\Program` Files\Git\usr\bin"
 RUN powershell Copy-Item -Path "C:\RSync\usr\bin\*" -Destination "C:\Program` Files\Git\usr\bin"
 
-# Install Python 3.10.11.
-RUN powershell Invoke-WebRequest -Uri https://www.python.org/ftp/python/3.10.11/python-3.10.11-embed-amd64.zip -OutFile $Env:TEMP\python-3.10.11-embed-amd64.zip
-RUN powershell Expand-Archive -Path $Env:TEMP\python-3.10.11-embed-amd64.zip -DestinationPath "C:\Python3"
-RUN powershell New-Item -ItemType HardLink -Value "C:\Python3\python.exe" -Path "C:\Python3\python3.exe"
+COPY pkg-resolver pkg-resolver
+
+## Install Python 3.11.8.
+# The Python installation steps below are derived from -
+# https://github.com/docker-library/python/blob/105d6f34e7d70aad6f8c3e249b8208efa591916a/3.11/windows/windowsservercore-ltsc2022/Dockerfile
+ENV PYTHONIOENCODING UTF-8
+ENV PYTHON_VERSION 3.11.8
+ENV PYTHON_PIP_VERSION 24.0
+ENV PYTHON_SETUPTOOLS_VERSION 65.5.1
+ENV PYTHON_GET_PIP_URL https://github.com/pypa/get-pip/raw/dbf0c85f76fb6e1ab42aa672ffca6f0a675d9ee4/public/get-pip.py
+ENV PYTHON_GET_PIP_SHA256 dfe9fd5c28dc98b5ac17979a953ea550cec37ae1b47a5116007395bfacff2ab9
+RUN powershell Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser
+RUN powershell pkg-resolver\install-python.ps1
+RUN powershell pkg-resolver\install-pip.ps1
+RUN powershell pip install python-dateutil
 
 # Create a user HadoopBuilder with basic privileges and use it for building Hadoop on Windows.
 RUN powershell New-LocalUser -Name 'HadoopBuilder' -Description 'User account for building Apache Hadoop' -Password ([securestring]::new()) -AccountNeverExpires -PasswordNeverExpires
@@ -121,12 +132,12 @@ USER HadoopBuilder
 ENV PROTOBUF_HOME "C:\vcpkg\installed\x64-windows"
 ENV JAVA_HOME "C:\Java\zulu8.62.0.19-ca-jdk8.0.332-win_x64"
 ENV MAVEN_OPTS '-Xmx2048M -Xss128M'
+ENV IS_WINDOWS 1
 RUN setx PATH "%PATH%;%ALLUSERSPROFILE%\chocolatey\bin"
 RUN setx PATH "%PATH%;%JAVA_HOME%\bin"
-RUN setx PATH "%PATH%;C:\Maven\apache-maven-3.8.6\bin"
+RUN setx PATH "%PATH%;C:\Maven\apache-maven-3.8.8\bin"
 RUN setx PATH "%PATH%;C:\CMake\cmake-3.19.0-win64-x64\bin"
 RUN setx PATH "%PATH%;C:\ZStd"
-RUN setx path "%PATH%;C:\Python3"
 RUN setx PATH "%PATH%;C:\Program Files\Git\usr\bin"
 
 # We get strange Javadoc errors without this.

dev-support/docker/pkg-resolver/install-pip.ps1

@@ -0,0 +1,44 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# The code lines below are derived from -
+# https://github.com/docker-library/python/blob/105d6f34e7d70aad6f8c3e249b8208efa591916a/3.11/windows/windowsservercore-ltsc2022/Dockerfile
+
+Write-Host ('Downloading get-pip.py ({0}) ...' -f $Env:PYTHON_GET_PIP_URL)
+[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+Invoke-WebRequest -Uri $Env:PYTHON_GET_PIP_URL -OutFile 'get-pip.py'
+Write-Host ('Verifying sha256 ({0}) ...' -f $Env:PYTHON_GET_PIP_SHA256)
+if ((Get-FileHash 'get-pip.py' -Algorithm sha256).Hash -ne $Env:PYTHON_GET_PIP_SHA256) {
+    Write-Host 'FAILED!'
+    exit 1
+}
+
+$Env:PYTHONDONTWRITEBYTECODE = '1'
+
+Write-Host ('Installing pip=={0} ...' -f $Env:PYTHON_PIP_VERSION)
+python get-pip.py `
+    --disable-pip-version-check `
+    --no-cache-dir `
+    --no-compile `
+('pip=={0}' -f $Env:PYTHON_PIP_VERSION) `
+('setuptools=={0}' -f $Env:PYTHON_SETUPTOOLS_VERSION)
+
+Remove-Item get-pip.py -Force
+
+Write-Host 'Verifying pip install ...'
+pip --version
+
+Write-Host 'Complete.'