Merge branch 'trunk' into HADOOP-17596

Author: sumangala17

BUILDING.txt

@@ -51,14 +51,13 @@ Known issues:
   and run your IDE and Docker etc inside that VM.
 
 ----------------------------------------------------------------------------------
-Installing required packages for clean install of Ubuntu 14.04 LTS Desktop:
+Installing required packages for clean install of Ubuntu 18.04 LTS Desktop.
+(For Ubuntu 20.04, gcc/g++ and cmake bundled with Ubuntu can be used.
+Refer to  dev-support/docker/Dockerfile):
 
-* Oracle JDK 1.8 (preferred)
-  $ sudo apt-get purge openjdk*
-  $ sudo apt-get install software-properties-common
-  $ sudo add-apt-repository ppa:webupd8team/java
+* Open JDK 1.8
   $ sudo apt-get update
-  $ sudo apt-get install oracle-java8-installer
+  $ sudo apt-get -y install java-8-openjdk
 * Maven
   $ sudo apt-get -y install maven
 * Native libraries
@@ -86,12 +85,13 @@ Installing required packages for clean install of Ubuntu 14.04 LTS Desktop:
   $ curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download > boost_1_72_0.tar.bz2
   $ tar --bzip2 -xf boost_1_72_0.tar.bz2 && cd boost_1_72_0
   $ ./bootstrap.sh --prefix=/usr/
-  $ ./b2 --without-python install
+  $ ./b2 --without-python
+  $ sudo ./b2 --without-python install
 
 Optional packages:
 
 * Snappy compression (only used for hadoop-mapreduce-client-nativetask)
-  $ sudo apt-get install snappy libsnappy-dev
+  $ sudo apt-get install libsnappy-dev
 * Intel ISA-L library for erasure coding
   Please refer to https://01.org/intel%C2%AE-storage-acceleration-library-open-source-version
   (OR https://github.com/01org/isa-l)

LICENSE-binary

@@ -240,7 +240,7 @@ com.google.guava:guava:20.0
 com.google.guava:guava:27.0-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.microsoft.azure:azure-storage:7.0.0
-com.nimbusds:nimbus-jose-jwt:4.41.1
+com.nimbusds:nimbus-jose-jwt:9.8.1
 com.squareup.okhttp:okhttp:2.7.5
 com.squareup.okio:okio:1.6.0
 com.zaxxer:HikariCP-java7:2.4.12
@@ -283,7 +283,7 @@ javax.inject:javax.inject:1
 log4j:log4j:1.2.17
 net.java.dev.jna:jna:5.2.0
 net.minidev:accessors-smart:1.2
-net.minidev:json-smart:2.3
+net.minidev:json-smart:2.4.2
 org.apache.avro:avro:1.7.7
 org.apache.commons:commons-collections4:4.2
 org.apache.commons:commons-compress:1.19

dev-support/Jenkinsfile

@@ -35,7 +35,7 @@ pipeline {
         DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile"
         YETUS='yetus'
         // Branch or tag name.  Yetus release tags are 'rel/X.Y.Z'
-        YETUS_VERSION='11eb9b09786e401fbdeaa3be83a19a4066fd7813'
+        YETUS_VERSION='f9ba0170a5787a5f4662d3769804fef0226a182f'
     }
 
     parameters {

dev-support/docker/Dockerfile_centos_7

@@ -0,0 +1,169 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Dockerfile for installing the necessary dependencies for building Hadoop.
+# See BUILDING.txt.
+
+FROM centos:7
+
+WORKDIR /root
+
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+RUN yum update -y \
+    && yum install -y centos-release-scl \
+    && yum install -y devtoolset-9 \
+    && yum install -y \
+    ant \
+    build-essential \
+    bzip2 \
+    bzip2-devel \
+    clang \
+    curl \
+    cyrus-sasl-devel \
+    doxygen \
+    fuse \
+    fuse-libs \
+    fuse-devel \
+    git \
+    libcurl-devel \
+    libtirpc-devel \
+    libpmem-devel \
+    libtool \
+    lz4-devel \
+    make \
+    openssl-devel \
+    pinentry-curses \
+    python3 \
+    python3-pip \
+    python3-setuptools \
+    python3-wheel \
+    rsync \
+    snappy-devel \
+    sudo \
+    valgrind \
+    zlib-devel
+
+# Set GCC 9 as the default C/C++ compiler
+RUN echo "source /opt/rh/devtoolset-9/enable" >> /etc/bashrc
+SHELL ["/bin/bash", "--login", "-c"]
+
+####
+# Install Maven 3.6.3
+####
+RUN mkdir -p /opt/maven /tmp/maven \
+    && curl -L -s -S https://mirrors.estointernet.in/apache/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz \
+        -o /tmp/maven/apache-maven-3.6.3-bin.tar.gz \
+    && tar xzf /tmp/maven/apache-maven-3.6.3-bin.tar.gz --strip-components 1 -C /opt/maven
+
+####
+# Install CMake 3.19
+####
+# hadolint ignore=DL3003
+RUN mkdir -p /tmp/cmake /opt/cmake \
+    && curl -L -s -S https://cmake.org/files/v3.19/cmake-3.19.0.tar.gz -o /tmp/cmake/cmake-3.19.0.tar.gz \
+    && tar xzf /tmp/cmake/cmake-3.19.0.tar.gz --strip-components 1 -C /opt/cmake \
+    && cd /opt/cmake || exit && ./bootstrap \
+    && make "-j$(nproc)" \
+    && make install \
+    && cd /root || exit
+
+####
+# Install zstandard
+####
+# hadolint ignore=DL3003
+RUN mkdir -p /opt/zstd /tmp/zstd \
+    && curl -L -s -S https://github.com/facebook/zstd/archive/refs/tags/v1.4.9.tar.gz -o /tmp/zstd/v1.4.9.tar.gz \
+    && tar xzf /tmp/zstd/v1.4.9.tar.gz --strip-components 1 -C /opt/zstd \
+    && cd /opt/zstd || exit \
+    && make "-j$(nproc)" \
+    && make install \
+    && cd /root || exit
+
+######
+# Set env vars required to build Hadoop
+######
+ENV MAVEN_HOME /opt/maven
+ENV PATH "${PATH}:${MAVEN_HOME}/bin"
+# JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003)
+ENV JAVA_HOME /usr/lib/jvm/java-1.8.0
+
+#######
+# Install SpotBugs 4.2.2
+#######
+RUN mkdir -p /opt/spotbugs \
+    && curl -L -s -S https://github.com/spotbugs/spotbugs/releases/download/4.2.2/spotbugs-4.2.2.tgz \
+      -o /opt/spotbugs.tgz \
+    && tar xzf /opt/spotbugs.tgz --strip-components 1 -C /opt/spotbugs \
+    && chmod +x /opt/spotbugs/bin/*
+ENV SPOTBUGS_HOME /opt/spotbugs
+
+#######
+# Install Boost 1.72 (1.71 ships with Focal)
+#######
+# hadolint ignore=DL3003
+RUN mkdir -p /opt/boost-library \
+    && curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download > boost_1_72_0.tar.bz2 \
+    && mv boost_1_72_0.tar.bz2 /opt/boost-library \
+    && cd /opt/boost-library \
+    && tar --bzip2 -xf boost_1_72_0.tar.bz2 \
+    && cd /opt/boost-library/boost_1_72_0 \
+    && ./bootstrap.sh --prefix=/usr/ \
+    && ./b2 --without-python install \
+    && cd /root \
+    && rm -rf /opt/boost-library
+
+######
+# Install Google Protobuf 3.7.1 (3.6.1 ships with Focal)
+######
+# hadolint ignore=DL3003
+RUN mkdir -p /opt/protobuf-src \
+    && curl -L -s -S \
+      https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \
+      -o /opt/protobuf.tar.gz \
+    && tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src \
+    && cd /opt/protobuf-src \
+    && ./configure --prefix=/opt/protobuf \
+    && make "-j$(nproc)" \
+    && make install \
+    && cd /root \
+    && rm -rf /opt/protobuf-src
+ENV PROTOBUF_HOME /opt/protobuf
+ENV PATH "${PATH}:/opt/protobuf/bin"
+
+####
+# Install Node.js
+####
+# hadolint ignore=DL3003
+RUN mkdir -p /tmp/node \
+    && curl -L -s -S https://nodejs.org/dist/v14.16.1/node-v14.16.1.tar.gz -o /tmp/node-v14.16.1.tar.gz \
+    && tar xzf /tmp/node-v14.16.1.tar.gz --strip-components 1 -C /tmp/node \
+    && cd /tmp/node || exit \
+    && ./configure \
+    && make "-j$(nproc)" \
+    && make install \
+    && cd /root || exit
+
+####
+# Install pylint and python-dateutil
+####
+RUN pip3 install pylint==2.6.0 python-dateutil==2.8.1
+
+####
+# Install bower
+####
+# hadolint ignore=DL3008
+RUN npm install -g bower@1.8.8

hadoop-client-modules/hadoop-client-check-test-invariants/src/test/resources/ensure-jars-have-correct-contents.sh

@@ -58,13 +58,6 @@ allowed_expr+="|^org.apache.hadoop.application-classloader.properties$"
 allowed_expr+="|^java.policy$"
 #   * Used by javax.annotation
 allowed_expr+="|^jndi.properties$"
-# * allowing native libraries from rocksdb. Leaving native libraries as it is.
-allowed_expr+="|^librocksdbjni-linux32.so"
-allowed_expr+="|^librocksdbjni-linux64.so"
-allowed_expr+="|^librocksdbjni-osx.jnilib"
-allowed_expr+="|^librocksdbjni-win64.dll"
-allowed_expr+="|^librocksdbjni-linux-ppc64le.so"
-
 
 allowed_expr+=")"
 declare -i bad_artifacts=0

hadoop-client-modules/hadoop-client-minicluster/pom.xml

@@ -760,13 +760,6 @@
                         <exclude>xml.xsd</exclude>
                       </excludes>
                     </filter>
-                    <!-- filtering HISTORY-JAVA.md from rocksdb jar -->
-                    <filter>
-                    <artifact>org.rocksdb:rocksdbjni</artifact>
-                    <excludes>
-                      <exclude>HISTORY-JAVA.md</exclude>
-                    </excludes>
-                    </filter>
                     <filter>
                       <!-- skip jetty license info already incorporated into LICENSE/NOTICE -->
                       <artifact>org.eclipse.jetty:*</artifact>

hadoop-cloud-storage-project/hadoop-huaweicloud/pom.xml

@@ -60,10 +60,9 @@
   <build>
     <plugins>
       <plugin>
-        <groupId>org.codehaus.mojo</groupId>
-        <artifactId>findbugs-maven-plugin</artifactId>
+        <groupId>com.github.spotbugs</groupId>
+        <artifactId>spotbugs-maven-plugin</artifactId>
         <configuration>
-          <findbugsXmlOutput>true</findbugsXmlOutput>
           <xmlOutput>true</xmlOutput>
           <excludeFilterFile>${basedir}/dev-support/findbugs-exclude.xml
           </excludeFilterFile>
@@ -116,7 +115,7 @@
     <dependency>
       <groupId>junit</groupId>
       <artifactId>junit</artifactId>
-      <version>4.12</version>
+      <version>${junit.version}</version>
       <scope>test</scope>
     </dependency>
     <dependency>

hadoop-common-project/hadoop-auth/pom.xml

@@ -193,11 +193,7 @@
       <artifactId>guava</artifactId>
       <scope>test</scope>
     </dependency>
-    <dependency>
-      <groupId>org.eclipse.jetty</groupId>
-      <artifactId>jetty-server</artifactId>
-    </dependency>
-  </dependencies>
+ </dependencies>
 
   <build>
     <plugins>

hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java

@@ -19,7 +19,6 @@
 import org.apache.hadoop.security.authentication.client.AuthenticationException;
 import org.apache.hadoop.security.authentication.client.KerberosAuthenticator;
 import org.apache.hadoop.security.authentication.util.*;
-import org.eclipse.jetty.server.Response;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -621,18 +620,15 @@ && getMaxInactiveInterval() > 0) {
           errCode = HttpServletResponse.SC_FORBIDDEN;
         }
         // After Jetty 9.4.21, sendError() no longer allows a custom message.
-        // use setStatusWithReason() to set a custom message.
+        // use setStatus() to set a custom message.
         String reason;
         if (authenticationEx == null) {
           reason = "Authentication required";
         } else {
           reason = authenticationEx.getMessage();
         }
 
-        if (httpResponse instanceof Response) {
-          ((Response)httpResponse).setStatusWithReason(errCode, reason);
-        }
-
+        httpResponse.setStatus(errCode, reason);
         httpResponse.sendError(errCode, reason);
       }
     }

hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosUtil.java

@@ -79,11 +79,16 @@ private static Oid getNumericOidInstance(String oidName) {
    *
    * @return Oid instance
    * @param oidName The oid Name
+   * @throws ClassNotFoundException for backward compatibility.
+   * @throws GSSException for backward compatibility.
    * @throws NoSuchFieldException if the input is not supported.
+   * @throws IllegalAccessException for backward compatibility.
+   *
    */
   @Deprecated
   public static Oid getOidInstance(String oidName)
-      throws NoSuchFieldException {
+      throws ClassNotFoundException, GSSException, NoSuchFieldException,
+      IllegalAccessException {
     switch (oidName) {
     case "GSS_SPNEGO_MECH_OID":
       return GSS_SPNEGO_MECH_OID;