fix merge conflict

Author: sumangala17

BUILDING.txt

@@ -57,7 +57,7 @@ Refer to  dev-support/docker/Dockerfile):
 
 * Open JDK 1.8
   $ sudo apt-get update
-  $ sudo apt-get -y install java-8-openjdk
+  $ sudo apt-get -y install openjdk-8-jdk
 * Maven
   $ sudo apt-get -y install maven
 * Native libraries
@@ -326,40 +326,35 @@ to update SNAPSHOTs from external repos.
 ----------------------------------------------------------------------------------
 Importing projects to eclipse
 
-When you import the project to eclipse, install hadoop-maven-plugins at first.
+At first, install artifacts including hadoop-maven-plugins at the top of the source tree.
 
-  $ cd hadoop-maven-plugins
-  $ mvn install
+  $ mvn clean install -DskipTests -DskipShade
 
-Then, generate eclipse project files.
-
-  $ mvn eclipse:eclipse -DskipTests
-
-At last, import to eclipse by specifying the root directory of the project via
-[File] > [Import] > [Existing Projects into Workspace].
+Then, import to eclipse by specifying the root directory of the project via
+[File] > [Import] > [Maven] > [Existing Maven Projects].
 
 ----------------------------------------------------------------------------------
 Building distributions:
 
-Create binary distribution without native code and without documentation:
+Create binary distribution without native code and without Javadocs:
 
   $ mvn package -Pdist -DskipTests -Dtar -Dmaven.javadoc.skip=true
 
-Create binary distribution with native code and with documentation:
+Create binary distribution with native code:
 
-  $ mvn package -Pdist,native,docs -DskipTests -Dtar
+  $ mvn package -Pdist,native -DskipTests -Dtar
 
 Create source distribution:
 
   $ mvn package -Psrc -DskipTests
 
-Create source and binary distributions with native code and documentation:
+Create source and binary distributions with native code:
 
-  $ mvn package -Pdist,native,docs,src -DskipTests -Dtar
+  $ mvn package -Pdist,native,src -DskipTests -Dtar
 
 Create a local staging version of the website (in /tmp/hadoop-site)
 
-  $ mvn clean site -Preleasedocs; mvn site:stage -DstagingDirectory=/tmp/hadoop-site
+  $ mvn site site:stage -Preleasedocs,docs -DstagingDirectory=/tmp/hadoop-site
 
 Note that the site needs to be built in a second pass after other artifacts.
 

LICENSE-binary

@@ -218,12 +218,12 @@ com.aliyun.oss:aliyun-sdk-oss:3.13.0
 com.amazonaws:aws-java-sdk-bundle:1.11.901
 com.cedarsoftware:java-util:1.9.0
 com.cedarsoftware:json-io:2.5.1
-com.fasterxml.jackson.core:jackson-annotations:2.9.9
-com.fasterxml.jackson.core:jackson-core:2.9.9
-com.fasterxml.jackson.core:jackson-databind:2.9.9.2
-com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:2.9.9
-com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.9.9
-com.fasterxml.jackson.module:jackson-module-jaxb-annotations:2.9.9
+com.fasterxml.jackson.core:jackson-annotations:2.13.0
+com.fasterxml.jackson.core:jackson-core:2.13.0
+com.fasterxml.jackson.core:jackson-databind:2.13.0
+com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:2.13.0
+com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.13.0
+com.fasterxml.jackson.module:jackson-module-jaxb-annotations:2.13.0
 com.fasterxml.uuid:java-uuid-generator:3.1.4
 com.fasterxml.woodstox:woodstox-core:5.3.0
 com.github.davidmoten:rxjava-extras:0.8.0.17
@@ -250,7 +250,6 @@ commons-codec:commons-codec:1.11
 commons-collections:commons-collections:3.2.2
 commons-daemon:commons-daemon:1.0.13
 commons-io:commons-io:2.8.0
-commons-lang:commons-lang:2.6
 commons-logging:commons-logging:1.1.3
 commons-net:commons-net:3.6
 de.ruedigermoeller:fst:2.50
@@ -290,23 +289,23 @@ org.apache.commons:commons-compress:1.21
 org.apache.commons:commons-configuration2:2.1.1
 org.apache.commons:commons-csv:1.0
 org.apache.commons:commons-digester:1.8.1
-org.apache.commons:commons-lang3:3.7
+org.apache.commons:commons-lang3:3.12.0
 org.apache.commons:commons-math3:3.1.1
 org.apache.commons:commons-text:1.4
 org.apache.commons:commons-validator:1.6
 org.apache.curator:curator-client:5.2.0
 org.apache.curator:curator-framework:5.2.0
 org.apache.curator:curator-recipes:5.2.0
 org.apache.geronimo.specs:geronimo-jcache_1.0_spec:1.0-alpha-1
-org.apache.hbase:hbase-annotations:1.4.8
-org.apache.hbase:hbase-client:1.4.8
-org.apache.hbase:hbase-common:1.4.8
-org.apache.hbase:hbase-protocol:1.4.8
+org.apache.hbase:hbase-annotations:1.7.1
+org.apache.hbase:hbase-client:1.7.1
+org.apache.hbase:hbase-common:1.7.1
+org.apache.hbase:hbase-protocol:1.7.1
 org.apache.htrace:htrace-core:3.1.0-incubating
 org.apache.htrace:htrace-core4:4.1.0-incubating
 org.apache.httpcomponents:httpclient:4.5.6
 org.apache.httpcomponents:httpcore:4.4.10
-org.apache.kafka:kafka-clients:2.4.0
+org.apache.kafka:kafka-clients:2.8.1
 org.apache.kerby:kerb-admin:1.0.1
 org.apache.kerby:kerb-client:1.0.1
 org.apache.kerby:kerb-common:1.0.1
@@ -329,22 +328,22 @@ org.codehaus.jackson:jackson-jaxrs:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.codehaus.jackson:jackson-xc:1.9.13
 org.codehaus.jettison:jettison:1.1
-org.eclipse.jetty:jetty-annotations:9.3.27.v20190418
-org.eclipse.jetty:jetty-http:9.3.27.v20190418
-org.eclipse.jetty:jetty-io:9.3.27.v20190418
-org.eclipse.jetty:jetty-jndi:9.3.27.v20190418
-org.eclipse.jetty:jetty-plus:9.3.27.v20190418
-org.eclipse.jetty:jetty-security:9.3.27.v20190418
-org.eclipse.jetty:jetty-server:9.3.27.v20190418
-org.eclipse.jetty:jetty-servlet:9.3.27.v20190418
-org.eclipse.jetty:jetty-util:9.3.27.v20190418
-org.eclipse.jetty:jetty-util-ajax:9.3.27.v20190418
-org.eclipse.jetty:jetty-webapp:9.3.27.v20190418
-org.eclipse.jetty:jetty-xml:9.3.27.v20190418
-org.eclipse.jetty.websocket:javax-websocket-client-impl:9.3.27.v20190418
-org.eclipse.jetty.websocket:javax-websocket-server-impl:9.3.27.v20190418
+org.eclipse.jetty:jetty-annotations:9.4.44.v20210927
+org.eclipse.jetty:jetty-http:9.4.44.v20210927
+org.eclipse.jetty:jetty-io:9.4.44.v20210927
+org.eclipse.jetty:jetty-jndi:9.4.44.v20210927
+org.eclipse.jetty:jetty-plus:9.4.44.v20210927
+org.eclipse.jetty:jetty-security:9.4.44.v20210927
+org.eclipse.jetty:jetty-server:9.4.44.v20210927
+org.eclipse.jetty:jetty-servlet:9.4.44.v20210927
+org.eclipse.jetty:jetty-util:9.4.44.v20210927
+org.eclipse.jetty:jetty-util-ajax:9.4.44.v20210927
+org.eclipse.jetty:jetty-webapp:9.4.44.v20210927
+org.eclipse.jetty:jetty-xml:9.4.44.v20210927
+org.eclipse.jetty.websocket:javax-websocket-client-impl:9.4.44.v20210927
+org.eclipse.jetty.websocket:javax-websocket-server-impl:9.4.44.v20210927
 org.ehcache:ehcache:3.3.1
-org.lz4:lz4-java:1.6.0
+org.lz4:lz4-java:1.7.1
 org.objenesis:objenesis:2.6
 org.xerial.snappy:snappy-java:1.0.5
 org.yaml:snakeyaml:1.16:
@@ -364,7 +363,7 @@ hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/com
 hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/util/tree.h
 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/compat/{fstatat|openat|unlinkat}.h
 
-com.github.luben:zstd-jni:1.4.3-1
+com.github.luben:zstd-jni:1.4.9-1
 dnsjava:dnsjava:2.1.7
 org.codehaus.woodstox:stax2-api:4.2.1
 
@@ -495,6 +494,7 @@ javax.annotation:javax.annotation-api:1.3.2
 javax.servlet:javax.servlet-api:3.1.0
 javax.servlet.jsp:jsp-api:2.1
 javax.websocket:javax.websocket-api:1.0
+javax.ws.rs:javax.ws.rs-api:2.1.1
 javax.ws.rs:jsr311-api:1.1.1
 javax.xml.bind:jaxb-api:2.2.11
 

dev-support/Jenkinsfile

@@ -15,13 +15,10 @@
 // specific language governing permissions and limitations
 // under the License.
 
-def getGithubAndJiraCreds() {
+def getGithubCreds() {
     return [usernamePassword(credentialsId: 'apache-hadoop-at-github.com',
                                 passwordVariable: 'GITHUB_TOKEN',
-                                usernameVariable: 'GITHUB_USER'),
-            usernamePassword(credentialsId: 'hadoopqa-at-asf-jira',
-                                passwordVariable: 'JIRA_PASSWORD',
-                                usernameVariable: 'JIRA_USER')]
+                                usernameVariable: 'GITHUB_USER')]
 }
 
 // Publish JUnit results only if there are XML files under surefire-reports
@@ -30,8 +27,10 @@ def publishJUnitResults() {
     boolean surefireReportsExist = findCmdExitCode == 0
     if (surefireReportsExist) {
         echo "XML files found under surefire-reports, running junit"
+        // The path should be relative to WORKSPACE for the junit.
+        SRC = "${SOURCEDIR}/**/target/surefire-reports/*.xml".replace("$WORKSPACE/","")
         try {
-            junit "${SOURCEDIR}/**/target/surefire-reports/*.xml"
+            junit "${SRC}"
         } catch(e) {
             echo 'junit processing: ' + e.toString()
         }
@@ -128,7 +127,7 @@ pipeline {
             }
 
             steps {
-                withCredentials(getGithubAndJiraCreds()) {
+                withCredentials(getGithubCreds()) {
                     sh '''#!/usr/bin/env bash
 
                     chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
@@ -174,7 +173,7 @@ pipeline {
             }
 
             steps {
-                withCredentials(getGithubAndJiraCreds()) {
+                withCredentials(getGithubCreds()) {
                         sh '''#!/usr/bin/env bash
 
                         chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
@@ -220,7 +219,7 @@ pipeline {
             }
 
             steps {
-                withCredentials(getGithubAndJiraCreds()) {
+                withCredentials(getGithubCreds()) {
                     sh '''#!/usr/bin/env bash
 
                     chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
@@ -263,7 +262,7 @@ pipeline {
             }
 
             steps {
-                withCredentials(getGithubAndJiraCreds()) {
+                withCredentials(getGithubCreds()) {
                     sh '''#!/usr/bin/env bash
 
                     chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"

dev-support/bin/hadoop.sh

@@ -513,7 +513,7 @@ function shadedclient_initialize
   maven_add_install shadedclient
 }
 
-## @description build client facing shaded artifacts and test them
+## @description build client facing shaded and non-shaded artifacts and test them
 ## @audience private
 ## @stability evolving
 ## @param repostatus
@@ -546,13 +546,20 @@ function shadedclient_rebuild
     return 0
   fi
 
-  big_console_header "Checking client artifacts on ${repostatus}"
+  big_console_header "Checking client artifacts on ${repostatus} with shaded clients"
 
   echo_and_redirect "${logfile}" \
     "${MAVEN}" "${MAVEN_ARGS[@]}" verify -fae --batch-mode -am \
       "${modules[@]}" \
       -Dtest=NoUnitTests -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dspotbugs.skip=true
 
+  big_console_header "Checking client artifacts on ${repostatus} with non-shaded clients"
+
+  echo_and_redirect "${logfile}" \
+    "${MAVEN}" "${MAVEN_ARGS[@]}" verify -fae --batch-mode -am \
+      "${modules[@]}" \
+      -DskipShade -Dtest=NoUnitTests -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dspotbugs.skip=true
+
   count=$("${GREP}" -c '\[ERROR\]' "${logfile}")
   if [[ ${count} -gt 0 ]]; then
     add_vote_table -1 shadedclient "${repostatus} has errors when building and testing our client artifacts."

dev-support/code-formatter/hadoop_idea_formatter.xml

@@ -0,0 +1,75 @@
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<code_scheme name="Hadoop" version="173">
+  <option name="RIGHT_MARGIN" value="100" />
+  <JavaCodeStyleSettings>
+    <option name="CLASS_COUNT_TO_USE_IMPORT_ON_DEMAND" value="999" />
+    <option name="NAMES_COUNT_TO_USE_IMPORT_ON_DEMAND" value="999" />
+    <option name="PACKAGES_TO_USE_IMPORT_ON_DEMAND">
+      <value />
+    </option>
+    <option name="IMPORT_LAYOUT_TABLE">
+      <value>
+        <package name="java" withSubpackages="true" static="false" />
+        <package name="javax" withSubpackages="true" static="false" />
+        <emptyLine />
+        <package name="org.apache.hadoop.thirdparty" withSubpackages="true" static="false" />
+        <package name="" withSubpackages="true" static="false" />
+        <emptyLine />
+        <package name="org.apache" withSubpackages="true" static="false" />
+        <emptyLine />
+        <package name="" withSubpackages="true" static="true" />
+      </value>
+    </option>
+  </JavaCodeStyleSettings>
+  <codeStyleSettings language="JAVA">
+    <option name="RIGHT_MARGIN" value="100" />
+    <option name="KEEP_LINE_BREAKS" value="false" />
+    <option name="KEEP_FIRST_COLUMN_COMMENT" value="false" />
+    <option name="KEEP_CONTROL_STATEMENT_IN_ONE_LINE" value="false" />
+    <option name="KEEP_BLANK_LINES_IN_DECLARATIONS" value="1" />
+    <option name="KEEP_BLANK_LINES_IN_CODE" value="1" />
+    <option name="KEEP_BLANK_LINES_BEFORE_RBRACE" value="1" />
+    <option name="INDENT_CASE_FROM_SWITCH" value="false" />
+    <option name="ALIGN_MULTILINE_PARAMETERS" value="false" />
+    <option name="ALIGN_MULTILINE_RESOURCES" value="false" />
+    <option name="SPACE_BEFORE_ARRAY_INITIALIZER_LBRACE" value="true" />
+    <option name="CALL_PARAMETERS_WRAP" value="1" />
+    <option name="METHOD_PARAMETERS_WRAP" value="1" />
+    <option name="RESOURCE_LIST_WRAP" value="5" />
+    <option name="EXTENDS_LIST_WRAP" value="1" />
+    <option name="THROWS_LIST_WRAP" value="1" />
+    <option name="EXTENDS_KEYWORD_WRAP" value="1" />
+    <option name="THROWS_KEYWORD_WRAP" value="1" />
+    <option name="METHOD_CALL_CHAIN_WRAP" value="1" />
+    <option name="BINARY_OPERATION_WRAP" value="1" />
+    <option name="BINARY_OPERATION_SIGN_ON_NEXT_LINE" value="true" />
+    <option name="TERNARY_OPERATION_WRAP" value="5" />
+    <option name="ARRAY_INITIALIZER_WRAP" value="1" />
+    <option name="ASSIGNMENT_WRAP" value="1" />
+    <option name="METHOD_ANNOTATION_WRAP" value="2" />
+    <option name="CLASS_ANNOTATION_WRAP" value="2" />
+    <option name="FIELD_ANNOTATION_WRAP" value="2" />
+    <option name="VARIABLE_ANNOTATION_WRAP" value="2" />
+    <indentOptions>
+      <option name="INDENT_SIZE" value="2" />
+      <option name="CONTINUATION_INDENT_SIZE" value="4" />
+      <option name="TAB_SIZE" value="2" />
+    </indentOptions>
+  </codeStyleSettings>
+</code_scheme>

dev-support/docker/Dockerfile_debian_10

@@ -29,28 +29,25 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 RUN echo APT::Install-Recommends "0"\; > /etc/apt/apt.conf.d/10disableextras
 RUN echo APT::Install-Suggests "0"\; >>  /etc/apt/apt.conf.d/10disableextras
 
-#####
-# For installing the latest packages
-#####
-RUN echo 'deb http://deb.debian.org/debian testing main' >> /etc/apt/sources.list
-
 ENV DEBIAN_FRONTEND noninteractive
 ENV DEBCONF_TERSE true
 
 ######
 # Platform package dependency resolver
 ######
 COPY pkg-resolver pkg-resolver
-RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \
-    && chmod a+r pkg-resolver/*.json
+RUN chmod a+x pkg-resolver/install-pkg-resolver.sh
+RUN pkg-resolver/install-pkg-resolver.sh debian:10
 
 ######
 # Install packages from apt
 ######
 # hadolint ignore=DL3008,SC2046
 RUN apt-get -q update \
-    && apt-get -q install -y --no-install-recommends python3 \
     && apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py debian:10) \
+    && echo 'deb http://deb.debian.org/debian bullseye main' >> /etc/apt/sources.list \
+    && apt-get -q update \
+    && apt-get -q install -y --no-install-recommends -t bullseye $(pkg-resolver/resolve.py --release=bullseye debian:10) \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
@@ -85,7 +82,6 @@ ENV HADOOP_SKIP_YETUS_VERIFICATION true
 ####
 # Install packages
 ####
-RUN pkg-resolver/install-common-pkgs.sh
 RUN pkg-resolver/install-spotbugs.sh debian:10
 RUN pkg-resolver/install-boost.sh debian:10
 RUN pkg-resolver/install-protobuf.sh debian:10