Merge branch 'apache:trunk' into HADOOP-19052

Author: liangyu-1

.github/labeler.yml

@@ -0,0 +1,57 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Pull Request Labeler Github Action Configuration: https://github.com/marketplace/actions/labeler
+
+trunk:
+  - '**'
+INFRA:
+  - .asf.yaml
+  - .gitattributes
+  - .gitignore
+  - .github/**
+  - dev-support/**
+  - start-build-env.sh
+BUILD:
+  - '**/pom.xml'
+COMMON:
+  - hadoop-common-project/**
+HDFS:
+  - hadoop-hdfs-project/**
+RBF:
+  - hadoop-hdfs-project/hadoop-hdfs-rbf/**
+NATIVE:
+  - hadoop-hdfs-project/hadoop-hdfs-native-client/**
+  - hadoop-common-project/hadoop-common/src/main/native/**
+YARN:
+  - hadoop-yarn-project/**
+MAPREDUCE:
+  - hadoop-mapreduce-project/**
+DISTCP:
+  - hadoop-tools/hadoop-distcp/**
+TOOLS:
+  - hadoop-tools/**
+AWS:
+  - hadoop-tools/hadoop-aws/**
+ABFS:
+  - hadoop-tools/hadoop-azure/**
+DYNAMOMETER:
+  - hadoop-tools/hadoop-dynamometer/**
+MAVEN-PLUGINS:
+  - hadoop-maven-plugins/**

.github/workflows/labeler.yml

@@ -0,0 +1,40 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: "Pull Request Labeler"
+on: pull_request_target
+
+permissions:
+  contents: read
+  pull-requests: write
+
+jobs:
+  triage:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          sparse-checkout: |
+            .github
+      - uses: actions/labeler@v4.3.0
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          sync-labels: true
+          configuration-path: .github/labeler.yml
+          dot: true

.gitignore

@@ -1,4 +1,4 @@
-.DS_Store
+*.DS_Store
 *.iml
 *.ipr
 *.iws

BUILDING.txt

@@ -7,7 +7,7 @@ Requirements:
 * JDK 1.8
 * Maven 3.3 or later
 * Boost 1.72 (if compiling native code)
-* Protocol Buffers 3.7.1 (if compiling native code)
+* Protocol Buffers 3.21.12 (if compiling native code)
 * CMake 3.19 or newer (if compiling native code)
 * Zlib devel (if compiling native code)
 * Cyrus SASL devel (if compiling native code)
@@ -74,10 +74,10 @@ Refer to  dev-support/docker/Dockerfile):
   $ ./bootstrap
   $ make -j$(nproc)
   $ sudo make install
-* Protocol Buffers 3.7.1 (required to build native code)
-  $ curl -L -s -S https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz -o protobuf-3.7.1.tar.gz
-  $ mkdir protobuf-3.7-src
-  $ tar xzf protobuf-3.7.1.tar.gz --strip-components 1 -C protobuf-3.7-src && cd protobuf-3.7-src
+* Protocol Buffers 3.21.12 (required to build native code)
+  $ curl -L https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.21.12.tar.gz > protobuf-3.21.12.tar.gz
+  $ tar -zxvf protobuf-3.21.12.tar.gz && cd protobuf-3.21.12
+  $ ./autogen.sh
   $ ./configure
   $ make -j$(nproc)
   $ sudo make install
@@ -163,13 +163,13 @@ Maven build goals:
  YARN Application Timeline Service V2 build options:
 
    YARN Timeline Service v.2 chooses Apache HBase as the primary backing storage. The supported
-   versions of Apache HBase are 1.2.6 (default) and 2.0.0-beta1.
+   versions of Apache HBase are 1.7.1 (default) and 2.2.4.
 
-  * HBase 1.2.6 is used by default to build Hadoop. The official releases are ready to use if you
-    plan on running Timeline Service v2 with HBase 1.2.6.
+  * HBase 1.7.1 is used by default to build Hadoop. The official releases are ready to use if you
+    plan on running Timeline Service v2 with HBase 1.7.1.
 
-  * Use -Dhbase.profile=2.0 to build Hadoop with HBase 2.0.0-beta1. Provide this option if you plan
-    on running Timeline Service v2 with HBase 2.0.
+  * Use -Dhbase.profile=2.0 to build Hadoop with HBase 2.2.4. Provide this option if you plan
+    on running Timeline Service v2 with HBase 2.x.
 
 
  Snappy build options:
@@ -311,6 +311,36 @@ Maven build goals:
     package. This option requires that -Dpmdk.lib is specified. With -Dbundle.pmdk provided,
     the build will fail if -Dpmdk.lib is not specified.
 
+Controlling the redistribution of the protobuf-2.5 dependency
+
+    The protobuf 2.5.0 library is used at compile time to compile the class
+    org.apache.hadoop.ipc.ProtobufHelper; this class known to have been used by
+    external projects in the past. Protobuf 2.5 is not used elsewhere in
+    the Hadoop codebase; alongside the move to Protobuf 3.x a private successor
+    class, org.apache.hadoop.ipc.internal.ShadedProtobufHelper is now used.
+
+    The hadoop-common module no longer exports its compile-time dependency on
+    protobuf-2.5. Hadoop distributions no longer include it.
+    Any application declaring a dependency on hadoop-commmon will no longer get
+    the artifact added to their classpath.
+    If is still required, then they must explicitly declare it:
+
+      <dependency>
+        <groupId>com.google.protobuf</groupId>
+        <artifactId>protobuf-java</artifactId>
+        <version>2.5.0</version>
+      </dependency>
+
+    In Hadoop builds the scope of the dependency can be set with the
+    option "common.protobuf2.scope".
+    This can be upgraded from "provided" to "compile" on the maven command line:
+
+           -Dcommon.protobuf2.scope=compile
+
+    If this is done then protobuf-2.5.0.jar will again be exported as a
+    hadoop-common dependency, and included in the share/hadoop/common/lib/
+    directory of any Hadoop distribution built.
+
 ----------------------------------------------------------------------------------
 Building components separately
 
@@ -403,10 +433,10 @@ Installing required dependencies for clean install of macOS 10.14:
 * Install native libraries, only openssl is required to compile native code,
 you may optionally install zlib, lz4, etc.
   $ brew install openssl
-* Protocol Buffers 3.7.1 (required to compile native code)
-  $ wget https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz
-  $ mkdir -p protobuf-3.7 && tar zxvf protobuf-java-3.7.1.tar.gz --strip-components 1 -C protobuf-3.7
-  $ cd protobuf-3.7
+* Protocol Buffers 3.21.12 (required to compile native code)
+  $ curl -L https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.21.12.tar.gz > protobuf-3.21.12.tar.gz
+  $ tar -zxvf protobuf-3.21.12.tar.gz && cd protobuf-3.21.12
+  $ ./autogen.sh
   $ ./configure
   $ make
   $ make check
@@ -442,11 +472,10 @@ Building on CentOS 8
 * Install python2 for building documentation.
   $ sudo dnf install python2
 
-* Install Protocol Buffers v3.7.1.
-  $ git clone https://github.com/protocolbuffers/protobuf
-  $ cd protobuf
-  $ git checkout v3.7.1
-  $ autoreconf -i
+* Install Protocol Buffers v3.21.12.
+  $ curl -L https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.21.12.tar.gz > protobuf-3.21.12.tar.gz
+  $ tar -zxvf protobuf-3.21.12.tar.gz && cd protobuf-3.21.12
+  $ ./autogen.sh
   $ ./configure --prefix=/usr/local
   $ make
   $ sudo make install
@@ -501,7 +530,7 @@ Requirements:
 * JDK 1.8
 * Maven 3.0 or later (maven.apache.org)
 * Boost 1.72 (boost.org)
-* Protocol Buffers 3.7.1 (https://github.com/protocolbuffers/protobuf/releases)
+* Protocol Buffers 3.21.12 (https://github.com/protocolbuffers/protobuf/tags)
 * CMake 3.19 or newer (cmake.org)
 * Visual Studio 2019 (visualstudio.com)
 * Windows SDK 8.1 (optional, if building CPU rate control for the container executor. Get this from