Change Python kokoro configs to test only their specified version. (#…

…7927)

Using non-versioned scripts for `build_file` means every Python version is
tested (under Tox), and this is multiplied by each Python version running Tox.

For example, when the `python38` config is used to run Tox, the
Tox layer runs on Python 3.8. This then tests all of the Python versions
individually, including, for example, python27 tests.

This change fixes the `build_file` paths to point to the same-pathed build
script. For example, `kokoro/linux/python27/presubmit.cfg` now uses
`kokoro/linux/python27/build.sh`.

Some additional fixes:

*  Use `python -m tox` in tests.sh instead of just `tox`. This helps non-site
   installations of tox, where the `tox` script may not be on `$PATH`.
*  Ensure tox (and other Python build-related packages) are available in
   Python testing images. (New images have been pushed.)
*  Disable `--warnings_as_errors` due to deprecated function.
*  Remove apt lists per [Docker best practices][1].

[1]: https://docs.docker.com/develop/develop-images/dockerfile_best-practices/

kokoro/linux/dockerfile/test/python27/Dockerfile

@@ -20,4 +20,12 @@ RUN apt-get update && apt-get install -y \
   parallel \
   time \
   wget \
-  && apt-get clean
+  && apt-get clean \
+  && rm -rf /var/lib/apt/lists/*
+
+# Install Python libraries.
+RUN python -m pip install --no-cache-dir --upgrade \
+  pip \
+  setuptools \
+  tox \
+  wheel

kokoro/linux/dockerfile/test/python35/Dockerfile

@@ -20,4 +20,12 @@ RUN apt-get update && apt-get install -y \
   parallel \
   time \
   wget \
-  && apt-get clean
+  && apt-get clean \
+  && rm -rf /var/lib/apt/lists/*
+
+# Install Python libraries.
+RUN python -m pip install --no-cache-dir --upgrade \
+  pip \
+  setuptools \
+  tox \
+  wheel

kokoro/linux/dockerfile/test/python36/Dockerfile

@@ -20,4 +20,12 @@ RUN apt-get update && apt-get install -y \
   parallel \
   time \
   wget \
-  && apt-get clean
+  && apt-get clean \
+  && rm -rf /var/lib/apt/lists/*
+
+# Install Python libraries.
+RUN python -m pip install --no-cache-dir --upgrade \
+  pip \
+  setuptools \
+  tox \
+  wheel

kokoro/linux/dockerfile/test/python37/Dockerfile

@@ -20,4 +20,12 @@ RUN apt-get update && apt-get install -y \
   parallel \
   time \
   wget \
-  && apt-get clean
+  && apt-get clean \
+  && rm -rf /var/lib/apt/lists/*
+
+# Install Python libraries.
+RUN python -m pip install --no-cache-dir --upgrade \
+  pip \
+  setuptools \
+  tox \
+  wheel

kokoro/linux/dockerfile/test/python38/Dockerfile

@@ -20,4 +20,12 @@ RUN apt-get update && apt-get install -y \
   parallel \
   time \
   wget \
-  && apt-get clean
+  && apt-get clean \
+  && rm -rf /var/lib/apt/lists/*
+
+# Install Python libraries.
+RUN python -m pip install --no-cache-dir --upgrade \
+  pip \
+  setuptools \
+  tox \
+  wheel

kokoro/linux/python27/continuous.cfg

@@ -1,7 +1,7 @@
 # Config file for running tests in Kokoro
 
 # Location of the build script in repository
-build_file: "protobuf/kokoro/linux/python/build.sh"
+build_file: "protobuf/kokoro/linux/python27/build.sh"
 timeout_mins: 120
 
 action {

kokoro/linux/python27/presubmit.cfg

@@ -1,7 +1,7 @@
 # Config file for running tests in Kokoro
 
 # Location of the build script in repository
-build_file: "protobuf/kokoro/linux/python/build.sh"
+build_file: "protobuf/kokoro/linux/python27/build.sh"
 timeout_mins: 120
 
 action {

kokoro/linux/python27_cpp/continuous.cfg

@@ -1,7 +1,7 @@
 # Config file for running tests in Kokoro
 
 # Location of the build script in repository
-build_file: "protobuf/kokoro/linux/python_cpp/build.sh"
+build_file: "protobuf/kokoro/linux/python27_cpp/build.sh"
 timeout_mins: 120
 
 action {

kokoro/linux/python27_cpp/presubmit.cfg

@@ -1,7 +1,7 @@
 # Config file for running tests in Kokoro
 
 # Location of the build script in repository
-build_file: "protobuf/kokoro/linux/python_cpp/build.sh"
+build_file: "protobuf/kokoro/linux/python27_cpp/build.sh"
 timeout_mins: 120
 
 action {

kokoro/linux/python35/continuous.cfg

@@ -1,7 +1,7 @@
 # Config file for running tests in Kokoro
 
 # Location of the build script in repository
-build_file: "protobuf/kokoro/linux/python/build.sh"
+build_file: "protobuf/kokoro/linux/python35/build.sh"
 timeout_mins: 120
 
 action {

kokoro/linux/python35/presubmit.cfg

@@ -1,7 +1,7 @@
 # Config file for running tests in Kokoro
 
 # Location of the build script in repository
-build_file: "protobuf/kokoro/linux/python/build.sh"
+build_file: "protobuf/kokoro/linux/python35/build.sh"
 timeout_mins: 120
 
 action {

kokoro/linux/python35_cpp/continuous.cfg

@@ -1,7 +1,7 @@
 # Config file for running tests in Kokoro
 
 # Location of the build script in repository
-build_file: "protobuf/kokoro/linux/python_cpp/build.sh"
+build_file: "protobuf/kokoro/linux/python35_cpp/build.sh"
 timeout_mins: 120
 
 action {

kokoro/linux/python35_cpp/presubmit.cfg

@@ -1,7 +1,7 @@
 # Config file for running tests in Kokoro
 
 # Location of the build script in repository
-build_file: "protobuf/kokoro/linux/python_cpp/build.sh"
+build_file: "protobuf/kokoro/linux/python35_cpp/build.sh"
 timeout_mins: 120
 
 action {

kokoro/linux/python36/continuous.cfg

@@ -1,7 +1,7 @@
 # Config file for running tests in Kokoro
 
 # Location of the build script in repository
-build_file: "protobuf/kokoro/linux/python/build.sh"
+build_file: "protobuf/kokoro/linux/python36/build.sh"
 timeout_mins: 120
 
 action {

kokoro/linux/python36/presubmit.cfg

@@ -1,7 +1,7 @@
 # Config file for running tests in Kokoro
 
 # Location of the build script in repository
-build_file: "protobuf/kokoro/linux/python/build.sh"
+build_file: "protobuf/kokoro/linux/python36/build.sh"
 timeout_mins: 120
 
 action {

kokoro/linux/python36_cpp/continuous.cfg

@@ -1,7 +1,7 @@
 # Config file for running tests in Kokoro
 
 # Location of the build script in repository
-build_file: "protobuf/kokoro/linux/python_cpp/build.sh"
+build_file: "protobuf/kokoro/linux/python36_cpp/build.sh"
 timeout_mins: 120
 
 action {

kokoro/linux/python36_cpp/presubmit.cfg

@@ -1,7 +1,7 @@
 # Config file for running tests in Kokoro
 
 # Location of the build script in repository
-build_file: "protobuf/kokoro/linux/python_cpp/build.sh"
+build_file: "protobuf/kokoro/linux/python36_cpp/build.sh"
 timeout_mins: 120
 
 action {

kokoro/linux/python37/continuous.cfg

@@ -1,7 +1,7 @@
 # Config file for running tests in Kokoro
 
 # Location of the build script in repository
-build_file: "protobuf/kokoro/linux/python/build.sh"
+build_file: "protobuf/kokoro/linux/python37/build.sh"
 timeout_mins: 120
 
 action {

kokoro/linux/python37/presubmit.cfg

@@ -1,7 +1,7 @@
 # Config file for running tests in Kokoro
 
 # Location of the build script in repository
-build_file: "protobuf/kokoro/linux/python/build.sh"
+build_file: "protobuf/kokoro/linux/python37/build.sh"
 timeout_mins: 120
 
 action {

kokoro/linux/python37_cpp/continuous.cfg

@@ -1,7 +1,7 @@
 # Config file for running tests in Kokoro
 
 # Location of the build script in repository
-build_file: "protobuf/kokoro/linux/python_cpp/build.sh"
+build_file: "protobuf/kokoro/linux/python37_cpp/build.sh"
 timeout_mins: 120
 
 action {

kokoro/linux/python37_cpp/presubmit.cfg

@@ -1,7 +1,7 @@
 # Config file for running tests in Kokoro
 
 # Location of the build script in repository
-build_file: "protobuf/kokoro/linux/python_cpp/build.sh"
+build_file: "protobuf/kokoro/linux/python37_cpp/build.sh"
 timeout_mins: 120
 
 action {

kokoro/linux/python38/continuous.cfg

@@ -1,7 +1,7 @@
 # Config file for running tests in Kokoro
 
 # Location of the build script in repository
-build_file: "protobuf/kokoro/linux/python/build.sh"
+build_file: "protobuf/kokoro/linux/python38/build.sh"
 timeout_mins: 120
 
 action {

kokoro/linux/python38/presubmit.cfg

@@ -1,7 +1,7 @@
 # Config file for running tests in Kokoro
 
 # Location of the build script in repository
-build_file: "protobuf/kokoro/linux/python/build.sh"
+build_file: "protobuf/kokoro/linux/python38/build.sh"
 timeout_mins: 120
 
 action {

kokoro/linux/python38_cpp/continuous.cfg

@@ -1,7 +1,7 @@
 # Config file for running tests in Kokoro
 
 # Location of the build script in repository
-build_file: "protobuf/kokoro/linux/python_cpp/build.sh"
+build_file: "protobuf/kokoro/linux/python38_cpp/build.sh"
 timeout_mins: 120
 
 action {

kokoro/linux/python38_cpp/presubmit.cfg

@@ -1,7 +1,7 @@
 # Config file for running tests in Kokoro
 
 # Location of the build script in repository
-build_file: "protobuf/kokoro/linux/python_cpp/build.sh"
+build_file: "protobuf/kokoro/linux/python38_cpp/build.sh"
 timeout_mins: 120
 
 action {

python/tox.ini

@@ -14,7 +14,10 @@ setenv =
 commands =
     python setup.py -q build_py
     python: python setup.py -q build
-    cpp: python setup.py -q build --cpp_implementation --warnings_as_errors --compile_static_extension
+    # --warnings_as_errors disabled until we update the Python C extension. See:
+    # https://github.com/protocolbuffers/protobuf/issues/7930
+    # cpp: python setup.py -q build --cpp_implementation --warnings_as_errors --compile_static_extension
+    cpp: python setup.py -q build --cpp_implementation --compile_static_extension
     python: python setup.py -q test -q
     cpp: python setup.py -q test -q --cpp_implementation
     python: python setup.py -q test_conformance

tests.sh

@@ -316,15 +316,15 @@ build_python() {
   else
     envlist=py\{27,36\}-python
   fi
-  tox -e $envlist
+  python -m tox -e $envlist
   cd ..
 }
 
 build_python_version() {
   internal_build_cpp
   cd python
   envlist=$1
-  tox -e $envlist
+  python -m tox -e $envlist
   cd ..
 }