[v1.x] Cherry-pick apache#17776 apache#17681 (apache#18465)

* Fix CD (apache#17776) * Fix cd/mxnet_lib/dynamic/Jenkins_pipeline.groovy Fixes a regression in CD introduced by apache#17645 * Fix whitespace * Add NATIVE_ADDITIONAL.md Fixes a regression in CD introduced by apache#16899 * Update other $TYPE_ADDITIONAL.MD * Fix cd/python/docker Fixes regression introduced by apache#15990 * [CD] update pypi description, setup.py (apache#17681) * update pypi description, setup.py, use manylinux2010, use unified dist link for nightly * Use manylinux2014 Co-authored-by: Leonard Lausen <leonard@lausen.nl> * reverting .so path as per MAKE flow Co-authored-by: Leonard Lausen <lausen@amazon.com> Co-authored-by: Sheng Zha <szha@users.noreply.github.com> Co-authored-by: Leonard Lausen <leonard@lausen.nl>
ChaiBapchya · Aug 15, 2020 · afa1398 · afa1398
1 parent 2472d4a
commit afa1398
Show file tree

Hide file tree

Showing 23 changed files with 92 additions and 457 deletions.
diff --git a/cd/mxnet_lib/mxnet_lib_pipeline.groovy b/cd/mxnet_lib/mxnet_lib_pipeline.groovy
@@ -52,7 +52,7 @@ def get_pipeline(mxnet_variant, build_fn) {
             }
           }
         }
-        
+
         parallel tests
       }
 

diff --git a/cd/python/docker/Dockerfile b/cd/python/docker/Dockerfile
@@ -23,7 +23,7 @@
 ARG BASE_IMAGE
 FROM ${BASE_IMAGE}
 
-ARG PYTHON_CMD=python
+ARG PYTHON_CMD=python3
 RUN apt-get update && \
     apt-get install -y wget ${PYTHON_CMD}-dev gcc && \
     wget https://bootstrap.pypa.io/get-pip.py && \

diff --git a/cd/python/docker/Jenkins_pipeline.groovy b/cd/python/docker/Jenkins_pipeline.groovy
@@ -50,24 +50,21 @@ def build(mxnet_variant) {
     ci_utils.docker_run(environment, "cd_package_pypi ${mxnet_variant}", nvidia_docker)
 
     // build python docker images
-    sh "./cd/python/docker/python_images.sh build ${mxnet_variant} py3"
-    sh "./cd/python/docker/python_images.sh build ${mxnet_variant} py2"
+    sh "./cd/python/docker/python_images.sh build ${mxnet_variant}"
   }
 }
 
 def test(mxnet_variant) {
   ws("workspace/python_docker/${mxnet_variant}/${env.BUILD_NUMBER}") {
     // test python docker images
-    sh "./cd/python/docker/python_images.sh test ${mxnet_variant} py3"
-    sh "./cd/python/docker/python_images.sh test ${mxnet_variant} py2"
+    sh "./cd/python/docker/python_images.sh test ${mxnet_variant}"
   }
 }
 
 def push(mxnet_variant) {
   ws("workspace/python_docker/${mxnet_variant}/${env.BUILD_NUMBER}") {
     // push python docker images
-    sh "./cd/python/docker/python_images.sh push ${mxnet_variant} py3"
-    sh "./cd/python/docker/python_images.sh push ${mxnet_variant} py2"
+    sh "./cd/python/docker/python_images.sh push ${mxnet_variant}"
   }
 }
 

diff --git a/cd/python/docker/python_images.sh b/cd/python/docker/python_images.sh
@@ -23,30 +23,16 @@
 
 set -xe
 
-usage="Usage: python_images.sh <build|test|publish> MXNET-VARIANT <py2|py3>"
+usage="Usage: python_images.sh <build|test|publish> MXNET-VARIANT"
 
 command=${1:?$usage}
 mxnet_variant=${2:?$usage}
-python_version=${3:?usage}
 
 cd_utils='cd/utils'
 ci_utils='ci/'
 
-case ${python_version} in
-    py3)
-        python_cmd="python3"
-        ;;
-    py2)
-        python_cmd="python"
-        ;;
-    *)
-        echo "Error: specify python version with either 'py2' or 'py3'"
-        exit 1
-        ;;
-esac
-
 docker_tags=($(./${cd_utils}/docker_tag.sh ${mxnet_variant}))
-main_tag="${docker_tags[0]}_${python_version}"
+main_tag="${docker_tags[0]}_py3"
 base_image=$(./${cd_utils}/mxnet_base_image.sh ${mxnet_variant})
 repository="python"
 image_name="${repository}:${main_tag}"
@@ -59,7 +45,7 @@ fi
 
 build() {
     # NOTE: Ensure the correct context root is passed in when building - Dockerfile expects ./wheel_build
-    docker build -t "${image_name}" --build-arg PYTHON_CMD=${python_cmd} --build-arg BASE_IMAGE="${base_image}" --build-arg MXNET_COMMIT_ID=${GIT_COMMIT} -f ${resources_path}/Dockerfile ./wheel_build
+    docker build -t "${image_name}" --build-arg --build-arg BASE_IMAGE="${base_image}" --build-arg MXNET_COMMIT_ID=${GIT_COMMIT} -f ${resources_path}/Dockerfile ./wheel_build
 }
 
 test() {
@@ -68,10 +54,10 @@ test() {
         runtime_param="--runtime=nvidia"
     fi
     local test_image_name="${image_name}_test"
-    
+
     # Ensure the correct context root is passed in when building - Dockerfile.test expects ci directory
     docker build -t "${test_image_name}" --build-arg USER_ID=`id -u` --build-arg GROUP_ID=`id -g` --build-arg BASE_IMAGE="${image_name}" -f ${resources_path}/Dockerfile.test ./ci
-    ./ci/safe_docker_run.py ${runtime_param} --cap-add "SYS_PTRACE" -u `id -u`:`id -g` -v `pwd`:/work/mxnet "${test_image_name}" ${resources_path}/test_python_image.sh "${mxnet_variant}" "${python_cmd}"
+    ./ci/safe_docker_run.py ${runtime_param} --cap-add "SYS_PTRACE" -u `id -u`:`id -g` -v `pwd`:/work/mxnet "${test_image_name}" ${resources_path}/test_python_image.sh "${mxnet_variant}"
 }
 
 push() {
@@ -90,23 +76,14 @@ push() {
     # Iterate over remaining tags, if any
     for ((i=1;i<${#docker_tags[@]};i++)); do
         local docker_tag="${docker_tags[${i}]}"
-        local latest_image_name="${RELEASE_DOCKERHUB_REPOSITORY}/${repository}:${docker_tag}"
-
-        # latest and latest gpu should only be pushed for py3
-        if [[ ${docker_tag} == "latest" || ${docker_tag} == "latest_gpu" ]]; then
-            if [[ ${python_version} == "py2" ]]; then
-                continue
-            fi
-        else
-            latest_image_name="${latest_image_name}_${python_version}"
-        fi
+        local latest_image_name="${RELEASE_DOCKERHUB_REPOSITORY}/${repository}:${docker_tag}_py3"
 
         docker tag "${image_name}" "${latest_image_name}"
         docker push "${latest_image_name}"
         echo "Successfully pushed ${latest_image_name}. Pull it with:"
         echo "docker pull ${latest_image_name}"
         echo "For a complete list of tags see https://hub.docker.com/u/${RELEASE_DOCKERHUB_REPOSITORY}/${repository}"
-    done    
+    done
 }
 
 case ${command} in

diff --git a/cd/python/docker/test_python_image.sh b/cd/python/docker/test_python_image.sh
@@ -25,7 +25,6 @@ set -ex
 
 # Variant parameter should be passed in
 mxnet_variant=${1:?"Missing mxnet variant"}
-python_cmd=${2:?"Missing python version (python or python3)"}
 
 if [ -z "${MXNET_COMMIT_ID}" ]; then
     echo "MXNET_COMMIT_ID environment variable is empty. Please rebuild the image with MXNET_COMMIT_ID build-arg specified."
@@ -39,9 +38,9 @@ if [[ $mxnet_variant == cu* ]]; then
 fi
 
 if [[ $mxnet_variant == cpu ]]; then
-    ${python_cmd} tests/python/mkl/test_mkldnn.py
+    python3 tests/python/mkl/test_mkldnn.py
 fi
 
-${python_cmd} tests/python/train/test_conv.py ${test_conv_params}
-${python_cmd} example/image-classification/train_mnist.py ${mnist_params}
+python3 tests/python/train/test_conv.py ${test_conv_params}
+python3 example/image-classification/train_mnist.py ${mnist_params}
 
diff --git a/tools/pip/doc/CPU_ADDITIONAL.md b/tools/pip/doc/CPU_ADDITIONAL.md
@@ -18,19 +18,14 @@
 Prerequisites
 -------------
 This package supports Linux, Mac OSX, and Windows platforms. You may also want to check:
-- [mxnet-cu101](https://pypi.python.org/pypi/mxnet-cu101mkl/) with CUDA-10.1 support.
-- [mxnet-cu101mkl](https://pypi.python.org/pypi/mxnet-cu101mkl/) with CUDA-10.1 support and MKLDNN support.
-- [mxnet-cu100](https://pypi.python.org/pypi/mxnet-cu100mkl/) with CUDA-10.0 support.
-- [mxnet-cu100mkl](https://pypi.python.org/pypi/mxnet-cu100mkl/) with CUDA-10.0 support and MKLDNN support.
+- [mxnet-cu102](https://pypi.python.org/pypi/mxnet-cu101/) with CUDA-10.2 support.
+- [mxnet-cu101](https://pypi.python.org/pypi/mxnet-cu101/) with CUDA-10.1 support.
+- [mxnet-cu100](https://pypi.python.org/pypi/mxnet-cu100/) with CUDA-10.0 support.
 - [mxnet-cu92](https://pypi.python.org/pypi/mxnet-cu92/) with CUDA-9.2 support.
-- [mxnet-cu92mkl](https://pypi.python.org/pypi/mxnet-cu92mkl/) with CUDA-9.2 support and MKLDNN support.
 - [mxnet-cu90](https://pypi.python.org/pypi/mxnet-cu90/) with CUDA-9.0 support.
-- [mxnet-cu90mkl](https://pypi.python.org/pypi/mxnet-cu90mkl/) with CUDA-9.0 support and MKLDNN support.
 - [mxnet-cu80](https://pypi.python.org/pypi/mxnet-cu80/) with CUDA-8.0 support.
-- [mxnet-cu80mkl](https://pypi.python.org/pypi/mxnet-cu80mkl/) with CUDA-8.0 support and MKLDNN support.
 - [mxnet-cu75](https://pypi.python.org/pypi/mxnet-cu75/) with CUDA-7.5 support.
-- [mxnet-cu75mkl](https://pypi.python.org/pypi/mxnet-cu75mkl/) with CUDA-7.5 support and MKLDNN support.
-- [mxnet-mkl](https://pypi.python.org/pypi/mxnet-mkl/) with MKLDNN support.
+- [mxnet-native](https://pypi.python.org/pypi/mxnet-native/) CPU variant without MKLDNN.
 
 To install for other platforms (e.g. Windows, Raspberry Pi/ARM) or other versions, check [Installing MXNet](https://mxnet.apache.org/versions/master/install/index.html) for instructions on building from source.
 
@@ -40,3 +35,10 @@ To install, use:
 ```bash
 pip install mxnet
 ```
+
+Nightly Builds
+--------------
+To install the latest nightly build, use:
+```bash
+pip install --pre mxnet -f https://dist.mxnet.io/python
+```
diff --git a/tools/pip/doc/CU100MKL_ADDITIONAL.md b/tools/pip/doc/CU100MKL_ADDITIONAL.md
diff --git a/tools/pip/doc/CU100_ADDITIONAL.md b/tools/pip/doc/CU100_ADDITIONAL.md
@@ -18,18 +18,11 @@
 Prerequisites
 -------------
 This package supports Linux and Windows platforms. You may also want to check:
-- [mxnet-cu101](https://pypi.python.org/pypi/mxnet-cu101mkl/) with CUDA-10.1 support.
-- [mxnet-cu101mkl](https://pypi.python.org/pypi/mxnet-cu101mkl/) with CUDA-10.1 support and MKLDNN support.
-- [mxnet-cu100mkl](https://pypi.python.org/pypi/mxnet-cu100mkl/) with CUDA-10.0 support and MKLDNN support.
+- [mxnet-cu102](https://pypi.python.org/pypi/mxnet-cu101/) with CUDA-10.2 support.
 - [mxnet-cu92](https://pypi.python.org/pypi/mxnet-cu92/) with CUDA-9.2 support.
-- [mxnet-cu92mkl](https://pypi.python.org/pypi/mxnet-cu92mkl/) with CUDA-9.2 support and MKLDNN support.
 - [mxnet-cu90](https://pypi.python.org/pypi/mxnet-cu90/) with CUDA-9.0 support.
-- [mxnet-cu90mkl](https://pypi.python.org/pypi/mxnet-cu90mkl/) with CUDA-9.0 support and MKLDNN support.
 - [mxnet-cu80](https://pypi.python.org/pypi/mxnet-cu80/) with CUDA-8.0 support.
-- [mxnet-cu80mkl](https://pypi.python.org/pypi/mxnet-cu80mkl/) with CUDA-8.0 support and MKLDNN support.
 - [mxnet-cu75](https://pypi.python.org/pypi/mxnet-cu75/) with CUDA-7.5 support.
-- [mxnet-cu75mkl](https://pypi.python.org/pypi/mxnet-cu75mkl/) with CUDA-7.5 support and MKLDNN support.
-- [mxnet-mkl](https://pypi.python.org/pypi/mxnet-mkl/) with MKLDNN support.
 - [mxnet](https://pypi.python.org/pypi/mxnet/).
 
 To download CUDA, check [CUDA download](https://developer.nvidia.com/cuda-downloads). For more instructions, check [CUDA Toolkit online documentation](http://docs.nvidia.com/cuda/index.html).
@@ -42,3 +35,10 @@ To install:
 ```bash
 pip install mxnet-cu100
 ```
+
+Nightly Builds
+--------------
+To install the latest nightly build, use:
+```bash
+pip install --pre mxnet-cu100 -f https://dist.mxnet.io/python
+```
diff --git a/tools/pip/doc/CU101MKL_ADDITIONAL.md b/tools/pip/doc/CU101MKL_ADDITIONAL.md
diff --git a/tools/pip/doc/CU101_ADDITIONAL.md b/tools/pip/doc/CU101_ADDITIONAL.md
@@ -18,18 +18,11 @@
 Prerequisites
 -------------
 This package supports Linux and Windows platforms. You may also want to check:
-- [mxnet-cu101mkl](https://pypi.python.org/pypi/mxnet-cu101mkl/) with CUDA-10.1 support and MKLDNN support.
-- [mxnet-cu100](https://pypi.python.org/pypi/mxnet-cu100mkl/) with CUDA-10.0 support.
-- [mxnet-cu100mkl](https://pypi.python.org/pypi/mxnet-cu100mkl/) with CUDA-10.0 support and MKLDNN support.
+- [mxnet-cu102](https://pypi.python.org/pypi/mxnet-cu101/) with CUDA-10.2 support.
 - [mxnet-cu92](https://pypi.python.org/pypi/mxnet-cu92/) with CUDA-9.2 support.
-- [mxnet-cu92mkl](https://pypi.python.org/pypi/mxnet-cu92mkl/) with CUDA-9.2 support and MKLDNN support.
 - [mxnet-cu90](https://pypi.python.org/pypi/mxnet-cu90/) with CUDA-9.0 support.
-- [mxnet-cu90mkl](https://pypi.python.org/pypi/mxnet-cu90mkl/) with CUDA-9.0 support and MKLDNN support.
 - [mxnet-cu80](https://pypi.python.org/pypi/mxnet-cu80/) with CUDA-8.0 support.
-- [mxnet-cu80mkl](https://pypi.python.org/pypi/mxnet-cu80mkl/) with CUDA-8.0 support and MKLDNN support.
 - [mxnet-cu75](https://pypi.python.org/pypi/mxnet-cu75/) with CUDA-7.5 support.
-- [mxnet-cu75mkl](https://pypi.python.org/pypi/mxnet-cu75mkl/) with CUDA-7.5 support and MKLDNN support.
-- [mxnet-mkl](https://pypi.python.org/pypi/mxnet-mkl/) with MKLDNN support.
 - [mxnet](https://pypi.python.org/pypi/mxnet/).
 
 To download CUDA, check [CUDA download](https://developer.nvidia.com/cuda-downloads). For more instructions, check [CUDA Toolkit online documentation](http://docs.nvidia.com/cuda/index.html).
@@ -42,3 +35,10 @@ To install:
 ```bash
 pip install mxnet-cu101
 ```
+
+Nightly Builds
+--------------
+To install the latest nightly build, use:
+```bash
+pip install --pre mxnet-cu101 -f https://dist.mxnet.io/python
+```
diff --git a/tools/pip/doc/CU102MKL_ADDITIONAL.md b/tools/pip/doc/CU102MKL_ADDITIONAL.md