[v1.x] use centos7 base image for aarch64 build (apache#20392)

* add centos7 aarch64 build * enable rh packages * update openssl curl version * install rhel7 armpl * find armpl cmake * test mkldnn build * test ninja * use armpl compiled with gcc-8 * enable py38 for unittests * check previously failing tests * create symlink for ninja * fix cd pypi * fix cd docker * update ssl for mac
waytrue17 · Jul 7, 2021 · a142898 · a142898
1 parent d0c2a94
commit a142898
Show file tree

Hide file tree

Showing 18 changed files with 150 additions and 164 deletions.
diff --git a/cd/mxnet_lib/Jenkins_pipeline.groovy b/cd/mxnet_lib/Jenkins_pipeline.groovy
@@ -47,7 +47,7 @@ def build(mxnet_variant) {
     node(NODE_LINUX_AARCH64_CPU) {
       ws("workspace/mxnet_${libtype}/${mxnet_variant}/${env.BUILD_NUMBER}") {
         ci_utils.init_git()
-        ci_utils.docker_run('publish.ubuntu1804_aarch64_cpu', "build_static_libmxnet ${mxnet_variant}", false)
+        ci_utils.docker_run('centos7_aarch64_cpu', "build_static_libmxnet ${mxnet_variant}", false)
         ci_utils.pack_lib("mxnet_${mxnet_variant}", libmxnet_pipeline.get_stash(mxnet_variant))
       }
     }

diff --git a/cd/mxnet_lib/mxnet_lib_pipeline.groovy b/cd/mxnet_lib/mxnet_lib_pipeline.groovy
@@ -76,7 +76,7 @@ def get_stash(mxnet_variant) {
 // The environment corresponds to the docker files in the 'docker' directory
 def get_environment(mxnet_variant) {
   if (mxnet_variant.startsWith("aarch64")) {
-    return "publish.ubuntu1804_aarch64_cpu"
+    return "centos7_aarch64_cpu"
   } else if (mxnet_variant.startsWith("cu")) {
     // Remove 'mkl' suffix from variant to properly format test environment
     return "ubuntu_gpu_${mxnet_variant.replace('mkl', '')}"

diff --git a/cd/python/docker/Dockerfile b/cd/python/docker/Dockerfile
@@ -36,12 +36,12 @@ RUN apt-get install -y libgomp1
 
 ARG MXNET_VARIANT
 RUN if [ "$MXNET_VARIANT" = "aarch64_cpu" ] ; then echo "not installing libquadmath0 on aarch64" ; else apt-get install -y libquadmath0 ; fi
-RUN if [ "$MXNET_VARIANT" = "aarch64_cpu" ] ; \
-    then wget https://armkeil.blob.core.windows.net/developer/Files/downloads/hpc/arm-performance-libraries/21-0-0/Ubuntu18.04/arm-performance-libraries_21.0_Ubuntu-18.04_gcc-8.2.tar && \
+RUN if [ "$MXNET_VARIANT" = "aarch64_cpu" ] ; then \
+    wget https://armkeil.blob.core.windows.net/developer/Files/downloads/hpc/arm-performance-libraries/21-0-0/Ubuntu18.04/arm-performance-libraries_21.0_Ubuntu-18.04_gcc-8.2.tar && \
     tar -xvf arm-performance-libraries_21.0_Ubuntu-18.04_gcc-8.2.tar && \
     arm-performance-libraries_21.0_Ubuntu-18.04_gcc-8.2/arm-performance-libraries_21.0_Ubuntu-18.04.sh -a; \
     fi
-ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/opt/arm/armpl_21.0_gcc-8.2/lib/
+ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/opt/arm/armpl_21.0_gcc-8.2/lib
 
 ARG MXNET_COMMIT_ID
 ENV MXNET_COMMIT_ID=${MXNET_COMMIT_ID}

diff --git a/cd/python/docker/Jenkins_pipeline.groovy b/cd/python/docker/Jenkins_pipeline.groovy
@@ -42,7 +42,7 @@ def get_pipeline(mxnet_variant) {
 // The environment corresponds to the docker files in the 'docker' directory
 def get_environment(mxnet_variant) {
   if (mxnet_variant.startsWith('aarch64')) {
-    return "publish.ubuntu1804_aarch64_cpu"
+    return "centos7_aarch64_cpu"
   }
   if (mxnet_variant.startsWith('cu')) {
     return "ubuntu_gpu_${mxnet_variant}"

diff --git a/cd/python/pypi/Jenkins_pipeline.groovy b/cd/python/pypi/Jenkins_pipeline.groovy
@@ -48,7 +48,7 @@ def get_pipeline(mxnet_variant) {
 // The environment corresponds to the docker files in the 'docker' directory
 def get_environment(mxnet_variant) {
   if (mxnet_variant.startsWith('aarch64')) {
-    return "publish.ubuntu1804_aarch64_cpu"
+    return "centos7_aarch64_cpu"
   }
   if (mxnet_variant.startsWith('cu')) {
     return "ubuntu_gpu_${mxnet_variant}"
@@ -73,7 +73,7 @@ def test(mxnet_variant) {
     // test wheel file
     def environment = get_environment(mxnet_variant)
     def nvidia_docker = mxnet_variant.startsWith('cu')
-    ci_utils.docker_run(environment, "cd_integration_test_pypi python3 ${nvidia_docker}", nvidia_docker)
+    ci_utils.docker_run(environment, "cd_integration_test_pypi ${mxnet_variant} ${nvidia_docker}", nvidia_docker)
   }
 }
 

diff --git a/ci/docker/Dockerfile.build.centos7_aarch64_cpu b/ci/docker/Dockerfile.build.centos7_aarch64_cpu
@@ -0,0 +1,111 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile for CentOS 7 AArch64 CPU build.
+# Via the CentOS 7 Dockerfiles, we ensure MXNet continues to run fine on older systems.
+
+FROM arm64v8/centos:7
+
+WORKDIR /work/deps
+
+RUN yum -y check-update || true && \
+    yum -y install epel-release centos-release-scl centos-release-scl-rh && \
+    yum install -y \
+        # Utilities
+        wget \
+        unzip \
+        patchelf \
+        pandoc \
+        # Development tools
+        git \
+        make \
+        ninja-build \
+        automake \
+        autoconf \
+        libtool \
+        protobuf-compiler \
+        protobuf-devel \
+        # CentOS Software Collections https://www.softwarecollections.org
+        devtoolset-10 \
+        devtoolset-10-gcc \
+        devtoolset-10-gcc-c++ \
+        devtoolset-10-gcc-gfortran \
+        rh-python38 \
+        rh-python38-python-numpy \
+        rh-python38-python-scipy \
+        # Libraries
+        opencv-devel \
+        openssl-devel \
+        zeromq-devel \
+        # Build-dependencies for ccache 3.7.9
+        gperf \
+        libb2-devel \
+        libzstd-devel && \
+    yum clean all
+
+# Make Red Hat Developer Toolset 10.0 and Python 3.8 Software Collections available by default
+# during the following build steps in this Dockerfile
+SHELL [ "/usr/bin/scl", "enable", "devtoolset-10", "rh-python38" ]
+
+# Install minimum required cmake version
+RUN cd /usr/local/src && \
+    wget -nv https://cmake.org/files/v3.20/cmake-3.20.5-linux-aarch64.sh && \
+    sh cmake-3.20.5-linux-aarch64.sh --prefix=/usr/local --skip-license && \
+    rm cmake-3.20.5-linux-aarch64.sh
+
+# ccache 3.7.9 has fixes for caching nvcc outputs
+RUN cd /usr/local/src && \
+    git clone --recursive https://github.com/ccache/ccache.git && \
+    cd ccache && \
+    git checkout v3.7.9 && \
+    ./autogen.sh && \
+    ./configure --disable-man && \
+    make -j$(nproc) && \
+    make install && \
+    cd /usr/local/src && \
+    rm -rf ccache
+
+# Arm Performance Libraries 21.0
+RUN cd /usr/local/src && \
+    wget https://armkeil.blob.core.windows.net/developer/Files/downloads/hpc/arm-performance-libraries/21-0-0/RHEL7/arm-performance-libraries_21.0_RHEL-7_gcc-8.2.tar && \
+    tar -xvf arm-performance-libraries_21.0_RHEL-7_gcc-8.2.tar && \
+    arm-performance-libraries_21.0_RHEL-7_gcc-8.2/arm-performance-libraries_21.0_RHEL-7.sh -a && \
+    rm -rf arm-performance-libraries_21.0_RHEL-7_gcc-8.2.tar arm-performance-libraries_21.0_RHEL-7_gcc-8.2
+ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/opt/arm/armpl_21.0_gcc-8.2/lib
+
+# Fix the en_DK.UTF-8 locale to test locale invariance
+RUN localedef -i en_DK -f UTF-8 en_DK.UTF-8
+
+# Python dependencies
+RUN python3 -m pip install --upgrade pip
+COPY install/requirements_aarch64 /work/
+RUN python3 -m pip install -r /work/requirements_aarch64
+
+ARG USER_ID=0
+COPY install/centos7_adduser.sh /work/
+RUN /work/centos7_adduser.sh
+
+ENV PYTHONPATH=./python/
+# Verify that MXNet works correctly when the C locale is set to a locale that uses a comma as the
+# decimal separator. Please see #16134 for an example of a bug caused by incorrect handling of
+# number serialization and deserialization.
+ENV LC_NUMERIC=en_DK.UTF-8
+WORKDIR /work/mxnet
+
+COPY runtime_functions.sh /work/
+
diff --git a/ci/docker/Dockerfile.publish.ubuntu1804_aarch64_cpu b/ci/docker/Dockerfile.publish.ubuntu1804_aarch64_cpu
diff --git a/ci/docker/install/requirements_aarch64 b/ci/docker/install/requirements_aarch64
@@ -25,10 +25,8 @@ decorator==4.4.0
 mock==2.0.0
 nose==1.3.7
 nose-timer==0.7.3
-numpy
 pylint==2.3.1  # pylint and astroid need to be aligned
 astroid==2.3.3  # pylint and astroid need to be aligned
 requests<2.19.0,>=2.18.4
-scipy
 setuptools
 coverage
diff --git a/ci/docker/install/ubuntu_aarch64_publish.sh b/ci/docker/install/ubuntu_aarch64_publish.sh
diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
@@ -984,6 +984,10 @@ cd_unittest_ubuntu() {
 
     local nose_cmd="nosetests-3.4"
 
+    if [[ ${mxnet_variant} = aarch64_cpu ]]; then
+        source /opt/rh/rh-python38/enable
+    fi
+
     $nose_cmd $NOSE_TIMER_ARGUMENTS --verbose tests/python/unittest
     $nose_cmd $NOSE_TIMER_ARGUMENTS --verbose tests/python/quantization
 
@@ -2007,6 +2011,9 @@ build_static_libmxnet() {
     set -ex
     pushd .
     local mxnet_variant=${1:?"This function requires a python command as the first argument"}
+    if [[ ${mxnet_variant} = aarch64_cpu ]]; then
+        source /opt/rh/devtoolset-10/enable
+    fi
     CMAKE_STATICBUILD=1
     source tools/staticbuild/build.sh ${mxnet_variant}
     popd
@@ -2017,32 +2024,37 @@ cd_package_pypi() {
     set -ex
     pushd .
     local mxnet_variant=${1:?"This function requires a python command as the first argument"}
+    if [[ ${mxnet_variant} = aarch64_cpu ]]; then
+        source /opt/rh/rh-python38/enable
+    fi
     ./cd/python/pypi/pypi_package.sh ${mxnet_variant}
     popd
 }
 
 # Sanity checks wheel file
 cd_integration_test_pypi() {
     set -ex
-    local python_cmd=${1:?"This function requires a python command as the first argument"}
+    local mxnet_variant=${1:?"This function requires a python command as the first argument"}
     local gpu_enabled=${2:-"false"}
 
     local test_conv_params=''
     local mnist_params=''
 
-    local pip_cmd='pip3'
-
     if [ "${gpu_enabled}" = "true" ]; then
         mnist_params="--gpu 0"
         test_conv_params="--gpu"
     fi
 
+    if [[ ${mxnet_variant} = aarch64_cpu ]]; then
+        source /opt/rh/rh-python38/enable
+    fi
+
     # install mxnet wheel package
-    ${pip_cmd} install --user ./wheel_build/dist/*.whl
+    python3 -m pip install --user ./wheel_build/dist/*.whl
 
     # execute tests
-    ${python_cmd} /work/mxnet/tests/python/train/test_conv.py ${test_conv_params}
-    ${python_cmd} /work/mxnet/example/image-classification/train_mnist.py ${mnist_params}
+    python3 /work/mxnet/tests/python/train/test_conv.py ${test_conv_params}
+    python3 /work/mxnet/example/image-classification/train_mnist.py ${mnist_params}
 }
 
 # Publishes wheel to PyPI

diff --git a/cmake/Modules/FindArmPL.cmake b/cmake/Modules/FindArmPL.cmake
@@ -42,8 +42,8 @@ SET(ArmPL_LIBRARIES
 	${ArmPL_LIB}
 	${MATH_LIB}
 	${STRING_LIB}
-	/usr/local/gcc-8.5.0/lib64/libgfortran.so
-	/usr/lib/aarch64-linux-gnu/libm.so
+	libgfortran.so
+	libm.so
 )
 
 SET(ArmPL_FOUND ON)