[v1.9.x] Use centos7 base image for CD pipeline and aarch64 build (#2…

…0423)

* [v1.x] use centos7 base image for aarch64 build (#20392)

* add centos7 aarch64 build

* enable rh packages

* update openssl curl version

* install rhel7 armpl

* find armpl cmake

* test mkldnn build

* test ninja

* use armpl compiled with gcc-8

* enable py38 for unittests

* check previously failing tests

* create symlink for ninja

* fix cd pypi

* fix cd docker

* update ssl for mac

* [v1.x] Update CD pipeline to build with centos7 (#20409)

Co-authored-by: Wei Chu <weichu@amazon.com>

Co-authored-by: Manu Seth <22492939+mseth10@users.noreply.github.com>
Co-authored-by: Wei Chu <weichu@amazon.com>

cd/mxnet_lib/Jenkins_pipeline.groovy

@@ -47,7 +47,7 @@ def build(mxnet_variant) {
     node(NODE_LINUX_AARCH64_CPU) {
       ws("workspace/mxnet_${libtype}/${mxnet_variant}/${env.BUILD_NUMBER}") {
         ci_utils.init_git()
-        ci_utils.docker_run('publish.ubuntu1804_aarch64_cpu', "build_static_libmxnet ${mxnet_variant}", false)
+        ci_utils.docker_run('centos7_aarch64_cpu', "build_static_libmxnet ${mxnet_variant}", false)
         ci_utils.pack_lib("mxnet_${mxnet_variant}", libmxnet_pipeline.get_stash(mxnet_variant))
       }
     }
@@ -57,7 +57,7 @@ def build(mxnet_variant) {
         ci_utils.init_git()
         // Compiling in Ubuntu14.04 due to glibc issues.
         // This should be updates once we have clarity on this issue.
-        ci_utils.docker_run('publish.ubuntu1404_cpu', "build_static_libmxnet ${mxnet_variant}", false)
+        ci_utils.docker_run('centos7_cpu', "build_static_libmxnet ${mxnet_variant}", false)
         ci_utils.pack_lib("mxnet_${mxnet_variant}", libmxnet_pipeline.get_stash(mxnet_variant))
       }
     }

cd/mxnet_lib/mxnet_lib_pipeline.groovy

@@ -76,12 +76,12 @@ def get_stash(mxnet_variant) {
 // The environment corresponds to the docker files in the 'docker' directory
 def get_environment(mxnet_variant) {
   if (mxnet_variant.startsWith("aarch64")) {
-    return "publish.ubuntu1804_aarch64_cpu"
+    return "centos7_aarch64_cpu"
   } else if (mxnet_variant.startsWith("cu")) {
     // Remove 'mkl' suffix from variant to properly format test environment
-    return "ubuntu_gpu_${mxnet_variant.replace('mkl', '')}"
+    return "centos7_gpu_${mxnet_variant.replace('mkl', '')}"
   }
-  return "ubuntu_cpu"
+  return "centos7_cpu"
 }
 
 // Returns the variant appropriate jenkins node test in which

cd/python/docker/Dockerfile

@@ -36,12 +36,12 @@ RUN apt-get install -y libgomp1
 
 ARG MXNET_VARIANT
 RUN if [ "$MXNET_VARIANT" = "aarch64_cpu" ] ; then echo "not installing libquadmath0 on aarch64" ; else apt-get install -y libquadmath0 ; fi
-RUN if [ "$MXNET_VARIANT" = "aarch64_cpu" ] ; \
-    then wget https://armkeil.blob.core.windows.net/developer/Files/downloads/hpc/arm-performance-libraries/21-0-0/Ubuntu18.04/arm-performance-libraries_21.0_Ubuntu-18.04_gcc-8.2.tar && \
+RUN if [ "$MXNET_VARIANT" = "aarch64_cpu" ] ; then \
+    wget https://armkeil.blob.core.windows.net/developer/Files/downloads/hpc/arm-performance-libraries/21-0-0/Ubuntu18.04/arm-performance-libraries_21.0_Ubuntu-18.04_gcc-8.2.tar && \
     tar -xvf arm-performance-libraries_21.0_Ubuntu-18.04_gcc-8.2.tar && \
     arm-performance-libraries_21.0_Ubuntu-18.04_gcc-8.2/arm-performance-libraries_21.0_Ubuntu-18.04.sh -a; \
     fi
-ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/opt/arm/armpl_21.0_gcc-8.2/lib/
+ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/opt/arm/armpl_21.0_gcc-8.2/lib
 
 ARG MXNET_COMMIT_ID
 ENV MXNET_COMMIT_ID=${MXNET_COMMIT_ID}

cd/python/docker/Jenkins_pipeline.groovy

@@ -42,12 +42,12 @@ def get_pipeline(mxnet_variant) {
 // The environment corresponds to the docker files in the 'docker' directory
 def get_environment(mxnet_variant) {
   if (mxnet_variant.startsWith('aarch64')) {
-    return "publish.ubuntu1804_aarch64_cpu"
+    return "centos7_aarch64_cpu"
   }
   if (mxnet_variant.startsWith('cu')) {
-    return "ubuntu_gpu_${mxnet_variant}"
+    return "centos7_gpu_${mxnet_variant}"
   }
-  return "ubuntu_cpu"
+  return "centos7_cpu"
 }
 
 

cd/python/pypi/Jenkins_pipeline.groovy

@@ -48,12 +48,12 @@ def get_pipeline(mxnet_variant) {
 // The environment corresponds to the docker files in the 'docker' directory
 def get_environment(mxnet_variant) {
   if (mxnet_variant.startsWith('aarch64')) {
-    return "publish.ubuntu1804_aarch64_cpu"
+    return "centos7_aarch64_cpu"
   }
   if (mxnet_variant.startsWith('cu')) {
-    return "ubuntu_gpu_${mxnet_variant}"
+    return "centos7_gpu_${mxnet_variant}"
   }
-  return "ubuntu_cpu"
+  return "centos7_cpu"
 }
 
 def build(mxnet_variant) {
@@ -73,7 +73,7 @@ def test(mxnet_variant) {
     // test wheel file
     def environment = get_environment(mxnet_variant)
     def nvidia_docker = mxnet_variant.startsWith('cu')
-    ci_utils.docker_run(environment, "cd_integration_test_pypi python3 ${nvidia_docker}", nvidia_docker)
+    ci_utils.docker_run(environment, "cd_integration_test_pypi ${mxnet_variant} ${nvidia_docker}", nvidia_docker)
   }
 }
 

ci/Jenkinsfile_docker_cache

@@ -38,6 +38,7 @@ core_logic: {
         timeout(time: total_timeout, unit: 'MINUTES') {
           utils.init_git()
           sh "ci/docker_cache.py --docker-registry ${env.DOCKER_ECR_REGISTRY}"
+          sh "cd ci && python3 ./docker_login.py --secret-name ${env.DOCKERHUB_SECRET_NAME} && docker-compose -f docker/docker-compose.yml build --parallel && docker-compose -f docker/docker-compose.yml push && docker logout"
         }
       }
     }

ci/Jenkinsfile_utils.groovy

@@ -147,9 +147,10 @@ def collect_test_results_windows(original_file_name, new_file_name) {
 }
 
 
-def docker_run(platform, function_name, use_nvidia, shared_mem = '500m', env_vars = "") {
-  def command = "ci/build.py %ENV_VARS% --docker-registry ${env.DOCKER_ECR_REGISTRY} %USE_NVIDIA% --platform %PLATFORM% --docker-build-retries 3 --shm-size %SHARED_MEM% /work/runtime_functions.sh %FUNCTION_NAME%"
+def docker_run(platform, function_name, use_nvidia, shared_mem = '500m', env_vars = "", build_args = "") {
+  def command = "ci/build.py %ENV_VARS% %BUILD_ARGS% --docker-registry ${env.DOCKER_CACHE_REGISTRY} %USE_NVIDIA% --platform %PLATFORM% --docker-build-retries 3 --shm-size %SHARED_MEM% /work/runtime_functions.sh %FUNCTION_NAME%"
   command = command.replaceAll('%ENV_VARS%', env_vars.length() > 0 ? "-e ${env_vars}" : '')
+  command = command.replaceAll('%BUILD_ARGS%', env_vars.length() > 0 ? "${build_args}" : '')
   command = command.replaceAll('%USE_NVIDIA%', use_nvidia ? '--nvidiadocker' : '')
   command = command.replaceAll('%PLATFORM%', platform)
   command = command.replaceAll('%FUNCTION_NAME%', function_name)

ci/build.py

@@ -38,18 +38,28 @@
 from subprocess import check_call, check_output
 from typing import *
 
+import yaml
+
 from util import *
 
+DOCKER_COMPOSE_WHITELIST = ('centos7_cpu', 'centos7_gpu_cu92', 'centos7_gpu_cu100',
+                            'centos7_gpu_cu101', 'centos7_gpu_cu102', 'centos7_gpu_cu110',
+                            'centos7_gpu_cu112')
+
+# Files for docker compose
+DOCKER_COMPOSE_FILES = set(('docker/build.centos7'))
 
 def get_dockerfiles_path():
     return "docker"
 
 
-def get_platforms(path: str = get_dockerfiles_path()) -> List[str]:
+def get_platforms(path: str = get_dockerfiles_path(), legacy_only=False) -> List[str]:
     """Get a list of architectures given our dockerfiles"""
     dockerfiles = glob.glob(os.path.join(path, "Dockerfile.*"))
-    dockerfiles = list(filter(lambda x: x[-1] != '~', dockerfiles))
-    files = list(map(lambda x: re.sub(r"Dockerfile.(.*)", r"\1", x), dockerfiles))
+    dockerfiles = set(filter(lambda x: x[-1] != '~', dockerfiles))
+    files = set(map(lambda x: re.sub(r"Dockerfile.(.*)", r"\1", x), dockerfiles))
+    if legacy_only:
+        files = files - DOCKER_COMPOSE_FILES
     platforms = list(map(lambda x: os.path.split(x)[1], sorted(files)))
     return platforms
 
@@ -79,6 +89,11 @@ def _hash_file(ctx, filename):
 
 def get_docker_tag(platform: str, registry: str) -> str:
     """:return: docker tag to be used for the container"""
+    if platform in DOCKER_COMPOSE_WHITELIST:
+        with open("docker/docker-compose.yml", "r") as f:
+            compose_config = yaml.load(f.read(), yaml.SafeLoader)
+            return compose_config["services"][platform]["image"].replace('${DOCKER_CACHE_REGISTRY}', registry)
+
     platform = platform if any(x in platform for x in ['build.', 'publish.']) else 'build.{}'.format(platform)
     if not registry:
         registry = "mxnet_local"
@@ -106,41 +121,58 @@ def build_docker(platform: str, registry: str, num_retries: int, no_cache: bool,
     :return: Id of the top level image
     """
     tag = get_docker_tag(platform=platform, registry=registry)
-    logging.info("Building docker container tagged '%s'", tag)
-    #
-    # We add a user with the same group as the executing non-root user so files created in the
-    # container match permissions of the local user. Same for the group.
-    #
-    # These variables are used in the docker files to create user and group with these ids.
-    # see: docker/install/ubuntu_adduser.sh
-    #
-    # cache-from is needed so we use the cached images tagged from the remote via
-    # docker pull see: docker_cache.load_docker_cache
-    #
-    # This also prevents using local layers for caching: https://github.com/moby/moby/issues/33002
-    # So to use local caching, we should omit the cache-from by using --no-dockerhub-cache argument to this
-    # script.
-    #
-    # This doesn't work with multi head docker files.
-    #
-    cmd = ["docker", "build",
-           "-f", get_dockerfile(platform),
-           "--build-arg", "USER_ID={}".format(os.getuid()),
-           "--build-arg", "GROUP_ID={}".format(os.getgid())]
-    if no_cache:
-        cmd.append("--no-cache")
-    if cache_intermediate:
-        cmd.append("--rm=false")
-    elif registry:
-        cmd.extend(["--cache-from", tag])
-    cmd.extend(["-t", tag, get_dockerfiles_path()])
+
+    # docker-compose
+    if platform in DOCKER_COMPOSE_WHITELIST:
+        logging.info('Building docker container tagged \'%s\' based on ci/docker/docker-compose.yml', tag)
+        # We add a user with the same group as the executing non-root user so files created in the
+        # container match permissions of the local user. Same for the group.
+        cmd = ['docker-compose', '-f', 'docker/docker-compose.yml', 'build',
+               "--build-arg", "USER_ID={}".format(os.getuid()),
+               "--build-arg", "GROUP_ID={}".format(os.getgid())]
+        if cache_intermediate:
+            cmd.append('--no-rm')
+        cmd.append(platform)
+    else:
+        logging.info("Building docker container tagged '%s'", tag)
+        #
+        # We add a user with the same group as the executing non-root user so files created in the
+        # container match permissions of the local user. Same for the group.
+        #
+        # These variables are used in the docker files to create user and group with these ids.
+        # see: docker/install/ubuntu_adduser.sh
+        #
+        # cache-from is needed so we use the cached images tagged from the remote via
+        # docker pull see: docker_cache.load_docker_cache
+        #
+        # This also prevents using local layers for caching: https://github.com/moby/moby/issues/33002
+        # So to use local caching, we should omit the cache-from by using --no-dockerhub-cache argument to this
+        # script.
+        #
+        # This doesn't work with multi head docker files.
+        #
+        cmd = ["docker", "build",
+            "-f", get_dockerfile(platform),
+            "--build-arg", "USER_ID={}".format(os.getuid()),
+            "--build-arg", "GROUP_ID={}".format(os.getgid())]
+        if no_cache:
+            cmd.append("--no-cache")
+        if cache_intermediate:
+            cmd.append("--rm=false")
+        elif registry:
+            cmd.extend(["--cache-from", tag])
+        cmd.extend(["-t", tag, get_dockerfiles_path()])
+
+    env = os.environ.copy()
+    env["DOCKER_CACHE_REGISTRY"] = registry
 
     @retry(subprocess.CalledProcessError, tries=num_retries)
-    def run_cmd():
+    def run_cmd(env=None):
         logging.info("Running command: '%s'", ' '.join(cmd))
-        check_call(cmd)
+        check_call(cmd, env=env)
+
+    run_cmd(env=env)
 
-    run_cmd()
     # Get image id by reading the tag. It's guaranteed (except race condition) that the tag exists. Otherwise, the
     # check_call would have failed
     image_id = _get_local_image_id(docker_tag=tag)
@@ -258,9 +290,19 @@ def list_platforms() -> str:
     return "\nSupported platforms:\n{}".format('\n'.join(get_platforms()))
 
 
-def load_docker_cache(tag, docker_registry) -> None:
+def load_docker_cache(platform, tag, docker_registry) -> None:
     """Imports tagged container from the given docker registry"""
     if docker_registry:
+        if platform in DOCKER_COMPOSE_WHITELIST:
+            env = os.environ.copy()
+            env["DOCKER_CACHE_REGISTRY"] = docker_registry
+            cmd = ['docker-compose', '-f', 'docker/docker-compose.yml', 'pull', platform]
+            logging.info("Running command: 'DOCKER_CACHE_REGISTRY=%s %s'", docker_registry, ' '.join(cmd))
+            check_call(cmd, env=env)
+            return
+
+        env = os.environ.copy()
+        env["DOCKER_CACHE_REGISTRY"] = docker_registry
         # noinspection PyBroadException
         try:
             import docker_cache
@@ -363,8 +405,8 @@ def main() -> int:
     elif args.platform:
         platform = args.platform
         tag = get_docker_tag(platform=platform, registry=args.docker_registry)
-        if args.docker_registry:
-            load_docker_cache(tag=tag, docker_registry=args.docker_registry)
+        if args.docker_registry:   
+            load_docker_cache(platform=platform, tag=tag, docker_registry=args.docker_registry)
         if not args.run_only:
             build_docker(platform=platform, registry=args.docker_registry,
                          num_retries=args.docker_build_retries, no_cache=args.no_cache,
@@ -409,7 +451,7 @@ def main() -> int:
         logging.info("Artifacts will be produced in the build/ directory.")
         for platform in platforms:
             tag = get_docker_tag(platform=platform, registry=args.docker_registry)
-            load_docker_cache(tag=tag, docker_registry=args.docker_registry)
+            load_docker_cache(platform=platform, tag=tag, docker_registry=args.docker_registry)
             build_docker(platform, registry=args.docker_registry,
                          num_retries=args.docker_build_retries, no_cache=args.no_cache,
                          cache_intermediate=args.cache_intermediate)

ci/docker/Dockerfile.build.centos7_cpu → ci/docker/Dockerfile.build.centos7

@@ -16,9 +16,22 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-# Dockerfile to build and run MXNet on CentOS 7 for CPU
+#
+# Dockerfile for CentOS 7 based builds.
+# Via the CentOS 7 Dockerfiles, we ensure MXNet continues to run fine on older systems.
+#
+# See docker-compose.yml for supported BASE_IMAGE ARGs and targets.
 
-FROM centos:7
+####################################################################################################
+# The Dockerfile uses a dynamic BASE_IMAGE (for example centos:7,
+# nvidia/cuda:10.2-cudnn7-devel-centos7 etc).
+# On top of BASE_IMAGE we install all dependencies shared by all MXNet build
+# environments into a "base" target. At the end of this file, we specialize
+# "base" for specific usecases. The target built by docker can be selected via
+# "--target" option or docker-compose.yml
+####################################################################################################
+ARG BASE_IMAGE
+FROM $BASE_IMAGE AS base
 
 WORKDIR /work/deps
 
@@ -39,3 +52,4 @@ ENV PYTHONPATH=./python/
 WORKDIR /work/mxnet
 
 COPY runtime_functions.sh /work/
+