Disable OpenMP offloading support for 3rdparty/openmp (apache#17098)

* Disable OpenMP offloading support for 3rdparty/openmp

OpenMP offloading was introduced some time during the past two years and is
enabled by default. With upgrading 3rdparty/openmp in
apache#17012
it was made part of the MXNet CMake build.

But we don't use OpenMP offloading and the Cuda target in the llvm OpenMP
Offloading build is broken in our setting.

* Update CMake on CI

CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.0.2)
+cmake_minimum_required(VERSION 3.13)
 
 # workaround to store CMAKE_CROSSCOMPILING because is getting reset by the project command
 if(CMAKE_CROSSCOMPILING)
@@ -442,6 +442,7 @@ if(USE_OPENMP)
     set(OPENMP_STANDALONE_BUILD TRUE)
     set(LIBOMP_ENABLE_SHARED TRUE)
     set(CMAKE_BUILD_TYPE Release)
+    set(OPENMP_ENABLE_LIBOMPTARGET OFF CACHE BOOL "LLVM OpenMP offloading support")  # Requires CMP0077 CMake 3.13
     add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/openmp)
   endfunction()
 

ci/build_windows.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
 # Licensed to the Apache Software Foundation (ASF) under one
@@ -28,7 +28,9 @@
 import platform
 import shutil
 import sys
+import tempfile
 import time
+import zipfile
 from distutils.dir_util import copy_tree
 from enum import Enum
 from subprocess import check_call
@@ -147,22 +149,33 @@ def windows_build(args):
     mxnet_root = get_mxnet_root()
     logging.info("Found MXNet root: {}".format(mxnet_root))
 
-    with remember_cwd():
-        os.chdir(path)
-        cmd = "\"{}\" && cmake -G \"NMake Makefiles JOM\" {} {}".format(args.vcvars,
-                                                                        CMAKE_FLAGS[args.flavour],
-                                                                        mxnet_root)
-        logging.info("Generating project with CMake:\n{}".format(cmd))
-        check_call(cmd, shell=True)
-
-        cmd = "\"{}\" && jom".format(args.vcvars)
-        logging.info("Building with jom:\n{}".format(cmd))
-
-        t0 = int(time.time())
-        check_call(cmd, shell=True)
-
-        logging.info("Build flavour: {} complete in directory: \"{}\"".format(args.flavour, os.path.abspath(path)))
-        logging.info("Build took {}".format(datetime.timedelta(seconds=int(time.time() - t0))))
+    url = 'https://github.com/Kitware/CMake/releases/download/v3.16.1/cmake-3.16.1-win64-x64.zip'
+    with tempfile.TemporaryDirectory() as tmpdir:
+        cmake_file_path = download_file(url, tmpdir)
+        with zipfile.ZipFile(cmake_file_path, 'r') as zip_ref:
+            # Create $tmpdir\cmake-3.16.1-win64-x64\bin\cmake.exe
+            zip_ref.extractall(tmpdir)
+
+        with remember_cwd():
+            os.chdir(path)
+            cmd = "\"{}\" && {} -G \"NMake Makefiles JOM\" {} {}".format(
+                args.vcvars,
+                os.path.join(tmpdir, 'cmake-3.16.1-win64-x64', 'bin', 'cmake.exe'),
+                CMAKE_FLAGS[args.flavour], mxnet_root)
+            logging.info("Generating project with CMake:\n{}".format(cmd))
+            check_call(cmd, shell=True)
+
+            cmd = "\"{}\" && jom".format(args.vcvars)
+            logging.info("Building with jom:\n{}".format(cmd))
+
+            t0 = int(time.time())
+            check_call(cmd, shell=True)
+
+            logging.info(
+                "Build flavour: {} complete in directory: \"{}\"".format(
+                    args.flavour, os.path.abspath(path)))
+            logging.info("Build took {}".format(
+                datetime.timedelta(seconds=int(time.time() - t0))))
     windows_package(args)
 
 
@@ -262,4 +275,3 @@ def main():
 
 if __name__ == '__main__':
     sys.exit(main())
-

ci/docker/Dockerfile.build.android_armv7

@@ -18,7 +18,7 @@
 #
 # Dockerfile to build MXNet for Android ARMv7
 
-FROM mxnetcipinned/dockcross-base:11262018
+FROM dockcross/base
 MAINTAINER Pedro Larroy "pllarroy@amazon.com"
 
 # The cross-compiling emulator

ci/docker/Dockerfile.build.android_armv8

@@ -18,7 +18,7 @@
 #
 # Dockerfile to build MXNet for Android ARM64/ARMv8
 
-FROM mxnetcipinned/dockcross-base:11262018
+FROM dockcross/base
 MAINTAINER Pedro Larroy "pllarroy@amazon.com"
 
 RUN apt-get update && apt-get install -y \
@@ -82,4 +82,4 @@ RUN /work/ubuntu_adduser.sh
 
 COPY runtime_functions.sh /work/
 
-WORKDIR /work/build
+WORKDIR /work/build

ci/docker/Dockerfile.build.armv6

@@ -18,7 +18,7 @@
 #
 # Dockerfile to build MXNet for ARMv6
 
-FROM mxnetcipinned/dockcross-linux-armv6:11262018
+FROM dockcross/linux-armv6
 
 ENV ARCH armv6l
 ENV HOSTCC gcc

ci/docker/Dockerfile.build.armv7

@@ -18,7 +18,7 @@
 #
 # Dockerfile to build MXNet for Android ARMv7
 
-FROM mxnetcipinned/dockcross-linux-armv7:11262018
+FROM dockcross/linux-armv7
 
 ENV ARCH armv7l
 ENV HOSTCC gcc

ci/docker/Dockerfile.build.armv8

@@ -18,7 +18,7 @@
 #
 # Dockerfile to build MXNet for ARM64/ARMv8
 
-FROM mxnetcipinned/dockcross-linux-arm64:11262018
+FROM dockcross/linux-arm64
 
 ENV ARCH aarch64
 ENV HOSTCC gcc

ci/docker/Dockerfile.build.jetson

@@ -22,7 +22,7 @@
 
 FROM nvidia/cuda:9.0-cudnn7-devel as cudabuilder
 
-FROM mxnetcipinned/dockcross-linux-arm64:11262018
+FROM dockcross/linux-arm64
 
 ENV ARCH aarch64
 ENV HOSTCC gcc

ci/docker/install/ubuntu_core.sh

@@ -53,11 +53,10 @@ apt-get install -y \
 ln -s /usr/lib/x86_64-linux-gnu/libturbojpeg.so.0.1.0 /usr/lib/x86_64-linux-gnu/libturbojpeg.so
 
 
-# Note: we specify an exact cmake version to work around a cmake 3.10 CUDA 10 issue.
-# Reference: https://github.com/clab/dynet/issues/1457
+# CMake 3.13.2+ is required
 mkdir /opt/cmake && cd /opt/cmake
-wget -nv https://cmake.org/files/v3.12/cmake-3.12.4-Linux-x86_64.sh
-sh cmake-3.12.4-Linux-x86_64.sh --prefix=/opt/cmake --skip-license
+wget -nv https://cmake.org/files/v3.13/cmake-3.13.5-Linux-x86_64.sh
+sh cmake-3.13.5-Linux-x86_64.sh --prefix=/opt/cmake --skip-license
 ln -s /opt/cmake/bin/cmake /usr/local/bin/cmake
-rm cmake-3.12.4-Linux-x86_64.sh
+rm cmake-3.13.5-Linux-x86_64.sh
 cmake --version

ci/util.py

@@ -15,12 +15,15 @@
 # specific language governing permissions and limitations
 # under the License.
 
-import os
 import contextlib
 import logging
 import logging.config
+import os
+import subprocess
 import sys
 
+import requests
+
 
 def get_mxnet_root() -> str:
     curpath = os.path.abspath(os.path.dirname(__file__))
@@ -130,3 +133,31 @@ def config_logging():
     # or sensitive information
     logging.getLogger("botocore").setLevel(logging.WARNING)
     logging.getLogger("requests").setLevel(logging.WARNING)
+
+
+# Takes url and downloads it to the dest_path directory on Windows.
+def download_file(url, dest_path):
+    file_name = url.split('/')[-1]
+    full_path = "{}\\{}".format(dest_path, file_name)
+    logging.info("Downloading: {}".format(full_path))
+    r = requests.get(url, stream=True)
+    if r.status_code == 404:
+        return r.status_code
+    elif r.status_code != 200:
+        logging.error("{} returned status code {}".format(url, r.status_code))
+    with open(full_path, 'wb') as f:
+        for chunk in r.iter_content(chunk_size=1024):
+            if chunk: # filter out keep-alive new chunks
+                f.write(chunk)
+    return full_path
+
+
+# Takes arguments and runs command on host.  Shell is disabled by default.
+def run_command(args, shell=False):
+    try:
+        logging.info("Issuing command: {}".format(args))
+        res = subprocess.check_output(args, shell=shell, timeout=1800).decode("utf-8").replace("\r\n", "")
+        logging.info("Output: {}".format(res))
+    except subprocess.CalledProcessError as e:
+        raise RuntimeError("command '{}' return with error (code {}): {}".format(e.cmd, e.returncode, e.output))
+    return res