Remove Makefile build support (apache#18721)

Replaced by cmake buildsystem as per apache#16167

ci/dev_menu.py

@@ -122,7 +122,7 @@ def provision_virtualenv(venv_path=DEFAULT_PYENV):
         "pytest -v tests/python/unittest/"
     ),
     ('[Docker] Build the MXNet binary - outputs to "lib/"',
-        "ci/build.py --platform ubuntu_cpu_lite /work/runtime_functions.sh build_ubuntu_cpu_docs"),
+        "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh build_ubuntu_cpu_docs"),
     ('[Docker] Build the Jekyll website - outputs to "docs/static_site/build/html/"',
         "ci/build.py --platform ubuntu_cpu_jekyll /work/runtime_functions.sh build_jekyll_docs"),
     ('[Docker] Build the Python API docs - outputs to "docs/python_docs/python/build/_build/html/"',

ci/docker/runtime_functions.sh

@@ -321,21 +321,6 @@ build_centos7_cpu() {
     ninja
 }
 
-build_centos7_cpu_make() {
-    set -ex
-    cd /work/mxnet
-    source /opt/rh/devtoolset-7/enable
-    make \
-        DEV=1 \
-        USE_LAPACK=1 \
-        USE_LAPACK_PATH=/usr/lib64/liblapack.so \
-        USE_BLAS=openblas \
-        USE_MKLDNN=0 \
-        USE_DIST_KVSTORE=1 \
-        USE_SIGNAL_HANDLER=1 \
-        -j$(nproc)
-}
-
 build_centos7_mkldnn() {
     set -ex
     cd /work/build
@@ -384,24 +369,6 @@ build_ubuntu_cpu_openblas() {
     ninja
 }
 
-build_ubuntu_cpu_openblas_make() {
-    set -ex
-    export CC=gcc-7
-    export CXX=g++-7
-    build_ccache_wrappers
-    make \
-        DEV=1                         \
-        USE_TVM_OP=1                  \
-        USE_CPP_PACKAGE=1             \
-        USE_BLAS=openblas             \
-        USE_MKLDNN=0                  \
-        USE_DIST_KVSTORE=1            \
-        USE_LIBJPEG_TURBO=1           \
-        USE_SIGNAL_HANDLER=1          \
-        -j$(nproc)
-    make cython PYTHON=python3
-}
-
 build_ubuntu_cpu_mkl() {
     set -ex
     cd /work/build
@@ -589,22 +556,6 @@ build_ubuntu_cpu_clang100_mkldnn() {
     ninja
 }
 
-build_ubuntu_cpu_mkldnn_make() {
-    set -ex
-
-    export CC=gcc-7
-    export CXX=g++-7
-    build_ccache_wrappers
-
-    make  \
-        DEV=1                         \
-        USE_CPP_PACKAGE=1             \
-        USE_TVM_OP=1                  \
-        USE_BLAS=openblas             \
-        USE_SIGNAL_HANDLER=1          \
-        -j$(nproc)
-}
-
 build_ubuntu_cpu_mkldnn() {
     set -ex
     cd /work/build
@@ -754,45 +705,6 @@ build_ubuntu_gpu_cuda101_cudnn7_debug() {
     ninja
 }
 
-build_ubuntu_gpu_cuda101_cudnn7_make() {
-    set -ex
-    export CC=gcc-7
-    export CXX=g++-7
-    build_ccache_wrappers
-    make \
-        USE_BLAS=openblas                         \
-        USE_MKLDNN=0                              \
-        USE_CUDA=1                                \
-        USE_CUDA_PATH=/usr/local/cuda             \
-        USE_CUDNN=1                               \
-        USE_CPP_PACKAGE=1                         \
-        USE_DIST_KVSTORE=1                        \
-        CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \
-        USE_SIGNAL_HANDLER=1                      \
-        -j$(nproc)
-    make cython PYTHON=python3
-}
-
-build_ubuntu_gpu_cuda101_cudnn7_mkldnn_cpp_test() {
-    set -ex
-    export CC=gcc-7
-    export CXX=g++-7
-    build_ccache_wrappers
-    make \
-        USE_BLAS=openblas                         \
-        USE_MKLDNN=1                              \
-        USE_CUDA=1                                \
-        USE_CUDA_PATH=/usr/local/cuda             \
-        USE_CUDNN=1                               \
-        USE_CPP_PACKAGE=1                         \
-        USE_DIST_KVSTORE=1                        \
-        CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \
-        USE_SIGNAL_HANDLER=1                      \
-        -j$(nproc)
-    make test USE_CPP_PACKAGE=1 -j$(nproc)
-    make cython PYTHON=python3
-}
-
 build_ubuntu_gpu_cmake() {
     set -ex
     cd /work/build
@@ -878,9 +790,9 @@ build_ubuntu_blc() {
 sanity_check() {
     set -ex
     tools/license_header.py check
-    make cpplint
-    make pylint
-    pytest -n 4 tests/tutorials/test_sanity_tutorials.py
+    3rdparty/dmlc-core/scripts/lint.py mxnet cpp include src plugin tests --exclude_path src/operator/contrib/ctc_include include/mkldnn
+    python3 -m pylint --rcfile=ci/other/pylintrc --ignore-patterns=".*\.so$$,.*\.dll$$,.*\.dylib$$" python/mxnet
+    OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -n 4 tests/tutorials/test_sanity_tutorials.py
 }
 
 # Tests libmxnet
@@ -1267,19 +1179,7 @@ build_docs_setup() {
 }
 
 build_ubuntu_cpu_docs() {
-    set -ex
-    export CC="gcc-7"
-    export CXX="g++-7"
-    build_ccache_wrappers
-    make \
-        DEV=1                         \
-        USE_CPP_PACKAGE=1             \
-        USE_BLAS=openblas             \
-        USE_MKLDNN=0                  \
-        USE_DIST_KVSTORE=1            \
-        USE_LIBJPEG_TURBO=1           \
-        USE_SIGNAL_HANDLER=1          \
-        -j$(nproc)
+    build_ubuntu_cpu_openblas
 }
 
 
@@ -1409,8 +1309,6 @@ build_static_libmxnet() {
     pushd .
     source /opt/rh/devtoolset-7/enable
     source /opt/rh/rh-python36/enable
-    export USE_SYSTEM_CUDA=1
-    export CMAKE_STATICBUILD=1
     local mxnet_variant=${1:?"This function requires a python command as the first argument"}
     source tools/staticbuild/build.sh ${mxnet_variant}
     popd
@@ -1484,7 +1382,6 @@ build_static_python_cu92() {
     set -ex
     pushd .
     export mxnet_variant=cu92
-    export USE_SYSTEM_CUDA=1
     source /opt/rh/devtoolset-7/enable
     source /opt/rh/rh-python36/enable
     ./ci/publish/python/build.sh

ci/jenkins/Jenkins_steps.groovy

@@ -25,7 +25,6 @@ utils = load('ci/Jenkinsfile_utils.groovy')
 // mxnet libraries
 mx_lib = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, build/3rdparty/openmp/runtime/src/libomp.so'
 mx_lib_cython = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, python/mxnet/_cy3/*.so, build/3rdparty/openmp/runtime/src/libomp.so, python/mxnet/_ffi/_cy3/*.so'
-mx_lib_make = 'lib/libmxnet.so, lib/libmxnet.a, lib/libtvm_runtime.so, lib/libtvmop.so, lib/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a'
 
 // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default.
 mx_cmake_lib = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so'
@@ -34,11 +33,8 @@ mx_cmake_lib_cython = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so,
 // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default.
 mx_cmake_lib_debug = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, build/tests/mxnet_unit_tests'
 mx_mkldnn_lib = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/3rdparty/openmp/runtime/src/libomp.so, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so'
-mx_mkldnn_lib_make = 'lib/libmxnet.so, lib/libmxnet.a, lib/libtvm_runtime.so, lib/libtvmop.so, lib/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a'
 mx_tensorrt_lib = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/3rdparty/openmp/runtime/src/libomp.so, lib/libnvonnxparser_runtime.so.0, lib/libnvonnxparser.so.0, lib/libonnx_proto.so, lib/libonnx.so'
 mx_lib_cpp_examples = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/3rdparty/openmp/runtime/src/libomp.so, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, python/mxnet/_cy3/*.so, python/mxnet/_ffi/_cy3/*.so'
-mx_lib_cpp_examples_make = 'lib/libmxnet.so, lib/libmxnet.a, lib/libtvm_runtime.so, lib/libtvmop.so, lib/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, python/mxnet/_cy3/*.so, python/mxnet/_ffi/_cy3/*.so'
-mx_lib_cpp_capi_make = 'lib/libmxnet.so, lib/libmxnet.a, lib/libtvm_runtime.so, lib/libtvmop.so, lib/tvmop.conf, libsample_lib.so, lib/libmkldnn.so.1, lib/libmklml_intel.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, python/mxnet/_cy3/*.so, python/mxnet/_ffi/_cy3/*.so, build/tests/cpp/mxnet_unit_tests'
 mx_lib_cpp_examples_no_tvm_op = 'build/libmxnet.so, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, build/3rdparty/openmp/runtime/src/libomp.so, python/mxnet/_cy3/*.so, python/mxnet/_ffi/_cy3/*.so'
 mx_lib_cpp_examples_cpu = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/3rdparty/openmp/runtime/src/libomp.so'
 mx_cd_lib = 'lib/libmxnet.so, licenses/*, lib/libgfortran.so.4, lib/libquadmath.so.0, lib/libopenblas.so.0, include/mkldnn/dnnl_version.h, include/mkldnn/dnnl_config.h'
@@ -101,20 +97,6 @@ def compile_unix_cpu_openblas(lib_name) {
     }]
 }
 
-def compile_unix_cpu_openblas_make(lib_name) {
-    return ['CPU: Openblas Makefile': {
-      node(NODE_LINUX_CPU) {
-        ws('workspace/build-cpu-openblas') {
-          timeout(time: max_time, unit: 'MINUTES') {
-            utils.init_git()
-            utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_openblas_make', false)
-            utils.pack_lib(lib_name, mx_lib_make)
-          }
-        }
-      }
-    }]
-}
-
 def compile_unix_openblas_debug_cpu(lib_name) {
     return ['CPU: Openblas, cmake, debug': {
       node(NODE_LINUX_CPU) {
@@ -198,20 +180,6 @@ def compile_unix_mkldnn_cpu(lib_name) {
     }]
 }
 
-def compile_unix_mkldnn_cpu_make(lib_name) {
-    return ['CPU: MKLDNN Makefile': {
-      node(NODE_LINUX_CPU) {
-        ws('workspace/build-mkldnn-cpu') {
-          timeout(time: max_time, unit: 'MINUTES') {
-            utils.init_git()
-            utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_mkldnn_make', false)
-            utils.pack_lib(lib_name, mx_mkldnn_lib_make)
-          }
-        }
-      }
-    }]
-}
-
 def compile_unix_mkldnn_mkl_cpu(lib_name) {
     return ['CPU: MKLDNN_MKL': {
       node(NODE_LINUX_CPU) {
@@ -268,21 +236,6 @@ def compile_unix_full_gpu(lib_name) {
     }]
 }
 
-def compile_unix_full_gpu_make(lib_name) {
-    return ['GPU: CUDA10.1+cuDNN7 Makefile': {
-      node(NODE_LINUX_CPU) {
-        ws('workspace/build-gpu') {
-          timeout(time: max_time, unit: 'MINUTES') {
-            utils.init_git()
-            utils.docker_run('ubuntu_build_cuda', 'build_ubuntu_gpu_cuda101_cudnn7_make', false)
-            utils.pack_lib(lib_name, mx_lib_cpp_examples_make)
-          }
-        }
-      }
-    }]
-}
-
-
 def compile_unix_full_gpu_debug(lib_name) {
     return ['GPU: CUDA10.1+cuDNN7, debug': {
       node(NODE_LINUX_CPU) {
@@ -297,20 +250,6 @@ def compile_unix_full_gpu_debug(lib_name) {
     }]
 }
 
-def compile_unix_full_gpu_mkldnn_cpp_test(lib_name) {
-    return ['GPU: CUDA10.1+cuDNN7+MKLDNN+CPPTEST Makefile': {
-      node(NODE_LINUX_CPU) {
-        ws('workspace/build-gpu-mkldnn-cpp') {
-          timeout(time: max_time, unit: 'MINUTES') {
-            utils.init_git()
-            utils.docker_run('ubuntu_build_cuda', 'build_ubuntu_gpu_cuda101_cudnn7_mkldnn_cpp_test', false)
-            utils.pack_lib(lib_name, mx_lib_cpp_capi_make)
-          }
-        }
-      }
-    }]
-}
-
 def compile_unix_cmake_gpu(lib_name) {
     return ['GPU: CMake': {
       node(NODE_LINUX_CPU) {
@@ -367,20 +306,6 @@ def compile_centos7_cpu(lib_name) {
     }]
 }
 
-def compile_centos7_cpu_make(lib_name) {
-    return ['CPU: CentOS 7 Makefile': {
-      node(NODE_LINUX_CPU) {
-        ws('workspace/build-centos7-cpu') {
-          timeout(time: max_time, unit: 'MINUTES') {
-            utils.init_git()
-            utils.docker_run('centos7_cpu', 'build_centos7_cpu_make', false)
-            utils.pack_lib(lib_name, mx_lib_make)
-          }
-        }
-      }
-    }]
-}
-
 def compile_centos7_cpu_mkldnn() {
     return ['CPU: CentOS 7 MKLDNN': {
       node(NODE_LINUX_CPU) {
@@ -903,7 +828,7 @@ def test_unix_onnx_cpu(lib_name) {
       node(NODE_LINUX_CPU) {
         ws('workspace/it-onnx-cpu') {
           timeout(time: max_time, unit: 'MINUTES') {
-            utils.unpack_and_init(lib_name, mx_lib_make)
+            utils.unpack_and_init(lib_name, mx_lib)
             utils.docker_run('ubuntu_cpu', 'integrationtest_ubuntu_cpu_onnx', false)
             utils.publish_test_coverage()
           }
@@ -1132,8 +1057,8 @@ def compile_unix_lite(lib_name) {
         ws('workspace/docs') {
           timeout(time: max_time, unit: 'MINUTES') {
             utils.init_git()
-            utils.docker_run('ubuntu_cpu_lite', 'build_ubuntu_cpu_docs', false)
-            utils.pack_lib(lib_name, 'lib/libmxnet.so', false)
+            utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_docs', false)
+            utils.pack_lib(lib_name, mx_lib, false)
           }
         }
       }
@@ -1161,7 +1086,7 @@ def docs_python(lib_name) {
       node(NODE_LINUX_CPU) {
         ws('workspace/docs') {
           timeout(time: max_time, unit: 'MINUTES') {
-            utils.unpack_and_init(lib_name, 'lib/libmxnet.so', false)
+            utils.unpack_and_init(lib_name, mx_lib, false)
             utils.docker_run('ubuntu_cpu_python', 'build_python_docs', false)
             if (should_pack_website()) {
               utils.pack_lib('python-artifacts', 'docs/_build/python-artifacts.tgz', false)

ci/jenkins/Jenkinsfile_centos_cpu

@@ -35,7 +35,6 @@ utils.main_wrapper(
 core_logic: {
   utils.parallel_stage('Build', [
     custom_steps.compile_centos7_cpu('centos7_cpu'),
-    custom_steps.compile_centos7_cpu_make('centos7_cpu_make'),
     custom_steps.compile_centos7_cpu_mkldnn(),
     custom_steps.compile_static_python_cpu(),
     custom_steps.compile_static_cd_cpu('centos7_cpu_cd')