From 8e6d52ec8a6888e110b258dffd1a407e2bb930e8 Mon Sep 17 00:00:00 2001 From: Leonard Lausen Date: Tue, 28 Jan 2020 17:37:34 +0000 Subject: [PATCH 01/21] Fix staticbuild pipeline --- ci/docker/install/ubuntu_publish.sh | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/ci/docker/install/ubuntu_publish.sh b/ci/docker/install/ubuntu_publish.sh index 65982eead389..e62efd5e7579 100755 --- a/ci/docker/install/ubuntu_publish.sh +++ b/ci/docker/install/ubuntu_publish.sh @@ -57,14 +57,13 @@ mv apache-maven-3.3.9/ /usr/local/maven/ update-alternatives --install /usr/bin/mvn mvn /usr/local/maven/apache-maven-3.3.9/bin/mvn 1 update-ca-certificates -f -apt-get install -y python python3 +apt-get install -y python python-pip python3 python3-pip # the version of the pip shipped with ubuntu may be too lower, install a recent version here -wget -nv https://bootstrap.pypa.io/get-pip.py -python3 get-pip.py -python2 get-pip.py +# Restrict pip version to <19 due to use of Python 3.4 on Ubuntu 14.04 +python2 -m pip install --upgrade 'pip<19' +python3 -m pip install --upgrade 'pip<19' -apt-get remove -y python3-urllib3 - -pip2 install nose cpplint==1.3.0 'numpy>1.16.0,<2.0.0' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3 -pip3 install nose cpplint==1.3.0 pylint==2.3.1 'numpy>1.16.0,<2.0.0' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3 +# Restrict numpy version to <1.18 due to use of Python 3.4 on Ubuntu 14.04 +python2 -m pip install --upgrade --ignore-installed nose cpplint==1.3.0 'numpy>1.16.0,<1.17' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3 +python3 -m pip install --upgrade --ignore-installed nose cpplint==1.3.0 pylint==2.3.1 'numpy>1.16.0,<1.18' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3 From 91cfcdbcc3e7e447c73aab330846c229c531099a Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Fri, 23 Aug 2019 18:50:26 -0700 Subject: [PATCH 02/21] Remove python2 from CI --- ci/docker/runtime_functions.sh | 97 ------------------- ci/jenkins/Jenkins_steps.groovy | 143 ----------------------------- ci/jenkins/Jenkinsfile_unix_cpu | 2 - ci/jenkins/Jenkinsfile_unix_gpu | 3 - ci/jenkins/Jenkinsfile_windows_cpu | 1 - ci/jenkins/Jenkinsfile_windows_gpu | 1 - ci/windows/test_py2_cpu.ps1 | 34 ------- ci/windows/test_py2_gpu.ps1 | 38 -------- 8 files changed, 319 deletions(-) delete mode 100644 ci/windows/test_py2_cpu.ps1 delete mode 100644 ci/windows/test_py2_gpu.ps1 diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index ca2878700252..a95ae8940a31 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -469,7 +469,6 @@ build_ubuntu_cpu_openblas() { USE_LIBJPEG_TURBO=1 \ USE_SIGNAL_HANDLER=1 \ -j$(nproc) - make cython PYTHON=python2 make cython PYTHON=python3 } @@ -808,7 +807,6 @@ build_ubuntu_gpu_cuda101_cudnn7_no_tvm_op() { USE_SIGNAL_HANDLER=1 \ -j$(nproc) - make cython PYTHON=python2 make cython PYTHON=python3 } @@ -998,34 +996,10 @@ cd_unittest_ubuntu() { fi } -unittest_ubuntu_python2_cpu_cython() { - set -ex - export PYTHONPATH=./python/ - export MXNET_MKLDNN_DEBUG=1 - export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 export MXNET_SUBGRAPH_VERBOSE=0 - export MXNET_ENABLE_CYTHON=1 - export MXNET_ENFORCE_CYTHON=1 export DMLC_LOG_STACK_TRACE_DEPTH=10 - check_cython 2 - nosetests-2.7 $NOSE_COVERAGE_ARGUMENTS $NOSE_TIMER_ARGUMENTS --with-xunit --xunit-file nosetests_unittest.xml --verbose tests/python/unittest - nosetests-2.7 $NOSE_COVERAGE_ARGUMENTS $NOSE_TIMER_ARGUMENTS --with-xunit --xunit-file nosetests_train.xml --verbose tests/python/train - nosetests-2.7 $NOSE_COVERAGE_ARGUMENTS $NOSE_TIMER_ARGUMENTS --with-xunit --xunit-file nosetests_quantization.xml --verbose tests/python/quantization -} - -unittest_ubuntu_python2_cpu() { - set -ex - export PYTHONPATH=./python/ - export MXNET_MKLDNN_DEBUG=0 - export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 export MXNET_SUBGRAPH_VERBOSE=0 - export MXNET_ENABLE_CYTHON=0 export DMLC_LOG_STACK_TRACE_DEPTH=10 - nosetests-2.7 $NOSE_COVERAGE_ARGUMENTS $NOSE_TIMER_ARGUMENTS --with-xunit --xunit-file nosetests_unittest.xml --verbose tests/python/unittest - nosetests-2.7 $NOSE_COVERAGE_ARGUMENTS $NOSE_TIMER_ARGUMENTS --with-xunit --xunit-file nosetests_train.xml --verbose tests/python/train - nosetests-2.7 $NOSE_COVERAGE_ARGUMENTS $NOSE_TIMER_ARGUMENTS --with-xunit --xunit-file nosetests_quantization.xml --verbose tests/python/quantization -} - unittest_ubuntu_python3_cpu() { set -ex export PYTHONPATH=./python/ @@ -1050,17 +1024,8 @@ unittest_ubuntu_python3_cpu_mkldnn() { nosetests-3.4 $NOSE_COVERAGE_ARGUMENTS $NOSE_TIMER_ARGUMENTS --with-xunit --xunit-file nosetests_mkl.xml --verbose tests/python/mkl } -unittest_ubuntu_python2_gpu() { - set -ex - export PYTHONPATH=./python/ - export MXNET_MKLDNN_DEBUG=0 # Ignored if not present - export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 export MXNET_SUBGRAPH_VERBOSE=0 - export CUDNN_VERSION=${CUDNN_VERSION:-7.0.3} export DMLC_LOG_STACK_TRACE_DEPTH=10 - nosetests-2.7 $NOSE_COVERAGE_ARGUMENTS $NOSE_TIMER_ARGUMENTS --with-xunit --xunit-file nosetests_gpu.xml --verbose tests/python/gpu -} - unittest_ubuntu_python3_gpu() { set -ex export PYTHONPATH=./python/ @@ -1112,20 +1077,8 @@ unittest_ubuntu_tensorrt_gpu() { } # quantization gpu currently only runs on P3 instances -# need to separte it from unittest_ubuntu_python2_gpu() -unittest_ubuntu_python2_quantization_gpu() { - set -ex - export PYTHONPATH=./python/ - export MXNET_MKLDNN_DEBUG=0 # Ignored if not present - export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 export MXNET_SUBGRAPH_VERBOSE=0 - export CUDNN_VERSION=${CUDNN_VERSION:-7.0.3} - export MXNET_ENABLE_CYTHON=0 export DMLC_LOG_STACK_TRACE_DEPTH=10 - nosetests-2.7 $NOSE_COVERAGE_ARGUMENTS $NOSE_TIMER_ARGUMENTS --with-xunit --xunit-file nosetests_quantization_gpu.xml --verbose tests/python/quantization_gpu -} - -# quantization gpu currently only runs on P3 instances # need to separte it from unittest_ubuntu_python3_gpu() unittest_ubuntu_python3_quantization_gpu() { set -ex @@ -1375,24 +1328,7 @@ integrationtest_ubuntu_gpu_dist_kvstore() { popd } -test_ubuntu_cpu_python2() { - set -ex - pushd . - export MXNET_LIBRARY_PATH=/work/build/libmxnet.so export DMLC_LOG_STACK_TRACE_DEPTH=10 - - VENV=mxnet_py2_venv - virtualenv -p `which python2` $VENV - source $VENV/bin/activate - pip install nose nose-timer - - cd /work/mxnet/python - pip install -e . - cd /work/mxnet - python -m "nose" $NOSE_COVERAGE_ARGUMENTS $NOSE_TIMER_ARGUMENTS --verbose tests/python/unittest - popd -} - test_ubuntu_cpu_python3() { set -ex pushd . @@ -1544,17 +1480,7 @@ nightly_model_backwards_compat_train() { ./tests/nightly/model_backwards_compatibility_check/train_mxnet_legacy_models.sh } -# Nightly 'MXNet: The Straight Dope' Single-GPU Tests -nightly_straight_dope_python2_single_gpu_tests() { - set -ex - cd /work/mxnet/tests/nightly/straight_dope - export PYTHONPATH=/work/mxnet/python/ - export MXNET_TEST_KERNEL=python2 export DMLC_LOG_STACK_TRACE_DEPTH=10 - nosetests-2.7 $NOSE_TIMER_ARGUMENTS --with-xunit --xunit-file nosetests_straight_dope_python2_single_gpu.xml \ - test_notebooks_single_gpu.py --nologcapture -} - nightly_straight_dope_python3_single_gpu_tests() { set -ex cd /work/mxnet/tests/nightly/straight_dope @@ -1565,17 +1491,7 @@ nightly_straight_dope_python3_single_gpu_tests() { test_notebooks_single_gpu.py --nologcapture } -# Nightly 'MXNet: The Straight Dope' Multi-GPU Tests -nightly_straight_dope_python2_multi_gpu_tests() { - set -ex - cd /work/mxnet/tests/nightly/straight_dope - export PYTHONPATH=/work/mxnet/python/ - export MXNET_TEST_KERNEL=python2 export DMLC_LOG_STACK_TRACE_DEPTH=10 - nosetests-2.7 $NOSE_TIMER_ARGUMENTS --with-xunit --xunit-file nosetests_straight_dope_python2_multi_gpu.xml \ - test_notebooks_multi_gpu.py --nologcapture -} - nightly_straight_dope_python3_multi_gpu_tests() { set -ex cd /work/mxnet/tests/nightly/straight_dope @@ -1601,21 +1517,8 @@ nightly_tutorial_test_ubuntu_python3_gpu() { nosetests-3.4 $NOSE_TIMER_ARGUMENTS --with-xunit --xunit-file nosetests_tutorials.xml test_tutorials.py --nologcapture } -nightly_tutorial_test_ubuntu_python2_gpu() { - set -ex - cd /work/mxnet/docs - export BUILD_VER=tutorial - export MXNET_DOCS_BUILD_MXNET=0 export DMLC_LOG_STACK_TRACE_DEPTH=10 - make html - export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 export MXNET_SUBGRAPH_VERBOSE=0 - export PYTHONPATH=/work/mxnet/python/ - export MXNET_TUTORIAL_TEST_KERNEL=python2 - cd /work/mxnet/tests/tutorials - nosetests-3.4 $NOSE_TIMER_ARGUMENTS --with-xunit --xunit-file nosetests_tutorials.xml test_tutorials.py --nologcapture -} - nightly_java_demo_test_cpu() { set -ex cd /work/mxnet/scala-package/mxnet-demo/java-demo diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy index c697e1e58788..a29d5a59ccb0 100644 --- a/ci/jenkins/Jenkins_steps.groovy +++ b/ci/jenkins/Jenkins_steps.groovy @@ -43,19 +43,6 @@ mx_lib_cpp_examples_no_tvm_op = 'lib/libmxnet.so, lib/libmxnet.a, libsample_lib. mx_lib_cpp_examples_cpu = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/cpp-package/example/*' // Python unittest for CPU -// Python 2 -def python2_ut(docker_container_name) { - timeout(time: max_time, unit: 'MINUTES') { - utils.docker_run(docker_container_name, 'unittest_ubuntu_python2_cpu', false) - } -} - -def python2_ut_cython(docker_container_name) { - timeout(time: max_time, unit: 'MINUTES') { - utils.docker_run(docker_container_name, 'unittest_ubuntu_python2_cpu_cython', false) - } -} - // Python 3 def python3_ut(docker_container_name) { timeout(time: max_time, unit: 'MINUTES') { @@ -78,13 +65,6 @@ def python3_ut_mkldnn(docker_container_name) { // GPU test has two parts. 1) run unittest on GPU, 2) compare the results on // both CPU and GPU -// Python 2 -def python2_gpu_ut(docker_container_name) { - timeout(time: max_time, unit: 'MINUTES') { - utils.docker_run(docker_container_name, 'unittest_ubuntu_python2_gpu', true) - } -} - // Python 3 def python3_gpu_ut(docker_container_name) { timeout(time: max_time, unit: 'MINUTES') { @@ -681,74 +661,6 @@ def test_static_python_gpu() { }] } -def test_unix_python2_cpu() { - return ['Python2: CPU': { - node(NODE_LINUX_CPU) { - ws('workspace/ut-python2-cpu') { - try { - utils.unpack_and_init('cpu', mx_lib_cython) - python2_ut_cython('ubuntu_cpu') - utils.publish_test_coverage() - } finally { - utils.collect_test_results_unix('nosetests_unittest.xml', 'nosetests_python2_cpu_unittest.xml') - utils.collect_test_results_unix('nosetests_train.xml', 'nosetests_python2_cpu_train.xml') - utils.collect_test_results_unix('nosetests_quantization.xml', 'nosetests_python2_cpu_quantization.xml') - } - } - } - }] -} - -def test_unix_python2_gpu() { - return ['Python2: GPU': { - node(NODE_LINUX_GPU) { - ws('workspace/ut-python2-gpu') { - try { - utils.unpack_and_init('gpu', mx_lib) - python2_gpu_ut('ubuntu_gpu_cu101') - utils.publish_test_coverage() - } finally { - utils.collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python2_gpu.xml') - } - } - } - }] -} - -def test_unix_python2_quantize_gpu() { - return ['Python2: Quantize GPU': { - node(NODE_LINUX_GPU_P3) { - ws('workspace/ut-python2-quantize-gpu') { - timeout(time: max_time, unit: 'MINUTES') { - try { - utils.unpack_and_init('gpu', mx_lib) - utils.docker_run('ubuntu_gpu_cu101', 'unittest_ubuntu_python2_quantization_gpu', true) - utils.publish_test_coverage() - } finally { - utils.collect_test_results_unix('nosetests_quantization_gpu.xml', 'nosetests_python2_quantize_gpu.xml') - } - } - } - } - }] -} - -def test_unix_python2_mkldnn_gpu() { - return ['Python2: MKLDNN-GPU': { - node(NODE_LINUX_GPU) { - ws('workspace/ut-python2-mkldnn-gpu') { - try { - utils.unpack_and_init('mkldnn_gpu', mx_mkldnn_lib) - python2_gpu_ut('ubuntu_gpu_cu101') - utils.publish_test_coverage() - } finally { - utils.collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python2_mkldnn_gpu.xml') - } - } - } - }] -} - def test_unix_python3_cpu() { return ['Python3: CPU': { node(NODE_LINUX_CPU) { @@ -865,24 +777,6 @@ def test_unix_python3_cpu_no_tvm_op() { }] } -def test_unix_python2_mkldnn_cpu() { - return ['Python2: MKLDNN-CPU': { - node(NODE_LINUX_CPU) { - ws('workspace/ut-python2-mkldnn-cpu') { - try { - utils.unpack_and_init('mkldnn_cpu', mx_mkldnn_lib) - python2_ut('ubuntu_cpu') - utils.publish_test_coverage() - } finally { - utils.collect_test_results_unix('nosetests_unittest.xml', 'nosetests_python2_mkldnn_cpu_unittest.xml') - utils.collect_test_results_unix('nosetests_train.xml', 'nosetests_python2_mkldnn_cpu_train.xml') - utils.collect_test_results_unix('nosetests_quantization.xml', 'nosetests_python2_mkldnn_cpu_quantization.xml') - } - } - } - }] -} - def test_unix_python3_mkldnn_cpu() { return ['Python3: MKLDNN-CPU': { node(NODE_LINUX_CPU) { @@ -1310,43 +1204,6 @@ def test_centos7_scala_cpu() { }] } -def test_windows_python2_cpu() { - return ['Python 2: CPU Win':{ - node(NODE_WINDOWS_CPU) { - timeout(time: max_time, unit: 'MINUTES') { - ws('workspace/ut-python-cpu') { - try { - utils.init_git_win() - unstash 'windows_package_cpu' - powershell 'ci/windows/test_py2_cpu.ps1' - } finally { - utils.collect_test_results_windows('nosetests_unittest.xml', 'nosetests_unittest_windows_python2_cpu.xml') - } - } - } - } - }] -} - -def test_windows_python2_gpu() { - return ['Python 2: GPU Win':{ - node(NODE_WINDOWS_GPU) { - timeout(time: max_time, unit: 'MINUTES') { - ws('workspace/ut-python-gpu') { - try { - utils.init_git_win() - unstash 'windows_package_gpu' - powershell 'ci/windows/test_py2_gpu.ps1' - } finally { - utils.collect_test_results_windows('nosetests_forward.xml', 'nosetests_gpu_forward_windows_python2_gpu.xml') - utils.collect_test_results_windows('nosetests_operator.xml', 'nosetests_gpu_operator_windows_python2_gpu.xml') - } - } - } - } - }] -} - def test_windows_python3_gpu() { return ['Python 3: GPU Win':{ node(NODE_WINDOWS_GPU) { diff --git a/ci/jenkins/Jenkinsfile_unix_cpu b/ci/jenkins/Jenkinsfile_unix_cpu index 931db9dce77f..0b6ad661a3eb 100644 --- a/ci/jenkins/Jenkinsfile_unix_cpu +++ b/ci/jenkins/Jenkinsfile_unix_cpu @@ -44,11 +44,9 @@ core_logic: { ]) utils.parallel_stage('Tests', [ - custom_steps.test_unix_python2_cpu(), custom_steps.test_unix_python3_cpu(), custom_steps.test_unix_python3_debug_cpu(), custom_steps.test_unix_python3_mkl_cpu(), - custom_steps.test_unix_python2_mkldnn_cpu(), custom_steps.test_unix_python3_mkldnn_cpu(), custom_steps.test_unix_python3_mkldnn_mkl_cpu(), custom_steps.test_unix_scala_cpu(), diff --git a/ci/jenkins/Jenkinsfile_unix_gpu b/ci/jenkins/Jenkinsfile_unix_gpu index 18e27198c330..a407052ad3b9 100644 --- a/ci/jenkins/Jenkinsfile_unix_gpu +++ b/ci/jenkins/Jenkinsfile_unix_gpu @@ -46,11 +46,8 @@ core_logic: { ]) utils.parallel_stage('Tests', [ - custom_steps.test_unix_python2_gpu(), custom_steps.test_unix_python3_gpu(), - custom_steps.test_unix_python2_quantize_gpu(), custom_steps.test_unix_python3_quantize_gpu(), - custom_steps.test_unix_python2_mkldnn_gpu(), custom_steps.test_unix_python3_mkldnn_gpu(), custom_steps.test_unix_python3_mkldnn_nocudnn_gpu(), custom_steps.test_unix_python3_tensorrt_gpu(), diff --git a/ci/jenkins/Jenkinsfile_windows_cpu b/ci/jenkins/Jenkinsfile_windows_cpu index 4475796771d1..3de8f7d4330b 100644 --- a/ci/jenkins/Jenkinsfile_windows_cpu +++ b/ci/jenkins/Jenkinsfile_windows_cpu @@ -41,7 +41,6 @@ core_logic: { ]) utils.parallel_stage('Tests', [ - custom_steps.test_windows_python2_cpu(), custom_steps.test_windows_python3_cpu(), custom_steps.test_windows_julia07_cpu(), custom_steps.test_windows_julia10_cpu() diff --git a/ci/jenkins/Jenkinsfile_windows_gpu b/ci/jenkins/Jenkinsfile_windows_gpu index 2319f25942de..631d9e947002 100644 --- a/ci/jenkins/Jenkinsfile_windows_gpu +++ b/ci/jenkins/Jenkinsfile_windows_gpu @@ -39,7 +39,6 @@ core_logic: { ]) utils.parallel_stage('Tests', [ - custom_steps.test_windows_python2_gpu(), custom_steps.test_windows_python3_gpu(), custom_steps.test_windows_python3_gpu_mkldnn() ]) diff --git a/ci/windows/test_py2_cpu.ps1 b/ci/windows/test_py2_cpu.ps1 deleted file mode 100644 index c39d1fa45328..000000000000 --- a/ci/windows/test_py2_cpu.ps1 +++ /dev/null @@ -1,34 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -7z x -y windows_package.7z - -$env:MXNET_LIBRARY_PATH=join-path $pwd.Path windows_package\lib\libmxnet.dll -$env:PYTHONPATH=join-path $pwd.Path windows_package\python -$env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 -$env:MXNET_SUBGRAPH_VERBOSE=0 -$env:MXNET_HOME=[io.path]::combine($PSScriptRoot, 'mxnet_home') - -C:\Python27\Scripts\pip install -r tests\requirements.txt -C:\Python27\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest -if (! $?) { Throw ("Error running unittest, python exited with status code " + ('{0:X}' -f $LastExitCode)) } -C:\Python27\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_train.xml tests\python\train -if (! $?) { Throw ("Error running train tests, python exited with status code " + ('{0:X}' -f $LastExitCode)) } -# Adding this extra test since it's not possible to set env var on the fly in Windows. -$env:MXNET_SAFE_ACCUMULATION=1 -C:\Python27\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest\test_operator.py:test_norm -if (! $?) { Throw ("Error running unittest, python exited with status code " + ('{0:X}' -f $LastExitCode)) } diff --git a/ci/windows/test_py2_gpu.ps1 b/ci/windows/test_py2_gpu.ps1 deleted file mode 100644 index b2ea62fc7cd4..000000000000 --- a/ci/windows/test_py2_gpu.ps1 +++ /dev/null @@ -1,38 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -7z x -y windows_package.7z - -$env:MXNET_LIBRARY_PATH=join-path $pwd.Path windows_package\lib\libmxnet.dll -$env:PYTHONPATH=join-path $pwd.Path windows_package\python -$env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 -$env:MXNET_SUBGRAPH_VERBOSE=0 -$env:MXNET_HOME=[io.path]::combine($PSScriptRoot, 'mxnet_home') - -C:\Python27\Scripts\pip install -r tests\requirements.txt -C:\Python27\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest -if (! $?) { Throw ("Error running unittest, python exited with status code " + ('{0:X}' -f $LastExitCode)) } -C:\Python27\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_operator.xml tests\python\gpu\test_operator_gpu.py -if (! $?) { Throw ("Error running tests, python exited with status code " + ('{0:X}' -f $LastExitCode)) } -C:\Python27\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_forward.xml tests\python\gpu\test_forward.py -if (! $?) { Throw ("Error running tests, python exited with status code " + ('{0:X}' -f $LastExitCode)) } -C:\Python27\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error tests\python\train -if (! $?) { Throw ("Error running tests, python exited with status code " + ('{0:X}' -f $LastExitCode)) } -# Adding this extra test since it's not possible to set env var on the fly in Windows. -$env:MXNET_SAFE_ACCUMULATION=1 -C:\Python27\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_operator.xml tests\python\gpu\test_operator_gpu.py:test_norm -if (! $?) { Throw ("Error running tests, python exited with status code " + ('{0:X}' -f $LastExitCode)) } From 2b91f1e358f7fff959eeda28324246f9025cdf5e Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Tue, 28 Jan 2020 13:42:42 -0800 Subject: [PATCH 03/21] fix merge issues --- 3rdparty/ps-lite | 2 +- ci/docker/runtime_functions.sh | 13 ------------- 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/3rdparty/ps-lite b/3rdparty/ps-lite index f60105427e4f..60b826e4422f 160000 --- a/3rdparty/ps-lite +++ b/3rdparty/ps-lite @@ -1 +1 @@ -Subproject commit f60105427e4f626244056c8d527dca54402d0a9c +Subproject commit 60b826e4422fee0df00b892c66ffffea11e5da3f diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index a95ae8940a31..d24957ac4343 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -996,10 +996,6 @@ cd_unittest_ubuntu() { fi } - export MXNET_SUBGRAPH_VERBOSE=0 - export DMLC_LOG_STACK_TRACE_DEPTH=10 - export MXNET_SUBGRAPH_VERBOSE=0 - export DMLC_LOG_STACK_TRACE_DEPTH=10 unittest_ubuntu_python3_cpu() { set -ex export PYTHONPATH=./python/ @@ -1024,8 +1020,6 @@ unittest_ubuntu_python3_cpu_mkldnn() { nosetests-3.4 $NOSE_COVERAGE_ARGUMENTS $NOSE_TIMER_ARGUMENTS --with-xunit --xunit-file nosetests_mkl.xml --verbose tests/python/mkl } - export MXNET_SUBGRAPH_VERBOSE=0 - export DMLC_LOG_STACK_TRACE_DEPTH=10 unittest_ubuntu_python3_gpu() { set -ex export PYTHONPATH=./python/ @@ -1077,8 +1071,6 @@ unittest_ubuntu_tensorrt_gpu() { } # quantization gpu currently only runs on P3 instances - export MXNET_SUBGRAPH_VERBOSE=0 - export DMLC_LOG_STACK_TRACE_DEPTH=10 # need to separte it from unittest_ubuntu_python3_gpu() unittest_ubuntu_python3_quantization_gpu() { set -ex @@ -1328,7 +1320,6 @@ integrationtest_ubuntu_gpu_dist_kvstore() { popd } - export DMLC_LOG_STACK_TRACE_DEPTH=10 test_ubuntu_cpu_python3() { set -ex pushd . @@ -1480,7 +1471,6 @@ nightly_model_backwards_compat_train() { ./tests/nightly/model_backwards_compatibility_check/train_mxnet_legacy_models.sh } - export DMLC_LOG_STACK_TRACE_DEPTH=10 nightly_straight_dope_python3_single_gpu_tests() { set -ex cd /work/mxnet/tests/nightly/straight_dope @@ -1491,7 +1481,6 @@ nightly_straight_dope_python3_single_gpu_tests() { test_notebooks_single_gpu.py --nologcapture } - export DMLC_LOG_STACK_TRACE_DEPTH=10 nightly_straight_dope_python3_multi_gpu_tests() { set -ex cd /work/mxnet/tests/nightly/straight_dope @@ -1517,8 +1506,6 @@ nightly_tutorial_test_ubuntu_python3_gpu() { nosetests-3.4 $NOSE_TIMER_ARGUMENTS --with-xunit --xunit-file nosetests_tutorials.xml test_tutorials.py --nologcapture } - export DMLC_LOG_STACK_TRACE_DEPTH=10 - export MXNET_SUBGRAPH_VERBOSE=0 nightly_java_demo_test_cpu() { set -ex cd /work/mxnet/scala-package/mxnet-demo/java-demo From b8faf46dc4ff94c88690a5a9e73922aff67707a0 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Tue, 28 Jan 2020 14:55:11 -0800 Subject: [PATCH 04/21] fix botched merge --- 3rdparty/ps-lite | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty/ps-lite b/3rdparty/ps-lite index 60b826e4422f..f60105427e4f 160000 --- a/3rdparty/ps-lite +++ b/3rdparty/ps-lite @@ -1 +1 @@ -Subproject commit 60b826e4422fee0df00b892c66ffffea11e5da3f +Subproject commit f60105427e4f626244056c8d527dca54402d0a9c From 34dec6ac53558baa8d1d8b0965a555310e2f6011 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 29 Jan 2020 00:08:19 -0800 Subject: [PATCH 05/21] Nuke more python2 --- ci/docker/install/centos7_python.sh | 3 --- ci/docker/install/ubuntu_publish.sh | 3 +-- ci/docker/install/ubuntu_python.sh | 3 --- ci/docker/runtime_functions.sh | 1 - 4 files changed, 1 insertion(+), 9 deletions(-) diff --git a/ci/docker/install/centos7_python.sh b/ci/docker/install/centos7_python.sh index 8521cde1acca..7e8a8df8ece5 100755 --- a/ci/docker/install/centos7_python.sh +++ b/ci/docker/install/centos7_python.sh @@ -28,8 +28,5 @@ yum -y install python36u # Install PIP curl "https://bootstrap.pypa.io/get-pip.py" -o "get-pip.py" -python2.7 get-pip.py python3.6 get-pip.py - -pip2 install nose pylint numpy nose-timer requests h5py scipy==1.0.1 pip3 install nose pylint numpy nose-timer requests h5py scipy==1.0.1 diff --git a/ci/docker/install/ubuntu_publish.sh b/ci/docker/install/ubuntu_publish.sh index e62efd5e7579..06b0ccb9357b 100755 --- a/ci/docker/install/ubuntu_publish.sh +++ b/ci/docker/install/ubuntu_publish.sh @@ -61,9 +61,8 @@ apt-get install -y python python-pip python3 python3-pip # the version of the pip shipped with ubuntu may be too lower, install a recent version here # Restrict pip version to <19 due to use of Python 3.4 on Ubuntu 14.04 -python2 -m pip install --upgrade 'pip<19' python3 -m pip install --upgrade 'pip<19' # Restrict numpy version to <1.18 due to use of Python 3.4 on Ubuntu 14.04 -python2 -m pip install --upgrade --ignore-installed nose cpplint==1.3.0 'numpy>1.16.0,<1.17' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3 python3 -m pip install --upgrade --ignore-installed nose cpplint==1.3.0 pylint==2.3.1 'numpy>1.16.0,<1.18' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3 + diff --git a/ci/docker/install/ubuntu_python.sh b/ci/docker/install/ubuntu_python.sh index 12149121ba0b..b6792a286fad 100755 --- a/ci/docker/install/ubuntu_python.sh +++ b/ci/docker/install/ubuntu_python.sh @@ -28,7 +28,4 @@ apt-get install -y python-dev python3-dev virtualenv wget # the version of the pip shipped with ubuntu may be too lower, install a recent version here wget -nv https://bootstrap.pypa.io/get-pip.py python3 get-pip.py -python2 get-pip.py - pip3 install -r /work/requirements -pip2 install -r /work/requirements diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index d24957ac4343..7d7641e43319 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -786,7 +786,6 @@ build_ubuntu_gpu_cuda101_cudnn7() { USE_SIGNAL_HANDLER=1 \ -j$(nproc) - make cython PYTHON=python2 make cython PYTHON=python3 } From b7fdd0267675b4a0d79fb980673d8068c092606f Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 29 Jan 2020 21:19:38 +0000 Subject: [PATCH 06/21] use aws mirror --- ci/docker/install/ubuntu_core.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/docker/install/ubuntu_core.sh b/ci/docker/install/ubuntu_core.sh index 70bc285d163b..0b0e90276040 100755 --- a/ci/docker/install/ubuntu_core.sh +++ b/ci/docker/install/ubuntu_core.sh @@ -21,6 +21,7 @@ # the whole docker cache for the image set -ex +perl -pi -e 's/archive.ubuntu.com/us-west-2.ec2.archive.ubuntu.com/' /etc/apt/sources.list apt-get update || true # Avoid interactive package installers such as tzdata. From 3a59deaa293fd5388b9876357b56d39ed9a6915c Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 29 Jan 2020 21:19:49 +0000 Subject: [PATCH 07/21] remove pip2 installs --- ci/docker/install/ubuntu_caffe.sh | 2 +- ci/docker/install/ubuntu_onnx.sh | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/ci/docker/install/ubuntu_caffe.sh b/ci/docker/install/ubuntu_caffe.sh index eaa8ab9b18ac..bda1c0b8aef3 100755 --- a/ci/docker/install/ubuntu_caffe.sh +++ b/ci/docker/install/ubuntu_caffe.sh @@ -55,4 +55,4 @@ ln -s /usr/lib/x86_64-linux-gnu/libhdf5_serial_hl.so.10.0.2 /usr/lib/x86_64-linu make all pycaffe -j$(nproc) cd python -for req in $(cat requirements.txt); do pip2 install $req; done +for req in $(cat requirements.txt); do pip3 install $req; done diff --git a/ci/docker/install/ubuntu_onnx.sh b/ci/docker/install/ubuntu_onnx.sh index 307028925b6e..44d6b9ed52dc 100755 --- a/ci/docker/install/ubuntu_onnx.sh +++ b/ci/docker/install/ubuntu_onnx.sh @@ -31,5 +31,4 @@ apt-get update || true apt-get install -y libprotobuf-dev protobuf-compiler echo "Installing pytest, pytest-cov, protobuf, Pillow, ONNX and tabulate ..." -pip2 install pytest==3.6.3 pytest-cov==2.5.1 protobuf==3.5.2 onnx==1.3.0 Pillow==5.0.0 tabulate==0.7.5 pip3 install pytest==3.6.3 pytest-cov==2.5.1 protobuf==3.5.2 onnx==1.3.0 Pillow==5.0.0 tabulate==0.7.5 From a1c0c6eb6758269837f5174ed5d38a945acda9d1 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 29 Jan 2020 23:03:51 +0000 Subject: [PATCH 08/21] Fix python3 unicode error in Julia docs --- ci/docker/runtime_functions.sh | 1 + julia/docs/Makefile | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 7d7641e43319..8f6b1245a8cc 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -1748,6 +1748,7 @@ build_julia_docs() { echo "Julia will check for MXNet in $MXNET_HOME/lib" + make -C julia/docs GZIP=-9 tar -zcvf $julia_doc_artifact -C $julia_doc_path . diff --git a/julia/docs/Makefile b/julia/docs/Makefile index e42b8cdccb93..5e14d8f33a2b 100644 --- a/julia/docs/Makefile +++ b/julia/docs/Makefile @@ -21,7 +21,11 @@ all: Pkg.develop(PackageSpec(name="MXNet", path = joinpath(pwd(), "..")))' julia --color=yes --project=./ ./make.jl pip install --user pygments mkdocs mkdocs-material python-markdown-math - ~/.local/bin/mkdocs build + export LC_ALL="C.UTF-8" + export LANG="C.UTF-8" + echo $(LC_ALL) + echo $(LANG) + LC_ALL="C.UTF-8" ~/.local/bin/mkdocs build clean: rm -rvf venv From 122f9f0de9c033a54db46800b9231e43dfb426da Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Thu, 30 Jan 2020 09:04:20 +0000 Subject: [PATCH 09/21] Fix centos --- 3rdparty/ps-lite | 2 +- tests/python/unittest/test_numpy_ndarray.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/3rdparty/ps-lite b/3rdparty/ps-lite index f60105427e4f..60b826e4422f 160000 --- a/3rdparty/ps-lite +++ b/3rdparty/ps-lite @@ -1 +1 @@ -Subproject commit f60105427e4f626244056c8d527dca54402d0a9c +Subproject commit 60b826e4422fee0df00b892c66ffffea11e5da3f diff --git a/tests/python/unittest/test_numpy_ndarray.py b/tests/python/unittest/test_numpy_ndarray.py index 4d6f48e225b7..a1196160903c 100644 --- a/tests/python/unittest/test_numpy_ndarray.py +++ b/tests/python/unittest/test_numpy_ndarray.py @@ -30,7 +30,6 @@ from mxnet.test_utils import verify_generator, gen_buckets_probs_with_ppf, assert_exception, is_op_runnable, collapse_sum_like from mxnet.ndarray.ndarray import py_slice from mxnet.base import integer_types -import scipy.stats as ss @with_seed() From 87208b850ec12e19c0b21a409a3ab46bfdfca289 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Thu, 30 Jan 2020 09:59:53 +0000 Subject: [PATCH 10/21] ps-lite synced with master --- 3rdparty/ps-lite | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty/ps-lite b/3rdparty/ps-lite index 60b826e4422f..f60105427e4f 160000 --- a/3rdparty/ps-lite +++ b/3rdparty/ps-lite @@ -1 +1 @@ -Subproject commit 60b826e4422fee0df00b892c66ffffea11e5da3f +Subproject commit f60105427e4f626244056c8d527dca54402d0a9c From aa87a91ccf2306244dc69242a564522902669580 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Thu, 30 Jan 2020 11:01:41 +0000 Subject: [PATCH 11/21] centos fix, scipy was too old, incompatible with nose --- ci/docker/install/centos7_python.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/docker/install/centos7_python.sh b/ci/docker/install/centos7_python.sh index 7e8a8df8ece5..06c53bea48c1 100755 --- a/ci/docker/install/centos7_python.sh +++ b/ci/docker/install/centos7_python.sh @@ -29,4 +29,4 @@ yum -y install python36u # Install PIP curl "https://bootstrap.pypa.io/get-pip.py" -o "get-pip.py" python3.6 get-pip.py -pip3 install nose pylint numpy nose-timer requests h5py scipy==1.0.1 +pip3 install nose pylint numpy nose-timer requests h5py scipy==1.2.3 From 37572aaa3beb58a835b64d55f20a4b644cce4006 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Thu, 30 Jan 2020 11:01:53 +0000 Subject: [PATCH 12/21] remove pip2 --- ci/docker/install/tensorrt.sh | 1 - ci/docker/install/ubuntu_nightly_tests.sh | 1 - ci/docker/install/ubuntu_tutorials.sh | 1 - 3 files changed, 3 deletions(-) diff --git a/ci/docker/install/tensorrt.sh b/ci/docker/install/tensorrt.sh index 674a3315eb09..e98c7643f923 100755 --- a/ci/docker/install/tensorrt.sh +++ b/ci/docker/install/tensorrt.sh @@ -18,7 +18,6 @@ # under the License. # Install gluoncv since we're testing Gluon models as well -pip2 install gluoncv==0.2.0 pip3 install gluoncv==0.2.0 # Install Protobuf diff --git a/ci/docker/install/ubuntu_nightly_tests.sh b/ci/docker/install/ubuntu_nightly_tests.sh index 80028d87c572..c80efed6c377 100755 --- a/ci/docker/install/ubuntu_nightly_tests.sh +++ b/ci/docker/install/ubuntu_nightly_tests.sh @@ -32,5 +32,4 @@ apt-get -y install time apt-get install -y subversion maven -y #>/dev/null # Packages needed for the Straight Dope Nightly tests. -pip2 install pandas scikit-image prompt_toolkit pip3 install pandas scikit-image prompt_toolkit diff --git a/ci/docker/install/ubuntu_tutorials.sh b/ci/docker/install/ubuntu_tutorials.sh index d82763e8fd3e..469df6190ea4 100755 --- a/ci/docker/install/ubuntu_tutorials.sh +++ b/ci/docker/install/ubuntu_tutorials.sh @@ -25,5 +25,4 @@ apt-get update || true apt-get install graphviz python-opencv # sckit-learn past version 0.20 does not support python version 2 and 3.4 -pip2 install jupyter matplotlib Pillow opencv-python "scikit-learn<0.21.0" graphviz==0.8.4 tqdm mxboard scipy gluoncv pip3 install jupyter matplotlib Pillow opencv-python scikit-learn graphviz==0.8.4 tqdm mxboard scipy gluoncv From bb095397719e40a0c557aa1700ddafd390b49855 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Thu, 30 Jan 2020 18:17:38 +0000 Subject: [PATCH 13/21] remove python2 from integration tests --- ci/docker/runtime_functions.sh | 12 ++++++------ example/image-classification/test_score.py | 3 +++ tests/nightly/test_kvstore.py | 5 +++-- tests/python-pytest/onnx/backend_test.py | 3 +++ tests/python/tensorrt/lenet5_train.py | 3 +++ tools/caffe_converter/test_converter.py | 3 +++ 6 files changed, 21 insertions(+), 8 deletions(-) mode change 100644 => 100755 tests/nightly/test_kvstore.py mode change 100644 => 100755 tests/python-pytest/onnx/backend_test.py mode change 100644 => 100755 tests/python/tensorrt/lenet5_train.py mode change 100644 => 100755 tools/caffe_converter/test_converter.py diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 8f6b1245a8cc..3127476b0539 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -124,7 +124,7 @@ build_wheel() { export MXNET_LIBRARY_PATH=${BUILD_DIR}/libmxnet.so cd ${PYTHON_DIR} - python setup.py bdist_wheel --universal + python3 setup.py bdist_wheel --universal # repackage @@ -1065,7 +1065,7 @@ unittest_ubuntu_tensorrt_gpu() { export CUDNN_VERSION=${CUDNN_VERSION:-7.0.3} export MXNET_ENABLE_CYTHON=0 export DMLC_LOG_STACK_TRACE_DEPTH=10 - python tests/python/tensorrt/lenet5_train.py + tests/python/tensorrt/lenet5_train.py nosetests-3.4 $NOSE_COVERAGE_ARGUMENTS $NOSE_TIMER_ARGUMENTS --with-xunit --xunit-file nosetests_trt_gpu.xml --verbose --nocapture tests/python/tensorrt/ } @@ -1229,7 +1229,7 @@ integrationtest_ubuntu_cpu_onnx() { set -ex export PYTHONPATH=./python/ export DMLC_LOG_STACK_TRACE_DEPTH=10 - python tests/python-pytest/onnx/backend_test.py + tests/python-pytest/onnx/backend_test.py pytest tests/python-pytest/onnx/mxnet_export_test.py pytest tests/python-pytest/onnx/test_models.py pytest tests/python-pytest/onnx/test_node.py @@ -1241,14 +1241,14 @@ integrationtest_ubuntu_gpu_python() { export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 export MXNET_SUBGRAPH_VERBOSE=0 export DMLC_LOG_STACK_TRACE_DEPTH=10 - python example/image-classification/test_score.py + example/image-classification/test_score.py } integrationtest_ubuntu_gpu_caffe() { set -ex export PYTHONPATH=/work/deps/caffe/python:./python export DMLC_LOG_STACK_TRACE_DEPTH=10 - python tools/caffe_converter/test_converter.py + tools/caffe_converter/test_converter.py } integrationtest_ubuntu_cpu_asan() { @@ -1400,7 +1400,7 @@ nightly_test_KVStore_singleNode() { set -ex export PYTHONPATH=./python/ export DMLC_LOG_STACK_TRACE_DEPTH=10 - python tests/nightly/test_kvstore.py + tests/nightly/test_kvstore.py } #Test Large Tensor Size diff --git a/example/image-classification/test_score.py b/example/image-classification/test_score.py index e41d4e62ea8e..8fbac68c9064 100644 --- a/example/image-classification/test_score.py +++ b/example/image-classification/test_score.py @@ -1,3 +1,6 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/tests/nightly/test_kvstore.py b/tests/nightly/test_kvstore.py old mode 100644 new mode 100755 index c087da1193fc..ced3ee1ef8cc --- a/tests/nightly/test_kvstore.py +++ b/tests/nightly/test_kvstore.py @@ -1,5 +1,6 @@ -#!/usr/bin/env python - +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/tests/python-pytest/onnx/backend_test.py b/tests/python-pytest/onnx/backend_test.py old mode 100644 new mode 100755 index 8eaa303a6c1b..5e8e1983c729 --- a/tests/python-pytest/onnx/backend_test.py +++ b/tests/python-pytest/onnx/backend_test.py @@ -1,3 +1,6 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/tests/python/tensorrt/lenet5_train.py b/tests/python/tensorrt/lenet5_train.py old mode 100644 new mode 100755 index 9e3f0a785516..a0ea447de5a0 --- a/tests/python/tensorrt/lenet5_train.py +++ b/tests/python/tensorrt/lenet5_train.py @@ -1,3 +1,6 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/tools/caffe_converter/test_converter.py b/tools/caffe_converter/test_converter.py old mode 100644 new mode 100755 index a4f744556fed..49f8bdb167c2 --- a/tools/caffe_converter/test_converter.py +++ b/tools/caffe_converter/test_converter.py @@ -1,3 +1,6 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information From b7da01f3fb31a1f144d7ecb3540f8cdf236c6718 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Thu, 30 Jan 2020 18:39:26 +0000 Subject: [PATCH 14/21] Add python3 to ARM --- ci/docker/install/ubuntu_arm.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/docker/install/ubuntu_arm.sh b/ci/docker/install/ubuntu_arm.sh index 1e4f371fbe6f..42788cb656e1 100755 --- a/ci/docker/install/ubuntu_arm.sh +++ b/ci/docker/install/ubuntu_arm.sh @@ -21,4 +21,5 @@ set -ex apt update || true apt install -y \ - unzip + unzip \ + python3 From 612ae5d09085a74de1a83139e2c889f26b65122a Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Thu, 30 Jan 2020 20:35:11 +0000 Subject: [PATCH 15/21] Fix arm --- ci/docker/install/ubuntu_arm.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ci/docker/install/ubuntu_arm.sh b/ci/docker/install/ubuntu_arm.sh index 42788cb656e1..608d0362f138 100755 --- a/ci/docker/install/ubuntu_arm.sh +++ b/ci/docker/install/ubuntu_arm.sh @@ -22,4 +22,7 @@ set -ex apt update || true apt install -y \ unzip \ - python3 + python3 \ + python3-pip + +pip3 install setuptools From 5445344f7f4617bcedd6898be1b1d94e353013fc Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Thu, 30 Jan 2020 20:37:17 +0000 Subject: [PATCH 16/21] Fix ubuntu_gpu --- ci/docker/install/requirements | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/docker/install/requirements b/ci/docker/install/requirements index 6fd89bc65663..2d5125e8e2da 100644 --- a/ci/docker/install/requirements +++ b/ci/docker/install/requirements @@ -31,3 +31,4 @@ pylint==2.3.1; python_version >= '3.0' requests<2.19.0,>=2.18.4 scipy==1.2.1 six==1.11.0 +setuptools From 242b72960d02194ec57170576bc1549c8643e996 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Fri, 31 Jan 2020 02:51:52 +0000 Subject: [PATCH 17/21] remove universal --- ci/docker/runtime_functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 3127476b0539..11a0dc7204df 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -124,7 +124,7 @@ build_wheel() { export MXNET_LIBRARY_PATH=${BUILD_DIR}/libmxnet.so cd ${PYTHON_DIR} - python3 setup.py bdist_wheel --universal + python3 setup.py bdist_wheel # repackage From a80ab145829b21b99e60a1c98e0a3a9770e5b597 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Fri, 31 Jan 2020 09:59:23 +0000 Subject: [PATCH 18/21] use python3 for staticbuild wheel --- tools/staticbuild/build_wheel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/staticbuild/build_wheel.sh b/tools/staticbuild/build_wheel.sh index 9c1803b3dad7..0ae2dde5d676 100755 --- a/tools/staticbuild/build_wheel.sh +++ b/tools/staticbuild/build_wheel.sh @@ -22,7 +22,7 @@ echo $(git rev-parse HEAD) >> python/mxnet/COMMIT_HASH cd python/ # Make wheel for testing -python setup.py bdist_wheel +python3 setup.py bdist_wheel wheel_name=$(ls -t dist | head -n 1) pip install -U --user --force-reinstall dist/$wheel_name From 68959268dba978f081970f4555dcb8a8d85409ba Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Fri, 31 Jan 2020 10:03:51 +0000 Subject: [PATCH 19/21] remove cython2 from Makefile --- Makefile | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 16d7d2393736..4fc14d3e4808 100644 --- a/Makefile +++ b/Makefile @@ -455,7 +455,7 @@ ifeq ($(USE_DIST_KVSTORE), 1) endif .PHONY: clean all extra-packages test lint clean_all rcpplint rcppexport roxygen\ - cython2 cython3 cython cyclean + cython3 cython cyclean all: lib/libmxnet.a lib/libmxnet.so $(BIN) extra-packages sample_lib subgraph_lib @@ -674,9 +674,6 @@ subgraph_lib: cython: cd python; $(PYTHON) setup.py build_ext --inplace --with-cython -cython2: - cd python; python2 setup.py build_ext --inplace --with-cython - cython3: cd python; python3 setup.py build_ext --inplace --with-cython From d0f9317cc268c682425c71b912696c7573e54775 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Sat, 1 Feb 2020 01:08:39 +0000 Subject: [PATCH 20/21] disable cython2 build, don't substitute for EC2 mirror as a part as this PR --- CMakeLists.txt | 1 - ci/docker/install/ubuntu_core.sh | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d6b348aea99f..915817fbce0b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -913,7 +913,6 @@ add_custom_target(mxnet_lint COMMAND ${CMAKE_COMMAND} -DMSVC=${MSVC} -DPYTHON_EX if(BUILD_CYTHON_MODULES) include(cmake/BuildCythonModules.cmake) - add_cython_modules(2) # Build cython module for python2 if python2 is found add_cython_modules(3) # Build cython module for python3 if python3 is found if((NOT ${PYTHON2_FOUND}) AND (NOT ${PYTHON3_FOUND})) message(FATAL_ERROR "No python interpreter found to build cython modules") diff --git a/ci/docker/install/ubuntu_core.sh b/ci/docker/install/ubuntu_core.sh index 0b0e90276040..2773aa26246c 100755 --- a/ci/docker/install/ubuntu_core.sh +++ b/ci/docker/install/ubuntu_core.sh @@ -21,7 +21,8 @@ # the whole docker cache for the image set -ex -perl -pi -e 's/archive.ubuntu.com/us-west-2.ec2.archive.ubuntu.com/' /etc/apt/sources.list +# FIXME(larroy) enable in a different PR +#perl -pi -e 's/archive.ubuntu.com/us-west-2.ec2.archive.ubuntu.com/' /etc/apt/sources.list apt-get update || true # Avoid interactive package installers such as tzdata. From fd7d65200a83ee6cdd8178516271410f130511e2 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Sat, 1 Feb 2020 09:48:12 +0000 Subject: [PATCH 21/21] Set executable bit --- example/image-classification/__init__.py | 0 example/image-classification/benchmark.py | 0 example/image-classification/benchmark_score.py | 0 example/image-classification/fine-tune.py | 0 example/image-classification/score.py | 0 example/image-classification/test_score.py | 0 example/image-classification/train_cifar10.py | 0 example/image-classification/train_imagenet.py | 0 example/image-classification/train_mnist.py | 0 9 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 example/image-classification/__init__.py mode change 100644 => 100755 example/image-classification/benchmark.py mode change 100644 => 100755 example/image-classification/benchmark_score.py mode change 100644 => 100755 example/image-classification/fine-tune.py mode change 100644 => 100755 example/image-classification/score.py mode change 100644 => 100755 example/image-classification/test_score.py mode change 100644 => 100755 example/image-classification/train_cifar10.py mode change 100644 => 100755 example/image-classification/train_imagenet.py mode change 100644 => 100755 example/image-classification/train_mnist.py diff --git a/example/image-classification/__init__.py b/example/image-classification/__init__.py old mode 100644 new mode 100755 diff --git a/example/image-classification/benchmark.py b/example/image-classification/benchmark.py old mode 100644 new mode 100755 diff --git a/example/image-classification/benchmark_score.py b/example/image-classification/benchmark_score.py old mode 100644 new mode 100755 diff --git a/example/image-classification/fine-tune.py b/example/image-classification/fine-tune.py old mode 100644 new mode 100755 diff --git a/example/image-classification/score.py b/example/image-classification/score.py old mode 100644 new mode 100755 diff --git a/example/image-classification/test_score.py b/example/image-classification/test_score.py old mode 100644 new mode 100755 diff --git a/example/image-classification/train_cifar10.py b/example/image-classification/train_cifar10.py old mode 100644 new mode 100755 diff --git a/example/image-classification/train_imagenet.py b/example/image-classification/train_imagenet.py old mode 100644 new mode 100755 diff --git a/example/image-classification/train_mnist.py b/example/image-classification/train_mnist.py old mode 100644 new mode 100755