Skip to content
This repository has been archived by the owner on Nov 17, 2023. It is now read-only.

Use x64 toolchain for CI windows build #17912

Closed
wants to merge 1 commit into from
Closed

Use x64 toolchain for CI windows build #17912

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 33 additions & 9 deletions ci/build_windows.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,10 +54,14 @@ class BuildFlavour(Enum):

CMAKE_FLAGS = {
'WIN_CPU': (
'-DCMAKE_C_COMPILER=cl '
'-DCMAKE_CXX_COMPILER=cl '
'-DOpenCV_RUNTIME=vc15 '
'-DOpenCV_ARCH=x64 '
'-DUSE_CUDA=OFF '
'-DUSE_CUDNN=OFF '
'-DENABLE_CUDA_RTC=OFF '
'-DUSE_OPENCV=ON '
'-DUSE_OPENCV=OFF '
'-DUSE_OPENMP=ON '
'-DUSE_BLAS=open '
'-DUSE_LAPACK=ON '
Expand All @@ -67,10 +71,14 @@ class BuildFlavour(Enum):
'-DCMAKE_BUILD_TYPE=Release')

, 'WIN_CPU_MKLDNN': (
'-DCMAKE_C_COMPILER=cl '
'-DCMAKE_CXX_COMPILER=cl '
'-DOpenCV_RUNTIME=vc15 '
'-DOpenCV_ARCH=x64 '
'-DUSE_CUDA=OFF '
'-DUSE_CUDNN=OFF '
'-DENABLE_CUDA_RTC=OFF '
'-DUSE_OPENCV=ON '
'-DUSE_OPENCV=OFF '
'-DUSE_OPENMP=ON '
'-DUSE_BLAS=open '
'-DUSE_LAPACK=ON '
Expand All @@ -80,10 +88,14 @@ class BuildFlavour(Enum):
'-DCMAKE_BUILD_TYPE=Release')

, 'WIN_CPU_MKLDNN_MKL': (
'-DCMAKE_C_COMPILER=cl '
'-DCMAKE_CXX_COMPILER=cl '
'-DOpenCV_RUNTIME=vc15 '
'-DOpenCV_ARCH=x64 '
'-DUSE_CUDA=OFF '
'-DUSE_CUDNN=OFF '
'-DENABLE_CUDA_RTC=OFF '
'-DUSE_OPENCV=ON '
'-DUSE_OPENCV=OFF '
'-DUSE_OPENMP=ON '
'-DUSE_BLAS=mkl '
'-DUSE_LAPACK=ON '
Expand All @@ -93,10 +105,14 @@ class BuildFlavour(Enum):
'-DCMAKE_BUILD_TYPE=Release')

, 'WIN_CPU_MKL': (
'-DCMAKE_C_COMPILER=cl '
'-DCMAKE_CXX_COMPILER=cl '
'-DOpenCV_RUNTIME=vc15 '
'-DOpenCV_ARCH=x64 '
'-DUSE_CUDA=OFF '
'-DUSE_CUDNN=OFF '
'-DENABLE_CUDA_RTC=OFF '
'-DUSE_OPENCV=ON '
'-DUSE_OPENCV=OFF '
'-DUSE_OPENMP=ON '
'-DUSE_BLAS=mkl '
'-DUSE_LAPACK=ON '
Expand All @@ -106,10 +122,14 @@ class BuildFlavour(Enum):
'-DCMAKE_BUILD_TYPE=Release')

, 'WIN_GPU': (
'-DCMAKE_C_COMPILER=cl '
'-DCMAKE_CXX_COMPILER=cl '
'-DOpenCV_RUNTIME=vc15 '
'-DOpenCV_ARCH=x64 '
'-DUSE_CUDA=ON '
'-DUSE_CUDNN=ON '
'-DENABLE_CUDA_RTC=ON '
'-DUSE_OPENCV=ON '
'-DUSE_OPENCV=OFF '
'-DUSE_OPENMP=ON '
'-DUSE_BLAS=open '
'-DUSE_LAPACK=ON '
Expand All @@ -120,10 +140,14 @@ class BuildFlavour(Enum):
'-DCMAKE_BUILD_TYPE=Release')

, 'WIN_GPU_MKLDNN': (
'-DCMAKE_C_COMPILER=cl '
'-DCMAKE_CXX_COMPILER=cl '
'-DOpenCV_RUNTIME=vc15 '
'-DOpenCV_ARCH=x64 '
'-DUSE_CUDA=ON '
'-DUSE_CUDNN=ON '
'-DENABLE_CUDA_RTC=ON '
'-DUSE_OPENCV=ON '
'-DUSE_OPENCV=OFF '
'-DUSE_OPENMP=ON '
'-DUSE_BLAS=open '
'-DUSE_LAPACK=ON '
Expand Down Expand Up @@ -154,15 +178,15 @@ def windows_build(args):

with remember_cwd():
os.chdir(path)
cmd = "\"{}\" && {} -G \"NMake Makefiles JOM\" {} {}".format(
cmd = "\"{}\" && {} -T host=x64 {} {}".format(
args.vcvars,
os.path.join(tmpdir, 'cmake-3.16.1-win64-x64', 'bin', 'cmake.exe'),
CMAKE_FLAGS[args.flavour], mxnet_root)
logging.info("Generating project with CMake:\n{}".format(cmd))
check_call(cmd, shell=True)

cmd = "\"{}\" && jom".format(args.vcvars)
logging.info("Building with jom:\n{}".format(cmd))
cmd = "\"{}\" && cmake --build".format(args.vcvars)
logging.info("Building:\n{}".format(cmd))

t0 = int(time.time())
check_call(cmd, shell=True)
Expand Down
8 changes: 0 additions & 8 deletions ci/jenkins/Jenkinsfile_centos_cpu
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,14 +34,6 @@ utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu')
utils.main_wrapper(
core_logic: {
utils.parallel_stage('Build', [
custom_steps.compile_centos7_cpu(),
custom_steps.compile_centos7_cpu_make(),
custom_steps.compile_centos7_cpu_mkldnn()
])

utils.parallel_stage('Tests', [
custom_steps.test_centos7_python3_cpu(),
custom_steps.test_centos7_scala_cpu()
])
}
,
Expand Down
7 changes: 1 addition & 6 deletions ci/jenkins/Jenkinsfile_centos_gpu
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,12 +34,7 @@ utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu', linux_
utils.main_wrapper(
core_logic: {
utils.parallel_stage('Build', [
custom_steps.compile_centos7_gpu()
])

utils.parallel_stage('Tests', [
custom_steps.test_centos7_python3_gpu()
])
])
}
,
failure_handler: {
Expand Down
37 changes: 0 additions & 37 deletions ci/jenkins/Jenkinsfile_unix_cpu
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,43 +34,6 @@ utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu')
utils.main_wrapper(
core_logic: {
utils.parallel_stage('Build', [
custom_steps.compile_unix_cpu_openblas(),
custom_steps.compile_unix_cpu_openblas_make(),
custom_steps.compile_unix_openblas_debug_cpu(),
custom_steps.compile_unix_mkl_cpu(),
custom_steps.compile_unix_mkldnn_cpu(),
custom_steps.compile_unix_mkldnn_cpu_make(),
custom_steps.compile_unix_mkldnn_mkl_cpu(),
custom_steps.compile_unix_int64_cpu(),
custom_steps.compile_unix_openblas_cpu_no_tvm_op(),
])

utils.parallel_stage('Tests', [
custom_steps.test_unix_python3_cpu(),
custom_steps.test_unix_python3_debug_cpu(),
custom_steps.test_unix_python3_mkl_cpu(),
custom_steps.test_unix_python3_mkldnn_cpu(),
custom_steps.test_unix_python3_mkldnn_mkl_cpu(),
custom_steps.test_unix_scala_cpu(),
custom_steps.test_unix_scala_mkldnn_cpu(),
custom_steps.test_unix_clojure_cpu(),
custom_steps.test_unix_clojure_integration_cpu(),
custom_steps.test_unix_perl_cpu(),
custom_steps.test_unix_r_cpu(),
custom_steps.test_unix_r_mkldnn_cpu(),
custom_steps.test_unix_julia07_cpu(),
custom_steps.test_unix_julia10_cpu(),
custom_steps.test_unix_onnx_cpu(),
custom_steps.test_unix_cpp_cpu(),
custom_steps.test_static_scala_cpu(),
custom_steps.test_static_python_cpu(),
custom_steps.test_static_python_cpu_cmake(),
/* Disabled due to master build failure:
* http://jenkins.mxnet-ci.amazon-ml.com/blue/organizations/jenkins/incubator-mxnet/detail/master/1221/pipeline/
* https://github.com/apache/incubator-mxnet/issues/11801
custom_steps.test_unix_distributed_kvstore_cpu()
*/
custom_steps.test_unix_python3_cpu_no_tvm_op(),
])
}
,
Expand Down
33 changes: 0 additions & 33 deletions ci/jenkins/Jenkinsfile_unix_gpu
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,40 +34,7 @@ utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu', linux_
utils.main_wrapper(
core_logic: {
utils.parallel_stage('Build', [
custom_steps.compile_unix_mkldnn_gpu(),
custom_steps.compile_unix_mkldnn_nocudnn_gpu(),
custom_steps.compile_unix_full_gpu(),
custom_steps.compile_unix_full_gpu_make(),
custom_steps.compile_unix_cmake_gpu(),
custom_steps.compile_unix_tensorrt_gpu(),
custom_steps.compile_unix_int64_gpu(),
custom_steps.compile_unix_full_gpu_no_tvm_op(),
custom_steps.compile_unix_cmake_gpu_no_tvm_op(),
custom_steps.compile_unix_cmake_gpu_no_rtc(),
custom_steps.compile_unix_full_gpu_mkldnn_cpp_test()
])

utils.parallel_stage('Tests', [
custom_steps.test_unix_python3_gpu(),
custom_steps.test_unix_python3_quantize_gpu(),
custom_steps.test_unix_python3_mkldnn_gpu(),
custom_steps.test_unix_python3_mkldnn_nocudnn_gpu(),
custom_steps.test_unix_python3_tensorrt_gpu(),
custom_steps.test_unix_perl_gpu(),
custom_steps.test_unix_r_gpu(),
custom_steps.test_unix_cpp_gpu(),
custom_steps.test_unix_python3_integration_gpu(),
custom_steps.test_unix_cpp_package_gpu(),
custom_steps.test_unix_scala_gpu(),
custom_steps.test_unix_distributed_kvstore_gpu(),
custom_steps.test_static_python_gpu(),
custom_steps.test_static_python_gpu_cmake(),
custom_steps.test_unix_python3_gpu_no_tvm_op(),
custom_steps.test_unix_capi_cpp_package(),

// Disabled due to: https://github.com/apache/incubator-mxnet/issues/11407
//custom_steps.test_unix_caffe_gpu()
])
}
,
failure_handler: {
Expand Down