Windows scripted build

Adjust Jenkins builds to use ci/build_windows.py

Issues:

    apache#8714
    apache#11100
    apache#10166
    apache#10049

CMakeLists.txt

@@ -372,13 +372,13 @@ endif()
 
 # ---[ LAPack
 if(USE_LAPACK)
+ message("USE_LAPACK is ON")
  add_definitions(-DMXNET_USE_LAPACK=1)
  if (NOT MSVC)
  list(APPEND mxnet_LINKER_LIBS lapack)
  endif()
 endif()
 
-message("USE LAPACK ${USE_LAPACK}")
 
 # ---[ jemalloc
 if(USE_JEMALLOC)

Jenkinsfile

@@ -378,28 +378,8 @@ try {
  ws('workspace/build-cpu') {
  withEnv(['OpenBLAS_HOME=C:\\mxnet\\openblas', 'OpenCV_DIR=C:\\mxnet\\opencv_vc14', 'CUDA_PATH=C:\\CUDA\\v8.0']) {
  init_git_win()
- bat """mkdir build_vc14_cpu
- call "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\bin\\x86_amd64\\vcvarsx86_amd64.bat"
- cd build_vc14_cpu
- cmake -G \"Visual Studio 14 2015 Win64\" -DUSE_CUDA=0 -DUSE_CUDNN=0 -DUSE_NVRTC=0 -DUSE_OPENCV=1 -DUSE_OPENMP=1 -DUSE_PROFILER=1 -DUSE_BLAS=open -DUSE_LAPACK=1 -DUSE_DIST_KVSTORE=0 -DUSE_MKL_IF_AVAILABLE=0 ${env.WORKSPACE}"""
- bat 'C:\\mxnet\\build_vc14_cpu.bat'
-
- bat '''rmdir /s/q pkg_vc14_cpu
- mkdir pkg_vc14_cpu\\lib
- mkdir pkg_vc14_cpu\\python
- mkdir pkg_vc14_cpu\\include
- mkdir pkg_vc14_cpu\\build
- copy build_vc14_cpu\\Release\\libmxnet.lib pkg_vc14_cpu\\lib
- copy build_vc14_cpu\\Release\\libmxnet.dll pkg_vc14_cpu\\build
- xcopy python pkg_vc14_cpu\\python /E /I /Y
- xcopy include pkg_vc14_cpu\\include /E /I /Y
- xcopy 3rdparty\\dmlc-core\\include pkg_vc14_cpu\\include /E /I /Y
- xcopy 3rdparty\\mshadow\\mshadow pkg_vc14_cpu\\include\\mshadow /E /I /Y
- xcopy 3rdparty\\nnvm\\include pkg_vc14_cpu\\nnvm\\include /E /I /Y
- del /Q *.7z
- 7z.exe a vc14_cpu.7z pkg_vc14_cpu\\
- '''
- stash includes: 'vc14_cpu.7z', name: 'vc14_cpu'
+ powershell 'python ci/build_windows.py -f WIN_CPU'
+ stash includes: 'windows_package.7z', name: 'windows_package_cpu'
  }
  }
  }
@@ -411,28 +391,9 @@ try {
  timeout(time: max_time, unit: 'MINUTES') {
  ws('workspace/build-gpu') {
  withEnv(['OpenBLAS_HOME=C:\\mxnet\\openblas', 'OpenCV_DIR=C:\\mxnet\\opencv_vc14', 'CUDA_PATH=C:\\CUDA\\v8.0']) {
- init_git_win()
- bat """mkdir build_vc14_gpu
- call "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\bin\\x86_amd64\\vcvarsx86_amd64.bat"
- cd build_vc14_gpu
- cmake -G \"NMake Makefiles JOM\" -DUSE_CUDA=1 -DUSE_CUDNN=1 -DUSE_NVRTC=1 -DUSE_OPENCV=1 -DUSE_OPENMP=1 -DUSE_PROFILER=1 -DUSE_BLAS=open -DUSE_LAPACK=1 -DUSE_DIST_KVSTORE=0 -DCUDA_ARCH_NAME=Manual -DCUDA_ARCH_BIN=52 -DCUDA_ARCH_PTX=52 -DCMAKE_CXX_FLAGS_RELEASE="/FS /MD /O2 /Ob2 /DNDEBUG" -DCMAKE_BUILD_TYPE=Release -DUSE_MKL_IF_AVAILABLE=0 ${env.WORKSPACE}"""
- bat 'C:\\mxnet\\build_vc14_gpu.bat'
- bat '''rmdir /s/q pkg_vc14_gpu
- mkdir pkg_vc14_gpu\\lib
- mkdir pkg_vc14_gpu\\python
- mkdir pkg_vc14_gpu\\include
- mkdir pkg_vc14_gpu\\build
- copy build_vc14_gpu\\libmxnet.lib pkg_vc14_gpu\\lib
- copy build_vc14_gpu\\libmxnet.dll pkg_vc14_gpu\\build
- xcopy python pkg_vc14_gpu\\python /E /I /Y
- xcopy include pkg_vc14_gpu\\include /E /I /Y
- xcopy 3rdparty\\dmlc-core\\include pkg_vc14_gpu\\include /E /I /Y
- xcopy 3rdparty\\mshadow\\mshadow pkg_vc14_gpu\\include\\mshadow /E /I /Y
- xcopy 3rdparty\\nnvm\\include pkg_vc14_gpu\\nnvm\\include /E /I /Y
- del /Q *.7z
- 7z.exe a vc14_gpu.7z pkg_vc14_gpu\\
- '''
- stash includes: 'vc14_gpu.7z', name: 'vc14_gpu'
+ init_git_win()
+ powershell 'python ci/build_windows.py -f WIN_GPU'
+ stash includes: 'windows_package.7z', name: 'windows_package_gpu'
  }
  }
  }
@@ -443,37 +404,9 @@ try {
  timeout(time: max_time, unit: 'MINUTES') {
  ws('workspace/build-gpu') {
  withEnv(['OpenBLAS_HOME=C:\\mxnet\\openblas', 'OpenCV_DIR=C:\\mxnet\\opencv_vc14', 'CUDA_PATH=C:\\CUDA\\v8.0','BUILD_NAME=vc14_gpu_mkldnn']) {
- init_git_win()
- bat """mkdir build_%BUILD_NAME%
- call "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\bin\\x86_amd64\\vcvarsx86_amd64.bat"
- cd build_%BUILD_NAME%
- copy ${env.WORKSPACE}\\3rdparty\\mkldnn\\config_template.vcxproj.user ${env.WORKSPACE}\\config_template.vcxproj.user /y
- cmake -G \"NMake Makefiles JOM\" -DUSE_CUDA=1 -DUSE_CUDNN=1 -DUSE_NVRTC=1 -DUSE_OPENCV=1 -DUSE_OPENMP=1 -DUSE_PROFILER=1 -DUSE_BLAS=open -DUSE_LAPACK=1 -DUSE_DIST_KVSTORE=0 -DCUDA_ARCH_NAME=Manual -DCUDA_ARCH_BIN=52 -DCUDA_ARCH_PTX=52 -DUSE_MKLDNN=1 -DCMAKE_CXX_FLAGS_RELEASE="/FS /MD /O2 /Ob2 /DNDEBUG" -DCMAKE_BUILD_TYPE=Release ${env.WORKSPACE}"""
- bat '''
- call "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\bin\\x86_amd64\\vcvarsx86_amd64.bat"
- cd build_%BUILD_NAME%
- set /a cores=%NUMBER_OF_PROCESSORS% * 2
- jom -j %cores%
- '''
- bat '''rmdir /s/q pkg_%BUILD_NAME%
- mkdir pkg_%BUILD_NAME%\\lib
- mkdir pkg_%BUILD_NAME%\\python
- mkdir pkg_%BUILD_NAME%\\include
- mkdir pkg_%BUILD_NAME%\\build
- copy build_%BUILD_NAME%\\libmxnet.lib pkg_%BUILD_NAME%\\lib
- copy build_%BUILD_NAME%\\libmxnet.dll pkg_%BUILD_NAME%\\build
- copy build_%BUILD_NAME%\\3rdparty\\mkldnn\\src\\mkldnn.dll pkg_%BUILD_NAME%\\build
- copy build_%BUILD_NAME%\\libiomp5md.dll pkg_%BUILD_NAME%\\build
- copy build_%BUILD_NAME%\\mklml.dll pkg_%BUILD_NAME%\\build
- xcopy python pkg_%BUILD_NAME%\\python /E /I /Y
- xcopy include pkg_%BUILD_NAME%\\include /E /I /Y
- xcopy 3rdparty\\dmlc-core\\include pkg_%BUILD_NAME%\\include /E /I /Y
- xcopy 3rdparty\\mshadow\\mshadow pkg_%BUILD_NAME%\\include\\mshadow /E /I /Y
- xcopy 3rdparty\\nnvm\\include pkg_%BUILD_NAME%\\nnvm\\include /E /I /Y
- del /Q *.7z
- 7z.exe a %BUILD_NAME%.7z pkg_%BUILD_NAME%\\
- '''
- stash includes: 'vc14_gpu_mkldnn.7z', name: 'vc14_gpu_mkldnn'
+ init_git_win()
+ powershell 'python ci/build_windows.py -f WIN_GPU_MKLDNN'
+ stash includes: 'windows_package.7z', name: 'windows_package_gpu_mkldnn'
  }
  }
  }
@@ -843,16 +776,8 @@ try {
  ws('workspace/ut-python-cpu') {
  try {
  init_git_win()
- unstash 'vc14_cpu'
- bat '''rmdir /s/q pkg_vc14_cpu
- 7z x -y vc14_cpu.7z'''
- bat """xcopy C:\\mxnet\\data data /E /I /Y
- xcopy C:\\mxnet\\model model /E /I /Y
- call activate py2
- pip install mock
- set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_cpu\\python
- del /S /Q ${env.WORKSPACE}\\pkg_vc14_cpu\\python\\*.pyc
- C:\\mxnet\\test_cpu.bat"""
+ unstash 'windows_package_cpu'
+ powershell 'ci/windows/test_py2_cpu.ps1'
  } finally {
  collect_test_results_windows('nosetests_unittest.xml', 'nosetests_unittest_windows_python2_cpu.xml')
  }
@@ -866,15 +791,8 @@ try {
  ws('workspace/ut-python-cpu') {
  try {
  init_git_win()
- unstash 'vc14_cpu'
- bat '''rmdir /s/q pkg_vc14_cpu
- 7z x -y vc14_cpu.7z'''
- bat """xcopy C:\\mxnet\\data data /E /I /Y
- xcopy C:\\mxnet\\model model /E /I /Y
- call activate py3
- set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_cpu\\python
- del /S /Q ${env.WORKSPACE}\\pkg_vc14_cpu\\python\\*.pyc
- C:\\mxnet\\test_cpu.bat"""
+ unstash 'windows_package_cpu'
+ powershell 'ci/windows/test_py3_cpu.ps1'
  } finally {
  collect_test_results_windows('nosetests_unittest.xml', 'nosetests_unittest_windows_python3_cpu.xml')
  }
@@ -888,16 +806,7 @@ try {
  ws('workspace/ut-python-gpu') {
  try {
  init_git_win()
- unstash 'vc14_gpu'
- bat '''rmdir /s/q pkg_vc14_gpu
- 7z x -y vc14_gpu.7z'''
- bat """xcopy C:\\mxnet\\data data /E /I /Y
- xcopy C:\\mxnet\\model model /E /I /Y
- call activate py2
- pip install mock
- set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_gpu\\python
- del /S /Q ${env.WORKSPACE}\\pkg_vc14_gpu\\python\\*.pyc
- C:\\mxnet\\test_gpu.bat"""
+ powershell 'ci/windows/test_py2_gpu.ps1'
  } finally {
  collect_test_results_windows('nosetests_gpu_forward.xml', 'nosetests_gpu_forward_windows_python2_gpu.xml')
  collect_test_results_windows('nosetests_gpu_operator.xml', 'nosetests_gpu_operator_windows_python2_gpu.xml')
@@ -912,15 +821,8 @@ try {
  ws('workspace/ut-python-gpu') {
  try {
  init_git_win()
- unstash 'vc14_gpu'
- bat '''rmdir /s/q pkg_vc14_gpu
- 7z x -y vc14_gpu.7z'''
- bat """xcopy C:\\mxnet\\data data /E /I /Y
- xcopy C:\\mxnet\\model model /E /I /Y
- call activate py3
- set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_gpu\\python
- del /S /Q ${env.WORKSPACE}\\pkg_vc14_gpu\\python\\*.pyc
- C:\\mxnet\\test_gpu.bat"""
+ unstash 'windows_package_gpu'
+ powershell 'ci/windows/test_py3_gpu.ps1'
  } finally {
  collect_test_results_windows('nosetests_gpu_forward.xml', 'nosetests_gpu_forward_windows_python3_gpu.xml')
  collect_test_results_windows('nosetests_gpu_operator.xml', 'nosetests_gpu_operator_windows_python3_gpu.xml')
@@ -935,15 +837,8 @@ try {
  ws('workspace/ut-python-gpu') {
  try {
  init_git_win()
- unstash 'vc14_gpu_mkldnn'
- bat '''rmdir /s/q pkg_vc14_gpu_mkldnn
- 7z x -y vc14_gpu_mkldnn.7z'''
- bat """xcopy C:\\mxnet\\data data /E /I /Y
- xcopy C:\\mxnet\\model model /E /I /Y
- call activate py3
- set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_gpu_mkldnn\\python
- del /S /Q ${env.WORKSPACE}\\pkg_vc14_gpu_mkldnn\\python\\*.pyc
- C:\\mxnet\\test_gpu.bat"""
+ unstash 'windows_package_gpu_mkldnn'
+ powershell 'ci/windows/test_py3_gpu.ps1'
  } finally {
  collect_test_results_windows('nosetests_gpu_forward.xml', 'nosetests_gpu_forward_windows_python3_gpu_mkldnn.xml')
  collect_test_results_windows('nosetests_gpu_operator.xml', 'nosetests_gpu_operator_windows_python3_gpu_mkldnn.xml')

ci/build.py

@@ -39,6 +39,7 @@
 from itertools import chain
 from subprocess import call, check_call
 from typing import *
+from util import *
 
 CCACHE_MAXSIZE = '500G'
 
@@ -138,24 +139,9 @@ def _get_local_image_id(docker_binary, docker_tag):
  return image_id
 
 
-def get_mxnet_root() -> str:
- curpath = os.path.abspath(os.path.dirname(__file__))
-
- def is_mxnet_root(path: str) -> bool:
- return os.path.exists(os.path.join(path, ".mxnet_root"))
-
- while not is_mxnet_root(curpath):
- parent = os.path.abspath(os.path.join(curpath, os.pardir))
- if parent == curpath:
- raise RuntimeError("Got to the root and couldn't find a parent folder with .mxnet_root")
- curpath = parent
- return curpath
-
-
 def buildir() -> str:
  return os.path.join(get_mxnet_root(), "build")
 
-
 def default_ccache_dir() -> str:
  # Share ccache across containers
  if 'CCACHE_DIR' in os.environ: