Skip to content
This repository has been archived by the owner on Nov 17, 2023. It is now read-only.

[CI] Upgrade unix gpu toolchain #18186

Merged
merged 8 commits into from
May 12, 2020
Merged

[CI] Upgrade unix gpu toolchain #18186

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
fe1fef4
update nvidiadocker command & remove cuda compat
ChaiBapchya Apr 27, 2020
f761b24
replace cu101 with cuda since compat is no longer to be used
ChaiBapchya Apr 28, 2020
1c720fa
skip flaky tests
ChaiBapchya Apr 28, 2020
ec5330d
get rid of ubuntu_build_cuda and point ubuntu_cu101 to base gpu inste…
ChaiBapchya Apr 28, 2020
36f5563
Revert "skip flaky tests"
ChaiBapchya May 7, 2020
b2428b9
Merge branch 'master' into update_unix_gpu_toolchain
ChaiBapchya May 10, 2020
386c42b
revert removal of ubuntu_build_cuda
ChaiBapchya May 10, 2020
2b05566
add linux gpu g4 node to all steps using g3 in unix-gpu pipeline
ChaiBapchya May 10, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions ci/Jenkinsfile_utils.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -257,6 +257,7 @@ def assign_node_labels(args) {
// knowing about the limitations.
NODE_LINUX_CPU = args.linux_cpu
NODE_LINUX_GPU = args.linux_gpu
NODE_LINUX_GPU_G4 = args.linux_gpu_g4
NODE_LINUX_GPU_P3 = args.linux_gpu_p3
NODE_WINDOWS_CPU = args.windows_cpu
NODE_WINDOWS_GPU = args.windows_gpu
Expand Down
3 changes: 2 additions & 1 deletion ci/build.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -225,8 +225,9 @@ def container_run(docker_client: SafeDockerClient,

# Equivalent command
docker_cmd_list = [
"nvidia-docker" if nvidia_runtime else "docker",
"docker",
'run',
"--gpus all" if nvidia_runtime else "",
"--cap-add",
"SYS_PTRACE", # Required by ASAN
'--rm',
Expand Down
6 changes: 0 additions & 6 deletions ci/docker/Dockerfile.build.ubuntu
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -164,9 +164,3 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
apt install -y --no-install-recommends \
cuda-10-1 && \
rm -rf /var/lib/apt/lists/*


FROM gpu as gpuwithcompatenv
# TVMOP requires /usr/local/cuda/compat is no LD_LIBRARY_PATH.
# This should be fixed and deleted.
ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/cuda/compat
2 changes: 1 addition & 1 deletion ci/docker/docker-compose.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ services:
build:
context: .
dockerfile: Dockerfile.build.ubuntu
target: gpuwithcompatenv
target: gpu
args:
BASE_IMAGE: nvidia/cuda:10.1-cudnn7-devel-ubuntu18.04
cache_from:
Expand Down
24 changes: 12 additions & 12 deletions ci/jenkins/Jenkins_steps.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,7 @@ def compile_unix_int64_cpu(lib_name) {

def compile_unix_int64_gpu(lib_name) {
return ['GPU: USE_INT64_TENSOR_SIZE': {
node(NODE_LINUX_GPU) {
node(NODE_LINUX_GPU_G4) {
ws('workspace/build-gpu-int64') {
timeout(time: max_time, unit: 'MINUTES') {
utils.init_git()
Expand Down Expand Up @@ -816,7 +816,7 @@ def test_unix_python3_mkl_cpu(lib_name) {

def test_unix_python3_gpu(lib_name) {
return ['Python3: GPU': {
node(NODE_LINUX_GPU) {
node(NODE_LINUX_GPU_G4) {
ws('workspace/ut-python3-gpu') {
try {
utils.unpack_and_init(lib_name, mx_lib_cython)
Expand Down Expand Up @@ -916,7 +916,7 @@ def test_unix_python3_mkldnn_mkl_cpu(lib_name) {

def test_unix_python3_mkldnn_gpu(lib_name) {
return ['Python3: MKLDNN-GPU': {
node(NODE_LINUX_GPU) {
node(NODE_LINUX_GPU_G4) {
ws('workspace/ut-python3-mkldnn-gpu') {
try {
utils.unpack_and_init(lib_name, mx_mkldnn_lib)
Expand All @@ -932,7 +932,7 @@ def test_unix_python3_mkldnn_gpu(lib_name) {

def test_unix_python3_mkldnn_nocudnn_gpu(lib_name) {
return ['Python3: MKLDNN-GPU-NOCUDNN': {
node(NODE_LINUX_GPU) {
node(NODE_LINUX_GPU_G4) {
ws('workspace/ut-python3-mkldnn-gpu-nocudnn') {
try {
utils.unpack_and_init(lib_name, mx_mkldnn_lib)
Expand Down Expand Up @@ -966,7 +966,7 @@ def test_unix_python3_tensorrt_gpu(lib_name) {

def test_unix_python3_integration_gpu(lib_name) {
return ['Python Integration GPU': {
node(NODE_LINUX_GPU) {
node(NODE_LINUX_GPU_G4) {
ws('workspace/it-python-gpu') {
timeout(time: max_time, unit: 'MINUTES') {
utils.unpack_and_init(lib_name, mx_lib)
Expand All @@ -980,7 +980,7 @@ def test_unix_python3_integration_gpu(lib_name) {

def test_unix_cpp_package_gpu(lib_name) {
return ['cpp-package GPU Makefile': {
node(NODE_LINUX_GPU) {
node(NODE_LINUX_GPU_G4) {
ws('workspace/it-cpp-package') {
timeout(time: max_time, unit: 'MINUTES') {
utils.unpack_and_init(lib_name, mx_lib_cpp_examples_make)
Expand All @@ -994,7 +994,7 @@ def test_unix_cpp_package_gpu(lib_name) {

def test_unix_capi_cpp_package(lib_name) {
return ['capi-cpp-package GPU Makefile': {
node(NODE_LINUX_GPU) {
node(NODE_LINUX_GPU_G4) {
ws('workspace/it-capi-cpp-package') {
timeout(time: max_time, unit: 'MINUTES') {
utils.unpack_and_init(lib_name, mx_lib_cpp_capi_make)
Expand Down Expand Up @@ -1036,7 +1036,7 @@ def test_unix_scala_mkldnn_cpu(lib_name){

def test_unix_scala_gpu(lib_name) {
return ['Scala: GPU Makefile': {
node(NODE_LINUX_GPU) {
node(NODE_LINUX_GPU_G4) {
ws('workspace/ut-scala-gpu') {
timeout(time: max_time, unit: 'MINUTES') {
utils.unpack_and_init(lib_name, mx_lib_make)
Expand Down Expand Up @@ -1119,7 +1119,7 @@ def test_unix_perl_cpu(lib_name) {

def test_unix_cpp_gpu(lib_name) {
return ['Cpp: GPU': {
node(NODE_LINUX_GPU) {
node(NODE_LINUX_GPU_G4) {
ws('workspace/ut-cpp-gpu') {
timeout(time: max_time, unit: 'MINUTES') {
utils.unpack_and_init(lib_name, mx_cmake_lib)
Expand Down Expand Up @@ -1147,7 +1147,7 @@ def test_unix_cpp_cpu(lib_name) {

def test_unix_perl_gpu(lib_name) {
return ['Perl: GPU Makefile': {
node(NODE_LINUX_GPU) {
node(NODE_LINUX_GPU_G4) {
ws('workspace/ut-perl-gpu') {
timeout(time: max_time, unit: 'MINUTES') {
utils.unpack_and_init(lib_name, mx_lib_make)
Expand All @@ -1161,7 +1161,7 @@ def test_unix_perl_gpu(lib_name) {

def test_unix_r_gpu(lib_name) {
return ['R: GPU': {
node(NODE_LINUX_GPU) {
node(NODE_LINUX_GPU_G4) {
ws('workspace/ut-r-gpu') {
timeout(time: max_time, unit: 'MINUTES') {
utils.unpack_and_init(lib_name, mx_lib)
Expand Down Expand Up @@ -1229,7 +1229,7 @@ def test_unix_distributed_kvstore_cpu(lib_name) {

def test_unix_distributed_kvstore_gpu(lib_name) {
return ['dist-kvstore tests GPU': {
node(NODE_LINUX_GPU) {
node(NODE_LINUX_GPU_G4) {
ws('workspace/it-dist-kvstore') {
timeout(time: max_time, unit: 'MINUTES') {
utils.unpack_and_init(lib_name, mx_lib)
Expand Down
2 changes: 1 addition & 1 deletion ci/jenkins/Jenkinsfile_unix_gpu
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ node('utility') {
utils = load('ci/Jenkinsfile_utils.groovy')
custom_steps = load('ci/jenkins/Jenkins_steps.groovy')
}
utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu', linux_gpu: 'mxnetlinux-gpu', linux_gpu_p3: 'mxnetlinux-gpu-p3')
utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu', linux_gpu: 'mxnetlinux-gpu', linux_gpu_p3: 'mxnetlinux-gpu-p3', linux_gpu_g4: 'mxnetlinux-gpu-g4')

utils.main_wrapper(
core_logic: {
Expand Down