From b3f442423601d9bc92f3efda0a3addf432201ea1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-No=C3=ABl=20Grad?= Date: Tue, 5 Apr 2022 02:42:12 +0200 Subject: [PATCH] CI: Use new CI infrastructure Improve CPU/GPU load balancing by restricting CI jobs that don't need a GPU on CI runners that don't provide a GPU. Use 4 cores instead of 2 cores when the runner allows it. Execute slow jobs on NUMA runners. --- .github/workflows/push_pull.yml | 4 +- .gitlab-ci.yml | 72 ++++++++++++++++++++++----------- maintainer/CI/build_cmake.sh | 8 +++- 3 files changed, 57 insertions(+), 27 deletions(-) diff --git a/.github/workflows/push_pull.yml b/.github/workflows/push_pull.yml index 3d8f0376700..26dfebcad9f 100644 --- a/.github/workflows/push_pull.yml +++ b/.github/workflows/push_pull.yml @@ -16,7 +16,7 @@ jobs: - name: Setup Python environment uses: actions/setup-python@v2 with: - python-version: '3.7' + python-version: '3.8' - name: Check without sanitizer uses: ./.github/actions/build_and_check with: @@ -33,7 +33,7 @@ jobs: - name: Setup Python environment uses: actions/setup-python@v2 with: - python-version: '3.7' + python-version: '3.8' - name: Check with sanitizer uses: ./.github/actions/build_and_check with: diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c8a54090ae1..2fec11330c3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -21,7 +21,9 @@ stages: timeout: 40m interruptible: false tags: - - linux + - docker + - espresso + - no-cuda variables: GIT_SUBMODULE_STRATEGY: recursive @@ -44,7 +46,8 @@ style: - sh maintainer/CI/fix_style.sh tags: - docker - - linux + - espresso + - no-cuda variables: GIT_SUBMODULE_STRATEGY: none artifacts: @@ -65,7 +68,8 @@ style_doxygen: - sh ../maintainer/CI/dox_warnings.sh tags: - docker - - linux + - espresso + - no-cuda ### Builds without CUDA @@ -85,7 +89,8 @@ default: - bash maintainer/CI/build_cmake.sh tags: - docker - - linux + - espresso + - no-cuda maxset: <<: *global_job_definition @@ -105,7 +110,9 @@ maxset: - bash maintainer/CI/build_cmake.sh tags: - docker - - linux + - espresso + - no-cuda + - numa no_rotation: <<: *global_job_definition @@ -122,7 +129,9 @@ no_rotation: - bash maintainer/CI/build_cmake.sh tags: - docker - - linux + - espresso + - no-cuda + - numa ubuntu:wo-dependencies: <<: *global_job_definition @@ -138,7 +147,8 @@ ubuntu:wo-dependencies: - bash maintainer/CI/build_cmake.sh tags: - docker - - linux + - espresso + - no-cuda ### Builds with different distributions @@ -155,7 +165,8 @@ debian:10: - bash maintainer/CI/build_cmake.sh tags: - docker - - linux + - espresso + - no-cuda fedora:34: <<: *global_job_definition @@ -170,7 +181,8 @@ fedora:34: - bash maintainer/CI/build_cmake.sh tags: - docker - - linux + - espresso + - no-cuda ### Builds with CUDA @@ -195,8 +207,9 @@ clang-sanitizer: timeout: 2h tags: - docker - - linux + - espresso - cuda + - numa fast_math: <<: *global_job_definition @@ -215,7 +228,7 @@ fast_math: - bash maintainer/CI/build_cmake.sh tags: - docker - - linux + - espresso - cuda when: manual @@ -238,8 +251,9 @@ cuda11-maxset: - bash maintainer/CI/build_cmake.sh tags: - docker - - linux + - espresso - cuda + - numa cuda10-maxset: <<: *global_job_definition @@ -263,8 +277,9 @@ cuda10-maxset: expire_in: 1 week tags: - docker - - linux + - espresso - cuda + - numa tutorials-samples-maxset: <<: *global_job_definition @@ -287,7 +302,7 @@ tutorials-samples-maxset: - bash maintainer/CI/build_cmake.sh tags: - docker - - linux + - espresso - cuda tutorials-samples-default: @@ -310,7 +325,7 @@ tutorials-samples-default: - bash maintainer/CI/build_cmake.sh tags: - docker - - linux + - espresso - cuda only: - schedules @@ -336,7 +351,7 @@ tutorials-samples-empty: - bash maintainer/CI/build_cmake.sh tags: - docker - - linux + - espresso - cuda only: - schedules @@ -362,7 +377,8 @@ tutorials-samples-no-gpu: - bash maintainer/CI/build_cmake.sh tags: - docker - - linux + - espresso + - no-cuda only: - schedules @@ -397,7 +413,7 @@ installation: - make -j2 check_tutorials tags: - docker - - linux + - espresso - cuda when: manual @@ -417,8 +433,9 @@ empty: - bash maintainer/CI/build_cmake.sh tags: - docker - - linux + - espresso - cuda + - numa check_sphinx: <<: *global_job_definition @@ -440,8 +457,9 @@ check_sphinx: expire_in: 1 week tags: - docker - - linux + - espresso - cuda + - numa run_tutorials: <<: *global_job_definition @@ -464,8 +482,9 @@ run_tutorials: expire_in: 1 week tags: - docker - - linux + - espresso - cuda + - numa only: - schedules @@ -486,7 +505,9 @@ run_doxygen: expire_in: 1 week tags: - docker - - linux + - espresso + - no-cuda + - numa maxset_no_gpu: <<: *global_job_definition @@ -500,7 +521,9 @@ maxset_no_gpu: - make -t && make check tags: - docker - - linux + - espresso + - no-cuda + - numa maxset_3_cores: <<: *global_job_definition @@ -514,8 +537,9 @@ maxset_3_cores: - make -t && make check_unit_tests && make check_python_parallel_odd tags: - docker - - linux + - espresso - cuda + - numa status_success: <<: *notification_job_definition diff --git a/maintainer/CI/build_cmake.sh b/maintainer/CI/build_cmake.sh index 91b44886c75..29a553b8f54 100755 --- a/maintainer/CI/build_cmake.sh +++ b/maintainer/CI/build_cmake.sh @@ -72,6 +72,12 @@ set_default_value() { fi } +# the number of available processors depends on the CI runner +if grep -q "i7-3820" /proc/cpuinfo; then + ci_procs=2 +else + ci_procs=4 +fi # handle environment variables set_default_value srcdir "$(pwd)" @@ -82,7 +88,7 @@ set_default_value with_ubsan false set_default_value with_asan false set_default_value with_static_analysis false set_default_value myconfig "default" -set_default_value build_procs 2 +set_default_value build_procs ${ci_procs} set_default_value check_procs ${build_procs} set_default_value check_odd_only false set_default_value check_gpu_only false