From a4f07114e46646914fb84d0858b077a6f65cdc4c Mon Sep 17 00:00:00 2001 From: aurianer Date: Mon, 15 Jan 2024 17:29:14 +0100 Subject: [PATCH] Add separate dockerfile for dependencies to avoid rebuild compiler --- .gitlab/docker/Dockerfile.spack_compiler | 7 ----- .gitlab/docker/Dockerfile.spack_dependencies | 9 ++++++ .gitlab/includes/clang11_pipeline.yml | 27 ++++++++++++++--- .gitlab/includes/clang12_pipeline.yml | 28 +++++++++++++++--- .gitlab/includes/clang13_pipeline.yml | 27 ++++++++++++++--- .gitlab/includes/clang14_cuda11_pipeline.yml | 27 ++++++++++++++--- .gitlab/includes/clang15_pipeline.yml | 27 ++++++++++++++--- .gitlab/includes/clang16_pipeline.yml | 27 ++++++++++++++--- .gitlab/includes/common_pipeline.yml | 3 +- .gitlab/includes/gcc10_apex_pipeline.yml | 27 ++++++++++++++--- .gitlab/includes/gcc11_pipeline.yml | 27 ++++++++++++++--- .gitlab/includes/gcc12_cuda12_pipeline.yml | 27 ++++++++++++++--- .gitlab/includes/gcc12_pipeline.yml | 27 ++++++++++++++--- .gitlab/includes/gcc13_pipeline.yml | 27 ++++++++++++++--- .gitlab/includes/gcc9_cuda11_pipeline.yml | 27 ++++++++++++++--- .gitlab/includes/gcc9_pipeline.yml | 27 ++++++++++++++--- .gitlab/includes/hip_pipeline.yml | 29 +++++++++++++++---- .../includes/performance_gcc13_pipeline.yml | 27 ++++++++++++++--- 18 files changed, 358 insertions(+), 69 deletions(-) create mode 100644 .gitlab/docker/Dockerfile.spack_dependencies diff --git a/.gitlab/docker/Dockerfile.spack_compiler b/.gitlab/docker/Dockerfile.spack_compiler index 15df896c4..5fa71a360 100644 --- a/.gitlab/docker/Dockerfile.spack_compiler +++ b/.gitlab/docker/Dockerfile.spack_compiler @@ -24,10 +24,3 @@ RUN spack compiler info $COMPILER > /dev/null 2> /dev/null; compiler_missing=$?; fi RUN echo "spack arch: $(spack arch)" - -# Install dependencies for this configuration -ARG SPACK_SPEC -ENV spack_spec $SPACK_SPEC -RUN spack spec -lI $spack_spec && \ - spack install --fail-fast --only dependencies $spack_spec && \ - spack clean --all diff --git a/.gitlab/docker/Dockerfile.spack_dependencies b/.gitlab/docker/Dockerfile.spack_dependencies new file mode 100644 index 000000000..b51e6d585 --- /dev/null +++ b/.gitlab/docker/Dockerfile.spack_dependencies @@ -0,0 +1,9 @@ +ARG BASE_IMAGE +FROM $BASE_IMAGE + +ARG SPACK_SPEC +ENV spack_spec $SPACK_SPEC +# Install dependencies for this configuration +RUN spack spec -lI $spack_spec && \ + spack install --fail-fast --only dependencies $spack_spec && \ + spack clean --all diff --git a/.gitlab/includes/clang11_pipeline.yml b/.gitlab/includes/clang11_pipeline.yml index c1577792a..91815a960 100644 --- a/.gitlab/includes/clang11_pipeline.yml +++ b/.gitlab/includes/clang11_pipeline.yml @@ -20,26 +20,45 @@ include: -DPIKA_WITH_MAX_CPU_COUNT=256 -DPIKA_WITH_MALLOC=system \ -DPIKA_WITH_SPINLOCK_DEADLOCK_DETECTION=ON" -clang11_spack_image: - stage: spack_configs +clang11_spack_compiler_image: + stage: spack_compiler needs: [base_spack_image] extends: - .container-builder - .variables_clang11_config before_script: - export DOCKERFILE_SHA=`sha256sum $DOCKERFILE | head -c 16` - - CONFIG_TAG=`echo $DOCKERFILE_SHA-$ARCH-$BASE_IMAGE-$COMPILER-$SPACK_SPEC | sha256sum - | head -c 16` + - CONFIG_TAG=`echo $DOCKERFILE_SHA-$ARCH-$BASE_IMAGE-$COMPILER | sha256sum - | head -c 16` - compiler=${COMPILER/@/-} - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/base/pika-$compiler:$CONFIG_TAG - echo -e "compiler=$compiler" >> compiler.env - echo -e "BASE_IMAGE=$PERSIST_IMAGE_NAME" >> compiler.env variables: DOCKERFILE: .gitlab/docker/Dockerfile.spack_compiler - DOCKER_BUILD_ARGS: '["BASE_IMAGE","ARCH","COMPILER","SPACK_SPEC"]' + DOCKER_BUILD_ARGS: '["BASE_IMAGE","ARCH","COMPILER"]' artifacts: reports: dotenv: compiler.env +clang11_spack_image: + stage: spack_dependencies + needs: [clang11_spack_compiler_image] + extends: + - .container-builder + - .variables_clang11_config + before_script: + - export DOCKERFILE_SHA=`sha256sum $DOCKERFILE | head -c 16` + - CONFIG_TAG=`echo $DOCKERFILE_SHA-$BASE_IMAGE-$SPACK_SPEC | sha256sum - | head -c 16` + - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/base/pika-$compiler-dependencies:$CONFIG_TAG + - echo -e "compiler=$compiler" >> dependencies.env + - echo -e "BASE_IMAGE=$PERSIST_IMAGE_NAME" >> dependencies.env + variables: + DOCKERFILE: .gitlab/docker/Dockerfile.spack_dependencies + DOCKER_BUILD_ARGS: '["BASE_IMAGE","SPACK_SPEC"]' + artifacts: + reports: + dotenv: dependencies.env + clang11_debug_build: stage: build extends: diff --git a/.gitlab/includes/clang12_pipeline.yml b/.gitlab/includes/clang12_pipeline.yml index c81e52b95..f66e6b49b 100644 --- a/.gitlab/includes/clang12_pipeline.yml +++ b/.gitlab/includes/clang12_pipeline.yml @@ -24,26 +24,46 @@ include: -DPIKA_WITH_THREAD_CUMULATIVE_COUNTS=ON -DPIKA_WITH_THREAD_QUEUE_WAITTIME=ON \ -DPIKA_WITH_THREAD_STEALING_COUNTS=ON" -clang12_spack_image: - stage: spack_configs +clang12_spack_compiler_image: + stage: spack_compiler needs: [base_spack_image] extends: - .container-builder - .variables_clang12_config before_script: - export DOCKERFILE_SHA=`sha256sum $DOCKERFILE | head -c 16` - - CONFIG_TAG=`echo $DOCKERFILE_SHA-$ARCH-$BASE_IMAGE-$COMPILER-$SPACK_SPEC | sha256sum - | head -c 16` + - CONFIG_TAG=`echo $DOCKERFILE_SHA-$ARCH-$BASE_IMAGE-$COMPILER | sha256sum - | head -c 16` - compiler=${COMPILER/@/-} - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/base/pika-$compiler:$CONFIG_TAG - echo -e "compiler=$compiler" >> compiler.env - echo -e "BASE_IMAGE=$PERSIST_IMAGE_NAME" >> compiler.env variables: DOCKERFILE: .gitlab/docker/Dockerfile.spack_compiler - DOCKER_BUILD_ARGS: '["BASE_IMAGE","ARCH","COMPILER","SPACK_SPEC"]' + DOCKER_BUILD_ARGS: '["BASE_IMAGE","ARCH","COMPILER"]' artifacts: reports: dotenv: compiler.env +clang12_spack_image: + stage: spack_dependencies + needs: [clang12_spack_compiler_image] + extends: + - .container-builder + - .variables_clang12_config + before_script: + - export DOCKERFILE_SHA=`sha256sum $DOCKERFILE | head -c 16` + - CONFIG_TAG=`echo $DOCKERFILE_SHA-$BASE_IMAGE-$SPACK_SPEC | sha256sum - | head -c 16` + - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/base/pika-$compiler-dependencies:$CONFIG_TAG + - echo -e "compiler=$compiler" >> dependencies.env + - echo -e "BASE_IMAGE=$PERSIST_IMAGE_NAME" >> dependencies.env + variables: + DOCKERFILE: .gitlab/docker/Dockerfile.spack_dependencies + DOCKER_BUILD_ARGS: '["BASE_IMAGE","SPACK_SPEC"]' + artifacts: + reports: + dotenv: dependencies.env + + clang12_debug_build: stage: build extends: diff --git a/.gitlab/includes/clang13_pipeline.yml b/.gitlab/includes/clang13_pipeline.yml index fd3f0b192..f882afa5b 100644 --- a/.gitlab/includes/clang13_pipeline.yml +++ b/.gitlab/includes/clang13_pipeline.yml @@ -21,26 +21,45 @@ include: -DCMAKE_CXX_FLAGS=-stdlib=libc++ -DPIKA_WITH_MALLOC=system \ -DPIKA_WITH_SPINLOCK_DEADLOCK_DETECTION=ON -DPIKA_WITH_STDEXEC=ON" -clang13_spack_image: - stage: spack_configs +clang13_spack_compiler_image: + stage: spack_compiler needs: [base_spack_image] extends: - .container-builder - .variables_clang13_config before_script: - export DOCKERFILE_SHA=`sha256sum $DOCKERFILE | head -c 16` - - CONFIG_TAG=`echo $DOCKERFILE_SHA-$ARCH-$BASE_IMAGE-$COMPILER-$SPACK_SPEC | sha256sum - | head -c 16` + - CONFIG_TAG=`echo $DOCKERFILE_SHA-$ARCH-$BASE_IMAGE-$COMPILER | sha256sum - | head -c 16` - compiler=${COMPILER/@/-} - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/base/pika-$compiler:$CONFIG_TAG - echo -e "compiler=$compiler" >> compiler.env - echo -e "BASE_IMAGE=$PERSIST_IMAGE_NAME" >> compiler.env variables: DOCKERFILE: .gitlab/docker/Dockerfile.spack_compiler - DOCKER_BUILD_ARGS: '["BASE_IMAGE","ARCH","COMPILER","SPACK_SPEC"]' + DOCKER_BUILD_ARGS: '["BASE_IMAGE","ARCH","COMPILER"]' artifacts: reports: dotenv: compiler.env +clang13_spack_image: + stage: spack_dependencies + needs: [clang13_spack_compiler_image] + extends: + - .container-builder + - .variables_clang13_config + before_script: + - export DOCKERFILE_SHA=`sha256sum $DOCKERFILE | head -c 16` + - CONFIG_TAG=`echo $DOCKERFILE_SHA-$BASE_IMAGE-$SPACK_SPEC | sha256sum - | head -c 16` + - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/base/pika-$compiler-dependencies:$CONFIG_TAG + - echo -e "compiler=$compiler" >> dependencies.env + - echo -e "BASE_IMAGE=$PERSIST_IMAGE_NAME" >> dependencies.env + variables: + DOCKERFILE: .gitlab/docker/Dockerfile.spack_dependencies + DOCKER_BUILD_ARGS: '["BASE_IMAGE","SPACK_SPEC"]' + artifacts: + reports: + dotenv: dependencies.env + clang13_debug_build: stage: build extends: diff --git a/.gitlab/includes/clang14_cuda11_pipeline.yml b/.gitlab/includes/clang14_cuda11_pipeline.yml index 8de003c35..10ada2eca 100644 --- a/.gitlab/includes/clang14_cuda11_pipeline.yml +++ b/.gitlab/includes/clang14_cuda11_pipeline.yml @@ -22,8 +22,8 @@ include: -DPIKA_WITH_SPINLOCK_DEADLOCK_DETECTION=ON -DCMAKE_CUDA_COMPILER=c++ \ -DCMAKE_CUDA_ARCHITECTURES=80 -DPIKA_WITH_CXX_LAMBDA_CAPTURE_DECLTYPE=OFF" -clang14_cuda11_spack_image: - stage: spack_configs +clang14_cuda11_spack_compiler_image: + stage: spack_compiler timeout: 2 hours needs: [base_spack_image] extends: @@ -31,18 +31,37 @@ clang14_cuda11_spack_image: - .variables_clang14_cuda11_config before_script: - export DOCKERFILE_SHA=`sha256sum $DOCKERFILE | head -c 16` - - CONFIG_TAG=`echo $DOCKERFILE_SHA-$ARCH-$BASE_IMAGE-$COMPILER-$SPACK_SPEC | sha256sum - | head -c 16` + - CONFIG_TAG=`echo $DOCKERFILE_SHA-$ARCH-$BASE_IMAGE-$COMPILER | sha256sum - | head -c 16` - compiler=${COMPILER/@/-} - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/base/pika-$compiler:$CONFIG_TAG - echo -e "compiler=$compiler" >> compiler.env - echo -e "BASE_IMAGE=$PERSIST_IMAGE_NAME" >> compiler.env variables: DOCKERFILE: .gitlab/docker/Dockerfile.spack_compiler - DOCKER_BUILD_ARGS: '["BASE_IMAGE","ARCH","COMPILER","SPACK_SPEC"]' + DOCKER_BUILD_ARGS: '["BASE_IMAGE","ARCH","COMPILER"]' artifacts: reports: dotenv: compiler.env +clang14_cuda11_spack_image: + stage: spack_dependencies + needs: [clang14_cuda11_spack_compiler_image] + extends: + - .container-builder + - .variables_clang14_cuda11_config + before_script: + - export DOCKERFILE_SHA=`sha256sum $DOCKERFILE | head -c 16` + - CONFIG_TAG=`echo $DOCKERFILE_SHA-$BASE_IMAGE-$SPACK_SPEC | sha256sum - | head -c 16` + - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/base/pika-$compiler-dependencies:$CONFIG_TAG + - echo -e "compiler=$compiler" >> dependencies.env + - echo -e "BASE_IMAGE=$PERSIST_IMAGE_NAME" >> dependencies.env + variables: + DOCKERFILE: .gitlab/docker/Dockerfile.spack_dependencies + DOCKER_BUILD_ARGS: '["BASE_IMAGE","SPACK_SPEC"]' + artifacts: + reports: + dotenv: dependencies.env + clang14_cuda11_debug_build: stage: build extends: diff --git a/.gitlab/includes/clang15_pipeline.yml b/.gitlab/includes/clang15_pipeline.yml index 07f729017..01ff9639d 100644 --- a/.gitlab/includes/clang15_pipeline.yml +++ b/.gitlab/includes/clang15_pipeline.yml @@ -20,26 +20,45 @@ include: -DPIKA_WITH_MALLOC=system -DPIKA_WITH_SPINLOCK_DEADLOCK_DETECTION=ON \ -DPIKA_WITH_UNITY_BUILD=OFF" -clang15_spack_image: - stage: spack_configs +clang15_spack_compiler_image: + stage: spack_compiler needs: [base_spack_image] extends: - .container-builder - .variables_clang15_config before_script: - export DOCKERFILE_SHA=`sha256sum $DOCKERFILE | head -c 16` - - CONFIG_TAG=`echo $DOCKERFILE_SHA-$ARCH-$BASE_IMAGE-$COMPILER-$SPACK_SPEC | sha256sum - | head -c 16` + - CONFIG_TAG=`echo $DOCKERFILE_SHA-$ARCH-$BASE_IMAGE-$COMPILER | sha256sum - | head -c 16` - compiler=${COMPILER/@/-} - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/base/pika-$compiler:$CONFIG_TAG - echo -e "compiler=$compiler" >> compiler.env - echo -e "BASE_IMAGE=$PERSIST_IMAGE_NAME" >> compiler.env variables: DOCKERFILE: .gitlab/docker/Dockerfile.spack_compiler - DOCKER_BUILD_ARGS: '["BASE_IMAGE","ARCH","COMPILER","SPACK_SPEC"]' + DOCKER_BUILD_ARGS: '["BASE_IMAGE","ARCH","COMPILER"]' artifacts: reports: dotenv: compiler.env +clang15_spack_image: + stage: spack_dependencies + needs: [clang15_spack_compiler_image] + extends: + - .container-builder + - .variables_clang15_config + before_script: + - export DOCKERFILE_SHA=`sha256sum $DOCKERFILE | head -c 16` + - CONFIG_TAG=`echo $DOCKERFILE_SHA-$BASE_IMAGE-$SPACK_SPEC | sha256sum - | head -c 16` + - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/base/pika-$compiler-dependencies:$CONFIG_TAG + - echo -e "compiler=$compiler" >> dependencies.env + - echo -e "BASE_IMAGE=$PERSIST_IMAGE_NAME" >> dependencies.env + variables: + DOCKERFILE: .gitlab/docker/Dockerfile.spack_dependencies + DOCKER_BUILD_ARGS: '["BASE_IMAGE","SPACK_SPEC"]' + artifacts: + reports: + dotenv: dependencies.env + clang15_debug_build: stage: build extends: diff --git a/.gitlab/includes/clang16_pipeline.yml b/.gitlab/includes/clang16_pipeline.yml index a00b62be5..d76baf74b 100644 --- a/.gitlab/includes/clang16_pipeline.yml +++ b/.gitlab/includes/clang16_pipeline.yml @@ -19,26 +19,45 @@ include: CMAKE_FLAGS: "-DCMAKE_BUILD_TYPE=$BUILD_TYPE -DPIKA_WITH_CXX_STANDARD=$CXXSTD \ -DPIKA_WITH_MALLOC=system -DPIKA_WITH_SPINLOCK_DEADLOCK_DETECTION=ON" -clang16_spack_image: - stage: spack_configs +clang16_spack_compiler_image: + stage: spack_compiler needs: [base_spack_image] extends: - .container-builder - .variables_clang16_config before_script: - export DOCKERFILE_SHA=`sha256sum $DOCKERFILE | head -c 16` - - CONFIG_TAG=`echo $DOCKERFILE_SHA-$ARCH-$BASE_IMAGE-$COMPILER-$SPACK_SPEC | sha256sum - | head -c 16` + - CONFIG_TAG=`echo $DOCKERFILE_SHA-$ARCH-$BASE_IMAGE-$COMPILER | sha256sum - | head -c 16` - compiler=${COMPILER/@/-} - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/base/pika-$compiler:$CONFIG_TAG - echo -e "compiler=$compiler" >> compiler.env - echo -e "BASE_IMAGE=$PERSIST_IMAGE_NAME" >> compiler.env variables: DOCKERFILE: .gitlab/docker/Dockerfile.spack_compiler - DOCKER_BUILD_ARGS: '["BASE_IMAGE","ARCH","COMPILER","SPACK_SPEC"]' + DOCKER_BUILD_ARGS: '["BASE_IMAGE","ARCH","COMPILER"]' artifacts: reports: dotenv: compiler.env +clang16_spack_image: + stage: spack_dependencies + needs: [clang16_spack_compiler_image] + extends: + - .container-builder + - .variables_clang16_config + before_script: + - export DOCKERFILE_SHA=`sha256sum $DOCKERFILE | head -c 16` + - CONFIG_TAG=`echo $DOCKERFILE_SHA-$BASE_IMAGE-$SPACK_SPEC | sha256sum - | head -c 16` + - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/base/pika-$compiler-dependencies:$CONFIG_TAG + - echo -e "compiler=$compiler" >> dependencies.env + - echo -e "BASE_IMAGE=$PERSIST_IMAGE_NAME" >> dependencies.env + variables: + DOCKERFILE: .gitlab/docker/Dockerfile.spack_dependencies + DOCKER_BUILD_ARGS: '["BASE_IMAGE","SPACK_SPEC"]' + artifacts: + reports: + dotenv: dependencies.env + clang16_debug_build: stage: build extends: diff --git a/.gitlab/includes/common_pipeline.yml b/.gitlab/includes/common_pipeline.yml index b1f3c2676..e02c67bce 100644 --- a/.gitlab/includes/common_pipeline.yml +++ b/.gitlab/includes/common_pipeline.yml @@ -9,7 +9,8 @@ include: stages: - spack_base - - spack_configs + - spack_compiler + - spack_dependencies - build - test diff --git a/.gitlab/includes/gcc10_apex_pipeline.yml b/.gitlab/includes/gcc10_apex_pipeline.yml index 08a8c5009..e086a7000 100644 --- a/.gitlab/includes/gcc10_apex_pipeline.yml +++ b/.gitlab/includes/gcc10_apex_pipeline.yml @@ -20,26 +20,45 @@ include: -DPIKA_WITH_APEX=ON -DPIKA_WITH_MALLOC=system \ -DPIKA_WITH_SPINLOCK_DEADLOCK_DETECTION=ON" -gcc10_apex_spack_image: - stage: spack_configs +gcc10_apex_spack_compiler_image: + stage: spack_compiler needs: [base_spack_image] extends: - .container-builder - .variables_gcc10_apex_config before_script: - export DOCKERFILE_SHA=`sha256sum $DOCKERFILE | head -c 16` - - CONFIG_TAG=`echo $DOCKERFILE_SHA-$ARCH-$BASE_IMAGE-$COMPILER-$SPACK_SPEC | sha256sum - | head -c 16` + - CONFIG_TAG=`echo $DOCKERFILE_SHA-$ARCH-$BASE_IMAGE-$COMPILER | sha256sum - | head -c 16` - compiler=${COMPILER/@/-} - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/base/pika-$compiler:$CONFIG_TAG - echo -e "compiler=$compiler" >> compiler.env - echo -e "BASE_IMAGE=$PERSIST_IMAGE_NAME" >> compiler.env variables: DOCKERFILE: .gitlab/docker/Dockerfile.spack_compiler - DOCKER_BUILD_ARGS: '["BASE_IMAGE","ARCH","COMPILER","SPACK_SPEC"]' + DOCKER_BUILD_ARGS: '["BASE_IMAGE","ARCH","COMPILER"]' artifacts: reports: dotenv: compiler.env +gcc10_apex_spack_image: + stage: spack_dependencies + needs: [gcc10_apex_spack_compiler_image] + extends: + - .container-builder + - .variables_gcc10_apex_config + before_script: + - export DOCKERFILE_SHA=`sha256sum $DOCKERFILE | head -c 16` + - CONFIG_TAG=`echo $DOCKERFILE_SHA-$BASE_IMAGE-$SPACK_SPEC | sha256sum - | head -c 16` + - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/base/pika-$compiler-dependencies:$CONFIG_TAG + - echo -e "compiler=$compiler" >> dependencies.env + - echo -e "BASE_IMAGE=$PERSIST_IMAGE_NAME" >> dependencies.env + variables: + DOCKERFILE: .gitlab/docker/Dockerfile.spack_dependencies + DOCKER_BUILD_ARGS: '["BASE_IMAGE","SPACK_SPEC"]' + artifacts: + reports: + dotenv: dependencies.env + gcc10_apex_release_build: stage: build extends: diff --git a/.gitlab/includes/gcc11_pipeline.yml b/.gitlab/includes/gcc11_pipeline.yml index a5060d801..e27552bb5 100644 --- a/.gitlab/includes/gcc11_pipeline.yml +++ b/.gitlab/includes/gcc11_pipeline.yml @@ -20,26 +20,45 @@ include: -DPIKA_WITH_MAX_CPU_COUNT=256 -DPIKA_WITH_MALLOC=system \ -DPIKA_WITH_SPINLOCK_DEADLOCK_DETECTION=ON" -gcc11_spack_image: - stage: spack_configs +gcc11_spack_compiler_image: + stage: spack_compiler needs: [base_spack_image] extends: - .container-builder - .variables_gcc11_config before_script: - export DOCKERFILE_SHA=`sha256sum $DOCKERFILE | head -c 16` - - CONFIG_TAG=`echo $DOCKERFILE_SHA-$ARCH-$BASE_IMAGE-$COMPILER-$SPACK_SPEC | sha256sum - | head -c 16` + - CONFIG_TAG=`echo $DOCKERFILE_SHA-$ARCH-$BASE_IMAGE-$COMPILER | sha256sum - | head -c 16` - compiler=${COMPILER/@/-} - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/base/pika-$compiler:$CONFIG_TAG - echo -e "compiler=$compiler" >> compiler.env - echo -e "BASE_IMAGE=$PERSIST_IMAGE_NAME" >> compiler.env variables: DOCKERFILE: .gitlab/docker/Dockerfile.spack_compiler - DOCKER_BUILD_ARGS: '["BASE_IMAGE","ARCH","COMPILER","SPACK_SPEC"]' + DOCKER_BUILD_ARGS: '["BASE_IMAGE","ARCH","COMPILER"]' artifacts: reports: dotenv: compiler.env +gcc11_spack_image: + stage: spack_dependencies + needs: [gcc11_spack_compiler_image] + extends: + - .container-builder + - .variables_gcc11_config + before_script: + - export DOCKERFILE_SHA=`sha256sum $DOCKERFILE | head -c 16` + - CONFIG_TAG=`echo $DOCKERFILE_SHA-$BASE_IMAGE-$SPACK_SPEC | sha256sum - | head -c 16` + - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/base/pika-$compiler-dependencies:$CONFIG_TAG + - echo -e "compiler=$compiler" >> dependencies.env + - echo -e "BASE_IMAGE=$PERSIST_IMAGE_NAME" >> dependencies.env + variables: + DOCKERFILE: .gitlab/docker/Dockerfile.spack_dependencies + DOCKER_BUILD_ARGS: '["BASE_IMAGE","SPACK_SPEC"]' + artifacts: + reports: + dotenv: dependencies.env + gcc11_debug_build: stage: build extends: diff --git a/.gitlab/includes/gcc12_cuda12_pipeline.yml b/.gitlab/includes/gcc12_cuda12_pipeline.yml index 23f7b62d4..ffa3a082b 100644 --- a/.gitlab/includes/gcc12_cuda12_pipeline.yml +++ b/.gitlab/includes/gcc12_cuda12_pipeline.yml @@ -23,26 +23,45 @@ include: -DCMAKE_CUDA_ARCHITECTURES=$GPU_TARGET \ -DPIKA_WITH_CXX_LAMBDA_CAPTURE_DECLTYPE=OFF" -gcc12_cuda12_spack_image: - stage: spack_configs +gcc12_cuda12_spack_compiler_image: + stage: spack_compiler needs: [base_spack_image] extends: - .container-builder - .variables_gcc12_cuda12_config before_script: - export DOCKERFILE_SHA=`sha256sum $DOCKERFILE | head -c 16` - - CONFIG_TAG=`echo $DOCKERFILE_SHA-$ARCH-$BASE_IMAGE-$COMPILER-$SPACK_SPEC | sha256sum - | head -c 16` + - CONFIG_TAG=`echo $DOCKERFILE_SHA-$ARCH-$BASE_IMAGE-$COMPILER | sha256sum - | head -c 16` - compiler=${COMPILER/@/-} - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/base/pika-$compiler:$CONFIG_TAG - echo -e "compiler=$compiler" >> compiler.env - echo -e "BASE_IMAGE=$PERSIST_IMAGE_NAME" >> compiler.env variables: DOCKERFILE: .gitlab/docker/Dockerfile.spack_compiler - DOCKER_BUILD_ARGS: '["BASE_IMAGE","ARCH","COMPILER","SPACK_SPEC"]' + DOCKER_BUILD_ARGS: '["BASE_IMAGE","ARCH","COMPILER"]' artifacts: reports: dotenv: compiler.env +gcc12_cuda12_spack_image: + stage: spack_dependencies + needs: [gcc12_cuda12_spack_compiler_image] + extends: + - .container-builder + - .variables_gcc12_cuda12_config + before_script: + - export DOCKERFILE_SHA=`sha256sum $DOCKERFILE | head -c 16` + - CONFIG_TAG=`echo $DOCKERFILE_SHA-$BASE_IMAGE-$SPACK_SPEC | sha256sum - | head -c 16` + - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/base/pika-$compiler-dependencies:$CONFIG_TAG + - echo -e "compiler=$compiler" >> dependencies.env + - echo -e "BASE_IMAGE=$PERSIST_IMAGE_NAME" >> dependencies.env + variables: + DOCKERFILE: .gitlab/docker/Dockerfile.spack_dependencies + DOCKER_BUILD_ARGS: '["BASE_IMAGE","SPACK_SPEC"]' + artifacts: + reports: + dotenv: dependencies.env + gcc12_cuda12_debug_build: stage: build extends: diff --git a/.gitlab/includes/gcc12_pipeline.yml b/.gitlab/includes/gcc12_pipeline.yml index 22b8d335d..486e16007 100644 --- a/.gitlab/includes/gcc12_pipeline.yml +++ b/.gitlab/includes/gcc12_pipeline.yml @@ -19,26 +19,45 @@ include: CMAKE_FLAGS: "-DCMAKE_BUILD_TYPE=$BUILD_TYPE -DPIKA_WITH_CXX_STANDARD=$CXXSTD \ -DPIKA_WITH_MALLOC=system -DPIKA_WITH_SPINLOCK_DEADLOCK_DETECTION=ON" -gcc12_spack_image: - stage: spack_configs +gcc12_spack_compiler_image: + stage: spack_compiler needs: [base_spack_image] extends: - .container-builder - .variables_gcc12_config before_script: - export DOCKERFILE_SHA=`sha256sum $DOCKERFILE | head -c 16` - - CONFIG_TAG=`echo $DOCKERFILE_SHA-$ARCH-$BASE_IMAGE-$COMPILER-$SPACK_SPEC | sha256sum - | head -c 16` + - CONFIG_TAG=`echo $DOCKERFILE_SHA-$ARCH-$BASE_IMAGE-$COMPILER | sha256sum - | head -c 16` - compiler=${COMPILER/@/-} - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/base/pika-$compiler:$CONFIG_TAG - echo -e "compiler=$compiler" >> compiler.env - echo -e "BASE_IMAGE=$PERSIST_IMAGE_NAME" >> compiler.env variables: DOCKERFILE: .gitlab/docker/Dockerfile.spack_compiler - DOCKER_BUILD_ARGS: '["BASE_IMAGE","ARCH","COMPILER","SPACK_SPEC"]' + DOCKER_BUILD_ARGS: '["BASE_IMAGE","ARCH","COMPILER"]' artifacts: reports: dotenv: compiler.env +gcc12_spack_image: + stage: spack_dependencies + needs: [gcc12_spack_compiler_image] + extends: + - .container-builder + - .variables_gcc12_config + before_script: + - export DOCKERFILE_SHA=`sha256sum $DOCKERFILE | head -c 16` + - CONFIG_TAG=`echo $DOCKERFILE_SHA-$BASE_IMAGE-$SPACK_SPEC | sha256sum - | head -c 16` + - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/base/pika-$compiler-dependencies:$CONFIG_TAG + - echo -e "compiler=$compiler" >> dependencies.env + - echo -e "BASE_IMAGE=$PERSIST_IMAGE_NAME" >> dependencies.env + variables: + DOCKERFILE: .gitlab/docker/Dockerfile.spack_dependencies + DOCKER_BUILD_ARGS: '["BASE_IMAGE","SPACK_SPEC"]' + artifacts: + reports: + dotenv: dependencies.env + gcc12_debug_build: stage: build extends: diff --git a/.gitlab/includes/gcc13_pipeline.yml b/.gitlab/includes/gcc13_pipeline.yml index f85d2a458..b925a543c 100644 --- a/.gitlab/includes/gcc13_pipeline.yml +++ b/.gitlab/includes/gcc13_pipeline.yml @@ -21,26 +21,45 @@ include: -DPIKA_WITH_MALLOC=system -DPIKA_WITH_STDEXEC=ON \ -DPIKA_WITH_SPINLOCK_DEADLOCK_DETECTION=ON" -gcc13_spack_image: - stage: spack_configs +gcc13_spack_compiler_image: + stage: spack_compiler needs: [base_spack_image] extends: - .container-builder - .variables_gcc13_config before_script: - export DOCKERFILE_SHA=`sha256sum $DOCKERFILE | head -c 16` - - CONFIG_TAG=`echo $DOCKERFILE_SHA-$ARCH-$BASE_IMAGE-$COMPILER-$SPACK_SPEC | sha256sum - | head -c 16` + - CONFIG_TAG=`echo $DOCKERFILE_SHA-$ARCH-$BASE_IMAGE-$COMPILER | sha256sum - | head -c 16` - compiler=${COMPILER/@/-} - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/base/pika-$compiler:$CONFIG_TAG - echo -e "compiler=$compiler" >> compiler.env - echo -e "BASE_IMAGE=$PERSIST_IMAGE_NAME" >> compiler.env variables: DOCKERFILE: .gitlab/docker/Dockerfile.spack_compiler - DOCKER_BUILD_ARGS: '["BASE_IMAGE","ARCH","COMPILER","SPACK_SPEC"]' + DOCKER_BUILD_ARGS: '["BASE_IMAGE","ARCH","COMPILER"]' artifacts: reports: dotenv: compiler.env +gcc13_spack_image: + stage: spack_dependencies + needs: [gcc13_spack_compiler_image] + extends: + - .container-builder + - .variables_gcc13_config + before_script: + - export DOCKERFILE_SHA=`sha256sum $DOCKERFILE | head -c 16` + - CONFIG_TAG=`echo $DOCKERFILE_SHA-$BASE_IMAGE-$SPACK_SPEC | sha256sum - | head -c 16` + - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/base/pika-$compiler-dependencies:$CONFIG_TAG + - echo -e "compiler=$compiler" >> dependencies.env + - echo -e "BASE_IMAGE=$PERSIST_IMAGE_NAME" >> dependencies.env + variables: + DOCKERFILE: .gitlab/docker/Dockerfile.spack_dependencies + DOCKER_BUILD_ARGS: '["BASE_IMAGE","SPACK_SPEC"]' + artifacts: + reports: + dotenv: dependencies.env + gcc13_debug_build: stage: build extends: diff --git a/.gitlab/includes/gcc9_cuda11_pipeline.yml b/.gitlab/includes/gcc9_cuda11_pipeline.yml index 7f1eb3190..3fccb192d 100644 --- a/.gitlab/includes/gcc9_cuda11_pipeline.yml +++ b/.gitlab/includes/gcc9_cuda11_pipeline.yml @@ -20,26 +20,45 @@ include: -DPIKA_WITH_CUDA=ON -DPIKA_WITH_MALLOC=system \ -DPIKA_WITH_EXAMPLES_OPENMP=ON" -gcc9_cuda11_spack_image: - stage: spack_configs +gcc9_cuda11_spack_compiler_image: + stage: spack_compiler needs: [base_spack_image] extends: - .container-builder - .variables_gcc9_cuda11_config before_script: - export DOCKERFILE_SHA=`sha256sum $DOCKERFILE | head -c 16` - - CONFIG_TAG=`echo $DOCKERFILE_SHA-$ARCH-$BASE_IMAGE-$COMPILER-$SPACK_SPEC | sha256sum - | head -c 16` + - CONFIG_TAG=`echo $DOCKERFILE_SHA-$ARCH-$BASE_IMAGE-$COMPILER | sha256sum - | head -c 16` - compiler=${COMPILER/@/-} - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/base/pika-$compiler:$CONFIG_TAG - echo -e "compiler=$compiler" >> compiler.env - echo -e "BASE_IMAGE=$PERSIST_IMAGE_NAME" >> compiler.env variables: DOCKERFILE: .gitlab/docker/Dockerfile.spack_compiler - DOCKER_BUILD_ARGS: '["BASE_IMAGE","ARCH","COMPILER","SPACK_SPEC"]' + DOCKER_BUILD_ARGS: '["BASE_IMAGE","ARCH","COMPILER"]' artifacts: reports: dotenv: compiler.env +gcc9_cuda11_spack_image: + stage: spack_dependencies + needs: [gcc9_cuda11_spack_compiler_image] + extends: + - .container-builder + - .variables_gcc9_cuda11_config + before_script: + - export DOCKERFILE_SHA=`sha256sum $DOCKERFILE | head -c 16` + - CONFIG_TAG=`echo $DOCKERFILE_SHA-$BASE_IMAGE-$SPACK_SPEC | sha256sum - | head -c 16` + - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/base/pika-$compiler-dependencies:$CONFIG_TAG + - echo -e "compiler=$compiler" >> dependencies.env + - echo -e "BASE_IMAGE=$PERSIST_IMAGE_NAME" >> dependencies.env + variables: + DOCKERFILE: .gitlab/docker/Dockerfile.spack_dependencies + DOCKER_BUILD_ARGS: '["BASE_IMAGE","SPACK_SPEC"]' + artifacts: + reports: + dotenv: dependencies.env + gcc9_cuda11_debug_build: stage: build extends: diff --git a/.gitlab/includes/gcc9_pipeline.yml b/.gitlab/includes/gcc9_pipeline.yml index acff69ca9..613616865 100644 --- a/.gitlab/includes/gcc9_pipeline.yml +++ b/.gitlab/includes/gcc9_pipeline.yml @@ -20,26 +20,45 @@ include: -DPIKA_WITH_MAX_CPU_COUNT=256 -DPIKA_WITH_MALLOC=system \ -DPIKA_WITH_BOOST_CONTEXT=ON -DPIKA_WITH_SPINLOCK_DEADLOCK_DETECTION=ON" -gcc9_spack_image: - stage: spack_configs +gcc9_spack_compiler_image: + stage: spack_compiler needs: [base_spack_image] extends: - .container-builder - .variables_gcc9_config before_script: - export DOCKERFILE_SHA=`sha256sum $DOCKERFILE | head -c 16` - - CONFIG_TAG=`echo $DOCKERFILE_SHA-$ARCH-$BASE_IMAGE-$COMPILER-$SPACK_SPEC | sha256sum - | head -c 16` + - CONFIG_TAG=`echo $DOCKERFILE_SHA-$ARCH-$BASE_IMAGE-$COMPILER | sha256sum - | head -c 16` - compiler=${COMPILER/@/-} - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/base/pika-$compiler:$CONFIG_TAG - echo -e "compiler=$compiler" >> compiler.env - echo -e "BASE_IMAGE=$PERSIST_IMAGE_NAME" >> compiler.env variables: DOCKERFILE: .gitlab/docker/Dockerfile.spack_compiler - DOCKER_BUILD_ARGS: '["BASE_IMAGE","ARCH","COMPILER","SPACK_SPEC"]' + DOCKER_BUILD_ARGS: '["BASE_IMAGE","ARCH","COMPILER"]' artifacts: reports: dotenv: compiler.env +gcc9_spack_image: + stage: spack_dependencies + needs: [gcc9_spack_compiler_image] + extends: + - .container-builder + - .variables_gcc9_config + before_script: + - export DOCKERFILE_SHA=`sha256sum $DOCKERFILE | head -c 16` + - CONFIG_TAG=`echo $DOCKERFILE_SHA-$BASE_IMAGE-$SPACK_SPEC | sha256sum - | head -c 16` + - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/base/pika-$compiler-dependencies:$CONFIG_TAG + - echo -e "compiler=$compiler" >> dependencies.env + - echo -e "BASE_IMAGE=$PERSIST_IMAGE_NAME" >> dependencies.env + variables: + DOCKERFILE: .gitlab/docker/Dockerfile.spack_dependencies + DOCKER_BUILD_ARGS: '["BASE_IMAGE","SPACK_SPEC"]' + artifacts: + reports: + dotenv: dependencies.env + gcc9_debug_build: stage: build extends: diff --git a/.gitlab/includes/hip_pipeline.yml b/.gitlab/includes/hip_pipeline.yml index 8a9595e86..8e960d4b5 100644 --- a/.gitlab/includes/hip_pipeline.yml +++ b/.gitlab/includes/hip_pipeline.yml @@ -22,27 +22,46 @@ include: -DPIKA_WITH_HIP=ON -DPIKA_WITH_MALLOC=system \ -DCMAKE_HIP_ARCHITECTURES=$GPU_TARGET -DPIKA_WITH_STDEXEC=ON" -gcc12_hip5_spack_image: - stage: spack_configs - timeout: 4 hours +gcc12_hip5_spack_compiler_image: + stage: spack_compiler needs: [base_spack_image] extends: - .container-builder - .variables_gcc12_hip5_config before_script: - export DOCKERFILE_SHA=`sha256sum $DOCKERFILE | head -c 16` - - CONFIG_TAG=`echo $DOCKERFILE_SHA-$ARCH-$BASE_IMAGE-$COMPILER-$SPACK_SPEC | sha256sum - | head -c 16` + - CONFIG_TAG=`echo $DOCKERFILE_SHA-$ARCH-$BASE_IMAGE-$COMPILER | sha256sum - | head -c 16` - compiler=${COMPILER/@/-} - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/base/pika-$compiler:$CONFIG_TAG - echo -e "compiler=$compiler" >> compiler.env - echo -e "BASE_IMAGE=$PERSIST_IMAGE_NAME" >> compiler.env variables: DOCKERFILE: .gitlab/docker/Dockerfile.spack_compiler - DOCKER_BUILD_ARGS: '["BASE_IMAGE","ARCH","COMPILER","SPACK_SPEC"]' + DOCKER_BUILD_ARGS: '["BASE_IMAGE","ARCH","COMPILER"]' artifacts: reports: dotenv: compiler.env +gcc12_hip5_spack_image: + stage: spack_dependencies + timeout: 4 hours + needs: [gcc12_hip5_spack_compiler_image] + extends: + - .container-builder + - .variables_gcc12_hip5_config + before_script: + - export DOCKERFILE_SHA=`sha256sum $DOCKERFILE | head -c 16` + - CONFIG_TAG=`echo $DOCKERFILE_SHA-$BASE_IMAGE-$SPACK_SPEC | sha256sum - | head -c 16` + - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/base/pika-$compiler-dependencies:$CONFIG_TAG + - echo -e "compiler=$compiler" >> dependencies.env + - echo -e "BASE_IMAGE=$PERSIST_IMAGE_NAME" >> dependencies.env + variables: + DOCKERFILE: .gitlab/docker/Dockerfile.spack_dependencies + DOCKER_BUILD_ARGS: '["BASE_IMAGE","SPACK_SPEC"]' + artifacts: + reports: + dotenv: dependencies.env + gcc12_hip5_debug_build: stage: build extends: diff --git a/.gitlab/includes/performance_gcc13_pipeline.yml b/.gitlab/includes/performance_gcc13_pipeline.yml index 78e713fc2..8602d796e 100644 --- a/.gitlab/includes/performance_gcc13_pipeline.yml +++ b/.gitlab/includes/performance_gcc13_pipeline.yml @@ -18,26 +18,45 @@ include: ^boost@1.83.0 ^hwloc@2.9.1" CMAKE_FLAGS: "-DCMAKE_BUILD_TYPE=$BUILD_TYPE -DPIKA_WITH_CXX_STANDARD=$CXXSTD" -performance_gcc13_spack_image: - stage: spack_configs +performance_gcc13_spack_compiler_image: + stage: spack_compiler needs: [base_spack_image] extends: - .container-builder - .variables_performance_gcc13_config before_script: - export DOCKERFILE_SHA=`sha256sum $DOCKERFILE | head -c 16` - - CONFIG_TAG=`echo $DOCKERFILE_SHA-$ARCH-$BASE_IMAGE-$COMPILER-$SPACK_SPEC | sha256sum - | head -c 16` + - CONFIG_TAG=`echo $DOCKERFILE_SHA-$ARCH-$BASE_IMAGE-$COMPILER | sha256sum - | head -c 16` - compiler=${COMPILER/@/-} - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/base/pika-$compiler:$CONFIG_TAG - echo -e "compiler=$compiler" >> compiler.env - echo -e "BASE_IMAGE=$PERSIST_IMAGE_NAME" >> compiler.env variables: DOCKERFILE: .gitlab/docker/Dockerfile.spack_compiler - DOCKER_BUILD_ARGS: '["BASE_IMAGE","ARCH","COMPILER","SPACK_SPEC"]' + DOCKER_BUILD_ARGS: '["BASE_IMAGE","ARCH","COMPILER"]' artifacts: reports: dotenv: compiler.env +performance_gcc13_spack_image: + stage: spack_dependencies + needs: [performance_gcc13_spack_compiler_image] + extends: + - .container-builder + - .variables_performance_gcc13_config + before_script: + - export DOCKERFILE_SHA=`sha256sum $DOCKERFILE | head -c 16` + - CONFIG_TAG=`echo $DOCKERFILE_SHA-$BASE_IMAGE-$SPACK_SPEC | sha256sum - | head -c 16` + - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/base/pika-$compiler-dependencies:$CONFIG_TAG + - echo -e "compiler=$compiler" >> dependencies.env + - echo -e "BASE_IMAGE=$PERSIST_IMAGE_NAME" >> dependencies.env + variables: + DOCKERFILE: .gitlab/docker/Dockerfile.spack_dependencies + DOCKER_BUILD_ARGS: '["BASE_IMAGE","SPACK_SPEC"]' + artifacts: + reports: + dotenv: dependencies.env + performance_gcc13_release_build: stage: build extends: