From db0bd1cb4bed6e939acb4116645a2aa00f330ab7 Mon Sep 17 00:00:00 2001 From: Lionel Untereiner Date: Fri, 2 Aug 2024 13:56:04 +0200 Subject: [PATCH 01/24] ci: split ci into atomic workflows --- .github/workflows/ci_tests.yml | 1 + .github/workflows/compliance.yml | 189 +++++++++++++++++++++++++ .github/workflows/cpu_builds.yml | 74 ++++++++++ .github/workflows/gpu_builds.yml | 106 ++++++++++++++ .github/workflows/integrated_tests.yml | 31 ++++ 5 files changed, 401 insertions(+) create mode 100644 .github/workflows/compliance.yml create mode 100644 .github/workflows/cpu_builds.yml create mode 100644 .github/workflows/gpu_builds.yml create mode 100644 .github/workflows/integrated_tests.yml diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_tests.yml index bf62843e07b..e8aa5c31ea0 100644 --- a/.github/workflows/ci_tests.yml +++ b/.github/workflows/ci_tests.yml @@ -397,3 +397,4 @@ jobs: needs.cuda_builds.result == 'success' && needs.run_integrated_tests.result == 'success' }} + diff --git a/.github/workflows/compliance.yml b/.github/workflows/compliance.yml new file mode 100644 index 00000000000..7de61b77980 --- /dev/null +++ b/.github/workflows/compliance.yml @@ -0,0 +1,189 @@ +name: Compliance Tests + +on: + pull_request: + types: [opened, synchronize, reopened, edited] + workflow_dispatch: + +# Cancels in-progress workflows for a PR when updated +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +# Please define `build.args.GEOS_TPL_TAG` in `.devcontainer/devcontainer.json` + +jobs: + # Checks if PR title follows conventional semantics + semantic_pull_request: + permissions: + pull-requests: write # for amannn/action-semantic-pull-request to analyze PRs and + statuses: write # for amannn/action-semantic-pull-request to mark status of analyzed PR + contents: read + runs-on: ubuntu-latest + + steps: + - name: Check if the PR name has conventional semantics + if: github.event_name == 'pull_request' + uses: amannn/action-semantic-pull-request@v5.5.3 + id: lint_pr_title + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + wip: true + # Configure that a scope doesn't need to be provided. + requireScope: false + + # Jobs will be cancelled if PR is a draft. + # PR status must be "Open" to run CI. + is_not_draft_pull_request: + needs: [semantic_pull_request] + # Everywhere in this workflow, we use the most recent ubuntu distribution available in Github Actions + # to ensure maximum support of google cloud's sdk. + runs-on: ubuntu-22.04 + outputs: + DOCKER_IMAGE_TAG: ${{ steps.extract_docker_image_tag.outputs.DOCKER_IMAGE_TAG }} + steps: + - name: Check that the PR is not a draft (cancel rest of jobs otherwise) + id: extract_pr_info + run: | + if [[ ${{github.event_name}} == 'pull_request' ]]; then + # We do not rely on the `github.event.pull_request.labels` information since it's cached at the job. + # Changing labels or assignee in the PR would not allow to simply re-run the job with a different outcome. + pr_json=$(curl -H "Accept: application/vnd.github+json" https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.number }}) + + # We stop the workflow if the pr is draft + draft_status=$(echo ${pr_json} | jq '.draft') + echo "Draft status of PR is ${draft_status}." + if [[ $draft_status == true ]]; then exit 1 ; fi + fi + + + # The TPL tag is contained in the codespaces configuration to avoid duplications. + - name: Checkout .devcontainer/devcontainer.json + uses: actions/checkout@v4.1.7 + with: + sparse-checkout: | + .devcontainer/devcontainer.json + sparse-checkout-cone-mode: false + submodules: false + lfs: false + fetch-depth: 1 + - name: Extract docker image tag + id: extract_docker_image_tag + run: | + echo "DOCKER_IMAGE_TAG=$(jq '.build.args.GEOS_TPL_TAG' -r .devcontainer/devcontainer.json)" >> "$GITHUB_OUTPUT" + + # PR must be assigned to be merged. + # This job will fail if this is not the case. + if_not_unassigned_pull_request: + needs: [is_not_draft_pull_request] + runs-on: ubuntu-22.04 + steps: + - name: If this is a PR, Check that it is assigned + run: | + if [[ ${{github.event_name}} != 'pull_request' ]]; then exit 0 ; fi + pr_json=$(curl -H "Accept: application/vnd.github+json" https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.number }}) + NUM_ASSIGNEES=$(echo ${pr_json} | jq '.assignees | length') + echo "There are ${NUM_ASSIGNEES} assignees on this PR." + if [[ $NUM_ASSIGNEES == 0 ]]; then exit 1 ; fi + + # Validates that the PR is still pointing to the HEAD of the main branch of the submodules repositories. + # (There are exceptions, read the script about those). + are_submodules_in_sync: + needs: [is_not_draft_pull_request] + runs-on: ubuntu-22.04 + steps: + # The integrated test submodule repository contains large data (using git lfs). + # To save time (and money) we do not let Github Actions automatically clone all our (lfs) subrepositories and do it by hand. + - name: Checkout Repository + uses: actions/checkout@v4.1.7 + with: + # Let script update submodules; Github Actions submodule history causes error + submodules: false + lfs: false + fetch-depth: 1 + - name: Check that submodules are up to date + run: "scripts/test_submodule_updated.sh" + + check_code_style_and_documentation: + name: ${{ matrix.name }} + needs: [is_not_draft_pull_request] + strategy: + fail-fast : false + matrix: + include: + # Validates the code-style using uncrustify + - name: Check code style + BUILD_AND_TEST_ARGS: --test-code-style + # Validates that the documentation generated using doxygen has no hole. + - name: Check documentation + BUILD_AND_TEST_ARGS: --test-documentation + uses: ./.github/workflows/build_and_test.yml + with: + BUILD_AND_TEST_CLI_ARGS: ${{ matrix.BUILD_AND_TEST_ARGS }} + CMAKE_BUILD_TYPE: Release + DOCKER_IMAGE_TAG: ${{ needs.is_not_draft_pull_request.outputs.DOCKER_IMAGE_TAG }} + DOCKER_REPOSITORY: geosx/ubuntu20.04-gcc9 + RUNS_ON: ubuntu-22.04 + USE_SCCACHE: false + + baseline_log: + needs: [is_not_draft_pull_request] + runs-on: ubuntu-22.04 + steps: + - name: Checkout Repository + uses: actions/checkout@v4.1.7 + with: + submodules: false + lfs: false + fetch-depth: 0 + sparse-checkout: | + scripts + - name: Check that the baseline logs are modified if rebaselines are detected + run: "scripts/check_baseline_log.sh" + + code_coverage: + needs: + - is_not_draft_pull_request + uses: ./.github/workflows/build_and_test.yml + secrets: inherit + with: + BUILD_AND_TEST_CLI_ARGS: "--no-run-unit-tests" + CMAKE_BUILD_TYPE: Debug + CODE_COVERAGE: true + DOCKER_IMAGE_TAG: ${{ needs.is_not_draft_pull_request.outputs.DOCKER_IMAGE_TAG }} + DOCKER_REPOSITORY: geosx/ubuntu22.04-gcc11 + ENABLE_HYPRE: ON + ENABLE_TRILINOS: OFF + GCP_BUCKET: geosx/ubuntu22.04-gcc11 + RUNS_ON: Runner_4core_16GB + + + + # Convenience job - passes when all other jobs have passed (must pass the CUDA jobs). + # check_that_all_jobs_succeeded: + # runs-on: ubuntu-22.04 + # needs: + # - if_not_unassigned_pull_request + # - are_submodules_in_sync + # - check_code_style_and_documentation + # - cpu_builds + # - cuda_builds + # - run_integrated_tests + # if: ${{ always() }} + # steps: + # - run: | + # echo "if_not_unassigned_pull_request: ${{needs.if_not_unassigned_pull_request.result}}" + # echo "are_submodules_in_sync: ${{needs.are_submodules_in_sync.result}}" + # echo "check_code_style_and_documentation: ${{needs.check_code_style_and_documentation.result}}" + # echo "cpu_builds: ${{needs.cpu_builds.result}}" + # echo "cuda_builds: ${{needs.cuda_builds.result}}" + # echo "run_integrated_tests: ${{needs.run_integrated_tests.result}} " + # ${{ + # needs.if_not_unassigned_pull_request.result == 'success' && + # needs.are_submodules_in_sync.result == 'success' && + # needs.check_code_style_and_documentation.result == 'success' && + # needs.cpu_builds.result == 'success' && + # needs.cuda_builds.result == 'success' && + # needs.run_integrated_tests.result == 'success' + # }} diff --git a/.github/workflows/cpu_builds.yml b/.github/workflows/cpu_builds.yml new file mode 100644 index 00000000000..809c5ce19b2 --- /dev/null +++ b/.github/workflows/cpu_builds.yml @@ -0,0 +1,74 @@ +name: CPU builds + +on: + pull_request: # Run workflow on PRs to the develop branch with modification on *.hpp or *.cpp files + branches: + - develop + paths: + - '**.hpp' + - '**.cpp' + push: # Run workflow on push to the develop branch. + branches: + - develop + workflow_dispatch: # Workflow can be run manually + +jobs: + # Matrix containing all the CPU build. + # Those are quite fast and can efficiently benefit from the `sccache' tool to make them even faster. + cpu_builds: + name: ${{ matrix.name }} + strategy: + # In-progress jobs will not be cancelled if there is a failure + fail-fast : false + matrix: + include: + - name: Ubuntu (20.04, gcc 9.4.0, open-mpi 4.0.3) + CMAKE_BUILD_TYPE: Release + DOCKER_REPOSITORY: geosx/ubuntu20.04-gcc9 + + - name: Ubuntu debug (20.04, gcc 10.5.0, open-mpi 4.0.3) - github codespaces + CMAKE_BUILD_TYPE: Debug + DOCKER_REPOSITORY: geosx/ubuntu20.04-gcc10 + + - name: Ubuntu (20.04, gcc 10.5.0, open-mpi 4.0.3) - github codespaces + CMAKE_BUILD_TYPE: Release + DOCKER_REPOSITORY: geosx/ubuntu20.04-gcc10 + + - name: Ubuntu (22.04, gcc 11.4.0, open-mpi 4.1.2) + CMAKE_BUILD_TYPE: Release + DOCKER_REPOSITORY: geosx/ubuntu22.04-gcc11 + ENABLE_HYPRE: ON + ENABLE_TRILINOS: OFF + GCP_BUCKET: geosx/ubuntu22.04-gcc11 + + - name: Ubuntu (22.04, gcc 12.3.0, open-mpi 4.1.2) + CMAKE_BUILD_TYPE: Release + DOCKER_REPOSITORY: geosx/ubuntu22.04-gcc12 + ENABLE_HYPRE: ON + ENABLE_TRILINOS: OFF + + - name: Ubuntu (22.04, clang 15.0.7, open-mpi 4.1.2) + CMAKE_BUILD_TYPE: Release + DOCKER_REPOSITORY: geosx/ubuntu22.04-clang15 + ENABLE_HYPRE: ON + ENABLE_TRILINOS: OFF + + - name: Sherlock CPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, openblas 0.3.10) + CMAKE_BUILD_TYPE: Release + DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-openblas0.3.10-zlib1.2.11 + ENABLE_HYPRE: ON + ENABLE_TRILINOS: OFF + GCP_BUCKET: geosx/Sherlock-CPU + HOST_CONFIG: host-configs/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10.cmake + + uses: ./.github/workflows/build_and_test.yml + with: + CMAKE_BUILD_TYPE: ${{ matrix.CMAKE_BUILD_TYPE }} + DOCKER_IMAGE_TAG: ${{ needs.is_not_draft_pull_request.outputs.DOCKER_IMAGE_TAG }} + DOCKER_REPOSITORY: ${{ matrix.DOCKER_REPOSITORY }} + ENABLE_HYPRE: ${{ matrix.ENABLE_HYPRE }} + ENABLE_TRILINOS: ${{ matrix.ENABLE_TRILINOS }} + GCP_BUCKET: ${{ matrix.GCP_BUCKET }} + HOST_CONFIG: ${{ matrix.HOST_CONFIG }} + RUNS_ON: ubuntu-22.04 + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/gpu_builds.yml b/.github/workflows/gpu_builds.yml new file mode 100644 index 00000000000..08d672490a8 --- /dev/null +++ b/.github/workflows/gpu_builds.yml @@ -0,0 +1,106 @@ +name: GPU builds + +on: + pull_request: # Run workflow on PRs to the develop branch on labeled event + branches: + - develop + types: [ labeled ] + push: # Run workflow on push to the develop branch. + branches: + - develop + workflow_dispatch: # Workflow can be run manually + +jobs: + # If the 'ci: run CUDA builds' PR label is found, the cuda jobs run immediately along side linux jobs. + # Note: CUDA jobs should only be run if PR is ready to merge. + cuda_builds: + # if the current added label is for GPUs + if: "${{ github.event.label.name == 'ci: run CUDA builds' }}" + name: ${{ matrix.name }} + strategy: + # In-progress jobs will not be cancelled if there is a failure + fail-fast : false + matrix: + include: + - name: Ubuntu CUDA debug (20.04, clang 10.0.0 + gcc 9.4.0, open-mpi 4.0.3, cuda-11.8.89) + BUILD_AND_TEST_CLI_ARGS: "--build-exe-only --no-install-schema" + CMAKE_BUILD_TYPE: Debug + DOCKER_REPOSITORY: geosx/ubuntu20.04-clang10.0.0-cuda11.8.89 + ENABLE_HYPRE_DEVICE: CUDA + ENABLE_HYPRE: ON + ENABLE_TRILINOS: OFF + RUNS_ON: streak2 + NPROC: 8 + DOCKER_RUN_ARGS: "--cpus=8 --memory=128g --runtime=nvidia -v /etc/pki/ca-trust/source/anchors/:/usr/local/share/ca-certificates/llnl:ro" + DOCKER_CERTS_DIR: "/usr/local/share/ca-certificates" + DOCKER_CERTS_UPDATE_COMMAND: "update-ca-certificates" + + - name: Ubuntu CUDA (20.04, clang 10.0.0 + gcc 9.4.0, open-mpi 4.0.3, cuda-11.8.89) + BUILD_AND_TEST_CLI_ARGS: "--no-install-schema" + CMAKE_BUILD_TYPE: Release + DOCKER_REPOSITORY: geosx/ubuntu20.04-clang10.0.0-cuda11.8.89 + ENABLE_HYPRE_DEVICE: CUDA + ENABLE_HYPRE: ON + ENABLE_TRILINOS: OFF + RUNS_ON: streak + NPROC: 8 + DOCKER_RUN_ARGS: "--cpus=8 --memory=256g --runtime=nvidia --gpus all -v /etc/pki/ca-trust/source/anchors/:/usr/local/share/ca-certificates/llnl:ro" + DOCKER_CERTS_DIR: "/usr/local/share/ca-certificates" + DOCKER_CERTS_UPDATE_COMMAND: "update-ca-certificates" + + # compiler error in ElasticFirstOrderWaveEquationSEMKernel::StressComputation::launch in call to FE_TYPE::computeFirstOrderStiffnessTermX + # - name: Rockylinux (8, clang 17.0.6, cuda 12.5) + # BUILD_AND_TEST_CLI_ARGS: "--no-run-unit-tests --no-install-schema" + # CMAKE_BUILD_TYPE: Release + # DOCKER_REPOSITORY: geosx/rockylinux8-clang17-cuda12.5 + # RUNS_ON: streak2 + # NPROC: 2 + # DOCKER_RUN_ARGS: "--cpus=1 --memory=128g --runtime=nvidia -v /etc/pki/ca-trust/source/anchors/:/etc/pki/ca-trust/source/anchors/llnl:ro" + # DOCKER_CERTS_DIR: "/etc/pki/ca-trust/source/anchors" + # DOCKER_CERTS_UPDATE_COMMAND: "update-ca-trust" + + # compiler error in ElasticFirstOrderWaveEquationSEMKernel::StressComputation::launch in call to FE_TYPE::computeFirstOrderStiffnessTermX + # - name: Rockylinux (8, gcc 8.5, cuda 12.5) + # BUILD_AND_TEST_CLI_ARGS: "--no-run-unit-tests --no-install-schema" + # CMAKE_BUILD_TYPE: Release + # DOCKER_REPOSITORY: geosx/rockylinux8-gcc8-cuda12.5 + # RUNS_ON: streak2 + # NPROC: 2 + # DOCKER_RUN_ARGS: "--cpus=1 --memory=128g --runtime=nvidia -v /etc/pki/ca-trust/source/anchors/:/etc/pki/ca-trust/source/anchors/llnl:ro" + # DOCKER_CERTS_DIR: "/etc/pki/ca-trust/source/anchors" + # DOCKER_CERTS_UPDATE_COMMAND: "update-ca-trust" + + # Below this line, jobs that deploy to Google Cloud. + - name: Sherlock GPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, openblas 0.3.10, cuda 11.7.1,) + BUILD_AND_TEST_CLI_ARGS: "--no-run-unit-tests --no-install-schema" + CMAKE_BUILD_TYPE: Release + DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-cuda11.7.1-openblas0.3.10-zlib1.2.11 + ENABLE_HYPRE_DEVICE: CUDA + ENABLE_HYPRE: ON + ENABLE_TRILINOS: OFF + GCP_BUCKET: geosx/Sherlock-GPU + HOST_CONFIG: host-configs/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10-cuda11.7.1-sm70.cmake + RUNS_ON: streak2 + NPROC: 8 + DOCKER_RUN_ARGS: "--cpus=8 --memory=128g --runtime=nvidia -v /etc/pki/ca-trust/source/anchors/:/etc/pki/ca-trust/source/anchors/llnl:ro" + DOCKER_CERTS_DIR: "/etc/pki/ca-trust/source/anchors" + DOCKER_CERTS_UPDATE_COMMAND: "update-ca-trust" + + uses: ./.github/workflows/build_and_test.yml + with: + BUILD_AND_TEST_CLI_ARGS: ${{ matrix.BUILD_AND_TEST_CLI_ARGS }} + CMAKE_BUILD_TYPE: ${{ matrix.CMAKE_BUILD_TYPE }} + DOCKER_CERTS_DIR: ${{ matrix.DOCKER_CERTS_DIR }} + DOCKER_CERTS_UPDATE_COMMAND: ${{ matrix.DOCKER_CERTS_UPDATE_COMMAND }} + DOCKER_IMAGE_TAG: ${{ needs.is_not_draft_pull_request.outputs.DOCKER_IMAGE_TAG }} + DOCKER_REPOSITORY: ${{ matrix.DOCKER_REPOSITORY }} + DOCKER_RUN_ARGS: ${{ matrix.DOCKER_RUN_ARGS }} + ENABLE_HYPRE_DEVICE: ${{ matrix.ENABLE_HYPRE_DEVICE }} + ENABLE_HYPRE: ${{ matrix.ENABLE_HYPRE }} + ENABLE_TRILINOS: ${{ matrix.ENABLE_TRILINOS }} + GCP_BUCKET: ${{ matrix.GCP_BUCKET }} + HOST_CONFIG: ${{ matrix.HOST_CONFIG }} + NPROC: ${{ matrix.NPROC }} + RUNS_ON: ${{ matrix.RUNS_ON }} + REQUIRED_LABEL: "ci: run CUDA builds" + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/integrated_tests.yml b/.github/workflows/integrated_tests.yml new file mode 100644 index 00000000000..24c65ed7606 --- /dev/null +++ b/.github/workflows/integrated_tests.yml @@ -0,0 +1,31 @@ +name: Integrated Tests + +on: + pull_request: + types: [ labeled ] + workflow_dispatch: + +jobs: + + # If the 'ci: run integrated tests' PR label is found, the integrated tests will be run immediately after the cpu jobs. + # Note: The integrated tests are optional and are (for the moment) run for convenience only. + run_integrated_tests: + if: "${{ github.event.label.name == 'ci: run integrated tests' }}" + uses: ./.github/workflows/build_and_test.yml + secrets: inherit + with: + BUILD_AND_TEST_CLI_ARGS: --build-exe-only + BUILD_TYPE: integrated_tests + CMAKE_BUILD_TYPE: Release + DOCKER_IMAGE_TAG: ${{ needs.is_not_draft_pull_request.outputs.DOCKER_IMAGE_TAG }} + DOCKER_REPOSITORY: geosx/ubuntu22.04-gcc11 + ENABLE_HYPRE: ON + ENABLE_TRILINOS: OFF + GCP_BUCKET: geosx/integratedTests + RUNS_ON: streak2-32core + NPROC: 32 + DOCKER_RUN_ARGS: "--cpus=32 --memory=384g -v /etc/pki/ca-trust/source/anchors/:/usr/local/share/ca-certificates/llnl:ro" + DOCKER_CERTS_DIR: "/usr/local/share/ca-certificates" + DOCKER_CERTS_UPDATE_COMMAND: "update-ca-certificates" + REQUIRED_LABEL: "ci: run integrated tests" + LOCAL_BASELINE_DIR: /data/GEOS/baselines \ No newline at end of file From 857573346f216547ceb27de8615988117628a242 Mon Sep 17 00:00:00 2001 From: Lionel Untereiner Date: Fri, 2 Aug 2024 15:21:24 +0200 Subject: [PATCH 02/24] wip: add tests for events --- .github/workflows/compliance.yml | 42 ++++++++++++-------------------- .github/workflows/cpu_builds.yml | 14 +++++++++-- .github/workflows/gpu_builds.yml | 2 +- 3 files changed, 29 insertions(+), 29 deletions(-) diff --git a/.github/workflows/compliance.yml b/.github/workflows/compliance.yml index 7de61b77980..5d6fd793d3b 100644 --- a/.github/workflows/compliance.yml +++ b/.github/workflows/compliance.yml @@ -161,29 +161,19 @@ jobs: # Convenience job - passes when all other jobs have passed (must pass the CUDA jobs). - # check_that_all_jobs_succeeded: - # runs-on: ubuntu-22.04 - # needs: - # - if_not_unassigned_pull_request - # - are_submodules_in_sync - # - check_code_style_and_documentation - # - cpu_builds - # - cuda_builds - # - run_integrated_tests - # if: ${{ always() }} - # steps: - # - run: | - # echo "if_not_unassigned_pull_request: ${{needs.if_not_unassigned_pull_request.result}}" - # echo "are_submodules_in_sync: ${{needs.are_submodules_in_sync.result}}" - # echo "check_code_style_and_documentation: ${{needs.check_code_style_and_documentation.result}}" - # echo "cpu_builds: ${{needs.cpu_builds.result}}" - # echo "cuda_builds: ${{needs.cuda_builds.result}}" - # echo "run_integrated_tests: ${{needs.run_integrated_tests.result}} " - # ${{ - # needs.if_not_unassigned_pull_request.result == 'success' && - # needs.are_submodules_in_sync.result == 'success' && - # needs.check_code_style_and_documentation.result == 'success' && - # needs.cpu_builds.result == 'success' && - # needs.cuda_builds.result == 'success' && - # needs.run_integrated_tests.result == 'success' - # }} + check_that_all_jobs_succeeded: + runs-on: ubuntu-22.04 + needs: + - if_not_unassigned_pull_request + - are_submodules_in_sync + - check_code_style_and_documentation + steps: + - run: | + echo "if_not_unassigned_pull_request: ${{needs.if_not_unassigned_pull_request.result}}" + echo "are_submodules_in_sync: ${{needs.are_submodules_in_sync.result}}" + echo "check_code_style_and_documentation: ${{needs.check_code_style_and_documentation.result}}" + echo "COMPLIANCE_SUCCESS = ${{ + needs.if_not_unassigned_pull_request.result == 'success' && + needs.are_submodules_in_sync.result == 'success' && + needs.check_code_style_and_documentation.result == 'success' + }}" >> $GITHUB_ENV \ No newline at end of file diff --git a/.github/workflows/cpu_builds.yml b/.github/workflows/cpu_builds.yml index 809c5ce19b2..f54ef433dd2 100644 --- a/.github/workflows/cpu_builds.yml +++ b/.github/workflows/cpu_builds.yml @@ -1,21 +1,31 @@ name: CPU builds on: - pull_request: # Run workflow on PRs to the develop branch with modification on *.hpp or *.cpp files + pull_request: # Run workflow on PRs to the develop branch with changes on *.hpp or *.cpp files when PR is opened, synchronized, reopented and ready fo review branches: - develop paths: - '**.hpp' - '**.cpp' + types: + - opened + - synchronize + - reopened + - ready_for_review push: # Run workflow on push to the develop branch. branches: - develop workflow_dispatch: # Workflow can be run manually + jobs: # Matrix containing all the CPU build. # Those are quite fast and can efficiently benefit from the `sccache' tool to make them even faster. cpu_builds: + if: | + vars.COMPLIANCE_SUCCESS && + github.event_name == 'push' || + (github.event_name == 'pull_request' && !github.event.pull_request.draft && github.event.pull_request.assignees > 0) name: ${{ matrix.name }} strategy: # In-progress jobs will not be cancelled if there is a failure @@ -64,7 +74,7 @@ jobs: uses: ./.github/workflows/build_and_test.yml with: CMAKE_BUILD_TYPE: ${{ matrix.CMAKE_BUILD_TYPE }} - DOCKER_IMAGE_TAG: ${{ needs.is_not_draft_pull_request.outputs.DOCKER_IMAGE_TAG }} + DOCKER_IMAGE_TAG: ${{ vars.DOCKER_IMAGE_TAG }} DOCKER_REPOSITORY: ${{ matrix.DOCKER_REPOSITORY }} ENABLE_HYPRE: ${{ matrix.ENABLE_HYPRE }} ENABLE_TRILINOS: ${{ matrix.ENABLE_TRILINOS }} diff --git a/.github/workflows/gpu_builds.yml b/.github/workflows/gpu_builds.yml index 08d672490a8..849a48c284c 100644 --- a/.github/workflows/gpu_builds.yml +++ b/.github/workflows/gpu_builds.yml @@ -15,7 +15,7 @@ jobs: # Note: CUDA jobs should only be run if PR is ready to merge. cuda_builds: # if the current added label is for GPUs - if: "${{ github.event.label.name == 'ci: run CUDA builds' }}" + if: "${{ github.event.label.name == 'ci: run CUDA builds' && vars.COMPLIANCE_SUCCESS}}" name: ${{ matrix.name }} strategy: # In-progress jobs will not be cancelled if there is a failure From 12d6f3c8a15c7762efba7e92a39d447b87c772b7 Mon Sep 17 00:00:00 2001 From: Lionel Untereiner Date: Fri, 2 Aug 2024 16:44:11 +0200 Subject: [PATCH 03/24] wip: reuse cpu workflow --- .github/workflows/compliance.yml | 80 +++++++++++++++++++------------- .github/workflows/cpu_builds.yml | 34 +++++++------- 2 files changed, 66 insertions(+), 48 deletions(-) diff --git a/.github/workflows/compliance.yml b/.github/workflows/compliance.yml index 5d6fd793d3b..b5ca0e963d4 100644 --- a/.github/workflows/compliance.yml +++ b/.github/workflows/compliance.yml @@ -35,7 +35,7 @@ jobs: # Jobs will be cancelled if PR is a draft. # PR status must be "Open" to run CI. - is_not_draft_pull_request: + get_docker_image_tag: needs: [semantic_pull_request] # Everywhere in this workflow, we use the most recent ubuntu distribution available in Github Actions # to ensure maximum support of google cloud's sdk. @@ -43,19 +43,19 @@ jobs: outputs: DOCKER_IMAGE_TAG: ${{ steps.extract_docker_image_tag.outputs.DOCKER_IMAGE_TAG }} steps: - - name: Check that the PR is not a draft (cancel rest of jobs otherwise) - id: extract_pr_info - run: | - if [[ ${{github.event_name}} == 'pull_request' ]]; then - # We do not rely on the `github.event.pull_request.labels` information since it's cached at the job. - # Changing labels or assignee in the PR would not allow to simply re-run the job with a different outcome. - pr_json=$(curl -H "Accept: application/vnd.github+json" https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.number }}) + # - name: Check that the PR is not a draft (cancel rest of jobs otherwise) + # id: extract_pr_info + # run: | + # if [[ ${{github.event_name}} == 'pull_request' ]]; then + # # We do not rely on the `github.event.pull_request.labels` information since it's cached at the job. + # # Changing labels or assignee in the PR would not allow to simply re-run the job with a different outcome. + # pr_json=$(curl -H "Accept: application/vnd.github+json" https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.number }}) - # We stop the workflow if the pr is draft - draft_status=$(echo ${pr_json} | jq '.draft') - echo "Draft status of PR is ${draft_status}." - if [[ $draft_status == true ]]; then exit 1 ; fi - fi + # # We stop the workflow if the pr is draft + # draft_status=$(echo ${pr_json} | jq '.draft') + # echo "Draft status of PR is ${draft_status}." + # if [[ $draft_status == true ]]; then exit 1 ; fi + # fi # The TPL tag is contained in the codespaces configuration to avoid duplications. @@ -76,7 +76,7 @@ jobs: # PR must be assigned to be merged. # This job will fail if this is not the case. if_not_unassigned_pull_request: - needs: [is_not_draft_pull_request] + needs: [semantic_pull_request] runs-on: ubuntu-22.04 steps: - name: If this is a PR, Check that it is assigned @@ -90,7 +90,7 @@ jobs: # Validates that the PR is still pointing to the HEAD of the main branch of the submodules repositories. # (There are exceptions, read the script about those). are_submodules_in_sync: - needs: [is_not_draft_pull_request] + needs: [if_not_unassigned_pull_request] runs-on: ubuntu-22.04 steps: # The integrated test submodule repository contains large data (using git lfs). @@ -107,7 +107,9 @@ jobs: check_code_style_and_documentation: name: ${{ matrix.name }} - needs: [is_not_draft_pull_request] + needs: + - if_not_unassigned_pull_request + - get_docker_image_tag strategy: fail-fast : false matrix: @@ -122,13 +124,13 @@ jobs: with: BUILD_AND_TEST_CLI_ARGS: ${{ matrix.BUILD_AND_TEST_ARGS }} CMAKE_BUILD_TYPE: Release - DOCKER_IMAGE_TAG: ${{ needs.is_not_draft_pull_request.outputs.DOCKER_IMAGE_TAG }} + DOCKER_IMAGE_TAG: ${{ needs.get_docker_image_tag.outputs.DOCKER_IMAGE_TAG }} DOCKER_REPOSITORY: geosx/ubuntu20.04-gcc9 RUNS_ON: ubuntu-22.04 USE_SCCACHE: false baseline_log: - needs: [is_not_draft_pull_request] + needs: [if_not_unassigned_pull_request] runs-on: ubuntu-22.04 steps: - name: Checkout Repository @@ -144,14 +146,15 @@ jobs: code_coverage: needs: - - is_not_draft_pull_request + - if_not_unassigned_pull_request + - get_docker_image_tag uses: ./.github/workflows/build_and_test.yml secrets: inherit with: BUILD_AND_TEST_CLI_ARGS: "--no-run-unit-tests" CMAKE_BUILD_TYPE: Debug CODE_COVERAGE: true - DOCKER_IMAGE_TAG: ${{ needs.is_not_draft_pull_request.outputs.DOCKER_IMAGE_TAG }} + DOCKER_IMAGE_TAG: ${{ needs.get_docker_image_tag.outputs.DOCKER_IMAGE_TAG }} DOCKER_REPOSITORY: geosx/ubuntu22.04-gcc11 ENABLE_HYPRE: ON ENABLE_TRILINOS: OFF @@ -164,16 +167,29 @@ jobs: check_that_all_jobs_succeeded: runs-on: ubuntu-22.04 needs: - - if_not_unassigned_pull_request - - are_submodules_in_sync - - check_code_style_and_documentation + # - is_not_draft_pull_request + - if_not_unassigned_pull_request + - are_submodules_in_sync + - check_code_style_and_documentation + - baseline_log + - code_coverage steps: - - run: | - echo "if_not_unassigned_pull_request: ${{needs.if_not_unassigned_pull_request.result}}" - echo "are_submodules_in_sync: ${{needs.are_submodules_in_sync.result}}" - echo "check_code_style_and_documentation: ${{needs.check_code_style_and_documentation.result}}" - echo "COMPLIANCE_SUCCESS = ${{ - needs.if_not_unassigned_pull_request.result == 'success' && - needs.are_submodules_in_sync.result == 'success' && - needs.check_code_style_and_documentation.result == 'success' - }}" >> $GITHUB_ENV \ No newline at end of file + - run: | + echo "if_not_unassigned_pull_request: ${{needs.if_not_unassigned_pull_request.result}}" + echo "are_submodules_in_sync: ${{needs.are_submodules_in_sync.result}}" + echo "check_code_style_and_documentation: ${{needs.check_code_style_and_documentation.result}}" + echo "COMPLIANCE_SUCCESS = ${{ + needs.if_not_unassigned_pull_request.result == 'success' && + needs.are_submodules_in_sync.result == 'success' && + needs.check_code_style_and_documentation.result == 'success' + }}" >> $GITHUB_ENV + + cpu_builds: + name: CPU builds + needs: + - get_docker_image_tag + - check_that_all_jobs_succeeded + uses: ./.github/workflows/cpu_builds.yml + with: + DOCKER_IMAGE_TAG: ${{ needs.get_docker_image_tag.outputs.DOCKER_IMAGE_TAG }} + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/cpu_builds.yml b/.github/workflows/cpu_builds.yml index f54ef433dd2..ae03ff6fcea 100644 --- a/.github/workflows/cpu_builds.yml +++ b/.github/workflows/cpu_builds.yml @@ -1,17 +1,23 @@ name: CPU builds on: - pull_request: # Run workflow on PRs to the develop branch with changes on *.hpp or *.cpp files when PR is opened, synchronized, reopented and ready fo review - branches: - - develop - paths: - - '**.hpp' - - '**.cpp' - types: - - opened - - synchronize - - reopened - - ready_for_review +# pull_request: # Run workflow on PRs to the develop branch with changes on *.hpp or *.cpp files when PR is opened, synchronized, reopented and ready fo review +# branches: +# - develop +# paths: +# - '**.hpp' +# - '**.cpp' +# types: +# - opened +# - synchronize +# - reopened +# - ready_for_review + # Allow this workflow to be called by the main workflow. + workflow_call: + inputs: + DOCKER_IMAGE_TAG: + required: true + type: string push: # Run workflow on push to the develop branch. branches: - develop @@ -22,10 +28,6 @@ jobs: # Matrix containing all the CPU build. # Those are quite fast and can efficiently benefit from the `sccache' tool to make them even faster. cpu_builds: - if: | - vars.COMPLIANCE_SUCCESS && - github.event_name == 'push' || - (github.event_name == 'pull_request' && !github.event.pull_request.draft && github.event.pull_request.assignees > 0) name: ${{ matrix.name }} strategy: # In-progress jobs will not be cancelled if there is a failure @@ -74,7 +76,7 @@ jobs: uses: ./.github/workflows/build_and_test.yml with: CMAKE_BUILD_TYPE: ${{ matrix.CMAKE_BUILD_TYPE }} - DOCKER_IMAGE_TAG: ${{ vars.DOCKER_IMAGE_TAG }} + DOCKER_IMAGE_TAG: ${{ inputs.DOCKER_IMAGE_TAG }} DOCKER_REPOSITORY: ${{ matrix.DOCKER_REPOSITORY }} ENABLE_HYPRE: ${{ matrix.ENABLE_HYPRE }} ENABLE_TRILINOS: ${{ matrix.ENABLE_TRILINOS }} From 26f9c27781d2b1549c865dd9e5ca4c05af946b89 Mon Sep 17 00:00:00 2001 From: Lionel Untereiner Date: Fri, 2 Aug 2024 16:46:21 +0200 Subject: [PATCH 04/24] wip: add pr type --- .github/workflows/compliance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/compliance.yml b/.github/workflows/compliance.yml index b5ca0e963d4..9b1036536a7 100644 --- a/.github/workflows/compliance.yml +++ b/.github/workflows/compliance.yml @@ -2,7 +2,7 @@ name: Compliance Tests on: pull_request: - types: [opened, synchronize, reopened, edited] + types: [opened, synchronize, reopened, edited, assigned] workflow_dispatch: # Cancels in-progress workflows for a PR when updated From fc2132144e6caf4a87421459698c9c51e98d0811 Mon Sep 17 00:00:00 2001 From: Lionel Untereiner Date: Fri, 2 Aug 2024 17:14:18 +0200 Subject: [PATCH 05/24] wip: change order --- .github/workflows/compliance.yml | 46 +++++++++++++------------- .github/workflows/cpu_builds.yml | 2 +- .github/workflows/gpu_builds.yml | 37 ++++++++++++++++++--- .github/workflows/integrated_tests.yml | 35 ++++++++++++++++++-- 4 files changed, 89 insertions(+), 31 deletions(-) diff --git a/.github/workflows/compliance.yml b/.github/workflows/compliance.yml index 9b1036536a7..6208fd4ac27 100644 --- a/.github/workflows/compliance.yml +++ b/.github/workflows/compliance.yml @@ -1,4 +1,4 @@ -name: Compliance Tests +name: Compliance Tests & CPU builds on: pull_request: @@ -14,7 +14,8 @@ concurrency: jobs: # Checks if PR title follows conventional semantics - semantic_pull_request: + semantic_pull_request: + name: Checks if PR title follows conventional semantics permissions: pull-requests: write # for amannn/action-semantic-pull-request to analyze PRs and statuses: write # for amannn/action-semantic-pull-request to mark status of analyzed PR @@ -36,6 +37,7 @@ jobs: # Jobs will be cancelled if PR is a draft. # PR status must be "Open" to run CI. get_docker_image_tag: + name: Get TPL tag needs: [semantic_pull_request] # Everywhere in this workflow, we use the most recent ubuntu distribution available in Github Actions # to ensure maximum support of google cloud's sdk. @@ -76,6 +78,7 @@ jobs: # PR must be assigned to be merged. # This job will fail if this is not the case. if_not_unassigned_pull_request: + name: Check PR assignement needs: [semantic_pull_request] runs-on: ubuntu-22.04 steps: @@ -144,25 +147,6 @@ jobs: - name: Check that the baseline logs are modified if rebaselines are detected run: "scripts/check_baseline_log.sh" - code_coverage: - needs: - - if_not_unassigned_pull_request - - get_docker_image_tag - uses: ./.github/workflows/build_and_test.yml - secrets: inherit - with: - BUILD_AND_TEST_CLI_ARGS: "--no-run-unit-tests" - CMAKE_BUILD_TYPE: Debug - CODE_COVERAGE: true - DOCKER_IMAGE_TAG: ${{ needs.get_docker_image_tag.outputs.DOCKER_IMAGE_TAG }} - DOCKER_REPOSITORY: geosx/ubuntu22.04-gcc11 - ENABLE_HYPRE: ON - ENABLE_TRILINOS: OFF - GCP_BUCKET: geosx/ubuntu22.04-gcc11 - RUNS_ON: Runner_4core_16GB - - - # Convenience job - passes when all other jobs have passed (must pass the CUDA jobs). check_that_all_jobs_succeeded: runs-on: ubuntu-22.04 @@ -172,7 +156,6 @@ jobs: - are_submodules_in_sync - check_code_style_and_documentation - baseline_log - - code_coverage steps: - run: | echo "if_not_unassigned_pull_request: ${{needs.if_not_unassigned_pull_request.result}}" @@ -184,10 +167,27 @@ jobs: needs.check_code_style_and_documentation.result == 'success' }}" >> $GITHUB_ENV + code_coverage: + needs: + - get_docker_image_tag + - check_that_all_jobs_succeeded + uses: ./.github/workflows/build_and_test.yml + secrets: inherit + with: + BUILD_AND_TEST_CLI_ARGS: "--no-run-unit-tests" + CMAKE_BUILD_TYPE: Debug + CODE_COVERAGE: true + DOCKER_IMAGE_TAG: ${{ needs.get_docker_image_tag.outputs.DOCKER_IMAGE_TAG }} + DOCKER_REPOSITORY: geosx/ubuntu22.04-gcc11 + ENABLE_HYPRE: ON + ENABLE_TRILINOS: OFF + GCP_BUCKET: geosx/ubuntu22.04-gcc11 + RUNS_ON: Runner_4core_16GB + cpu_builds: name: CPU builds needs: - - get_docker_image_tag + - get_docker_image_tag - check_that_all_jobs_succeeded uses: ./.github/workflows/cpu_builds.yml with: diff --git a/.github/workflows/cpu_builds.yml b/.github/workflows/cpu_builds.yml index ae03ff6fcea..3780e1cbee9 100644 --- a/.github/workflows/cpu_builds.yml +++ b/.github/workflows/cpu_builds.yml @@ -83,4 +83,4 @@ jobs: GCP_BUCKET: ${{ matrix.GCP_BUCKET }} HOST_CONFIG: ${{ matrix.HOST_CONFIG }} RUNS_ON: ubuntu-22.04 - secrets: inherit \ No newline at end of file + secrets: inherit diff --git a/.github/workflows/gpu_builds.yml b/.github/workflows/gpu_builds.yml index 849a48c284c..3cb1b86f811 100644 --- a/.github/workflows/gpu_builds.yml +++ b/.github/workflows/gpu_builds.yml @@ -11,11 +11,35 @@ on: workflow_dispatch: # Workflow can be run manually jobs: + get_docker_image_tag: + # if the current added label is for GPUs + if: "${{ github.event.label.name == 'ci: run CUDA builds' && vars.COMPLIANCE_SUCCESS}}" + # Everywhere in this workflow, we use the most recent ubuntu distribution available in Github Actions + # to ensure maximum support of google cloud's sdk. + runs-on: ubuntu-22.04 + outputs: + DOCKER_IMAGE_TAG: ${{ steps.extract_docker_image_tag.outputs.DOCKER_IMAGE_TAG }} + steps: + # The TPL tag is contained in the codespaces configuration to avoid duplications. + - name: Checkout .devcontainer/devcontainer.json + uses: actions/checkout@v4.1.7 + with: + sparse-checkout: | + .devcontainer/devcontainer.json + sparse-checkout-cone-mode: false + submodules: false + lfs: false + fetch-depth: 1 + - name: Extract docker image tag + id: extract_docker_image_tag + run: | + echo "DOCKER_IMAGE_TAG=$(jq '.build.args.GEOS_TPL_TAG' -r .devcontainer/devcontainer.json)" >> "$GITHUB_OUTPUT" + # If the 'ci: run CUDA builds' PR label is found, the cuda jobs run immediately along side linux jobs. # Note: CUDA jobs should only be run if PR is ready to merge. cuda_builds: - # if the current added label is for GPUs - if: "${{ github.event.label.name == 'ci: run CUDA builds' && vars.COMPLIANCE_SUCCESS}}" + needs: + - get_docker_image_tag name: ${{ matrix.name }} strategy: # In-progress jobs will not be cancelled if there is a failure @@ -92,7 +116,7 @@ jobs: CMAKE_BUILD_TYPE: ${{ matrix.CMAKE_BUILD_TYPE }} DOCKER_CERTS_DIR: ${{ matrix.DOCKER_CERTS_DIR }} DOCKER_CERTS_UPDATE_COMMAND: ${{ matrix.DOCKER_CERTS_UPDATE_COMMAND }} - DOCKER_IMAGE_TAG: ${{ needs.is_not_draft_pull_request.outputs.DOCKER_IMAGE_TAG }} + DOCKER_IMAGE_TAG: ${{ needs.get_docker_image_tag.outputs.DOCKER_IMAGE_TAG }} DOCKER_REPOSITORY: ${{ matrix.DOCKER_REPOSITORY }} DOCKER_RUN_ARGS: ${{ matrix.DOCKER_RUN_ARGS }} ENABLE_HYPRE_DEVICE: ${{ matrix.ENABLE_HYPRE_DEVICE }} @@ -103,4 +127,9 @@ jobs: NPROC: ${{ matrix.NPROC }} RUNS_ON: ${{ matrix.RUNS_ON }} REQUIRED_LABEL: "ci: run CUDA builds" - secrets: inherit \ No newline at end of file + secrets: inherit + + remove_label: + uses: actions-ecosystem/action-remove-labels@v1 + with: + labels: 'ci: run CUDA builds' \ No newline at end of file diff --git a/.github/workflows/integrated_tests.yml b/.github/workflows/integrated_tests.yml index 24c65ed7606..8231524d215 100644 --- a/.github/workflows/integrated_tests.yml +++ b/.github/workflows/integrated_tests.yml @@ -6,18 +6,42 @@ on: workflow_dispatch: jobs: + get_docker_image_tag: + if: "${{ github.event.label.name == 'ci: run integrated tests' }}" + # Everywhere in this workflow, we use the most recent ubuntu distribution available in Github Actions + # to ensure maximum support of google cloud's sdk. + runs-on: ubuntu-22.04 + outputs: + DOCKER_IMAGE_TAG: ${{ steps.extract_docker_image_tag.outputs.DOCKER_IMAGE_TAG }} + steps: + # The TPL tag is contained in the codespaces configuration to avoid duplications. + - name: Checkout .devcontainer/devcontainer.json + uses: actions/checkout@v4.1.7 + with: + sparse-checkout: | + .devcontainer/devcontainer.json + sparse-checkout-cone-mode: false + submodules: false + lfs: false + fetch-depth: 1 + - name: Extract docker image tag + id: extract_docker_image_tag + run: | + echo "DOCKER_IMAGE_TAG=$(jq '.build.args.GEOS_TPL_TAG' -r .devcontainer/devcontainer.json)" >> "$GITHUB_OUTPUT" + # If the 'ci: run integrated tests' PR label is found, the integrated tests will be run immediately after the cpu jobs. # Note: The integrated tests are optional and are (for the moment) run for convenience only. run_integrated_tests: - if: "${{ github.event.label.name == 'ci: run integrated tests' }}" + needs: + - get_docker_image_tag uses: ./.github/workflows/build_and_test.yml secrets: inherit with: BUILD_AND_TEST_CLI_ARGS: --build-exe-only BUILD_TYPE: integrated_tests CMAKE_BUILD_TYPE: Release - DOCKER_IMAGE_TAG: ${{ needs.is_not_draft_pull_request.outputs.DOCKER_IMAGE_TAG }} + DOCKER_IMAGE_TAG: ${{ needs.get_docker_image_tag.outputs.DOCKER_IMAGE_TAG }} DOCKER_REPOSITORY: geosx/ubuntu22.04-gcc11 ENABLE_HYPRE: ON ENABLE_TRILINOS: OFF @@ -28,4 +52,9 @@ jobs: DOCKER_CERTS_DIR: "/usr/local/share/ca-certificates" DOCKER_CERTS_UPDATE_COMMAND: "update-ca-certificates" REQUIRED_LABEL: "ci: run integrated tests" - LOCAL_BASELINE_DIR: /data/GEOS/baselines \ No newline at end of file + LOCAL_BASELINE_DIR: /data/GEOS/baselines + + remove_label: + uses: actions-ecosystem/action-remove-labels@v1 + with: + labels: 'ci: run integrated tests' \ No newline at end of file From 950cc356cfeb3a30abd8c358b3064ef2ff06d653 Mon Sep 17 00:00:00 2001 From: Lionel Untereiner Date: Fri, 2 Aug 2024 17:48:42 +0200 Subject: [PATCH 06/24] wip: --- .github/workflows/gpu_builds.yml | 9 ++++++--- .github/workflows/integrated_tests.yml | 9 ++++++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/gpu_builds.yml b/.github/workflows/gpu_builds.yml index 3cb1b86f811..670fed4bc64 100644 --- a/.github/workflows/gpu_builds.yml +++ b/.github/workflows/gpu_builds.yml @@ -130,6 +130,9 @@ jobs: secrets: inherit remove_label: - uses: actions-ecosystem/action-remove-labels@v1 - with: - labels: 'ci: run CUDA builds' \ No newline at end of file + runs-on: ubuntu-22.04 + steps: + - name: Remove the label + uses: actions-ecosystem/action-remove-labels@v1 + with: + labels: 'ci: run CUDA builds' \ No newline at end of file diff --git a/.github/workflows/integrated_tests.yml b/.github/workflows/integrated_tests.yml index 8231524d215..f2ef9d11fa1 100644 --- a/.github/workflows/integrated_tests.yml +++ b/.github/workflows/integrated_tests.yml @@ -55,6 +55,9 @@ jobs: LOCAL_BASELINE_DIR: /data/GEOS/baselines remove_label: - uses: actions-ecosystem/action-remove-labels@v1 - with: - labels: 'ci: run integrated tests' \ No newline at end of file + runs-on: ubuntu-22.04 + steps: + - name: Remove the label + uses: actions-ecosystem/action-remove-labels@v1 + with: + labels: 'ci: run CUDA builds' \ No newline at end of file From a8757ee266e0a95e3bd6beec16437c4c36c1fa56 Mon Sep 17 00:00:00 2001 From: Lionel Untereiner Date: Fri, 2 Aug 2024 17:54:12 +0200 Subject: [PATCH 07/24] wip: --- .github/workflows/gpu_builds.yml | 4 +++- .github/workflows/integrated_tests.yml | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/gpu_builds.yml b/.github/workflows/gpu_builds.yml index 670fed4bc64..d4df3a20d0d 100644 --- a/.github/workflows/gpu_builds.yml +++ b/.github/workflows/gpu_builds.yml @@ -13,7 +13,7 @@ on: jobs: get_docker_image_tag: # if the current added label is for GPUs - if: "${{ github.event.label.name == 'ci: run CUDA builds' && vars.COMPLIANCE_SUCCESS}}" + if: "${{ github.event.label.name == 'ci: run CUDA builds'}}" # Everywhere in this workflow, we use the most recent ubuntu distribution available in Github Actions # to ensure maximum support of google cloud's sdk. runs-on: ubuntu-22.04 @@ -130,6 +130,8 @@ jobs: secrets: inherit remove_label: + needs: + - cuda_builds runs-on: ubuntu-22.04 steps: - name: Remove the label diff --git a/.github/workflows/integrated_tests.yml b/.github/workflows/integrated_tests.yml index f2ef9d11fa1..c47f04e6141 100644 --- a/.github/workflows/integrated_tests.yml +++ b/.github/workflows/integrated_tests.yml @@ -2,6 +2,8 @@ name: Integrated Tests on: pull_request: + branches: + - develop types: [ labeled ] workflow_dispatch: @@ -55,6 +57,8 @@ jobs: LOCAL_BASELINE_DIR: /data/GEOS/baselines remove_label: + needs: + - run_integrated_tests runs-on: ubuntu-22.04 steps: - name: Remove the label From 99d3a78332658bb3239059529ec306bc0d9d5dbf Mon Sep 17 00:00:00 2001 From: Lionel Untereiner Date: Fri, 2 Aug 2024 18:37:46 +0200 Subject: [PATCH 08/24] wip: --- .github/workflows/compliance.yml | 2 -- .github/workflows/cpu_builds.yml | 41 ++++++++++++++++---------- .github/workflows/gpu_builds.yml | 1 + .github/workflows/integrated_tests.yml | 1 + 4 files changed, 27 insertions(+), 18 deletions(-) diff --git a/.github/workflows/compliance.yml b/.github/workflows/compliance.yml index 6208fd4ac27..c2a204ac95f 100644 --- a/.github/workflows/compliance.yml +++ b/.github/workflows/compliance.yml @@ -190,6 +190,4 @@ jobs: - get_docker_image_tag - check_that_all_jobs_succeeded uses: ./.github/workflows/cpu_builds.yml - with: - DOCKER_IMAGE_TAG: ${{ needs.get_docker_image_tag.outputs.DOCKER_IMAGE_TAG }} secrets: inherit \ No newline at end of file diff --git a/.github/workflows/cpu_builds.yml b/.github/workflows/cpu_builds.yml index 3780e1cbee9..a817224cdca 100644 --- a/.github/workflows/cpu_builds.yml +++ b/.github/workflows/cpu_builds.yml @@ -1,23 +1,8 @@ name: CPU builds on: -# pull_request: # Run workflow on PRs to the develop branch with changes on *.hpp or *.cpp files when PR is opened, synchronized, reopented and ready fo review -# branches: -# - develop -# paths: -# - '**.hpp' -# - '**.cpp' -# types: -# - opened -# - synchronize -# - reopened -# - ready_for_review # Allow this workflow to be called by the main workflow. workflow_call: - inputs: - DOCKER_IMAGE_TAG: - required: true - type: string push: # Run workflow on push to the develop branch. branches: - develop @@ -25,10 +10,34 @@ on: jobs: + get_docker_image_tag: + if: "${{ github.event.label.name == 'ci: run integrated tests' }}" + # Everywhere in this workflow, we use the most recent ubuntu distribution available in Github Actions + # to ensure maximum support of google cloud's sdk. + runs-on: ubuntu-22.04 + outputs: + DOCKER_IMAGE_TAG: ${{ steps.extract_docker_image_tag.outputs.DOCKER_IMAGE_TAG }} + steps: + # The TPL tag is contained in the codespaces configuration to avoid duplications. + - name: Checkout .devcontainer/devcontainer.json + uses: actions/checkout@v4.1.7 + with: + sparse-checkout: | + .devcontainer/devcontainer.json + sparse-checkout-cone-mode: false + submodules: false + lfs: false + fetch-depth: 1 + - name: Extract docker image tag + id: extract_docker_image_tag + run: | + echo "DOCKER_IMAGE_TAG=$(jq '.build.args.GEOS_TPL_TAG' -r .devcontainer/devcontainer.json)" >> "$GITHUB_OUTPUT" # Matrix containing all the CPU build. # Those are quite fast and can efficiently benefit from the `sccache' tool to make them even faster. cpu_builds: name: ${{ matrix.name }} + needs: + - get_docker_image_tag strategy: # In-progress jobs will not be cancelled if there is a failure fail-fast : false @@ -76,7 +85,7 @@ jobs: uses: ./.github/workflows/build_and_test.yml with: CMAKE_BUILD_TYPE: ${{ matrix.CMAKE_BUILD_TYPE }} - DOCKER_IMAGE_TAG: ${{ inputs.DOCKER_IMAGE_TAG }} + DOCKER_IMAGE_TAG: ${{ needs.get_docker_image_tag.outputs.DOCKER_IMAGE_TAG }} DOCKER_REPOSITORY: ${{ matrix.DOCKER_REPOSITORY }} ENABLE_HYPRE: ${{ matrix.ENABLE_HYPRE }} ENABLE_TRILINOS: ${{ matrix.ENABLE_TRILINOS }} diff --git a/.github/workflows/gpu_builds.yml b/.github/workflows/gpu_builds.yml index d4df3a20d0d..a9389bd7669 100644 --- a/.github/workflows/gpu_builds.yml +++ b/.github/workflows/gpu_builds.yml @@ -130,6 +130,7 @@ jobs: secrets: inherit remove_label: + if: always() needs: - cuda_builds runs-on: ubuntu-22.04 diff --git a/.github/workflows/integrated_tests.yml b/.github/workflows/integrated_tests.yml index c47f04e6141..ed06a91789a 100644 --- a/.github/workflows/integrated_tests.yml +++ b/.github/workflows/integrated_tests.yml @@ -57,6 +57,7 @@ jobs: LOCAL_BASELINE_DIR: /data/GEOS/baselines remove_label: + if: always() needs: - run_integrated_tests runs-on: ubuntu-22.04 From a5505b6172f822a5ae1d3aea2f2cdf1b6d67ab78 Mon Sep 17 00:00:00 2001 From: Lionel Untereiner Date: Fri, 2 Aug 2024 21:33:35 +0200 Subject: [PATCH 09/24] wip: --- .github/workflows/compliance.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/compliance.yml b/.github/workflows/compliance.yml index c2a204ac95f..fd010a32b97 100644 --- a/.github/workflows/compliance.yml +++ b/.github/workflows/compliance.yml @@ -187,7 +187,6 @@ jobs: cpu_builds: name: CPU builds needs: - - get_docker_image_tag - check_that_all_jobs_succeeded uses: ./.github/workflows/cpu_builds.yml secrets: inherit \ No newline at end of file From 432bf118e801c330978757effa7e306023b734c5 Mon Sep 17 00:00:00 2001 From: Lionel Untereiner Date: Fri, 2 Aug 2024 21:57:28 +0200 Subject: [PATCH 10/24] wip: --- .github/workflows/cpu_builds.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/cpu_builds.yml b/.github/workflows/cpu_builds.yml index a817224cdca..8a4007c5e52 100644 --- a/.github/workflows/cpu_builds.yml +++ b/.github/workflows/cpu_builds.yml @@ -11,7 +11,6 @@ on: jobs: get_docker_image_tag: - if: "${{ github.event.label.name == 'ci: run integrated tests' }}" # Everywhere in this workflow, we use the most recent ubuntu distribution available in Github Actions # to ensure maximum support of google cloud's sdk. runs-on: ubuntu-22.04 From 8f47576fe6018521c7c3c6d7fcaa9baf5aa2e793 Mon Sep 17 00:00:00 2001 From: Lionel Untereiner Date: Mon, 5 Aug 2024 10:32:09 +0200 Subject: [PATCH 11/24] wip: wrong label to remove --- .github/workflows/integrated_tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integrated_tests.yml b/.github/workflows/integrated_tests.yml index ed06a91789a..6784f1b2649 100644 --- a/.github/workflows/integrated_tests.yml +++ b/.github/workflows/integrated_tests.yml @@ -34,7 +34,7 @@ jobs: # If the 'ci: run integrated tests' PR label is found, the integrated tests will be run immediately after the cpu jobs. # Note: The integrated tests are optional and are (for the moment) run for convenience only. - run_integrated_tests: + run_integrated_tests: needs: - get_docker_image_tag uses: ./.github/workflows/build_and_test.yml @@ -65,4 +65,4 @@ jobs: - name: Remove the label uses: actions-ecosystem/action-remove-labels@v1 with: - labels: 'ci: run CUDA builds' \ No newline at end of file + labels: 'ci: run integrated tests' \ No newline at end of file From a7317288be2b6ad08e87fe5a431897665bc12f00 Mon Sep 17 00:00:00 2001 From: Lionel Untereiner Date: Mon, 5 Aug 2024 10:41:56 +0200 Subject: [PATCH 12/24] wip: erro rebase --- .github/workflows/ci_tests.yml | 400 --------------------------------- 1 file changed, 400 deletions(-) delete mode 100644 .github/workflows/ci_tests.yml diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_tests.yml deleted file mode 100644 index e8aa5c31ea0..00000000000 --- a/.github/workflows/ci_tests.yml +++ /dev/null @@ -1,400 +0,0 @@ -name: GEOS CI - -on: - push: - branches: - - develop - pull_request: - types: [opened, synchronize, reopened] - workflow_dispatch: - -# Cancels in-progress workflows for a PR when updated -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -# Please define `build.args.GEOS_TPL_TAG` in `.devcontainer/devcontainer.json` - -jobs: - # Checks if PR title follows conventional semantics - semantic_pull_request: - permissions: - pull-requests: write # for amannn/action-semantic-pull-request to analyze PRs and - statuses: write # for amannn/action-semantic-pull-request to mark status of analyzed PR - contents: read - runs-on: ubuntu-latest - - steps: - - name: Check if the PR name has conventional semantics - if: github.event_name == 'pull_request' - uses: amannn/action-semantic-pull-request@v5.5.3 - id: lint_pr_title - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - wip: true - # Configure that a scope doesn't need to be provided. - requireScope: false - - - name: Skip the check on develop branch - if: github.ref_name == 'develop' - run: | - echo "This in't a Pull-Request, skipping" - - # Jobs will be cancelled if PR is a draft. - # PR status must be "Open" to run CI. - is_not_draft_pull_request: - # if: ${{ always() }} - needs: [semantic_pull_request] - # Everywhere in this workflow, we use the most recent ubuntu distribution available in Github Actions - # to ensure maximum support of google cloud's sdk. - runs-on: ubuntu-22.04 - outputs: - DOCKER_IMAGE_TAG: ${{ steps.extract_docker_image_tag.outputs.DOCKER_IMAGE_TAG }} - steps: - - name: Check that the PR is not a draft (cancel rest of jobs otherwise) - id: extract_pr_info - run: | - if [[ ${{github.event_name}} == 'pull_request' ]]; then - # We do not rely on the `github.event.pull_request.labels` information since it's cached at the job. - # Changing labels or assignee in the PR would not allow to simply re-run the job with a different outcome. - pr_json=$(curl -H "Accept: application/vnd.github+json" https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.number }}) - - # We stop the workflow if the pr is draft - draft_status=$(echo ${pr_json} | jq '.draft') - echo "Draft status of PR is ${draft_status}." - if [[ $draft_status == true ]]; then exit 1 ; fi - fi - - - # The TPL tag is contained in the codespaces configuration to avoid duplications. - - name: Checkout .devcontainer/devcontainer.json - uses: actions/checkout@v4.1.7 - with: - sparse-checkout: | - .devcontainer/devcontainer.json - sparse-checkout-cone-mode: false - submodules: false - lfs: false - fetch-depth: 1 - - name: Extract docker image tag - id: extract_docker_image_tag - run: | - echo "DOCKER_IMAGE_TAG=$(jq '.build.args.GEOS_TPL_TAG' -r .devcontainer/devcontainer.json)" >> "$GITHUB_OUTPUT" - - # PR must be assigned to be merged. - # This job will fail if this is not the case. - if_not_unassigned_pull_request: - needs: [is_not_draft_pull_request] - runs-on: ubuntu-22.04 - steps: - - name: If this is a PR, Check that it is assigned - run: | - if [[ ${{github.event_name}} != 'pull_request' ]]; then exit 0 ; fi - pr_json=$(curl -H "Accept: application/vnd.github+json" https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.number }}) - NUM_ASSIGNEES=$(echo ${pr_json} | jq '.assignees | length') - echo "There are ${NUM_ASSIGNEES} assignees on this PR." - if [[ $NUM_ASSIGNEES == 0 ]]; then exit 1 ; fi - - # Validates that the PR is still pointing to the HEAD of the main branch of the submodules repositories. - # (There are exceptions, read the script about those). - are_submodules_in_sync: - needs: [is_not_draft_pull_request] - runs-on: ubuntu-22.04 - steps: - # The integrated test submodule repository contains large data (using git lfs). - # To save time (and money) we do not let Github Actions automatically clone all our (lfs) subrepositories and do it by hand. - - name: Checkout Repository - uses: actions/checkout@v4.1.7 - with: - # Let script update submodules; Github Actions submodule history causes error - submodules: false - lfs: false - fetch-depth: 1 - - name: Check that submodules are up to date - run: "scripts/test_submodule_updated.sh" - - check_code_style_and_documentation: - name: ${{ matrix.name }} - needs: [is_not_draft_pull_request] - strategy: - fail-fast : false - matrix: - include: - # Validates the code-style using uncrustify - - name: Check code style - BUILD_AND_TEST_ARGS: --test-code-style - # Validates that the documentation generated using doxygen has no hole. - - name: Check documentation - BUILD_AND_TEST_ARGS: --test-documentation - uses: ./.github/workflows/build_and_test.yml - with: - BUILD_AND_TEST_CLI_ARGS: ${{ matrix.BUILD_AND_TEST_ARGS }} - CMAKE_BUILD_TYPE: Release - DOCKER_IMAGE_TAG: ${{ needs.is_not_draft_pull_request.outputs.DOCKER_IMAGE_TAG }} - DOCKER_REPOSITORY: geosx/ubuntu20.04-gcc9 - RUNS_ON: ubuntu-22.04 - USE_SCCACHE: false - - # Matrix containing all the CPU build. - # Those are quite fast and can efficiently benefit from the `sccache' tool to make them even faster. - cpu_builds: - name: ${{ matrix.name }} - needs: [is_not_draft_pull_request] - strategy: - # In-progress jobs will not be cancelled if there is a failure - fail-fast : false - matrix: - include: - - name: Ubuntu (20.04, gcc 9.4.0, open-mpi 4.0.3) - CMAKE_BUILD_TYPE: Release - DOCKER_REPOSITORY: geosx/ubuntu20.04-gcc9 - BUILD_SHARED_LIBS: ON - - - name: Ubuntu debug (20.04, gcc 10.5.0, open-mpi 4.0.3) - github codespaces - CMAKE_BUILD_TYPE: Debug - DOCKER_REPOSITORY: geosx/ubuntu20.04-gcc10 - BUILD_SHARED_LIBS: ON - - - name: Ubuntu (20.04, gcc 10.5.0, open-mpi 4.0.3) - github codespaces - CMAKE_BUILD_TYPE: Release - DOCKER_REPOSITORY: geosx/ubuntu20.04-gcc10 - BUILD_SHARED_LIBS: ON - - - name: Ubuntu (22.04, gcc 11.4.0, open-mpi 4.1.2) - CMAKE_BUILD_TYPE: Release - DOCKER_REPOSITORY: geosx/ubuntu22.04-gcc11 - ENABLE_HYPRE: ON - ENABLE_TRILINOS: OFF - GCP_BUCKET: geosx/ubuntu22.04-gcc11 - BUILD_SHARED_LIBS: ON - - - name: Ubuntu (22.04, gcc 12.3.0, open-mpi 4.1.2) - CMAKE_BUILD_TYPE: Release - DOCKER_REPOSITORY: geosx/ubuntu22.04-gcc12 - ENABLE_HYPRE: ON - ENABLE_TRILINOS: OFF - BUILD_SHARED_LIBS: ON - - - name: Ubuntu (22.04, clang 15.0.7, open-mpi 4.1.2) - CMAKE_BUILD_TYPE: Release - DOCKER_REPOSITORY: geosx/ubuntu22.04-clang15 - ENABLE_HYPRE: ON - ENABLE_TRILINOS: OFF - BUILD_SHARED_LIBS: ON - - uses: ./.github/workflows/build_and_test.yml - with: - BUILD_SHARED_LIBS: ${{ matrix.BUILD_SHARED_LIBS }} - CMAKE_BUILD_TYPE: ${{ matrix.CMAKE_BUILD_TYPE }} - DOCKER_IMAGE_TAG: ${{ needs.is_not_draft_pull_request.outputs.DOCKER_IMAGE_TAG }} - DOCKER_REPOSITORY: ${{ matrix.DOCKER_REPOSITORY }} - ENABLE_HYPRE: ${{ matrix.ENABLE_HYPRE }} - ENABLE_TRILINOS: ${{ matrix.ENABLE_TRILINOS }} - GCP_BUCKET: ${{ matrix.GCP_BUCKET }} - RUNS_ON: ubuntu-22.04 - secrets: inherit - - # If the 'ci: run integrated tests' PR label is found, the integrated tests will be run immediately after the cpu jobs. - # Note: The integrated tests are optional and are (for the moment) run for convenience only. - run_integrated_tests: - needs: - - is_not_draft_pull_request - - cpu_builds - uses: ./.github/workflows/build_and_test.yml - secrets: inherit - with: - BUILD_AND_TEST_CLI_ARGS: --build-exe-only - BUILD_TYPE: integrated_tests - CMAKE_BUILD_TYPE: Release - DOCKER_IMAGE_TAG: ${{ needs.is_not_draft_pull_request.outputs.DOCKER_IMAGE_TAG }} - DOCKER_REPOSITORY: geosx/ubuntu22.04-gcc11 - ENABLE_HYPRE: ON - ENABLE_TRILINOS: OFF - GCP_BUCKET: geosx/integratedTests - RUNS_ON: streak2-32core - NPROC: 32 - DOCKER_RUN_ARGS: "--cpus=32 --memory=384g -v /etc/pki/ca-trust/source/anchors/:/usr/local/share/ca-certificates/llnl:ro" - DOCKER_CERTS_DIR: "/usr/local/share/ca-certificates" - DOCKER_CERTS_UPDATE_COMMAND: "update-ca-certificates" - REQUIRED_LABEL: "ci: run integrated tests" - LOCAL_BASELINE_DIR: /data/GEOS/baselines - - baseline_log: - needs: [is_not_draft_pull_request] - runs-on: ubuntu-22.04 - steps: - - name: Checkout Repository - uses: actions/checkout@v4.1.7 - with: - submodules: false - lfs: false - fetch-depth: 0 - sparse-checkout: | - scripts - - name: Check that the baseline logs are modified if rebaselines are detected - run: "scripts/check_baseline_log.sh" - - code_coverage: - needs: - - is_not_draft_pull_request - uses: ./.github/workflows/build_and_test.yml - secrets: inherit - with: - BUILD_AND_TEST_CLI_ARGS: "--no-run-unit-tests" - CMAKE_BUILD_TYPE: Debug - CODE_COVERAGE: true - DOCKER_IMAGE_TAG: ${{ needs.is_not_draft_pull_request.outputs.DOCKER_IMAGE_TAG }} - DOCKER_REPOSITORY: geosx/ubuntu22.04-gcc11 - ENABLE_HYPRE: ON - ENABLE_TRILINOS: OFF - GCP_BUCKET: geosx/ubuntu22.04-gcc11 - RUNS_ON: Runner_4core_16GB - REQUIRED_LABEL: "ci: run code coverage" - - - # mac_builds: - # needs: - # - is_not_draft_pull_request - # runs-on: macos-14-xlarge - # steps: - # - run: sysctl -n hw.physicalcpu - # - run: sysctl -h hw.memsize - # - run: sysctl -n machdep.cpu.brand_string - - - # If the 'ci: run CUDA builds' PR label is found, the cuda jobs run immediately along side linux jobs. - # Note: CUDA jobs should only be run if PR is ready to merge. - cuda_builds: - name: ${{ matrix.name }} - needs: - - is_not_draft_pull_request - strategy: - # In-progress jobs will not be cancelled if there is a failure - fail-fast : false - matrix: - include: - - name: Ubuntu CUDA debug (20.04, clang 10.0.0 + gcc 9.4.0, open-mpi 4.0.3, cuda-11.8.89) - BUILD_AND_TEST_CLI_ARGS: "--build-exe-only --no-install-schema" - CMAKE_BUILD_TYPE: Debug - BUILD_GENERATOR: "--ninja" - DOCKER_REPOSITORY: geosx/ubuntu20.04-clang10.0.0-cuda11.8.89 - ENABLE_HYPRE_DEVICE: CUDA - ENABLE_HYPRE: ON - ENABLE_TRILINOS: OFF - RUNS_ON: streak2 - NPROC: 8 - DOCKER_RUN_ARGS: "--cpus=8 --memory=128g --runtime=nvidia -v /etc/pki/ca-trust/source/anchors/:/usr/local/share/ca-certificates/llnl:ro" - DOCKER_CERTS_DIR: "/usr/local/share/ca-certificates" - DOCKER_CERTS_UPDATE_COMMAND: "update-ca-certificates" - - - name: Ubuntu CUDA (20.04, clang 10.0.0 + gcc 9.4.0, open-mpi 4.0.3, cuda-11.8.89) - BUILD_AND_TEST_CLI_ARGS: "--no-install-schema" - CMAKE_BUILD_TYPE: Release - BUILD_GENERATOR: "--ninja" - DOCKER_REPOSITORY: geosx/ubuntu20.04-clang10.0.0-cuda11.8.89 - ENABLE_HYPRE_DEVICE: CUDA - ENABLE_HYPRE: ON - ENABLE_TRILINOS: OFF - RUNS_ON: streak - NPROC: 8 - DOCKER_RUN_ARGS: "--cpus=8 --memory=256g --runtime=nvidia --gpus all -v /etc/pki/ca-trust/source/anchors/:/usr/local/share/ca-certificates/llnl:ro" - DOCKER_CERTS_DIR: "/usr/local/share/ca-certificates" - DOCKER_CERTS_UPDATE_COMMAND: "update-ca-certificates" - - - name: Rockylinux CUDA (8, clang 17.0.6, cuda 12.5.1) - BUILD_AND_TEST_CLI_ARGS: "--no-install-schema" - CMAKE_BUILD_TYPE: Release - BUILD_GENERATOR: "--ninja" - ENABLE_HYPRE_DEVICE: CUDA - ENABLE_HYPRE: ON - ENABLE_TRILINOS: OFF - DOCKER_REPOSITORY: geosx/rockylinux8-clang17-cuda12.5 - RUNS_ON: streak - NPROC: 8 - DOCKER_RUN_ARGS: "--cpus=8 --memory=256g --runtime=nvidia --gpus all -v /etc/pki/ca-trust/source/anchors/:/usr/local/share/ca-certificates/llnl:ro" - DOCKER_CERTS_DIR: "/usr/local/share/ca-certificates" - DOCKER_CERTS_UPDATE_COMMAND: "update-ca-trust" - - - name: Rockylinux CUDA (8, gcc 8.5, cuda 12.5.1) - BUILD_AND_TEST_CLI_ARGS: "--no-run-unit-tests --no-install-schema" - CMAKE_BUILD_TYPE: Release - BUILD_GENERATOR: "--ninja" - ENABLE_HYPRE_DEVICE: CUDA - ENABLE_HYPRE: ON - ENABLE_TRILINOS: OFF - DOCKER_REPOSITORY: geosx/rockylinux8-gcc8-cuda12.5 - RUNS_ON: streak2 - NPROC: 8 - DOCKER_RUN_ARGS: "--cpus=8 --memory=128g --runtime=nvidia -v /etc/pki/ca-trust/source/anchors/:/etc/pki/ca-trust/source/anchors/llnl:ro" - DOCKER_CERTS_DIR: "/etc/pki/ca-trust/source/anchors" - DOCKER_CERTS_UPDATE_COMMAND: "update-ca-trust" - - - name: Pangea 3 (AlmaLinux 8.8, gcc 9.4.0, open-mpi 4.1.2, cuda 11.5.0, openblas 0.3.10) - BUILD_AND_TEST_CLI_ARGS: "--build-exe-only --no-install-schema" - CMAKE_BUILD_TYPE: Release - BUILD_GENERATOR: "--makefile" - DOCKER_REPOSITORY: geosx/pangea3-almalinux8-gcc9.4-openmpi4.1.2-cuda11.5.0-openblas0.3.18 - HOST_CONFIG: host-configs/TOTAL/pangea3-gcc8.4.1-openmpi-4.1.2-wave-solver.cmake - ENABLE_HYPRE_DEVICE: CUDA - ENABLE_HYPRE: ON - ENABLE_TRILINOS: OFF - HOST_ARCH: ppc64le - RUNS_ON: streak2 - NPROC: 8 - DOCKER_RUN_ARGS: "--cpus=8 --memory=128g -v /etc/pki/ca-trust/source/anchors/:/etc/pki/ca-trust/source/anchors/llnl:ro" - DOCKER_CERTS_DIR: "/etc/pki/ca-trust/source/anchors" - DOCKER_CERTS_UPDATE_COMMAND: "update-ca-trust" - - - # Below this line, jobs that deploy to Google Cloud. - - uses: ./.github/workflows/build_and_test.yml - with: - BUILD_AND_TEST_CLI_ARGS: ${{ matrix.BUILD_AND_TEST_CLI_ARGS }} - CMAKE_BUILD_TYPE: ${{ matrix.CMAKE_BUILD_TYPE }} - BUILD_GENERATOR: ${{ matrix.BUILD_GENERATOR }} - DOCKER_CERTS_DIR: ${{ matrix.DOCKER_CERTS_DIR }} - DOCKER_CERTS_UPDATE_COMMAND: ${{ matrix.DOCKER_CERTS_UPDATE_COMMAND }} - DOCKER_IMAGE_TAG: ${{ needs.is_not_draft_pull_request.outputs.DOCKER_IMAGE_TAG }} - DOCKER_REPOSITORY: ${{ matrix.DOCKER_REPOSITORY }} - DOCKER_RUN_ARGS: ${{ matrix.DOCKER_RUN_ARGS }} - ENABLE_HYPRE_DEVICE: ${{ matrix.ENABLE_HYPRE_DEVICE }} - ENABLE_HYPRE: ${{ matrix.ENABLE_HYPRE }} - ENABLE_TRILINOS: ${{ matrix.ENABLE_TRILINOS }} - GCP_BUCKET: ${{ matrix.GCP_BUCKET }} - HOST_ARCH: ${{ matrix.HOST_ARCH }} - HOST_CONFIG: ${{ matrix.HOST_CONFIG }} - NPROC: ${{ matrix.NPROC }} - RUNS_ON: ${{ matrix.RUNS_ON }} - REQUIRED_LABEL: "ci: run CUDA builds" - secrets: inherit - - # Convenience job - passes when all other jobs have passed (must pass the CUDA jobs). - check_that_all_jobs_succeeded: - runs-on: ubuntu-22.04 - needs: - - if_not_unassigned_pull_request - - are_submodules_in_sync - - check_code_style_and_documentation - - cpu_builds - - cuda_builds - - run_integrated_tests - if: ${{ always() }} - steps: - - run: | - echo "if_not_unassigned_pull_request: ${{needs.if_not_unassigned_pull_request.result}}" - echo "are_submodules_in_sync: ${{needs.are_submodules_in_sync.result}}" - echo "check_code_style_and_documentation: ${{needs.check_code_style_and_documentation.result}}" - echo "cpu_builds: ${{needs.cpu_builds.result}}" - echo "cuda_builds: ${{needs.cuda_builds.result}}" - echo "run_integrated_tests: ${{needs.run_integrated_tests.result}} " - ${{ - needs.if_not_unassigned_pull_request.result == 'success' && - needs.are_submodules_in_sync.result == 'success' && - needs.check_code_style_and_documentation.result == 'success' && - needs.cpu_builds.result == 'success' && - needs.cuda_builds.result == 'success' && - needs.run_integrated_tests.result == 'success' - }} - From 8d5ee206e11f24e422ce84adf12f7f2ec4b27140 Mon Sep 17 00:00:00 2001 From: Lionel Untereiner Date: Mon, 5 Aug 2024 11:08:25 +0200 Subject: [PATCH 13/24] wip: remove sherlock --- .github/workflows/cpu_builds.yml | 8 -------- .github/workflows/gpu_builds.yml | 14 -------------- 2 files changed, 22 deletions(-) diff --git a/.github/workflows/cpu_builds.yml b/.github/workflows/cpu_builds.yml index 8a4007c5e52..8f7bc093f53 100644 --- a/.github/workflows/cpu_builds.yml +++ b/.github/workflows/cpu_builds.yml @@ -73,14 +73,6 @@ jobs: ENABLE_HYPRE: ON ENABLE_TRILINOS: OFF - - name: Sherlock CPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, openblas 0.3.10) - CMAKE_BUILD_TYPE: Release - DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-openblas0.3.10-zlib1.2.11 - ENABLE_HYPRE: ON - ENABLE_TRILINOS: OFF - GCP_BUCKET: geosx/Sherlock-CPU - HOST_CONFIG: host-configs/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10.cmake - uses: ./.github/workflows/build_and_test.yml with: CMAKE_BUILD_TYPE: ${{ matrix.CMAKE_BUILD_TYPE }} diff --git a/.github/workflows/gpu_builds.yml b/.github/workflows/gpu_builds.yml index a9389bd7669..9d50a81562e 100644 --- a/.github/workflows/gpu_builds.yml +++ b/.github/workflows/gpu_builds.yml @@ -95,20 +95,6 @@ jobs: # DOCKER_CERTS_UPDATE_COMMAND: "update-ca-trust" # Below this line, jobs that deploy to Google Cloud. - - name: Sherlock GPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, openblas 0.3.10, cuda 11.7.1,) - BUILD_AND_TEST_CLI_ARGS: "--no-run-unit-tests --no-install-schema" - CMAKE_BUILD_TYPE: Release - DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-cuda11.7.1-openblas0.3.10-zlib1.2.11 - ENABLE_HYPRE_DEVICE: CUDA - ENABLE_HYPRE: ON - ENABLE_TRILINOS: OFF - GCP_BUCKET: geosx/Sherlock-GPU - HOST_CONFIG: host-configs/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10-cuda11.7.1-sm70.cmake - RUNS_ON: streak2 - NPROC: 8 - DOCKER_RUN_ARGS: "--cpus=8 --memory=128g --runtime=nvidia -v /etc/pki/ca-trust/source/anchors/:/etc/pki/ca-trust/source/anchors/llnl:ro" - DOCKER_CERTS_DIR: "/etc/pki/ca-trust/source/anchors" - DOCKER_CERTS_UPDATE_COMMAND: "update-ca-trust" uses: ./.github/workflows/build_and_test.yml with: From 9caa469589310670c65c3eb2fcd27d7c2bbab322 Mon Sep 17 00:00:00 2001 From: Lionel Untereiner Date: Mon, 5 Aug 2024 11:23:46 +0200 Subject: [PATCH 14/24] wip: remove label on event != opened --- .github/workflows/compliance.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/compliance.yml b/.github/workflows/compliance.yml index fd010a32b97..c59a6effbc5 100644 --- a/.github/workflows/compliance.yml +++ b/.github/workflows/compliance.yml @@ -13,9 +13,22 @@ concurrency: # Please define `build.args.GEOS_TPL_TAG` in `.devcontainer/devcontainer.json` jobs: + # Alows remove labels at sync avoiding gpu / integrated tests run + remove_labels: + if: github.event.action != 'opened' + runs-on: ubuntu-22.04 + steps: + - name: Remove the label + uses: actions-ecosystem/action-remove-labels@v1 + with: + labels: | + 'ci: run CUDA builds' + 'ci: run integrated tests' + # Checks if PR title follows conventional semantics semantic_pull_request: name: Checks if PR title follows conventional semantics + needs: remove_labels permissions: pull-requests: write # for amannn/action-semantic-pull-request to analyze PRs and statuses: write # for amannn/action-semantic-pull-request to mark status of analyzed PR From 1a8f563bdd8a44d37e656165e06396f05e566dee Mon Sep 17 00:00:00 2001 From: Lionel Untereiner Date: Mon, 5 Aug 2024 11:26:36 +0200 Subject: [PATCH 15/24] wip: remove label on event != opened --- .github/workflows/compliance.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/compliance.yml b/.github/workflows/compliance.yml index c59a6effbc5..6a6f280362f 100644 --- a/.github/workflows/compliance.yml +++ b/.github/workflows/compliance.yml @@ -22,8 +22,8 @@ jobs: uses: actions-ecosystem/action-remove-labels@v1 with: labels: | - 'ci: run CUDA builds' - 'ci: run integrated tests' + ci: run CUDA builds + ci: run integrated tests # Checks if PR title follows conventional semantics semantic_pull_request: From 390dc3a99bed87a38ff7924e119e161b0d37807b Mon Sep 17 00:00:00 2001 From: Lionel Untereiner Date: Tue, 1 Oct 2024 16:00:10 +0200 Subject: [PATCH 16/24] ci: update to reflect recent changes --- .github/workflows/cpu_builds.yml | 8 +++- .github/workflows/gpu_builds.yml | 69 ++++++++++++++++++++++---------- 2 files changed, 54 insertions(+), 23 deletions(-) diff --git a/.github/workflows/cpu_builds.yml b/.github/workflows/cpu_builds.yml index 8f7bc093f53..8551a7f35c5 100644 --- a/.github/workflows/cpu_builds.yml +++ b/.github/workflows/cpu_builds.yml @@ -45,14 +45,17 @@ jobs: - name: Ubuntu (20.04, gcc 9.4.0, open-mpi 4.0.3) CMAKE_BUILD_TYPE: Release DOCKER_REPOSITORY: geosx/ubuntu20.04-gcc9 + BUILD_SHARED_LIBS: ON - name: Ubuntu debug (20.04, gcc 10.5.0, open-mpi 4.0.3) - github codespaces CMAKE_BUILD_TYPE: Debug DOCKER_REPOSITORY: geosx/ubuntu20.04-gcc10 + BUILD_SHARED_LIBS: ON - name: Ubuntu (20.04, gcc 10.5.0, open-mpi 4.0.3) - github codespaces CMAKE_BUILD_TYPE: Release DOCKER_REPOSITORY: geosx/ubuntu20.04-gcc10 + BUILD_SHARED_LIBS: ON - name: Ubuntu (22.04, gcc 11.4.0, open-mpi 4.1.2) CMAKE_BUILD_TYPE: Release @@ -60,27 +63,30 @@ jobs: ENABLE_HYPRE: ON ENABLE_TRILINOS: OFF GCP_BUCKET: geosx/ubuntu22.04-gcc11 + BUILD_SHARED_LIBS: ON - name: Ubuntu (22.04, gcc 12.3.0, open-mpi 4.1.2) CMAKE_BUILD_TYPE: Release DOCKER_REPOSITORY: geosx/ubuntu22.04-gcc12 ENABLE_HYPRE: ON ENABLE_TRILINOS: OFF + BUILD_SHARED_LIBS: ON - name: Ubuntu (22.04, clang 15.0.7, open-mpi 4.1.2) CMAKE_BUILD_TYPE: Release DOCKER_REPOSITORY: geosx/ubuntu22.04-clang15 ENABLE_HYPRE: ON ENABLE_TRILINOS: OFF + BUILD_SHARED_LIBS: ON uses: ./.github/workflows/build_and_test.yml with: + BUILD_SHARED_LIBS: ${{ matrix.BUILD_SHARED_LIBS }} CMAKE_BUILD_TYPE: ${{ matrix.CMAKE_BUILD_TYPE }} DOCKER_IMAGE_TAG: ${{ needs.get_docker_image_tag.outputs.DOCKER_IMAGE_TAG }} DOCKER_REPOSITORY: ${{ matrix.DOCKER_REPOSITORY }} ENABLE_HYPRE: ${{ matrix.ENABLE_HYPRE }} ENABLE_TRILINOS: ${{ matrix.ENABLE_TRILINOS }} GCP_BUCKET: ${{ matrix.GCP_BUCKET }} - HOST_CONFIG: ${{ matrix.HOST_CONFIG }} RUNS_ON: ubuntu-22.04 secrets: inherit diff --git a/.github/workflows/gpu_builds.yml b/.github/workflows/gpu_builds.yml index 9d50a81562e..f20ba5293fa 100644 --- a/.github/workflows/gpu_builds.yml +++ b/.github/workflows/gpu_builds.yml @@ -49,6 +49,7 @@ jobs: - name: Ubuntu CUDA debug (20.04, clang 10.0.0 + gcc 9.4.0, open-mpi 4.0.3, cuda-11.8.89) BUILD_AND_TEST_CLI_ARGS: "--build-exe-only --no-install-schema" CMAKE_BUILD_TYPE: Debug + BUILD_GENERATOR: "--ninja" DOCKER_REPOSITORY: geosx/ubuntu20.04-clang10.0.0-cuda11.8.89 ENABLE_HYPRE_DEVICE: CUDA ENABLE_HYPRE: ON @@ -62,6 +63,7 @@ jobs: - name: Ubuntu CUDA (20.04, clang 10.0.0 + gcc 9.4.0, open-mpi 4.0.3, cuda-11.8.89) BUILD_AND_TEST_CLI_ARGS: "--no-install-schema" CMAKE_BUILD_TYPE: Release + BUILD_GENERATOR: "--ninja" DOCKER_REPOSITORY: geosx/ubuntu20.04-clang10.0.0-cuda11.8.89 ENABLE_HYPRE_DEVICE: CUDA ENABLE_HYPRE: ON @@ -72,27 +74,49 @@ jobs: DOCKER_CERTS_DIR: "/usr/local/share/ca-certificates" DOCKER_CERTS_UPDATE_COMMAND: "update-ca-certificates" - # compiler error in ElasticFirstOrderWaveEquationSEMKernel::StressComputation::launch in call to FE_TYPE::computeFirstOrderStiffnessTermX - # - name: Rockylinux (8, clang 17.0.6, cuda 12.5) - # BUILD_AND_TEST_CLI_ARGS: "--no-run-unit-tests --no-install-schema" - # CMAKE_BUILD_TYPE: Release - # DOCKER_REPOSITORY: geosx/rockylinux8-clang17-cuda12.5 - # RUNS_ON: streak2 - # NPROC: 2 - # DOCKER_RUN_ARGS: "--cpus=1 --memory=128g --runtime=nvidia -v /etc/pki/ca-trust/source/anchors/:/etc/pki/ca-trust/source/anchors/llnl:ro" - # DOCKER_CERTS_DIR: "/etc/pki/ca-trust/source/anchors" - # DOCKER_CERTS_UPDATE_COMMAND: "update-ca-trust" - - # compiler error in ElasticFirstOrderWaveEquationSEMKernel::StressComputation::launch in call to FE_TYPE::computeFirstOrderStiffnessTermX - # - name: Rockylinux (8, gcc 8.5, cuda 12.5) - # BUILD_AND_TEST_CLI_ARGS: "--no-run-unit-tests --no-install-schema" - # CMAKE_BUILD_TYPE: Release - # DOCKER_REPOSITORY: geosx/rockylinux8-gcc8-cuda12.5 - # RUNS_ON: streak2 - # NPROC: 2 - # DOCKER_RUN_ARGS: "--cpus=1 --memory=128g --runtime=nvidia -v /etc/pki/ca-trust/source/anchors/:/etc/pki/ca-trust/source/anchors/llnl:ro" - # DOCKER_CERTS_DIR: "/etc/pki/ca-trust/source/anchors" - # DOCKER_CERTS_UPDATE_COMMAND: "update-ca-trust" + - name: Rockylinux CUDA (8, clang 17.0.6, cuda 12.5.1) + BUILD_AND_TEST_CLI_ARGS: "--no-install-schema" + CMAKE_BUILD_TYPE: Release + BUILD_GENERATOR: "--ninja" + ENABLE_HYPRE_DEVICE: CUDA + ENABLE_HYPRE: ON + ENABLE_TRILINOS: OFF + DOCKER_REPOSITORY: geosx/rockylinux8-clang17-cuda12.5 + RUNS_ON: streak + NPROC: 8 + DOCKER_RUN_ARGS: "--cpus=8 --memory=256g --runtime=nvidia --gpus all -v /etc/pki/ca-trust/source/anchors/:/usr/local/share/ca-certificates/llnl:ro" + DOCKER_CERTS_DIR: "/usr/local/share/ca-certificates" + DOCKER_CERTS_UPDATE_COMMAND: "update-ca-trust" + + - name: Rockylinux CUDA (8, gcc 8.5, cuda 12.5.1) + BUILD_AND_TEST_CLI_ARGS: "--no-run-unit-tests --no-install-schema" + CMAKE_BUILD_TYPE: Release + BUILD_GENERATOR: "--ninja" + ENABLE_HYPRE_DEVICE: CUDA + ENABLE_HYPRE: ON + ENABLE_TRILINOS: OFF + DOCKER_REPOSITORY: geosx/rockylinux8-gcc8-cuda12.5 + RUNS_ON: streak2 + NPROC: 8 + DOCKER_RUN_ARGS: "--cpus=8 --memory=128g --runtime=nvidia -v /etc/pki/ca-trust/source/anchors/:/etc/pki/ca-trust/source/anchors/llnl:ro" + DOCKER_CERTS_DIR: "/etc/pki/ca-trust/source/anchors" + DOCKER_CERTS_UPDATE_COMMAND: "update-ca-trust" + + - name: Pangea 3 (AlmaLinux 8.8, gcc 9.4.0, open-mpi 4.1.2, cuda 11.5.0, openblas 0.3.10) + BUILD_AND_TEST_CLI_ARGS: "--build-exe-only --no-install-schema" + CMAKE_BUILD_TYPE: Release + BUILD_GENERATOR: "--makefile" + DOCKER_REPOSITORY: geosx/pangea3-almalinux8-gcc9.4-openmpi4.1.2-cuda11.5.0-openblas0.3.18 + HOST_CONFIG: host-configs/TOTAL/pangea3-gcc8.4.1-openmpi-4.1.2-wave-solver.cmake + ENABLE_HYPRE_DEVICE: CUDA + ENABLE_HYPRE: ON + ENABLE_TRILINOS: OFF + HOST_ARCH: ppc64le + RUNS_ON: streak2 + NPROC: 8 + DOCKER_RUN_ARGS: "--cpus=8 --memory=128g -v /etc/pki/ca-trust/source/anchors/:/etc/pki/ca-trust/source/anchors/llnl:ro" + DOCKER_CERTS_DIR: "/etc/pki/ca-trust/source/anchors" + DOCKER_CERTS_UPDATE_COMMAND: "update-ca-trust" # Below this line, jobs that deploy to Google Cloud. @@ -100,6 +124,7 @@ jobs: with: BUILD_AND_TEST_CLI_ARGS: ${{ matrix.BUILD_AND_TEST_CLI_ARGS }} CMAKE_BUILD_TYPE: ${{ matrix.CMAKE_BUILD_TYPE }} + BUILD_GENERATOR: ${{ matrix.BUILD_GENERATOR }} DOCKER_CERTS_DIR: ${{ matrix.DOCKER_CERTS_DIR }} DOCKER_CERTS_UPDATE_COMMAND: ${{ matrix.DOCKER_CERTS_UPDATE_COMMAND }} DOCKER_IMAGE_TAG: ${{ needs.get_docker_image_tag.outputs.DOCKER_IMAGE_TAG }} @@ -108,7 +133,7 @@ jobs: ENABLE_HYPRE_DEVICE: ${{ matrix.ENABLE_HYPRE_DEVICE }} ENABLE_HYPRE: ${{ matrix.ENABLE_HYPRE }} ENABLE_TRILINOS: ${{ matrix.ENABLE_TRILINOS }} - GCP_BUCKET: ${{ matrix.GCP_BUCKET }} + HOST_ARCH: ${{ matrix.HOST_ARCH }} HOST_CONFIG: ${{ matrix.HOST_CONFIG }} NPROC: ${{ matrix.NPROC }} RUNS_ON: ${{ matrix.RUNS_ON }} From 7462e3630c324852b72484a32cce91fe9dad79a4 Mon Sep 17 00:00:00 2001 From: Lionel Untereiner Date: Wed, 2 Oct 2024 10:49:04 +0200 Subject: [PATCH 17/24] extract coverage to its own --- .github/workflows/compliance.yml | 17 --------- .github/workflows/coverage.yml | 64 ++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/coverage.yml diff --git a/.github/workflows/compliance.yml b/.github/workflows/compliance.yml index 6a6f280362f..85162b42608 100644 --- a/.github/workflows/compliance.yml +++ b/.github/workflows/compliance.yml @@ -180,23 +180,6 @@ jobs: needs.check_code_style_and_documentation.result == 'success' }}" >> $GITHUB_ENV - code_coverage: - needs: - - get_docker_image_tag - - check_that_all_jobs_succeeded - uses: ./.github/workflows/build_and_test.yml - secrets: inherit - with: - BUILD_AND_TEST_CLI_ARGS: "--no-run-unit-tests" - CMAKE_BUILD_TYPE: Debug - CODE_COVERAGE: true - DOCKER_IMAGE_TAG: ${{ needs.get_docker_image_tag.outputs.DOCKER_IMAGE_TAG }} - DOCKER_REPOSITORY: geosx/ubuntu22.04-gcc11 - ENABLE_HYPRE: ON - ENABLE_TRILINOS: OFF - GCP_BUCKET: geosx/ubuntu22.04-gcc11 - RUNS_ON: Runner_4core_16GB - cpu_builds: name: CPU builds needs: diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml new file mode 100644 index 00000000000..ebe23aafd84 --- /dev/null +++ b/.github/workflows/coverage.yml @@ -0,0 +1,64 @@ +name: Code coverage + +on: + pull_request: # Run workflow on PRs to the develop branch on labeled event + branches: + - develop + types: [ labeled ] + push: # Run workflow on push to the develop branch. + branches: + - develop + workflow_dispatch: # Workflow can be run manually + +jobs: + get_docker_image_tag: + # if the current added label is for GPUs + if: "${{ github.event.label.name == 'ci: run code coverage'}}" + # Everywhere in this workflow, we use the most recent ubuntu distribution available in Github Actions + # to ensure maximum support of google cloud's sdk. + runs-on: ubuntu-22.04 + outputs: + DOCKER_IMAGE_TAG: ${{ steps.extract_docker_image_tag.outputs.DOCKER_IMAGE_TAG }} + steps: + # The TPL tag is contained in the codespaces configuration to avoid duplications. + - name: Checkout .devcontainer/devcontainer.json + uses: actions/checkout@v4.1.7 + with: + sparse-checkout: | + .devcontainer/devcontainer.json + sparse-checkout-cone-mode: false + submodules: false + lfs: false + fetch-depth: 1 + - name: Extract docker image tag + id: extract_docker_image_tag + run: | + echo "DOCKER_IMAGE_TAG=$(jq '.build.args.GEOS_TPL_TAG' -r .devcontainer/devcontainer.json)" >> "$GITHUB_OUTPUT" + + # If the 'ci: run code coverage' PR label is found, the cuda jobs run immediately along side linux jobs. + code_coverage: + needs: + - get_docker_image_tag + uses: ./.github/workflows/build_and_test.yml + with: + BUILD_AND_TEST_CLI_ARGS: "--no-run-unit-tests" + CMAKE_BUILD_TYPE: Debug + CODE_COVERAGE: true + DOCKER_IMAGE_TAG: ${{ needs.get_docker_image_tag.outputs.DOCKER_IMAGE_TAG }} + DOCKER_REPOSITORY: geosx/ubuntu22.04-gcc11 + ENABLE_HYPRE: ON + ENABLE_TRILINOS: OFF + GCP_BUCKET: geosx/ubuntu22.04-gcc11 + RUNS_ON: Runner_4core_16GB + secrets: inherit + + remove_label: + if: always() + needs: + - code_coverage + runs-on: ubuntu-22.04 + steps: + - name: Remove the label + uses: actions-ecosystem/action-remove-labels@v1 + with: + labels: 'ci: run code coverage' \ No newline at end of file From 2c69a61d3393021096206cd6af2166f15bd631fc Mon Sep 17 00:00:00 2001 From: Lionel Untereiner Date: Wed, 2 Oct 2024 14:46:58 +0200 Subject: [PATCH 18/24] ci: test next version computation --- .github/workflows/release.yml | 52 +++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000000..50822a4c0d8 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,52 @@ +name: Release + +on: + workflow_dispatch: + +env: + BUMP_BEHAVIOR: Monday + +jobs: + deploy: + runs-on: ubuntu-latest + + steps: + - name: Checkout Code + uses: actions/checkout@v4.1.7 + with: + submodules: false + lfs: false + + # - name: Detect API Change + # id: changed-api + # uses: tj-actions/changed-files@v43 + # with: + # # Avoid using single or double quotes for multiline patterns + # files: | + # src/coreComponents/schema/**/*.{xsd,rst} + + # - name: Run step if test file(s) change + # if: steps.changed-api.outputs.any_changed == 'true' + # env: + # BUMP_BEHAVIOR: "major" + # run: | + # echo "Must be a major change" + + - name: Get Next Version + id: semver + uses: ietf-tools/semver-action@v1.8.0 + with: + token: ${{ secrets.GITHUB_TOKEN }} + branch: develop + noVersionBumpBehavior: patch + + # - name: Create Release + # uses: ncipollo/release-action@v1.14.0 + # with: + # allowUpdates: true + # draft: false + # makeLatest: true + # generateReleaseNotes: true + # name: ${{ steps.semver.outputs.next }} + # body: Changelog Contents + # token: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file From 6c4107f1ae17b9bad6dc051dd54d3ea56bf41d6c Mon Sep 17 00:00:00 2001 From: Lionel Untereiner Date: Wed, 2 Oct 2024 15:06:08 +0200 Subject: [PATCH 19/24] ci(draft): test next version computation in the PR --- .github/workflows/compliance.yml | 31 +------------------------- .github/workflows/coverage.yml | 29 ++++++++++++------------ .github/workflows/gpu_builds.yml | 30 ++++++++++++------------- .github/workflows/integrated_tests.yml | 24 ++++++++++---------- .github/workflows/release.yml | 3 +++ 5 files changed, 44 insertions(+), 73 deletions(-) diff --git a/.github/workflows/compliance.yml b/.github/workflows/compliance.yml index 85162b42608..a27094fbab3 100644 --- a/.github/workflows/compliance.yml +++ b/.github/workflows/compliance.yml @@ -13,28 +13,14 @@ concurrency: # Please define `build.args.GEOS_TPL_TAG` in `.devcontainer/devcontainer.json` jobs: - # Alows remove labels at sync avoiding gpu / integrated tests run - remove_labels: - if: github.event.action != 'opened' - runs-on: ubuntu-22.04 - steps: - - name: Remove the label - uses: actions-ecosystem/action-remove-labels@v1 - with: - labels: | - ci: run CUDA builds - ci: run integrated tests - # Checks if PR title follows conventional semantics semantic_pull_request: name: Checks if PR title follows conventional semantics - needs: remove_labels permissions: pull-requests: write # for amannn/action-semantic-pull-request to analyze PRs and statuses: write # for amannn/action-semantic-pull-request to mark status of analyzed PR contents: read runs-on: ubuntu-latest - steps: - name: Check if the PR name has conventional semantics if: github.event_name == 'pull_request' @@ -57,22 +43,7 @@ jobs: runs-on: ubuntu-22.04 outputs: DOCKER_IMAGE_TAG: ${{ steps.extract_docker_image_tag.outputs.DOCKER_IMAGE_TAG }} - steps: - # - name: Check that the PR is not a draft (cancel rest of jobs otherwise) - # id: extract_pr_info - # run: | - # if [[ ${{github.event_name}} == 'pull_request' ]]; then - # # We do not rely on the `github.event.pull_request.labels` information since it's cached at the job. - # # Changing labels or assignee in the PR would not allow to simply re-run the job with a different outcome. - # pr_json=$(curl -H "Accept: application/vnd.github+json" https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.number }}) - - # # We stop the workflow if the pr is draft - # draft_status=$(echo ${pr_json} | jq '.draft') - # echo "Draft status of PR is ${draft_status}." - # if [[ $draft_status == true ]]; then exit 1 ; fi - # fi - - + steps: # The TPL tag is contained in the codespaces configuration to avoid duplications. - name: Checkout .devcontainer/devcontainer.json uses: actions/checkout@v4.1.7 diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index ebe23aafd84..3144a054987 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -11,9 +11,20 @@ on: workflow_dispatch: # Workflow can be run manually jobs: - get_docker_image_tag: - # if the current added label is for GPUs + # If the 'ci: run code coverage' PR label is found, the coverage jobs run + remove_label: if: "${{ github.event.label.name == 'ci: run code coverage'}}" + runs-on: ubuntu-22.04 + steps: + - name: Remove the label + uses: actions-ecosystem/action-remove-labels@v1.3.0 + with: + labels: 'ci: run code coverage' + + # first get the docker image + get_docker_image_tag: + needs: + - remove_label # Everywhere in this workflow, we use the most recent ubuntu distribution available in Github Actions # to ensure maximum support of google cloud's sdk. runs-on: ubuntu-22.04 @@ -35,7 +46,6 @@ jobs: run: | echo "DOCKER_IMAGE_TAG=$(jq '.build.args.GEOS_TPL_TAG' -r .devcontainer/devcontainer.json)" >> "$GITHUB_OUTPUT" - # If the 'ci: run code coverage' PR label is found, the cuda jobs run immediately along side linux jobs. code_coverage: needs: - get_docker_image_tag @@ -50,15 +60,4 @@ jobs: ENABLE_TRILINOS: OFF GCP_BUCKET: geosx/ubuntu22.04-gcc11 RUNS_ON: Runner_4core_16GB - secrets: inherit - - remove_label: - if: always() - needs: - - code_coverage - runs-on: ubuntu-22.04 - steps: - - name: Remove the label - uses: actions-ecosystem/action-remove-labels@v1 - with: - labels: 'ci: run code coverage' \ No newline at end of file + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/gpu_builds.yml b/.github/workflows/gpu_builds.yml index f20ba5293fa..8a0b069f7a4 100644 --- a/.github/workflows/gpu_builds.yml +++ b/.github/workflows/gpu_builds.yml @@ -11,9 +11,19 @@ on: workflow_dispatch: # Workflow can be run manually jobs: - get_docker_image_tag: - # if the current added label is for GPUs - if: "${{ github.event.label.name == 'ci: run CUDA builds'}}" + # If the 'ci: run CUDA builds' PR label is found, the cuda jobs run. + remove_label: + if: "${{ github.event.label.name == 'ci: run CUDA builds'}}" + runs-on: ubuntu-22.04 + steps: + - name: Remove the label + uses: actions-ecosystem/action-remove-labels@v1.3.0 + with: + labels: 'ci: run CUDA builds' + + get_docker_image_tag: + needs: + - remove_label # Everywhere in this workflow, we use the most recent ubuntu distribution available in Github Actions # to ensure maximum support of google cloud's sdk. runs-on: ubuntu-22.04 @@ -35,7 +45,6 @@ jobs: run: | echo "DOCKER_IMAGE_TAG=$(jq '.build.args.GEOS_TPL_TAG' -r .devcontainer/devcontainer.json)" >> "$GITHUB_OUTPUT" - # If the 'ci: run CUDA builds' PR label is found, the cuda jobs run immediately along side linux jobs. # Note: CUDA jobs should only be run if PR is ready to merge. cuda_builds: needs: @@ -138,15 +147,4 @@ jobs: NPROC: ${{ matrix.NPROC }} RUNS_ON: ${{ matrix.RUNS_ON }} REQUIRED_LABEL: "ci: run CUDA builds" - secrets: inherit - - remove_label: - if: always() - needs: - - cuda_builds - runs-on: ubuntu-22.04 - steps: - - name: Remove the label - uses: actions-ecosystem/action-remove-labels@v1 - with: - labels: 'ci: run CUDA builds' \ No newline at end of file + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/integrated_tests.yml b/.github/workflows/integrated_tests.yml index 6784f1b2649..793f0e5411e 100644 --- a/.github/workflows/integrated_tests.yml +++ b/.github/workflows/integrated_tests.yml @@ -8,8 +8,19 @@ on: workflow_dispatch: jobs: + # If the 'ci: run integrated tests' PR label is found, the integrated tests jobs run. + remove_label: + if: "${{ github.event.label.name == 'ci: run integrated tests' }}" + runs-on: ubuntu-22.04 + steps: + - name: Remove the label + uses: actions-ecosystem/action-remove-labels@v1.3.0 + with: + labels: 'ci: run integrated tests' + get_docker_image_tag: - if: "${{ github.event.label.name == 'ci: run integrated tests' }}" + needs: + - remove_label # Everywhere in this workflow, we use the most recent ubuntu distribution available in Github Actions # to ensure maximum support of google cloud's sdk. runs-on: ubuntu-22.04 @@ -55,14 +66,3 @@ jobs: DOCKER_CERTS_UPDATE_COMMAND: "update-ca-certificates" REQUIRED_LABEL: "ci: run integrated tests" LOCAL_BASELINE_DIR: /data/GEOS/baselines - - remove_label: - if: always() - needs: - - run_integrated_tests - runs-on: ubuntu-22.04 - steps: - - name: Remove the label - uses: actions-ecosystem/action-remove-labels@v1 - with: - labels: 'ci: run integrated tests' \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 50822a4c0d8..e41884f8dc7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,6 +1,9 @@ name: Release on: + pull_request: # Run workflow on PRs to the develop branch + branches: + - develop workflow_dispatch: env: From 9b301e165e761ca6d7953f358a3a08b3318a1970 Mon Sep 17 00:00:00 2001 From: Lionel Untereiner Date: Wed, 2 Oct 2024 15:32:53 +0200 Subject: [PATCH 20/24] ci(draft): test release notes computation --- .github/workflows/release.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e41884f8dc7..3ab9e60e53a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -43,6 +43,11 @@ jobs: branch: develop noVersionBumpBehavior: patch + - name: Build Changelog + id: github_release + uses: mikepenz/release-changelog-builder-action@v5 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # - name: Create Release # uses: ncipollo/release-action@v1.14.0 # with: @@ -51,5 +56,5 @@ jobs: # makeLatest: true # generateReleaseNotes: true # name: ${{ steps.semver.outputs.next }} - # body: Changelog Contents + # body: ${{ steps.github_release.outputs.changelog }} # token: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file From 8fbc3b93324442ee0c87924f5e0dc57d920d59fc Mon Sep 17 00:00:00 2001 From: Lionel Untereiner Date: Wed, 2 Oct 2024 15:47:34 +0200 Subject: [PATCH 21/24] ci(draft): test release notes print in terminal --- .github/workflows/release.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3ab9e60e53a..d680618c2ff 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -46,8 +46,15 @@ jobs: - name: Build Changelog id: github_release uses: mikepenz/release-changelog-builder-action@v5 + with: + fromTag: ${{ steps.semver.outputs.current }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Print debug + run: | + echo "${{ steps.github_release.outputs.changelog }}" + # - name: Create Release # uses: ncipollo/release-action@v1.14.0 # with: From 77a650916dafd55276e5e6cee7c6d38923d5684b Mon Sep 17 00:00:00 2001 From: Lionel Untereiner Date: Wed, 2 Oct 2024 16:17:27 +0200 Subject: [PATCH 22/24] ci(draft): new deployment workflow --- .github/workflows/deploy.yml | 79 +++++++++++++++++++++++++++++++++++ .github/workflows/release.yml | 67 ----------------------------- 2 files changed, 79 insertions(+), 67 deletions(-) create mode 100644 .github/workflows/deploy.yml delete mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 00000000000..a263775abfb --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,79 @@ +name: Deploy + +on: + pull_request: # Run workflow on PRs to the develop branch + branches: + - develop + workflow_dispatch: + + +jobs: + deploy: + runs-on: ubuntu-latest + + steps: + - name: Checkout Code + uses: actions/checkout@v4.1.7 + with: + submodules: false + lfs: false + + # - name: Detect API Change + # id: changed-api + # uses: tj-actions/changed-files@v43 + # with: + # # Avoid using single or double quotes for multiline patterns + # files: | + # src/coreComponents/schema/**/*.{xsd,rst} + + # - name: Run step if test file(s) change + # if: steps.changed-api.outputs.any_changed == 'true' + # env: + # BUMP_BEHAVIOR: "major" + # run: | + # echo "Must be a major change" + + - name: Get Next Version + id: semver + uses: ietf-tools/semver-action@v1.8.0 + with: + token: ${{ secrets.GITHUB_TOKEN }} + branch: develop + noVersionBumpBehavior: patch + + - name: Create Draft Release + uses: ncipollo/release-action@v1.14.0 + with: + prerelease: true + draft: false + commit: ${{ github.sha }} + tag: ${{ steps.semver.outputs.nextStrict }} + name: ${{ steps.semver.outputs.nextStrict }} + body: '*pending*' + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Update CHANGELOG + id: changelog + uses: requarks/changelog-action@v1.10.2 + with: + token: ${{ github.token }} + fromTag: ${{ steps.semver.outputs.nextStrict }} + toTag: ${{ steps.semver.outputs.current }} + excludeTypes: '' + writeToFile: false + + - name: Extract docker image tag + id: extract_docker_image_tag + run: | + echo "${{ steps.changelog.outputs.changes }}" + + - name: Create Release + uses: ncipollo/release-action@v1.14.0 + with: + allowUpdates: true + makeLatest: true + draft: false + tag: ${{ steps.semver.outputs.nextStrict }} + name: ${{ steps.semver.outputs.nextStrict }} + body: ${{ steps.changelog.outputs.changes }} + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index d680618c2ff..00000000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,67 +0,0 @@ -name: Release - -on: - pull_request: # Run workflow on PRs to the develop branch - branches: - - develop - workflow_dispatch: - -env: - BUMP_BEHAVIOR: Monday - -jobs: - deploy: - runs-on: ubuntu-latest - - steps: - - name: Checkout Code - uses: actions/checkout@v4.1.7 - with: - submodules: false - lfs: false - - # - name: Detect API Change - # id: changed-api - # uses: tj-actions/changed-files@v43 - # with: - # # Avoid using single or double quotes for multiline patterns - # files: | - # src/coreComponents/schema/**/*.{xsd,rst} - - # - name: Run step if test file(s) change - # if: steps.changed-api.outputs.any_changed == 'true' - # env: - # BUMP_BEHAVIOR: "major" - # run: | - # echo "Must be a major change" - - - name: Get Next Version - id: semver - uses: ietf-tools/semver-action@v1.8.0 - with: - token: ${{ secrets.GITHUB_TOKEN }} - branch: develop - noVersionBumpBehavior: patch - - - name: Build Changelog - id: github_release - uses: mikepenz/release-changelog-builder-action@v5 - with: - fromTag: ${{ steps.semver.outputs.current }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Print debug - run: | - echo "${{ steps.github_release.outputs.changelog }}" - - # - name: Create Release - # uses: ncipollo/release-action@v1.14.0 - # with: - # allowUpdates: true - # draft: false - # makeLatest: true - # generateReleaseNotes: true - # name: ${{ steps.semver.outputs.next }} - # body: ${{ steps.github_release.outputs.changelog }} - # token: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file From baf47cacb2155953d5811d8cd6f4cbdeb28d2ba8 Mon Sep 17 00:00:00 2001 From: Lionel Untereiner Date: Wed, 2 Oct 2024 16:20:01 +0200 Subject: [PATCH 23/24] ci: new deployment worflow should be ready --- .github/workflows/deploy.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index a263775abfb..be4e252a23c 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -1,9 +1,6 @@ name: Deploy on: - pull_request: # Run workflow on PRs to the develop branch - branches: - - develop workflow_dispatch: @@ -62,11 +59,6 @@ jobs: excludeTypes: '' writeToFile: false - - name: Extract docker image tag - id: extract_docker_image_tag - run: | - echo "${{ steps.changelog.outputs.changes }}" - - name: Create Release uses: ncipollo/release-action@v1.14.0 with: From 98eebae864f5b57c9cb6e69be38db3fd0b3c72a0 Mon Sep 17 00:00:00 2001 From: Lionel Untereiner Date: Thu, 3 Oct 2024 10:59:19 +0200 Subject: [PATCH 24/24] ci: wip build number --- .github/workflows/build_number.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .github/workflows/build_number.yml diff --git a/.github/workflows/build_number.yml b/.github/workflows/build_number.yml new file mode 100644 index 00000000000..0c3125cac87 --- /dev/null +++ b/.github/workflows/build_number.yml @@ -0,0 +1,22 @@ +name: Compute a build number + +on: + pull_request: + branches: + - develop + workflow_dispatch: + + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Generate build number + id: buildnumber + uses: onyxmueller/build-tag-number@v1 + with: + token: ${{secrets.github_token}} + + - name: Echo + run: | + echo ${{ steps.buildnumber.outputs.build_number }} \ No newline at end of file