Merge pull request #1378 from nf-core/improve_cloud_tests_matrix

Improve cloud testing matrix strategy
nf-core · Apr 25, 2024 · 752494f · 752494f
2 parents 55e6839 + 1275645
commit 752494f
Show file tree

Hide file tree

Showing 2 changed files with 43 additions and 44 deletions.
diff --git a/.github/workflows/cloudtest.yml b/.github/workflows/cloudtest.yml
@@ -31,79 +31,76 @@ on:
         default: true
 
 jobs:
-  trigger-profile-test:
-    name: Run AWS tests
+  trigger-test:
+    name: launch
     runs-on: ubuntu-latest
+    if: ${{ github.repository == 'nf-core/sarek' }}
     strategy:
       fail-fast: false
       matrix:
         include:
           - profile: test_aws
-            enabled: ${{ ( github.repository == 'nf-core/sarek' ) && ( github.event_name != 'workflow_dispatch' || ( inputs.test && inputs.aws ) ) }}
+            test: test
+            cloud: aws
+            compute_env: TOWER_COMPUTE_ENV
+            workdir: TOWER_BUCKET_AWS
           - profile: test_azure
-            enabled: ${{ ( github.repository == 'nf-core/sarek' ) && ( github.event_name != 'workflow_dispatch' || ( inputs.test && inputs.azure ) ) }}
+            test: test
+            cloud: azure
+            compute_env: TOWER_CE_AZURE_CPU
+            workdir: TOWER_BUCKET_AZURE
           - profile: test_full_aws
-            enabled: ${{ ( github.repository == 'nf-core/sarek' ) && ( github.event_name != 'workflow_dispatch' || ( inputs.somatic && inputs.aws ) ) }}
+            test: somatic
+            cloud: aws
+            compute_env: TOWER_COMPUTE_ENV
+            workdir: TOWER_BUCKET_AWS
           - profile: test_full_azure
-            enabled: ${{ ( github.repository == 'nf-core/sarek' ) && ( github.event_name != 'workflow_dispatch' || ( inputs.somatic && inputs.azure ) ) }}
+            test: somatic
+            cloud: azure
+            compute_env: TOWER_CE_AZURE_CPU
+            workdir: TOWER_BUCKET_AZURE
           - profile: test_full_germline_aws
-            enabled: ${{ ( github.repository == 'nf-core/sarek' ) && ( github.event_name != 'workflow_dispatch' || ( inputs.germline && inputs.aws ) ) }}
+            test: germline
+            cloud: aws
+            compute_env: TOWER_COMPUTE_ENV
+            workdir: TOWER_BUCKET_AWS
           - profile: test_full_germline_azure
-            enabled: ${{ ( github.repository == 'nf-core/sarek' ) && ( github.event_name != 'workflow_dispatch' || ( inputs.germline && inputs.azure ) ) }}
-          - profile: test_full_germline_ncbench_agilent_aws
-            enabled: ${{ ( github.repository == 'nf-core/sarek' ) && ( github.event_name != 'workflow_dispatch' || ( inputs.germline_ncbench_agilent && inputs.aws ) ) }}
+            test: germline
+            cloud: azure
+            compute_env: TOWER_CE_AZURE_CPU
+            workdir: TOWER_BUCKET_AZURE
+          - profile: test_full_germline_ncbench_agilent
+            test: germline_ncbench_agilent
+            cloud: aws
+            compute_env: TOWER_COMPUTE_ENV
+            workdir: TOWER_BUCKET_AWS
 
     steps:
       # Launch workflow on AWS Batch
-      - name: AWS Launch
+      - name: Launch
         uses: seqeralabs/action-tower-launch@v2
-        if: ${{ matrix.enabled && ( github.event_name != 'workflow_dispatch' || inputs.aws ) }}
+        # If inputs item exists (i.e. workflow_dispatch), then we find matrix.test and check it is false
+        # If is false, we negate and run the job
+        if: ( !contains(inputs[matrix.test], 'false') && !contains(inputs[matrix.cloud], 'false') )
         with:
           run_name: sarek_${{ matrix.profile }}
           workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }}
           access_token: ${{ secrets.TOWER_ACCESS_TOKEN }}
-          compute_env: ${{ secrets.TOWER_COMPUTE_ENV }}
+          compute_env: ${{ secrets[matrix.compute_env] }}
           revision: ${{ github.sha }}
-          workdir: s3://${{ secrets.AWS_S3_BUCKET }}/work/sarek/work-${{ github.sha }}/${{ matrix.profile }}
+          workdir: ${{ secrets[matrix.workdir] }}/work/sarek/work-${{ github.sha }}/${{ matrix.profile }}
           parameters: |
             {
               "hook_url": "${{ secrets.MEGATESTS_ALERTS_SLACK_HOOK_URL }}",
-              "outdir": "s3://${{ secrets.AWS_S3_BUCKET }}/sarek/results-${{ github.sha }}/${{ matrix.profile }}/"
+              "outdir": "${{ secrets[matrix.workdir] }}/sarek/results-${{ github.sha }}/${{ matrix.profile }}/"
             }
           profiles: ${{ matrix.profile }}
 
       - uses: actions/upload-artifact@v3
-        name: Save AWS Logs
+        name: Save Logs
         if: success() || failure()
         with:
-          name: tower-aws-${{ matrix.profile }}-log
-          path: |
-            tower_action_*.log
-            tower_action_*.json
-
-      # Launch workflow using Tower CLI tool action
-      - name: Azure Launch
-        uses: seqeralabs/action-tower-launch@v2
-        if: ${{ matrix.enabled && ( github.event_name != 'workflow_dispatch' || inputs.azure ) }}
-        with:
-          run_name: sarek_${{ matrix.profile }}
-          workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }}
-          access_token: ${{ secrets.TOWER_ACCESS_TOKEN }}
-          compute_env: ${{ secrets.TOWER_CE_AZURE_CPU }}
-          revision: ${{ github.sha }}
-          workdir: ${{ secrets.TOWER_BUCKET_AZURE}}/sarek/work-${{ github.sha }}/${{ matrix.profile }}
-          parameters: |
-            {
-              "hook_url": "${{ secrets.MEGATESTS_ALERTS_SLACK_HOOK_URL }}",
-              "outdir": "${{ secrets.TOWER_BUCKET_AZURE }}/sarek/results-${{ github.sha }}/${{ matrix.profile }}/"
-            }
-          profiles: ${{ matrix.profile }}
-
-      - uses: actions/upload-artifact@v3
-        name: Save Azure Logs
-        if: success() || failure()
-        with:
-          name: tower-azure-${{ matrix.profile }}-log
+          name: tower-${{ matrix.profile }}-log
           path: |
             tower_action_*.log
             tower_action_*.json
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -16,6 +16,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed
 
+- [#1378](https://github.com/nf-core/sarek/pull/1378) - Improve cloud tests launch workflow to use matrix
+
 ### Removed
 
 ### Dependencies