Add pre filter and shard counting step

.github/actions/number-of-shards/action.yml

@@ -0,0 +1,46 @@
+name: "Get number of shards"
+description: "Get the number of nf-test shards for the current CI job"
+outputs:
+  shards:
+    description: "Array of shard numbers"
+    value: ${{ steps.shards.outputs.shards }}
+  total_shards:
+    description: "Total number of shards"
+    value: ${{ steps.shards.outputs.total_shards }}
+  tags:
+    description: "Tags to test (`<tags>[,<tags>...]`)"
+    value: ${{ steps.shards.outputs.tags }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Install nf-test
+      uses: nf-core/setup-nf-test@v1
+      with:
+        version: ${{ env.NFT_VER }}
+
+    - name: Get number of shards
+      id: shards
+      shell: bash
+      run: |
+        nftest_output=$(nf-test test --dry-run --changed-since HEAD^ --filter process --follow-dependencies)
+        echo "nftest_output: $nftest_output"
+        if echo "$nftest_output" | grep -q 'Found 0 related test(s)'; then
+          shards="[]"
+          total_shards=0
+          tags="[]"
+        else
+          number_of_shards=$(echo "$nftest_output" | grep -o 'Found [0-9]* related test' | tail -1 | awk '{print $2}')
+          shards=$(seq 1 "$number_of_shards" | jq -R . | jq -s .)
+          tags=$(echo "$nftest_output" | grep -o 'tag [^ ]*' | tail -1 | awk '{print $2}')
+          total_shards="$number_of_shards"
+        fi
+        echo "shards=${shards}" >> $GITHUB_OUTPUT
+        echo "total_shards=${total_shards}" >> $GITHUB_OUTPUT
+        echo "tags=${tags}" >> $GITHUB_OUTPUT
+
+        # Debugging output
+        echo "nftest_output: $nftest_output"
+        echo "shards: $shards"
+        echo "total_shards: $total_shards"
+        echo "tags: $tags"

.github/workflows/gpu-tests.yml

@@ -42,11 +42,9 @@ jobs:
     name: nf-test-changes
     runs-on: ubuntu-latest
     outputs:
-      # Expose detected tags as 'modules' and 'workflows' output variables
-      paths: ${{ steps.outputs.outputs.components }}
-      modules: ${{ steps.outputs.outputs.modules }}
-      subworkflows: ${{ steps.outputs.outputs.subworkflows}}
-      # Prod for version bumping
+      shards: ${{ steps.get-shards.outputs.shards }}
+      total_shards: ${{ steps.get-shards.outputs.total_shards }}
+      tags: ${{ steps.get-shards.outputs.tags }}
     steps:
       - name: Clean Workspace # Purge the workspace in case it's running on a self-hosted runner
         run: |
@@ -58,40 +56,31 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: List nf-test files
-        id: list
-        uses: adamrtalbot/detect-nf-test-changes@e114e734c3bfa4599151087dcbc457ff38e5bf2c
-        with:
-          head: ${{ github.sha }}
-          base: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }}
-          n_parents: 2
-          tags: "gpu"
+      - name: get number of shards
+        id: get-shards
+        uses: ./.github/actions/number-of-shards
+        env:
+          NFT_VER: ${{ env.NFT_VER }}
 
-      - name: Separate modules and subworkflows
-        id: outputs
-        run: |
-          echo modules=$(echo '${{ steps.list.outputs.components }}' | jq -c '. | map(select(contains("modules"))) | map(gsub("modules/nf-core/"; ""))') >> $GITHUB_OUTPUT
-          echo subworkflows=$(echo '${{ steps.list.outputs.components }}' | jq '. | map(select(contains("subworkflows"))) | map(gsub("subworkflows/nf-core/"; ""))') >> $GITHUB_OUTPUT
-      - name: debug
+      - name: list changes
         run: |
-          echo ${{ steps.outputs.outputs.components }}
-          echo ${{ steps.outputs.outputs.modules }}
-          echo ${{ steps.outputs.outputs.subworkflows }}
+          echo shards = ${{ steps.get-shards.outputs.shards }}
+          echo num_shards = ${{ steps.get-shards.outputs.total_shards }}
 
   nf-test-gpu:
     runs-on: "gpu"
     needs: nf-test-changes
-    if: ${{ fromJSON(needs.nf-test-changes.outputs.paths) != '[]' || needs.nf-test-changes.outputs.paths != '' }}
+    if: ${{ fromJSON(needs.nf-test-changes.outputs.total_shards) > 0 }}
     name: "GPU | ${{ matrix.tags}} | ${{ matrix.profile }} | ${{ matrix.shard }}"
     strategy:
       fail-fast: false
       matrix:
-        shard: [1, 2]
+        shard: ${{ needs.nf-test-changes.outputs.shards }}
         profile: [docker_self_hosted, singularity] # conda?
-        tags: ${{ fromJSON(needs.nf-test-changes.outputs.modules) && fromJSON(needs.nf-test-changes.outputs.subworkflows) }}
+        tags: ${{ needs.nf-test-changes.outputs.tags }}
     env:
       NXF_ANSI_LOG: false
-      TOTAL_SHARDS: 2
+      TOTAL_SHARDS: ${{ needs.nf-test-changes.outputs.total_shards }}
 
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4

.github/workflows/nf-test.yml

@@ -35,6 +35,29 @@ env:
   NXF_VER: "24.10.1"
 
 jobs:
+  nf-test-changes:
+    name: nf-test-changes
+    runs-on: ubuntu-latest
+    outputs:
+      shards: ${{ steps.get-shards.outputs.shards }}
+      total_shards: ${{ steps.get-shards.outputs.total_shards }}
+      tags: ${{ steps.get-shards.outputs.tags }}
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
+        with:
+          fetch-depth: 0
+
+      - name: get number of shards
+        id: get-shards
+        uses: ./.github/actions/number-of-shards
+        env:
+          NFT_VER: ${{ env.NFT_VER }}
+
+      - name: list changes
+        run: |
+          echo shards = ${{ steps.get-shards.outputs.shards }}
+          echo num_shards = ${{ steps.get-shards.outputs.total_shards }}
+
   nf-test:
     runs-on: ${{ github.event.inputs.runners || 'self-hosted' }}
     # NOTE I think this is the cleanest way to get them organized
@@ -45,20 +68,19 @@ jobs:
     # ...
     # workflow | singularity | 3
     name: "${{ matrix.profile }} | ${{ matrix.shard }}"
-    # TODO
-    # needs: get-number-of-shards
-    # if: ${{ fromJSON(needs.get-number-of-shards.outputs.shards) != fromJSON('["1", "0"]') }}
+    needs: nf-test-changes
+    if: ${{ fromJSON(needs.nf-test-changes.outputs.total_shards) > 0 }}
     strategy:
       fail-fast: false
       matrix:
         # NOTE We could split these, but there's probably going to be more process tests than workflow tests, so we're just going to combine them all and bump up the shards for now
         # NOTE The name of the test would be name: "${{ matrix.filter }} | ${{ matrix.profile }} | ${{ matrix.shard }}"
         # filter: [process, workflow]
         profile: [conda, docker_self_hosted, singularity]
-        shard: [1, 2, 3, 4, 5]
+        shard: ${{ needs.nf-test-changes.outputs.shards }}
     env:
       # FIXME Bumping them up to make the transition smooth, then we can throttle them back
-      TOTAL_SHARDS: 5
+      TOTAL_SHARDS: ${{ needs.nf-test-changes.outputs.total_shards }}
       SENTIEON_LICENSE_MESSAGE: ${{ secrets.SENTIEON_LICENSE_MESSAGE }}
       SENTIEON_ENCRYPTION_KEY: ${{ secrets.SENTIEON_ENCRYPTION_KEY }}
     steps:
@@ -76,4 +98,5 @@ jobs:
         with:
           profile: ${{ matrix.profile }}
           shard: ${{ matrix.shard }}
-          total_shards: ${{ env.TOTAL_SHARDS }}
+          total_shards: ${{ needs.nf-test-changes.outputs.total_shards }}
+          tags: ${{ needs.nf-test-changes.outputs.tags }}