Adding our nf-test tests to the GitHub Actions CI

.github/workflows/ci.yml

@@ -3,44 +3,161 @@ name: nf-core CI
 on:
   push:
     branches:
-      - dev
+      # - dev
+      - ci_nf-test
   pull_request:
   release:
     types: [published]
+  workflow_dispatch:
 
 env:
   NXF_ANSI_LOG: false
+  NFT_VER: "0.9.0"
+  NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity
+  NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity
+  # env var to force tests
+  # TEST_SET: '["tests/workflows/mcmicro.nf.test", "subworkflows/local/utils_nfcore_mcmicro_pipeline/tests/initialisation.nf.test", "tests/main.nf.test"]'
 
 concurrency:
   group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}"
   cancel-in-progress: true
 
 jobs:
+  nf-test-changes:
+    name: Check for changes
+    runs-on: ubuntu-latest
+    outputs:
+      nf_test_files: ${{ steps.list.outputs.components }}
+      # nf_test_files: ${{ env.TEST_SET }}
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: List nf-test files
+        id: list
+        uses: adamrtalbot/detect-nf-test-changes@v0.0.4
+        with:
+          head: ${{ github.sha }}
+          # TODO: check that the following works on pull_request
+          # base: origin/${{ github.ref_name }}
+          base: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || github.event.before }}
+
+      - name: print list of nf-test files
+        run: |
+          echo ${{ steps.list.outputs.components }}
+
   test:
-    name: Run pipeline with test data
-    # Only run on push if this is the nf-core dev branch (merged PRs)
-    if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/mcmicro') }}"
+    name: "Run tests (${{ matrix.nf_test_files }} ${{ matrix.profile }} NF-${{ matrix.NXF_VER }})"
+    needs: [nf-test-changes]
+    if: needs.nf-test-changes.outputs.nf_test_files != '[]'
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
         NXF_VER:
-          - "23.04.0"
+          - "24.04.2"
           - "latest-everything"
+        nf_test_files: ["${{ fromJson(needs.nf-test-changes.outputs.nf_test_files) }}"]
+        profile:
+          # - "conda"
+          - "docker"
+          - "singularity"
     steps:
+      - name: install curl
+        run: sudo apt update && sudo apt install -y curl wget
+
       - name: Check out pipeline code
         uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4
 
-      - name: Install Nextflow
+      - name: Set up Nextflow
         uses: nf-core/setup-nextflow@v2
         with:
           version: "${{ matrix.NXF_VER }}"
 
-      - name: Disk space cleanup
+      - name: Set up Apptainer
+        if: matrix.profile == 'singularity'
+        uses: eWaterCycle/setup-apptainer@main
+
+      - name: Set up Singularity
+        if: matrix.profile == 'singularity'
+        run: |
+          mkdir -p $NXF_SINGULARITY_CACHEDIR
+          mkdir -p $NXF_SINGULARITY_LIBRARYDIR
+
+      #      - name: Set up Miniconda
+      #        if: matrix.profile == 'conda'
+      #        uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3
+      #        with:
+      #          miniconda-version: "latest"
+      #          auto-update-conda: true
+      #          conda-solver: libmamba
+      #          channels: conda-forge,bioconda
+
+      #      - name: Set up Conda
+      #        if: matrix.profile == 'conda'
+      #        run: |
+      #          echo $(realpath $CONDA)/condabin >> $GITHUB_PATH
+      #          echo $(realpath python) >> $GITHUB_PATH
+
+      - name: Install nf-test
+        uses: nf-core/setup-nf-test@v1
+        with:
+          version: ${{ env.NFT_VER }}
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+          architecture: "x64"
+
+      - name: Install pdiff to see diff between nf-test snapshots
+        run: |
+          python -m pip install --upgrade pip
+          pip install pdiff
+
+      - name: Clean up Disk space
         uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
 
-      - name: Run pipeline with test data
-        # TODO nf-core: You can customise CI pipeline run tests as required
-        # For example: adding multiple test runs with different parameters
-        # Remember that you can parallelise this by using strategy.matrix
+      - name: Run nf-test
+        run: |
+          nf-test test \
+            --verbose ${{ matrix.nf_test_files }} \
+            --profile "+${{ matrix.profile }},ci" \
+            --junitxml=test.xml \
+            --tap=test.tap
+
+      - uses: pcolby/tap-summary@v1
+        with:
+          path: >-
+            test.tap
+
+      - name: Output log on failure
+        if: failure()
+        run: |
+          sudo apt install bat > /dev/null
+          batcat --decorations=always --color=always ${{ github.workspace }}/.nf-test/tests/*/meta/nextflow.log
+
+      - name: Publish Test Report
+        uses: mikepenz/action-junit-report@v3
+        if: always() # always run even if the previous step fails
+        with:
+          report_paths: test.xml
+
+  confirm-pass:
+    runs-on: ubuntu-latest
+    needs: [test]
+    if: always()
+    steps:
+      - name: All tests ok
+        if: ${{ !contains(needs.*.result, 'failure') }}
+        run: exit 0
+      - name: One or more tests failed
+        if: ${{ contains(needs.*.result, 'failure') }}
+        run: exit 1
+
+      - name: debug-print
+        if: always()
         run: |
-          nextflow run ${GITHUB_WORKSPACE} -profile test,docker --outdir ./results
+          echo "toJSON(needs) = ${{ toJSON(needs) }}"
+          echo "toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}"

conf/ci.config

@@ -0,0 +1,10 @@
+process {
+    withName: ".*:DEEPCELL_MESMER" {
+        memory = "6.GB"
+        cpus = 2
+    }
+    withName: ".*:CELLPOSE"{
+        memory = "12.GB"
+        cpus   = 4
+    }
+}

conf/modules.config

@@ -28,7 +28,6 @@ process {
     }
 
     withName: BASICPY {
-        containerOptions = '--entrypoint "" --user root'
         publishDir = [
             path: { "${params.outdir}/illumination_correction/basicpy" },
             mode: params.publish_dir_mode,

modules.json

@@ -7,7 +7,7 @@
                 "nf-core": {
                     "ashlar": {
                         "branch": "master",
-                        "git_sha": "a5a287db3590e7aa121e1251bf126c35e1418abf",
+                        "git_sha": "666652151335353eef2fcd58880bcef5bc2928e1",
                         "installed_by": ["modules"],
                         "patch": "modules/nf-core/ashlar/ashlar.diff"
                     },
@@ -18,7 +18,7 @@
                     },
                     "basicpy": {
                         "branch": "master",
-                        "git_sha": "716ef3019b66772a817b417078edce2f7b337858",
+                        "git_sha": "666652151335353eef2fcd58880bcef5bc2928e1",
                         "installed_by": ["modules"],
                         "patch": "modules/nf-core/basicpy/basicpy.diff"
                     },
@@ -35,7 +35,7 @@
                     },
                     "deepcell/mesmer": {
                         "branch": "master",
-                        "git_sha": "298406de7fb599b3a7acfed1dddb606ec182d4b6",
+                        "git_sha": "666652151335353eef2fcd58880bcef5bc2928e1",
                         "installed_by": ["modules"]
                     },
                     "ilastik/multicut": {