ci: Add pipeline, subworkflows, and modules test based on changes

.github/workflows/pytest-workflow.yml → .github/workflows/ci-changes.yml

@@ -1,4 +1,4 @@
-name: pytest-workflow
+name: nf-test on changes
 # This workflow runs the pipeline with the minimal test dataset to check that it completes without any syntax errors
 on:
   pull_request:
@@ -21,10 +21,10 @@ jobs:
       - uses: dorny/paths-filter@v2
         id: filter
         with:
-          filters: "tests/config/pytest_tags.yml"
+          filters: "tests/config/tags.yml"
 
   test_changes:
-    name: ${{ matrix.tags }} ${{ matrix.profile }}
+    name: ${{ matrix.tags }} ${{ matrix.component }} ${{ matrix.profile }}
     runs-on: ubuntu-20.04
     needs: changes
     if: needs.changes.outputs.tags != '[]'
@@ -33,28 +33,14 @@ jobs:
       matrix:
         tags: ["${{ fromJson(needs.changes.outputs.tags) }}"]
         profile: ["docker"] # TODO Add singularity/conda
+        component: ["pipeline", "subworkflows", "modules"]
     steps:
       - name: Check out pipeline code
         uses: actions/checkout@v2
 
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: "3.x"
-
       - uses: actions/cache@v2
         with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-
-
-      - name: Install Python dependencies
-        run: python -m pip install --upgrade pip pytest-workflow
-
-      - uses: actions/cache@v2
-        with:
-          path: /usr/local/bin/nextflow
+          path: /usr/local/bin/
           key: ${{ runner.os }}
           restore-keys: |
             ${{ runner.os }}-nextflow-
@@ -66,6 +52,11 @@ jobs:
           wget -qO- get.nextflow.io | bash
           sudo mv nextflow /usr/local/bin/
 
+      - name: Install nf-test
+        run: |
+          wget -qO- https://code.askimed.com/install/nf-test | bash
+          sudo mv nf-test /usr/local/bin/
+
       - name: Set up Singularity
         if: matrix.profile == 'singularity'
         uses: eWaterCycle/setup-singularity@v5
@@ -84,26 +75,10 @@ jobs:
         if: matrix.profile == 'conda'
         run: conda clean -a
 
-      # Test the module
-      - name: Run pytest-workflow
-        # only use one thread for pytest-workflow to avoid race condition on conda cache.
-        run: TMPDIR=~ PROFILE=${{ matrix.profile }} pytest --tag ${{ matrix.tags }} --symlink --kwdof --git-aware --color=yes
-
-      - name: Output log on failure
-        if: failure()
-        run: |
-          sudo apt-get install bat > /dev/null
-          batcat --decorations=always --color=always /home/runner/pytest_workflow_*/*/log.{out,err}
+      - name: Run nf-test
+        run: nf-test test --profile=${{ matrix.profile }} tests/${{ matrix.component }}/${{ matrix.tags }}/*.nf.test --tap=test.tap
 
-      - name: Upload logs on failure
-        if: failure()
-        uses: actions/upload-artifact@v2
+      - uses: pcolby/tap-summary@v1
         with:
-          name: logs-${{ matrix.profile }}
-          path: |
-            /home/runner/pytest_workflow_*/*/.nextflow.log
-            /home/runner/pytest_workflow_*/*/log.out
-            /home/runner/pytest_workflow_*/*/log.err
-            /home/runner/pytest_workflow_*/*/work
-            !/home/runner/pytest_workflow_*/*/work/conda
-            !/home/runner/pytest_workflow_*/*/work/singularity
+          path: >-
+            test.tap

tests/config/pytest_tags.yml → tests/config/tags.yml

@@ -1,23 +1,23 @@
-quantify_salmon:
-  - subworkflows/local/quantify_salmon.nf
-  - tests/subworkflows/local/quantify_salmon/**
-  - modules/nf-core/software/salmon/quant/**
-  - modules/local/salmon_tx2gene.nf
-  - modules/local/salmon_tximport.nf
-  - modules/local/salmon_merge_counts.nf
-  - modules/local/salmon_summarizedexperiment.nf
+# quantify_salmon:
+#   - subworkflows/local/quantify_salmon.nf
+#   - tests/subworkflows/local/quantify_salmon/**
+#   - modules/nf-core/software/salmon/quant/**
+#   - modules/local/salmon_tx2gene.nf
+#   - modules/local/salmon_tximport.nf
+#   - modules/local/salmon_merge_counts.nf
+#   - modules/local/salmon_summarizedexperiment.nf
 
 hisat2:
   - modules/nf-core/hisat2/**
   - subworkflows/nf-core/align_hisat2.nf
   - tests/pipeline/test_hisat2.yml
 
-quantify_rsem:
-  - subworkflows/local/quantify_rsem.nf
-  - subworkflows/nf-core/bam_sort_samtools.nf
-  - tests/subworkflows/local/quantify_rsem/**
-  - modules/nf-core/software/rsem/calculateexpression/**
-  - modules/local/rsem_merge_counts.nf
+# quantify_rsem:
+#   - subworkflows/local/quantify_rsem.nf
+#   - subworkflows/nf-core/bam_sort_samtools.nf
+#   - tests/subworkflows/local/quantify_rsem/**
+#   - modules/nf-core/software/rsem/calculateexpression/**
+#   - modules/local/rsem_merge_counts.nf
 
 salmon:
   - bin/salmon_summarizedexperiment.r
@@ -32,15 +32,15 @@ salmon:
   - tests/pipeline/test_star_salmon.yml
   # - tests/subworkflows/local/quantify_salmon/
 
-star:
+star_salmon:
   - modules/local/star_align_igenomes.nf
   - modules/local/star_genomegenerate_igenomes.nf
   - modules/nf-core/star/**
   - subworkflows/local/align_star.nf
   - tests/pipeline/test_star_rsem.yml
   - tests/pipeline/test_star_salmon.yml
 
-rsem:
+star_rsem:
   - bin/prepare-for-rsem.py
   - modules/local/rsem_merge_counts.nf
   - modules/local/umitools_prepareforrsem.nf