Merge branch 'dev' into minimap2

nf-core · Oct 4, 2024 · f2c1753 · f2c1753
2 parents e91e972 + e9a0c1e
commit f2c1753
Show file tree

Hide file tree

Showing 502 changed files with 22,734 additions and 8,608 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -12,8 +12,15 @@ on:
       - dev
 
 env:
+  NFT_DIFF: "pdiff"
+  NFT_DIFF_ARGS: "--line-numbers --width 120 --expand-tabs=2"
+  NFT_VER: "0.9.0"
+  NFT_WORKDIR: "~"
   NXF_ANSI_LOG: false
-  NFTEST_VER: "0.8.1"
+  NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity
+  NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity
+  SENTIEON_LICENSE_BASE64: ${{ secrets.SENTIEON_LICENSE_BASE64 }}
+  TEST_DATA_BASE: "${{ github.workspace }}/test-datasets"
 
 # Cancel if a newer run is started
 concurrency:
@@ -28,7 +35,7 @@ jobs:
       # Expose matched filters as job 'tags' output variable
       tags: ${{ steps.filter.outputs.changes }}
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
 
       - uses: frouioui/paths-filter@main
         id: filter
@@ -63,24 +70,20 @@ jobs:
           - tags: "sentieon/haplotyper_skip_filter"
           - NXF_VER: "latest-everything"
             tags: "joint_germline"
-    env:
-      NXF_ANSI_LOG: false
-      TEST_DATA_BASE: "${{ github.workspace }}/test-datasets"
-      SENTIEON_LICENSE_BASE64: ${{ secrets.SENTIEON_LICENSE_BASE64 }}
 
     steps:
       - name: Check out pipeline code
-        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
 
       - name: Hash Github Workspace
         id: hash_workspace
         run: |
           echo "digest=$(echo sarek3_${{ github.workspace }} | md5sum | cut -c 1-25)" >> $GITHUB_OUTPUT
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5
         with:
-          python-version: "3.x"
+          python-version: "3.11"
           cache: "pip"
           cache-dependency-path: |
             **/requirements.txt
@@ -93,30 +96,43 @@ jobs:
         with:
           version: "${{ matrix.NXF_VER }}"
 
+      - name: Setup apptainer
+        if: matrix.profile == 'singularity'
+        uses: eWaterCycle/setup-apptainer@main
+
       - name: Set up Singularity
         if: matrix.profile == 'singularity'
-        uses: eWaterCycle/setup-singularity@v5
-        with:
-          singularity-version: 3.7.1
+        run: |
+          mkdir -p $NXF_SINGULARITY_CACHEDIR
+          mkdir -p $NXF_SINGULARITY_LIBRARYDIR
 
       - name: Set up miniconda
         if: matrix.profile == 'conda'
-        uses: conda-incubator/setup-miniconda@v2
+        uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3
         with:
+          miniconda-version: "latest"
           auto-update-conda: true
-          channels: conda-forge,bioconda,defaults
-          python-version: ${{ matrix.python-version }}
+          channels: conda-forge,bioconda
+
+      - name: Conda setup
+        if: matrix.profile == 'conda'
+        run: |
+          conda clean -a
+          conda install -n base conda-libmamba-solver
+          conda config --set solver libmamba
+          echo $(realpath $CONDA)/condabin >> $GITHUB_PATH
+          echo $(realpath python) >> $GITHUB_PATH
 
       - name: Cache test data
         id: cache-testdata
-        uses: actions/cache@v3
+        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4
         with:
           path: test-datasets/
           key: ${{ steps.hash_workspace.outputs.digest }}
 
       - name: Check out test data
         if: steps.cache-testdata.outputs.cache-hit != 'true'
-        uses: actions/checkout@v4
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
         with:
           repository: nf-core/test-datasets
           ref: sarek3
@@ -164,16 +180,17 @@ jobs:
 
       - name: Upload logs on failure
         if: failure()
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4
         with:
           name: logs-${{ matrix.profile }}
           path: |
-            /home/runner/pytest_workflow_*/*/.nextflow.log
-            /home/runner/pytest_workflow_*/*/log.out
-            /home/runner/pytest_workflow_*/*/log.err
-            /home/runner/pytest_workflow_*/*/work
-            !/home/runner/pytest_workflow_*/*/work/conda
-            !/home/runner/pytest_workflow_*/*/work/singularity
+            /home/ubuntu/pytest_workflow_*/*/.nextflow.log
+            /home/ubuntu/pytest_workflow_*/*/log.out
+            /home/ubuntu/pytest_workflow_*/*/log.err
+            /home/ubuntu/pytest_workflow_*/*/work
+            !/home/ubuntu/pytest_workflow_*/*/work/conda
+            !/home/ubuntu/pytest_workflow_*/*/work/singularity
+            !${{ github.workspace }}/.singularity
 
   nftest-changes:
     name: Check for changes (nf-test)
@@ -182,15 +199,12 @@ jobs:
       tags: ${{ steps.filter.outputs.changes }}
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
 
       - name: Combine all tags.yml files
         id: get_tags
         run: find . -name "tags.yml" -not -path "./.github/*" -exec cat {} + > .github/tags.yml
 
-      - name: debug
-        run: cat .github/tags.yml
-
       - uses: frouioui/paths-filter@main
         id: filter
         with:
@@ -224,14 +238,19 @@ jobs:
           - tags: "bwamem2/mem"
           - tags: "cat/cat"
           - tags: "cat/fastq"
+          - tags: "cnvkit/antitarget"
           - tags: "cnvkit/batch"
-          - tags: "deepvariant"
+          - tags: "cnvkit/reference"
+          - tags: "controlfreec/assesssignificance"
+          - tags: "controlfreec/freec"
+          - tags: "deepvariant/rundeepvariant"
           - tags: "dragmap/align"
           - tags: "dragmap/hashtable"
           - tags: "ensemblvep/download"
           - tags: "ensemblvep/vep"
           - tags: "fastp"
           - tags: "fastqc"
+          - tags: "fgbio/callmolecularconsensusreads"
           - tags: "fgbio/fastqtobam"
           - tags: "freebayes"
           - tags: "gatk4/applybqsr"
@@ -245,35 +264,32 @@ jobs:
           - tags: "gatk4spark/applybqsr"
           - tags: "gatk4spark/markduplicates"
           - tags: "gawk"
+          - tags: "lofreq/callparallel"
           - tags: "minimap2/index"
           - tags: "minimap2/align"
           - tags: "mosdepth"
           - tags: "multiqc"
+          - tags: "ngscheckmate/ncm"
           - tags: "samblaster"
           - tags: "samtools/convert"
           - tags: "samtools/mpileup"
           - tags: "samtools/stats"
+          - tags: "sentieon/bwamem"
+          - tags: "sentieon/haplotyper"
           - tags: "snpeff/snpeff"
+          - tags: "spring/decompress"
           - tags: "strelka/germline"
           - tags: "strelka/somatic"
           - tags: "subworkflows/utils_nfvalidation_plugin"
           - tags: "tabix/bgziptabix"
           - tags: "tabix/tabix"
           - tags: "tiddit/sv"
           - tags: "untar"
-          - tags: "pipeline_sarek"
-        include:
-          - tags: "pipeline_sarek"
-            profile: "test,docker"
-    env:
-      NXF_ANSI_LOG: false
-      TEST_DATA_BASE: "${{ github.workspace }}/test-datasets"
-      SENTIEON_LICENSE_BASE64: ${{ secrets.SENTIEON_LICENSE_BASE64 }}
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
 
-      - uses: actions/setup-java@v3
+      - uses: actions/setup-java@2dfa2011c5b2a0f1489bf9e433881c92c1631f88 # v4
         with:
           distribution: "temurin"
           java-version: "17"
@@ -283,20 +299,10 @@ jobs:
         with:
           version: "${{ matrix.NXF_VER }}"
 
-      - name: Cache nf-test installation
-        id: cache-software
-        uses: actions/cache@v3
-        with:
-          path: |
-            /usr/local/bin/nf-test
-            /home/runner/.nf-test/nf-test.jar
-          key: ${{ runner.os }}-${{ env.NFTEST_VER }}-nftest
-
       - name: Install nf-test
-        if: steps.cache-software.outputs.cache-hit != 'true'
-        run: |
-          wget -qO- https://code.askimed.com/install/nf-test | bash
-          sudo mv nf-test /usr/local/bin/
+        uses: nf-core/setup-nf-test@v1
+        with:
+          version: ${{ env.NFT_VER }}
 
       - name: Setup apptainer
         if: matrix.profile == 'singularity'
@@ -309,22 +315,23 @@ jobs:
           mkdir -p $NXF_SINGULARITY_LIBRARYDIR
 
       - name: Set up miniconda
-        uses: conda-incubator/setup-miniconda@v2
+        if: matrix.profile == 'conda'
+        uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3
         with:
           miniconda-version: "latest"
           auto-update-conda: true
-          channels: conda-forge,bioconda,defaults
-          python-version: ${{ matrix.python-version }}
+          channels: conda-forge,bioconda
 
       - name: Conda setup
+        if: matrix.profile == 'conda'
         run: |
           conda clean -a
           conda install -n base conda-libmamba-solver
           conda config --set solver libmamba
           echo $(realpath $CONDA)/condabin >> $GITHUB_PATH
           echo $(realpath python) >> $GITHUB_PATH
 
-      # Set up secrets
+            # Set up secrets
       - name: Set up nextflow secrets
         if: env.SENTIEON_LICENSE_BASE64 != null
         run: |
@@ -339,13 +346,15 @@ jobs:
       - name: Disk space cleanup
         uses: jlumbroso/free-disk-space@v1.3.1
 
-      # Test the module
+      # Test the component
       - name: Run nf-test
         run: |
           nf-test test \
-            --profile=${{ matrix.profile }} \
+            --profile="+${{ matrix.profile }}" \
             --tag ${{ matrix.tags }} \
             --tap=test.tap \
+            --junitxml=test.xml \
+            --debug \
             --verbose
 
   confirm-pass:

diff --git a/.gitignore b/.gitignore
@@ -10,3 +10,5 @@ testing*
 .nf-test*
 .nf-test/
 test-datasets/
+test.tap
+test.xml
diff --git a/.nf-core.yml b/.nf-core.yml
@@ -6,16 +6,11 @@ lint:
     - .github/workflows/awsfulltest.yml
     - .github/workflows/awstest.yml
     - conf/modules.config
-    - lib/WorkflowMain.groovy
-    - lib/NfcoreTemplate.groovy
-    - lib/WorkflowSarek.groovy
   files_unchanged:
     - .gitignore
     - .github/PULL_REQUEST_TEMPLATE.md
     - assets/nf-core-sarek_logo_light.png
     - docs/images/nf-core-sarek_logo_dark.png
     - docs/images/nf-core-sarek_logo_light.png
-    - lib/NfcoreTemplate.groovy
-    - lib/NfcoreSchema.groovy
   modules_config: False
   template_strings: False