Added Minimap2 alignment

.devcontainer/devcontainer.json

@@ -10,15 +10,7 @@
         "vscode": {
             // Set *default* container specific settings.json values on container create.
             "settings": {
-                "python.defaultInterpreterPath": "/opt/conda/bin/python",
-                "python.linting.enabled": true,
-                "python.linting.pylintEnabled": true,
-                "python.formatting.autopep8Path": "/opt/conda/bin/autopep8",
-                "python.formatting.yapfPath": "/opt/conda/bin/yapf",
-                "python.linting.flake8Path": "/opt/conda/bin/flake8",
-                "python.linting.pycodestylePath": "/opt/conda/bin/pycodestyle",
-                "python.linting.pydocstylePath": "/opt/conda/bin/pydocstyle",
-                "python.linting.pylintPath": "/opt/conda/bin/pylint"
+                "python.defaultInterpreterPath": "/opt/conda/bin/python"
             },
 
             // Add the IDs of extensions you want installed when the container is created.

.editorconfig

@@ -18,7 +18,12 @@ end_of_line = unset
 insert_final_newline = unset
 trim_trailing_whitespace = unset
 indent_style = unset
-indent_size = unset
+[/subworkflows/nf-core/**]
+charset = unset
+end_of_line = unset
+insert_final_newline = unset
+trim_trailing_whitespace = unset
+indent_style = unset
 
 [/assets/email*]
 indent_size = unset

.github/CONTRIBUTING.md

@@ -9,9 +9,8 @@ Please use the pre-filled template to save time.
 However, don't be put off by this template - other more general issues and suggestions are welcome!
 Contributions to the code are even more welcome ;)
 
-:::info
-If you need help using or modifying nf-core/sarek then the best place to ask is on the nf-core Slack [#sarek](https://nfcore.slack.com/channels/sarek) channel ([join our Slack here](https://nf-co.re/join/slack)).
-:::
+> [!NOTE]
+> If you need help using or modifying nf-core/sarek then the best place to ask is on the nf-core Slack [#sarek](https://nfcore.slack.com/channels/sarek) channel ([join our Slack here](https://nf-co.re/join/slack)).
 
 ## Contribution workflow
 
@@ -27,8 +26,11 @@ If you're not used to this workflow with git, you can start with some [docs from
 
 ## Tests
 
-You can optionally test your changes by running the pipeline locally. Then it is recommended to use the `debug` profile to
-receive warnings about process selectors and other debug info. Example: `nextflow run . -profile debug,test,docker --outdir <OUTDIR>`.
+You have the option to test your changes locally by running the pipeline. For receiving warnings about process selectors and other `debug` information, it is recommended to use the debug profile. Execute all the tests with the following command:
+
+```bash
+nf-test test --profile debug,test,docker --verbose
+```
 
 When you create a pull request with changes, [GitHub Actions](https://github.com/features/actions) will run automatic tests.
 Typically, pull-requests are only fully reviewed when these tests are passing, though of course we can help out before then.
@@ -90,7 +92,7 @@ Once there, use `nf-core schema build` to add to `nextflow_schema.json`.
 
 Sensible defaults for process resource requirements (CPUs / memory / time) for a process should be defined in `conf/base.config`. These should generally be specified generic with `withLabel:` selectors so they can be shared across multiple processes/steps of the pipeline. A nf-core standard set of labels that should be followed where possible can be seen in the [nf-core pipeline template](https://github.com/nf-core/tools/blob/master/nf_core/pipeline-template/conf/base.config), which has the default process as a single core-process, and then different levels of multi-core configurations for increasingly large memory requirements defined with standardised labels.
 
-The process resources can be passed on to the tool dynamically within the process with the `${task.cpu}` and `${task.memory}` variables in the `script:` block.
+The process resources can be passed on to the tool dynamically within the process with the `${task.cpus}` and `${task.memory}` variables in the `script:` block.
 
 ### Naming schemes
 

.github/PULL_REQUEST_TEMPLATE.md

@@ -18,7 +18,7 @@ Learn more about contributing: [CONTRIBUTING.md](https://github.com/nf-core/sare
 - [ ] If you've added a new tool - have you followed the pipeline conventions in the [contribution docs](https://github.com/nf-core/sarek/tree/master/.github/CONTRIBUTING.md)
 - [ ] If necessary, also make a PR on the nf-core/sarek _branch_ on the [nf-core/test-datasets](https://github.com/nf-core/test-datasets) repository.
 - [ ] Make sure your code lints (`nf-core lint`).
-- [ ] Ensure the test suite passes (`nextflow run . -profile test,docker --outdir <OUTDIR>`).
+- [ ] Ensure the test suite passes (`nf-test test tests/ --verbose --profile +docker`).
 - [ ] Check for unexpected warnings in debug mode (`nextflow run . -profile debug,test,docker --outdir <OUTDIR>`).
 - [ ] Usage Documentation in `docs/usage.md` is updated.
 - [ ] Output Documentation in `docs/output.md` is updated.

.github/workflows/awsfulltest.yml

@@ -0,0 +1,39 @@
+name: nf-core AWS full size tests
+# This workflow is triggered on published releases.
+# It can be additionally triggered manually with GitHub actions workflow dispatch button.
+# It runs the -profile 'test_full' on AWS batch
+
+on:
+  release:
+    types: [published]
+  workflow_dispatch:
+jobs:
+  run-tower:
+    name: Run AWS full tests
+    if: github.repository == 'nf-core/sarek'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Launch workflow via tower
+        uses: seqeralabs/action-tower-launch@v2
+        # TODO nf-core: You can customise AWS full pipeline tests as required
+        # Add full size test data (but still relatively small datasets for few samples)
+        # on the `test_full.config` test runs with only one set of parameters
+        with:
+          workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }}
+          access_token: ${{ secrets.TOWER_ACCESS_TOKEN }}
+          compute_env: ${{ secrets.TOWER_COMPUTE_ENV }}
+          revision: ${{ github.sha }}
+          workdir: s3://${{ secrets.AWS_S3_BUCKET }}/work/sarek/work-${{ github.sha }}
+          parameters: |
+            {
+              "hook_url": "${{ secrets.MEGATESTS_ALERTS_SLACK_HOOK_URL }}",
+              "outdir": "s3://${{ secrets.AWS_S3_BUCKET }}/sarek/results-${{ github.sha }}"
+            }
+          profiles: test_full
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: Tower debug log file
+          path: |
+            tower_action_*.log
+            tower_action_*.json

.github/workflows/branch.yml

@@ -19,7 +19,7 @@ jobs:
       # NOTE - this doesn't currently work if the PR is coming from a fork, due to limitations in GitHub actions secrets
       - name: Post PR comment
         if: failure()
-        uses: mshick/add-pr-comment@v2
+        uses: mshick/add-pr-comment@b8f338c590a895d50bcbfa6c5859251edc8952fc # v2
         with:
           message: |
             ## This PR is against the `master` branch :x:

.github/workflows/ci.yml

@@ -1,4 +1,4 @@
-name: test
+name: nf-core CI
 # This workflow runs the pipeline with the minimal test dataset to check that it completes without any syntax errors
 on:
   pull_request:
@@ -52,14 +52,25 @@ jobs:
         NXF_VER:
           - "23.04.0"
           - "latest-everything"
+        exclude:
+          - tags: "sentieon/bwamem"
+          - tags: "sentieon/dedup"
+          - tags: "sentieon/dnascope"
+          - tags: "sentieon/dnascope_joint_germline"
+          - tags: "sentieon/dnascope_skip_filter"
+          - tags: "sentieon/haplotyper"
+          - tags: "sentieon/haplotyper_joint_germline"
+          - tags: "sentieon/haplotyper_skip_filter"
+          - NXF_VER: "latest-everything"
+            tags: "joint_germline"
     env:
       NXF_ANSI_LOG: false
       TEST_DATA_BASE: "${{ github.workspace }}/test-datasets"
       SENTIEON_LICENSE_BASE64: ${{ secrets.SENTIEON_LICENSE_BASE64 }}
 
     steps:
       - name: Check out pipeline code
-        uses: actions/checkout@v4
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
 
       - name: Hash Github Workspace
         id: hash_workspace
@@ -200,16 +211,47 @@ jobs:
           - "23.04.0"
           - "latest-everything"
         exclude:
+          - tags: "bcftools/concat"
+          - tags: "bcftools/mpileup"
+          - tags: "bcftools/sort"
           - tags: "bwa/index"
           - tags: "bwa/mem"
+          - tags: "bwamem2/index"
+          - tags: "bwamem2/mem"
           - tags: "cat/cat"
           - tags: "cat/fastq"
-          - tags: "custom/dumpsoftwareversions"
+          - tags: "cnvkit/batch"
+          - tags: "deepvariant"
           - tags: "dragmap/align"
+          - tags: "dragmap/hashtable"
+          - tags: "ensemblvep/download"
+          - tags: "ensemblvep/vep"
           - tags: "fastp"
           - tags: "fastqc"
+          - tags: "fgbio/fastqtobam"
+          - tags: "freebayes"
+          - tags: "gatk4/applybqsr"
+          - tags: "gatk4/baserecalibrator"
+          - tags: "gatk4/estimatelibrarycomplexity"
+          - tags: "gatk4/genomicsdbimport"
+          - tags: "gatk4/haplotypecaller"
+          - tags: "gatk4/markduplicates"
+          - tags: "gatk4/mergevcfs"
+          - tags: "gatk4/mutect2"
+          - tags: "gatk4spark/applybqsr"
+          - tags: "gatk4spark/markduplicates"
+          - tags: "mosdepth"
           - tags: "multiqc"
+          - tags: "samblaster"
+          - tags: "samtools/mpileup"
           - tags: "samtools/stats"
+          - tags: "snpeff/snpeff"
+          - tags: "strelka/germline"
+          - tags: "strelka/somatic"
+          - tags: "subworkflows/utils_nfvalidation_plugin"
+          - tags: "tabix/bgziptabix"
+          - tags: "tabix/tabix"
+          - tags: "tiddit/sv"
           - tags: "untar"
     env:
       NXF_ANSI_LOG: false

.github/workflows/clean-up.yml

@@ -10,7 +10,7 @@ jobs:
       issues: write
       pull-requests: write
     steps:
-      - uses: actions/stale@v9
+      - uses: actions/stale@28ca1036281a5e5922ead5184a1bbf96e5fc984e # v9
         with:
           stale-issue-message: "This issue has been tagged as awaiting-changes or awaiting-feedback by an nf-core contributor. Remove stale label or add a comment otherwise this issue will be closed in 20 days."
           stale-pr-message: "This PR has been tagged as awaiting-changes or awaiting-feedback by an nf-core contributor. Remove stale label or add a comment if it is still useful."

.github/workflows/cloudtest.yml

@@ -1,109 +1,33 @@
-name: nf-core cloud test
+name: nf-core AWS test
+# This workflow can be triggered manually with the GitHub actions workflow dispatch button.
+# It runs the -profile 'test' on AWS batch
 
 on:
-  release:
-    types: [created]
   workflow_dispatch:
-    inputs:
-      test:
-        description: "-profile test (smaller)"
-        type: boolean
-        default: true
-      somatic:
-        description: "Somatic full test"
-        type: boolean
-        default: false
-      germline:
-        description: "Germline full test"
-        type: boolean
-        default: false
-      germline_ncbench_agilent:
-        description: "Germline Agilent NCBench test"
-        type: boolean
-        default: false
-      aws:
-        description: "AWS Batch"
-        type: boolean
-        default: true
-      azure:
-        description: "Azure Batch"
-        type: boolean
-        default: true
-
 jobs:
-  trigger-profile-test:
+  run-tower:
     name: Run AWS tests
+    if: github.repository == 'nf-core/sarek'
     runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - profile: test_aws
-            enabled: ${{ ( github.repository == 'nf-core/sarek' ) && ( github.event_name != 'workflow_dispatch' || ( inputs.test && inputs.aws ) ) }}
-          - profile: test_azure
-            enabled: ${{ ( github.repository == 'nf-core/sarek' ) && ( github.event_name != 'workflow_dispatch' || ( inputs.test && inputs.azure ) ) }}
-          - profile: test_full_aws
-            enabled: ${{ ( github.repository == 'nf-core/sarek' ) && ( github.event_name != 'workflow_dispatch' || ( inputs.somatic && inputs.aws ) ) }}
-          - profile: test_full_azure
-            enabled: ${{ ( github.repository == 'nf-core/sarek' ) && ( github.event_name != 'workflow_dispatch' || ( inputs.somatic && inputs.azure ) ) }}
-          - profile: test_full_germline_aws
-            enabled: ${{ ( github.repository == 'nf-core/sarek' ) && ( github.event_name != 'workflow_dispatch' || ( inputs.germline && inputs.aws ) ) }}
-          - profile: test_full_germline_azure
-            enabled: ${{ ( github.repository == 'nf-core/sarek' ) && ( github.event_name != 'workflow_dispatch' || ( inputs.germline && inputs.azure ) ) }}
-          - profile: test_full_germline_ncbench_agilent_aws
-            enabled: ${{ ( github.repository == 'nf-core/sarek' ) && ( github.event_name != 'workflow_dispatch' || ( inputs.germline_ncbench_agilent && inputs.aws ) ) }}
-
     steps:
-      # Launch workflow on AWS Batch
-      - name: AWS Launch
-        uses: seqeralabs/action-tower-launch@v2
-        if: ${{ matrix.enabled && ( github.event_name != 'workflow_dispatch' || inputs.aws ) }}
-        with:
-          run_name: sarek_${{ matrix.profile }}
-          workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }}
-          access_token: ${{ secrets.TOWER_ACCESS_TOKEN }}
-          compute_env: ${{ secrets.TOWER_COMPUTE_ENV }}
-          revision: ${{ github.sha }}
-          workdir: s3://${{ secrets.AWS_S3_BUCKET }}/work/sarek/work-${{ github.sha }}/${{ matrix.profile }}
-          parameters: |
-            {
-              "hook_url": "${{ secrets.MEGATESTS_ALERTS_SLACK_HOOK_URL }}",
-              "outdir": "s3://${{ secrets.AWS_S3_BUCKET }}/sarek/results-${{ github.sha }}/${{ matrix.profile }}/"
-            }
-          profiles: ${{ matrix.profile }}
-
-      - uses: actions/upload-artifact@v3
-        name: Save AWS Logs
-        if: success() || failure()
-        with:
-          name: tower-aws-${{ matrix.profile }}-log
-          path: |
-            tower_action_*.log
-            tower_action_*.json
-
       # Launch workflow using Tower CLI tool action
-      - name: Azure Launch
+      - name: Launch workflow via tower
         uses: seqeralabs/action-tower-launch@v2
-        if: ${{ matrix.enabled && ( github.event_name != 'workflow_dispatch' || inputs.azure ) }}
         with:
-          run_name: sarek_${{ matrix.profile }}
           workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }}
           access_token: ${{ secrets.TOWER_ACCESS_TOKEN }}
-          compute_env: ${{ secrets.TOWER_CE_AZURE_CPU }}
+          compute_env: ${{ secrets[matrix.compute_env] }}
           revision: ${{ github.sha }}
-          workdir: ${{ secrets.TOWER_BUCKET_AZURE}}/sarek/work-${{ github.sha }}/${{ matrix.profile }}
+          workdir: s3://${{ secrets.AWS_S3_BUCKET }}/work/sarek/work-${{ github.sha }}
           parameters: |
             {
-              "hook_url": "${{ secrets.MEGATESTS_ALERTS_SLACK_HOOK_URL }}",
-              "outdir": "${{ secrets.TOWER_BUCKET_AZURE }}/sarek/results-${{ github.sha }}/${{ matrix.profile }}/"
+              "outdir": "s3://${{ secrets.AWS_S3_BUCKET }}/sarek/results-test-${{ github.sha }}"
             }
-          profiles: ${{ matrix.profile }}
+          profiles: test
 
-      - uses: actions/upload-artifact@v3
-        name: Save Azure Logs
-        if: success() || failure()
+      - uses: actions/upload-artifact@v4
         with:
-          name: tower-azure-${{ matrix.profile }}-log
+          name: Tower debug log file
           path: |
             tower_action_*.log
             tower_action_*.json

.github/workflows/download_pipeline.yml

@@ -6,6 +6,11 @@ name: Test successful pipeline download with 'nf-core download'
 #  - the head branch of the pull request is updated, i.e. if fixes for a release are pushed last minute to dev.
 on:
   workflow_dispatch:
+    inputs:
+      testbranch:
+        description: "The specific branch you wish to utilize for the test execution of nf-core download."
+        required: true
+        default: "dev"
   pull_request:
     types:
       - opened
@@ -25,11 +30,11 @@ jobs:
       - name: Install Nextflow
         uses: nf-core/setup-nextflow@v1
 
-      - uses: actions/setup-python@v5
+      - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5
         with:
           python-version: "3.11"
           architecture: "x64"
-      - uses: eWaterCycle/setup-singularity@v7
+      - uses: eWaterCycle/setup-singularity@931d4e31109e875b13309ae1d07c70ca8fbc8537 # v7
         with:
           singularity-version: 3.8.3
 
@@ -42,13 +47,13 @@ jobs:
         run: |
           echo "REPO_LOWERCASE=${GITHUB_REPOSITORY,,}" >> ${GITHUB_ENV}
           echo "REPOTITLE_LOWERCASE=$(basename ${GITHUB_REPOSITORY,,})" >> ${GITHUB_ENV}
-          echo "REPO_BRANCH=${GITHUB_REF#refs/heads/}" >> ${GITHUB_ENV}
+          echo "REPO_BRANCH=${{ github.event.inputs.testbranch || 'dev' }}" >> ${GITHUB_ENV}
 
       - name: Download the pipeline
         env:
           NXF_SINGULARITY_CACHEDIR: ./
         run: |
-          nf-core download  ${{ env.REPO_LOWERCASE }} \
+          nf-core download ${{ env.REPO_LOWERCASE }} \
           --revision ${{ env.REPO_BRANCH }} \
           --outdir ./${{ env.REPOTITLE_LOWERCASE }} \
           --compress "none" \
@@ -64,4 +69,4 @@ jobs:
         env:
           NXF_SINGULARITY_CACHEDIR: ./
           NXF_SINGULARITY_HOME_MOUNT: true
-        run: nextflow run ./${{ env.REPOTITLE_LOWERCASE }}/$( sed 's/\W/_/g' <<< ${{ env.REPO_BRANCH }}) -stub -profile test,singularity --outdir ./results
+        run: nextflow run ./${{ env.REPOTITLE_LOWERCASE }}/$( sed 's/\W/_/g' <<< ${{ env.REPO_BRANCH }}) -profile test,singularity --outdir ./results