diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 87f0345583a..877ac9c91da 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -76,8 +76,9 @@ jobs: name: pytest-changes runs-on: ubuntu-latest outputs: - # Expose matched filters as job 'modules' output variable - modules: ${{ steps.filter.outputs.changes }} + tags: ${{ steps.filter.outputs.changes }} + modules: ${{ steps.tags.outputs.modules }} + subworkflows: ${{ steps.tags.outputs.subworkflows }} steps: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 with: @@ -90,36 +91,56 @@ jobs: filters: "tests/config/pytest_modules.yml" token: "" + - name: Fetch module tags + id: tags + run: | + echo modules=$(echo '${{ steps.filter.outputs.changes }}' | jq -c '. | map(select(contains("modules"))) | map(gsub("modules/"; ""))') >> $GITHUB_OUTPUT + echo subworkflows=$(echo '${{ steps.filter.outputs.changes }}' | jq '. | map(select(contains("subworkflow"))) | map(gsub("subworkflows/"; ""))') >> $GITHUB_OUTPUT + + - name: debug + run: | + echo ${{ steps.tags.outputs.modules }} + echo ${{ steps.tags.outputs.subworkflows }} + nf-test-changes: name: nf-test-changes runs-on: ubuntu-latest outputs: - # Expose matched filters as job 'modules' output variable - modules: ${{ steps.filter.outputs.changes }} + # Expose detected tags as 'modules' and 'workflows' output variables + paths: ${{ steps.list.outputs.components }} + modules: ${{ steps.outputs.outputs.modules }} + subworkflows: ${{ steps.outputs.outputs.subworkflows}} + # Prod for version bumping + steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@v4 with: - fetch-depth: 2 # To retrieve the preceding commit. + fetch-depth: 0 - - name: Combine all tags.yml files - id: get_tags - run: find . -name "tags.yml" -not -path "./.github/*" -exec cat {} + > .github/tags.yml + - name: List nf-test files + id: list + uses: adamrtalbot/detect-nf-test-changes@v0.0.3 + with: + head: ${{ github.sha }} + base: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }} + n_parents: 2 - - name: debug - run: cat .github/tags.yml + - name: Separate modules and subworkflows + id: outputs + run: | + echo modules=$(echo '${{ steps.list.outputs.components }}' | jq -c '. | map(select(contains("modules"))) | map(gsub("modules/nf-core/"; ""))') >> $GITHUB_OUTPUT + echo subworkflows=$(echo '${{ steps.list.outputs.components }}' | jq '. | map(select(contains("subworkflows"))) | map(gsub("subworkflows/nf-core/"; ""))') >> $GITHUB_OUTPUT - # TODO: change back to using dorny/paths-filter when https://github.com/dorny/paths-filter/pull/133 is implemented - - uses: mirpedrol/paths-filter@main - id: filter - with: - filters: ".github/tags.yml" - token: "" + - name: debug + run: | + echo ${{ steps.outputs.outputs.modules }} + echo ${{ steps.outputs.outputs.subworkflows }} - nf-core-lint: + nf-core-lint-modules: runs-on: ${{ github.event.inputs.runners || 'self-hosted' }} - name: nf-core-lint + name: nf-core-lint-modules needs: [pytest-changes, nf-test-changes] - if: ${{ (needs.pytest-changes.outputs.modules != '[]') || (needs.nf-test-changes.outputs.modules != '[]') }} + if: ${{ (needs.pytest-changes.outputs.modules != '[]') || ( needs.nf-test-changes.outputs.modules != '[]') }} strategy: fail-fast: false matrix: @@ -151,6 +172,7 @@ jobs: with: distribution: "temurin" java-version: "17" + - name: Setup Nextflow uses: nf-core/setup-nextflow@v2 @@ -159,25 +181,54 @@ jobs: - name: Lint module ${{ matrix.tags }} run: nf-core modules lint ${{ matrix.tags }} - if: ${{ !startsWith(matrix.tags, 'subworkflows/') }} - - name: Remove substring from matrix.tags - id: remove_substring - run: echo subworkflow_names=$(echo "${{ matrix.tags }}" | sed 's/subworkflows\///g') >> $GITHUB_OUTPUT + nf-core-lint-subworkflows: + runs-on: ubuntu-latest + name: nf-core-lint-modules + needs: [pytest-changes, nf-test-changes] + if: ${{ (needs.pytest-changes.outputs.subworkflows != '[]') || ( needs.nf-test-changes.outputs.subworkflows != '[]') }} + strategy: + fail-fast: false + matrix: + tags: + [ + "${{ fromJson(needs.pytest-changes.outputs.subworkflows) }}", + "${{ fromJson(needs.nf-test-changes.outputs.subworkflows) }}", + ] + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.11" + + - name: Install pip + run: python -m pip install --upgrade pip + + - uses: actions/setup-java@v3 + with: + distribution: "temurin" + java-version: "17" + + - name: Setup Nextflow + uses: nf-core/setup-nextflow@v1 + + - name: Install nf-core tools development version + run: python -m pip install --upgrade --force-reinstall git+https://github.com/nf-core/tools.git@dev - - name: Lint subworkflow ${{ matrix.tags }} - run: nf-core subworkflows lint ${{steps.remove_substring.outputs.subworkflow_names}} - if: ${{ startsWith(matrix.tags, 'subworkflows/') }} + - name: Lint module ${{ matrix.tags }} + run: nf-core subworkflows lint ${{ matrix.tags }} pytest: runs-on: ${{ github.event.inputs.runners || 'self-hosted' }} name: pytest needs: [pytest-changes] - if: needs.pytest-changes.outputs.modules != '[]' + if: needs.pytest-changes.outputs.tags != '[]' strategy: fail-fast: false matrix: - tags: ["${{ fromJson(needs.pytest-changes.outputs.modules) }}"] + tags: ["${{ fromJson(needs.pytest-changes.outputs.tags) }}"] profile: [conda, docker, singularity] exclude: - tags: nf-test @@ -370,6 +421,7 @@ jobs: uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: python-version: "3.11" + - uses: actions/cache@ab5e6d0c87105b4c9c2047343972218f562e4319 # v4 id: cache-pip-pytest with: @@ -465,130 +517,134 @@ jobs: runs-on: ${{ github.event.inputs.runners || 'self-hosted' }} name: nf-test needs: [nf-test-changes] - if: needs.nf-test-changes.outputs.modules != '[]' + if: ( needs.nf-test-changes.outputs.paths != '[]' ) strategy: fail-fast: false matrix: - tags: ["${{ fromJson(needs.nf-test-changes.outputs.modules) }}"] - profile: [conda, docker, singularity] + path: ["${{ fromJson(needs.nf-test-changes.outputs.paths) }}"] + profile: [conda, docker_self_hosted, singularity] exclude: - - tags: nf-test + - path: modules/nf-core/nf-test - profile: conda - tags: angsd/gl + path: modules/nf-core/angsd/gl - profile: conda - tags: annotsv + path: modules/nf-core/annotsv - profile: conda - tags: happy/sompy + path: modules/nf-core/happy/sompy - profile: conda - tags: backsub + path: modules/nf-core/backsub - profile: conda - tags: bakta/bakta + path: modules/nf-core/bakta/bakta - profile: conda - tags: bakta/baktadbdownload + path: modules/nf-core/bakta/baktadbdownload - profile: conda - tags: bases2fastq + path: modules/nf-core/bases2fastq - profile: conda - tags: bcl2fastq + path: modules/nf-core/bcl2fastq - profile: conda - tags: bclconvert + path: modules/nf-core/bclconvert - profile: conda - tags: celesta + path: modules/nf-core/celesta - profile: conda - tags: cellpose + path: modules/nf-core/cellpose - profile: conda - tags: cellranger/count + path: modules/nf-core/cellranger/count - profile: conda - tags: cellranger/mkfastq + path: modules/nf-core/cellranger/mkfastq - profile: conda - tags: cellranger/mkgtf + path: modules/nf-core/cellranger/mkgtf - profile: conda - tags: cellranger/mkref + path: modules/nf-core/cellranger/mkref - profile: conda - tags: cellranger/mkvdjref + path: modules/nf-core/cellranger/mkvdjref - profile: conda - tags: cellranger/multi + path: modules/nf-core/cellranger/multi - profile: conda - tags: cellranger/vdj + path: modules/nf-core/cellranger/vdj - profile: conda - tags: custom/dumpsoftwareversions + path: modules/nf-core/custom/dumpsoftwareversions - profile: conda - tags: deepcell/mesmer + path: modules/nf-core/deepcell/mesmer - profile: conda - tags: deepvariant + path: modules/nf-core/deepvariant - profile: conda - tags: ensemblvep/vep + path: modules/nf-core/ensemblvep/vep - profile: conda - tags: fastk/fastk + path: modules/nf-core/fastk/fastk - profile: conda - tags: fastk/histex + path: modules/nf-core/fastk/histex - profile: conda - tags: fastk/merge + path: modules/nf-core/fastk/merge - profile: conda - tags: fcs/fcsadaptor + path: modules/nf-core/fcs/fcsadaptor - profile: conda - tags: fcs/fcsgx + path: modules/nf-core/fcs/fcsgx - profile: conda - tags: ganon/buildcustom + path: modules/nf-core/ganon/buildcustom - profile: conda - tags: ganon/classify + path: modules/nf-core/ganon/classify - profile: conda - tags: ganon/report + path: modules/nf-core/ganon/report - profile: conda - tags: ganon/table + path: modules/nf-core/ganon/table - profile: conda - tags: gatk4/cnnscorevariants + path: modules/nf-core/gatk4/cnnscorevariants - profile: conda - tags: gatk4/determinegermlinecontigploidy + path: modules/nf-core/gatk4/determinegermlinecontigploidy - profile: conda - tags: genescopefk + path: modules/nf-core/genescopefk - profile: conda - tags: ilastik/multicut + path: modules/nf-core/ilastik/multicut - profile: conda - tags: ilastik/pixelclassification + path: modules/nf-core/ilastik/pixelclassification - profile: conda - tags: imputeme/vcftoprs + path: modules/nf-core/imputeme/vcftoprs - profile: conda - tags: mcquant + path: modules/nf-core/mcquant - profile: conda - tags: merquryfk/katcomp + path: modules/nf-core/merquryfk/katcomp - profile: conda - tags: merquryfk/katgc + path: modules/nf-core/merquryfk/katgc - profile: conda - tags: merquryfk/merquryfk + path: modules/nf-core/merquryfk/merquryfk - profile: conda - tags: merquryfk/ploidyplot + path: modules/nf-core/merquryfk/ploidyplot - profile: conda - tags: molkartgarage/clahe + path: modules/nf-core/molkartgarage/clahe - profile: conda - tags: quartonotebook + path: modules/nf-core/quartonotebook - profile: conda - tags: scimap/spatiallda + path: modules/nf-core/scimap/spatiallda - profile: conda - tags: sentieon/bwaindex + path: modules/nf-core/sentieon/bwaindex - profile: conda - tags: sentieon/bwamem + path: modules/nf-core/sentieon/bwamem - profile: conda - tags: spaceranger/mkgtf + path: modules/nf-core/spaceranger/mkgtf - profile: conda - tags: spaceranger/mkref + path: modules/nf-core/spaceranger/mkref - profile: conda - tags: spaceranger/count + path: modules/nf-core/spaceranger/count - profile: conda - tags: svanalyzer/svbenchmark + path: modules/nf-core/svanalyzer/svbenchmark - profile: conda - tags: universc + path: modules/nf-core/universc - profile: singularity - tags: universc + path: modules/nf-core/universc - profile: conda - tags: vt/decompose - - profile: conda - tags: subworkflows/vcf_annotate_ensemblvep + path: modules/nf-core/vt/decompose - profile: singularity - tags: bases2fastq + path: modules/nf-core/bases2fastq - profile: conda - tags: wittyer + path: modules/nf-core/wittyer - profile: conda - tags: islandpath + path: modules/nf-core/islandpath + - profile: conda + path: subworkflows/nf-core/vcf_annotate_ensemblvep + - profile: conda + path: subworkflows/nf-core/fastq_align_bamcmp_bwa + - profile: conda + path: subworkflows/nf-core/fastq_align_bwa env: NXF_ANSI_LOG: false SENTIEON_LICENSE_BASE64: ${{ secrets.SENTIEON_LICENSE_BASE64 }} @@ -638,7 +694,6 @@ jobs: miniconda-version: "latest" auto-update-conda: true channels: conda-forge,bioconda,defaults - python-version: ${{ matrix.python-version }} - name: Conda setup run: | @@ -675,10 +730,10 @@ jobs: NFT_WORKDIR=~ \ nf-test test \ - --profile=$PROFILE \ - --tag ${{ matrix.tags }} \ + --profile=${{ matrix.profile }} \ --tap=test.tap \ - --verbose + --verbose \ + ${{ matrix.path }} - uses: pcolby/tap-summary@0959cbe1d4422e62afc65778cdaea6716c41d936 # v1 with: @@ -692,7 +747,17 @@ jobs: confirm-pass: runs-on: ubuntu-latest - needs: [prettier, editorconfig, pytest-changes, nf-core-lint, pytest, nf-test-changes, nf-test] + needs: + [ + prettier, + editorconfig, + pytest-changes, + nf-core-lint-modules, + nf-core-lint-subworkflows, + pytest, + nf-test-changes, + nf-test, + ] if: always() steps: - name: All tests ok