From 98601db99e166680a068f8db361ae021f31df37b Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 10 Oct 2022 11:48:30 +0200 Subject: [PATCH 01/49] ci: Yoink current ci from sarek --- .github/workflows/ci.yml | 222 ++++++++++++++++++++++++--------------- 1 file changed, 140 insertions(+), 82 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d71cadb70..8bde7bd21 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,117 +18,175 @@ jobs: if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/rnaseq') }}" runs-on: ubuntu-latest strategy: + # HACK Remove after DSL2 rewrite is done + fail-fast: false matrix: NXF_VER: - "21.10.3" - "latest-everything" + test: + - "default" + profile: ["docker", "singularity", "conda"] + exclude: + - NXF_VER: "latest-everything" + profile: "conda" steps: - name: Check out pipeline code uses: actions/checkout@v2 - - name: Install Nextflow - uses: nf-core/setup-nextflow@v1 + - name: Set up Python + uses: actions/setup-python@v2 with: - version: "${{ matrix.NXF_VER }}" + python-version: "3.x" - - name: Run pipeline with test data - run: | - nextflow run ${GITHUB_WORKSPACE} -profile test,docker --outdir ./results + - uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- - star_salmon: - name: Test STAR Salmon with workflow parameters - if: ${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/rnaseq') }} - runs-on: ubuntu-latest - strategy: - matrix: - parameters: - - "--skip_qc" - - "--skip_trimming" - - "--gtf false" - - "--star_index false" - - "--transcript_fasta false" - - "--min_mapped_reads 90" - - "--with_umi" - - "--with_umi --skip_trimming" - - "--remove_ribo_rna --skip_qualimap" - - "--bam_csi_index" - - "--save_align_intermeds --save_reference" - - "--featurecounts_group_type false" - steps: - - name: Check out pipeline code - uses: actions/checkout@v2 + - name: Install Python dependencies + run: python -m pip install --upgrade pip pytest-workflow + + - uses: actions/cache@v2 + with: + path: /usr/local/bin/nextflow + key: ${{ runner.os }} + restore-keys: | + ${{ runner.os }}-nextflow- - name: Install Nextflow - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ + uses: nf-core/setup-nextflow@v1 + with: + version: "${{ matrix.NXF_VER }}" - - name: Run pipeline with STAR and various parameters - run: | - nextflow run ${GITHUB_WORKSPACE} -profile test,docker --aligner star_salmon ${{ matrix.parameters }} --outdir ./results + - name: Set up Singularity + if: matrix.profile == 'singularity' + uses: eWaterCycle/setup-singularity@v5 + with: + singularity-version: 3.7.1 - star_rsem: - name: Test STAR RSEM with workflow parameters - if: ${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/rnaseq') }} - runs-on: ubuntu-latest - strategy: - matrix: - parameters: - - "--skip_qc" - - "--rsem_index false" - steps: - - name: Check out pipeline code - uses: actions/checkout@v2 + - name: Set up miniconda + if: matrix.profile == 'conda' + uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + channels: conda-forge,bioconda,defaults + python-version: ${{ matrix.python-version }} - - name: Install Nextflow - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ + - name: Conda clean + if: matrix.profile == 'conda' + run: conda clean -a + + - name: Run pipeline with tests settings + uses: Wandalen/wretry.action@v1.0.11 + with: + command: TMPDIR=~ PROFILE=${{ matrix.profile }} pytest --tag ${{ matrix.test }} --symlink --kwdof --git-aware --color=yes + attempt_limit: 3 - - name: Run pipeline with RSEM STAR and various parameters + - name: Output log on failure + if: failure() run: | - nextflow run ${GITHUB_WORKSPACE} -profile test,docker --aligner star_rsem ${{ matrix.parameters }} --outdir ./results + sudo apt install bat > /dev/null + batcat --decorations=always --color=always /tmp/pytest_workflow_*/*/log.{out,err} - hisat2: - name: Test HISAT2 with workflow parameters - if: ${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/rnaseq') }} + - name: Upload logs on failure + if: failure() + uses: actions/upload-artifact@v2 + with: + name: logs-${{ matrix.profile }} + path: | + /tmp/pytest_workflow_*/*/.nextflow.log + /tmp/pytest_workflow_*/*/log.out + /tmp/pytest_workflow_*/*/log.err + /tmp/pytest_workflow_*/*/work + /tmp/pytest_workflow_*/**/.command.log + !/tmp/pytest_workflow_*/*/work/conda + !/tmp/pytest_workflow_*/*/work/singularity + + test_all: + name: Run pipeline with test data (complete) + # Only run on push if this is the nf-core dev branch (merged PRs) + if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/rnaseq') }}" runs-on: ubuntu-latest strategy: + # HACK Remove after DSL2 rewrite is done + fail-fast: false matrix: - parameters: - - "--skip_qc" - - "--hisat2_index false" + NXF_VER: + - "21.10.3" + test: + - "aligner" + - "alignment_to_fastq" + - "annotation" + - "cnvkit" + - "controlfreec" + - "deepvariant" + - "freebayes" + - "gatk4_spark" + - "haplotypecaller" + - "intervals" + - "manta" + - "markduplicates" + - "mpileup" + - "msisensorpro" + - "mutect2" + - "prepare_recalibration" + - "recalibrate" + - "save_bam_mapped" + - "save_output_as_bam" + - "skip_markduplicates" + - "skip_qc" + - "split_fastq" + - "strelka" + - "strelkabp" + - "targeted" + - "tiddit" + - "trimming" + - "tumor_normal_pair" + - "umi" + - "variantcalling_channel" + profile: ["docker"] steps: - name: Check out pipeline code uses: actions/checkout@v2 - name: Install Nextflow - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ + uses: nf-core/setup-nextflow@v1 + with: + version: "${{ matrix.NXF_VER }}" - - name: Run pipeline with HISAT2 and various parameters - run: | - nextflow run ${GITHUB_WORKSPACE} -profile test,docker --aligner hisat2 ${{ matrix.parameters }} --outdir ./results + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: "3.x" - salmon: - name: Test Salmon with workflow parameters - if: ${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/rnaseq') }} - runs-on: ubuntu-latest - strategy: - matrix: - parameters: - - "--skip_qc --skip_alignment" - - "--salmon_index false --transcript_fasta false" - steps: - - name: Check out pipeline code - uses: actions/checkout@v2 + - name: Install dependencies + run: python -m pip install --upgrade pip pytest-workflow - - name: Install Nextflow - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ + - name: Run pipeline with tests settings + uses: Wandalen/wretry.action@v1.0.11 + with: + command: PROFILE=${{ matrix.profile }} pytest --tag ${{ matrix.test }} --symlink --kwdof --git-aware --color=yes + attempt_limit: 3 - - name: Run pipeline with Salmon and various parameters + - name: Output log on failure + if: failure() run: | - nextflow run ${GITHUB_WORKSPACE} -profile test,docker --pseudo_aligner salmon ${{ matrix.parameters }} --outdir ./results + sudo apt install bat > /dev/null + batcat --decorations=always --color=always /tmp/pytest_workflow_*/*/log.{out,err} + + - name: Upload logs on failure + if: failure() + uses: actions/upload-artifact@v2 + with: + name: logs-${{ matrix.profile }} + path: | + /tmp/pytest_workflow_*/*/.nextflow.log + /tmp/pytest_workflow_*/*/log.out + /tmp/pytest_workflow_*/*/log.err + /tmp/pytest_workflow_*/*/work + /tmp/pytest_workflow_*/**/.command.log + !/tmp/pytest_workflow_*/*/work/conda + !/tmp/pytest_workflow_*/*/work/singularity From 38747746c7826209d668331253fde85b1f1cb8b7 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 10 Oct 2022 11:49:38 +0200 Subject: [PATCH 02/49] test: Copy tests from old pytest-workflow branch --- .github/workflows/ci.yml | 32 +------ tests/config/nextflow.config | 38 ++++++++ tests/config/pytest_software.yml | 15 ++++ tests/config/test_data.config | 3 + .../subworkflows/local/quantify_rsem/main.nf | 25 ++++++ .../subworkflows/local/quantify_rsem/test.yml | 7 ++ .../local/quantify_salmon/main.nf | 25 ++++++ .../local/quantify_salmon/test.yml | 7 ++ tests/test_default.yml | 70 +++++++++++++++ tests/test_hisat2.yml | 12 +++ tests/test_salmon.yml | 16 ++++ tests/test_sra_download.yml | 5 ++ tests/test_star_rsem.yml | 14 +++ tests/test_star_salmon.yml | 87 +++++++++++++++++++ 14 files changed, 326 insertions(+), 30 deletions(-) create mode 100644 tests/config/nextflow.config create mode 100644 tests/config/pytest_software.yml create mode 100644 tests/config/test_data.config create mode 100644 tests/subworkflows/local/quantify_rsem/main.nf create mode 100644 tests/subworkflows/local/quantify_rsem/test.yml create mode 100644 tests/subworkflows/local/quantify_salmon/main.nf create mode 100644 tests/subworkflows/local/quantify_salmon/test.yml create mode 100644 tests/test_default.yml create mode 100644 tests/test_hisat2.yml create mode 100644 tests/test_salmon.yml create mode 100644 tests/test_sra_download.yml create mode 100644 tests/test_star_rsem.yml create mode 100644 tests/test_star_salmon.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8bde7bd21..c98a7af01 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -117,36 +117,8 @@ jobs: NXF_VER: - "21.10.3" test: - - "aligner" - - "alignment_to_fastq" - - "annotation" - - "cnvkit" - - "controlfreec" - - "deepvariant" - - "freebayes" - - "gatk4_spark" - - "haplotypecaller" - - "intervals" - - "manta" - - "markduplicates" - - "mpileup" - - "msisensorpro" - - "mutect2" - - "prepare_recalibration" - - "recalibrate" - - "save_bam_mapped" - - "save_output_as_bam" - - "skip_markduplicates" - - "skip_qc" - - "split_fastq" - - "strelka" - - "strelkabp" - - "targeted" - - "tiddit" - - "trimming" - - "tumor_normal_pair" - - "umi" - - "variantcalling_channel" + - "skip_alignment" + - "skip_qc skip_alignment" profile: ["docker"] steps: - name: Check out pipeline code diff --git a/tests/config/nextflow.config b/tests/config/nextflow.config new file mode 100644 index 000000000..3cd8ab3cb --- /dev/null +++ b/tests/config/nextflow.config @@ -0,0 +1,38 @@ +params { + outdir = "output/" + publish_dir_mode = "copy" + enable_conda = false + singularity_pull_docker_container = false + max_cpus = 2 + max_memory = 6.GB + max_time = 6.h +} + +process { + cpus = 2 + memory = 6.GB + time = 48.h +} + +if ("$PROFILE" == "singularity") { + singularity.enabled = true + singularity.autoMounts = true +} else if ("$PROFILE" == "conda") { + params.enable_conda = true +} else { + docker.enabled = true + docker.runOptions = '-u \$(id -u):\$(id -g)' +} + +// Load test_data.config containing paths to test data +includeConfig 'test_data.config' + +// Load default nextflow.config for default module params +includeConfig '../../nextflow.config' + +// Load modules.config for default module params +includeConfig '../../conf/modules.config' + +manifest { + nextflowVersion = '!>=21.04.0' +} diff --git a/tests/config/pytest_software.yml b/tests/config/pytest_software.yml new file mode 100644 index 000000000..84b3f181d --- /dev/null +++ b/tests/config/pytest_software.yml @@ -0,0 +1,15 @@ +quantify_salmon: + - subworkflows/local/quantify_salmon.nf + - tests/subworkflows/local/quantify_salmon/** + - modules/nf-core/software/salmon/quant/** + - modules/local/salmon_tx2gene.nf + - modules/local/salmon_tximport.nf + - modules/local/salmon_merge_counts.nf + - modules/local/salmon_summarizedexperiment.nf + +quantify_rsem: + - subworkflows/local/quantify_rsem.nf + - subworkflows/nf-core/bam_sort_samtools.nf + - tests/subworkflows/local/quantify_rsem/** + - modules/nf-core/software/rsem/calculateexpression/** + - modules/local/rsem_merge_counts.nf diff --git a/tests/config/test_data.config b/tests/config/test_data.config new file mode 100644 index 000000000..0d7304b12 --- /dev/null +++ b/tests/config/test_data.config @@ -0,0 +1,3 @@ +def nf_core_rnaseq_data = "https://raw.githubusercontent.com/nf-core/test-datasets/rnaseq/data/" + +includeConfig 'https://github.com/nf-core/modules/raw/master/tests/config/test_data.config' diff --git a/tests/subworkflows/local/quantify_rsem/main.nf b/tests/subworkflows/local/quantify_rsem/main.nf new file mode 100644 index 000000000..6da37f6e2 --- /dev/null +++ b/tests/subworkflows/local/quantify_rsem/main.nf @@ -0,0 +1,25 @@ +#!/usr/bin/env nextflow + +nextflow.enable.dsl = 2 + +def modules = params.modules.clone() + +include { QUANTIFY_RSEM } from '../../../../subworkflows/local/quantify_rsem' addParams( + calculateexpression_options: rsem_calculateexpression_options, + samtools_sort_options: samtools_sort_genome_options, + samtools_index_options: samtools_index_genome_options, + samtools_stats_options: samtools_index_genome_options, + merge_counts_options: modules['rsem_merge_counts'] +) + +workflow test_quantify_rsem { + input = [ [ id:'test', single_end:true ], + file(params.test_data['sarscov2']['illumina']['bam']['test_single_end_bam'], checkIfExists: true)] + + rsem_index = file('https://github.com/nf-core/test-datasets/raw/rnaseq/reference/rsem.tar.gz', checkIfExists: true) + + QUANTIFY_RSEM ( + input, + rsem_index + ) +} diff --git a/tests/subworkflows/local/quantify_rsem/test.yml b/tests/subworkflows/local/quantify_rsem/test.yml new file mode 100644 index 000000000..7ec7f3128 --- /dev/null +++ b/tests/subworkflows/local/quantify_rsem/test.yml @@ -0,0 +1,7 @@ +- name: subworkflow quantify_rsem + command: nextflow run ./tests/subworkflows/local/quantify_rsem/ -entry test_quantify_rsem -c tests/config/nextflow.config + tags: + - quantify_rsem + - failing + exit_code: 1 + # TODO files: diff --git a/tests/subworkflows/local/quantify_salmon/main.nf b/tests/subworkflows/local/quantify_salmon/main.nf new file mode 100644 index 000000000..cbc0be131 --- /dev/null +++ b/tests/subworkflows/local/quantify_salmon/main.nf @@ -0,0 +1,25 @@ +#!/usr/bin/env nextflow + +nextflow.enable.dsl = 2 + +def modules = params.modules.clone() + +include { SALMON_INDEX } from '../../../../modules/nf-core/software/salmon/index/main.nf' addParams( options: [:] ) +include { QUANTIFY_SALMON } from '../../../../subworkflows/local/quantify_salmon' addParams( + genome_options: '', + tximport_options: modules['salmon_tximport'], + salmon_quant_options: modules['salmon_quant'], + merge_counts_options: modules['salmon_merge_counts'] +) + +workflow test_quantify_salmon { + input = [ [ id:'test', single_end:true ], // meta map + file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true) + ] + genome_fasta = file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true) + transcript_fasta = file(params.test_data['sarscov2']['genome']['transcriptome_fasta'], checkIfExists: true) + gtf = file(params.test_data['sarscov2']['genome']['genome_gtf'], checkIfExists: true) + + SALMON_INDEX ( genome_fasta, transcript_fasta ) + QUANTIFY_SALMON ( input, SALMON_INDEX.out.index, transcript_fasta, gtf, false ) +} diff --git a/tests/subworkflows/local/quantify_salmon/test.yml b/tests/subworkflows/local/quantify_salmon/test.yml new file mode 100644 index 000000000..d1d487839 --- /dev/null +++ b/tests/subworkflows/local/quantify_salmon/test.yml @@ -0,0 +1,7 @@ +- name: subworkflow quantify_salmon + command: nextflow run ./tests/subworkflows/local/quantify_salmon/ -entry test_quantify_salmon -c tests/config/nextflow.config + tags: + - quantify_salmon + - failing + exit_code: 1 + # TODO files: diff --git a/tests/test_default.yml b/tests/test_default.yml new file mode 100644 index 000000000..5aa4b708c --- /dev/null +++ b/tests/test_default.yml @@ -0,0 +1,70 @@ +# NOTE For now these tests don't support any containers beside docker +- name: Run default pipeline with test data + command: nextflow run main.nf -profile test,docker + tags: + - default + files: + # fastqc + - path: results/fastqc/RAP1_IAA_30M_REP1_1_fastqc.html + - path: results/fastqc/RAP1_IAA_30M_REP1_1_fastqc.zip + - path: results/fastqc/RAP1_IAA_30M_REP1_2_fastqc.html + - path: results/fastqc/RAP1_IAA_30M_REP1_2_fastqc.zip + - path: results/fastqc/RAP1_UNINDUCED_REP1_fastqc.html + - path: results/fastqc/RAP1_UNINDUCED_REP1_fastqc.zip + - path: results/fastqc/RAP1_UNINDUCED_REP2_fastqc.html + - path: results/fastqc/RAP1_UNINDUCED_REP2_fastqc.zip + - path: results/fastqc/WT_REP1_1_fastqc.html + - path: results/fastqc/WT_REP1_1_fastqc.zip + - path: results/fastqc/WT_REP1_2_fastqc.html + - path: results/fastqc/WT_REP1_2_fastqc.zip + - path: results/fastqc/WT_REP2_1_fastqc.html + - path: results/fastqc/WT_REP2_1_fastqc.zip + - path: results/fastqc/WT_REP2_2_fastqc.html + - path: results/fastqc/WT_REP2_2_fastqc.zip + + # multiqc + - path: results/multiqc/star_salmon/multiqc_report.html + + # salmon + - path: results/salmon/salmon.merged.gene_counts_length_scaled.rds + - path: results/salmon/salmon.merged.gene_counts_length_scaled.tsv + - path: results/salmon/salmon.merged.gene_counts.rds + - path: results/salmon/salmon.merged.gene_counts_scaled.rds + - path: results/salmon/salmon.merged.gene_counts_scaled.tsv + - path: results/salmon/salmon.merged.gene_counts.tsv + - path: results/salmon/salmon.merged.gene_tpm.tsv + - path: results/salmon/salmon.merged.transcript_counts.rds + - path: results/salmon/salmon.merged.transcript_counts.tsv + - path: results/salmon/salmon.merged.transcript_tpm.tsv + + # star_salmon + - md5sum: c35b111750b6dec93b2605a4efd05ecf + path: results/star_salmon/RAP1_IAA_30M_REP1.markdup.sorted.bam + - md5sum: e66938e03e213321e25bbf71a59dc0f7 + path: results/star_salmon/RAP1_IAA_30M_REP1.markdup.sorted.bam.bai + - md5sum: a428c6152079a33c8835d1dbe02080e1 + path: results/star_salmon/RAP1_UNINDUCED_REP1.markdup.sorted.bam + - md5sum: 51d711d916e06fb27f24e93207b99648 + path: results/star_salmon/RAP1_UNINDUCED_REP1.markdup.sorted.bam.bai + - md5sum: 4f4f85c87e9212e8eb5f82accbe4a6ce + path: results/star_salmon/RAP1_UNINDUCED_REP2.markdup.sorted.bam + - md5sum: f5bf66d3fba93b98ef7326a30bc2fd0f + path: results/star_salmon/RAP1_UNINDUCED_REP2.markdup.sorted.bam.bai + - md5sum: 16f89e27cbaf438883dd1708c5dfe631 + path: results/star_salmon/WT_REP1.markdup.sorted.bam + - md5sum: 22e88cd06d331f84fb00d599851c0382 + path: results/star_salmon/WT_REP1.markdup.sorted.bam.bai + - md5sum: bbe49353f4b18ba6445503c3b7a23cc5 + path: results/star_salmon/WT_REP2.markdup.sorted.bam + - md5sum: 90df98e87262785a3776b9cf82b30457 + path: results/star_salmon/WT_REP2.markdup.sorted.bam.bai + + # Trimgalore + - path: results/trimgalore/RAP1_IAA_30M_REP1_1.fastq.gz_trimming_report.txt + - path: results/trimgalore/RAP1_IAA_30M_REP1_2.fastq.gz_trimming_report.txt + - path: results/trimgalore/RAP1_UNINDUCED_REP1.fastq.gz_trimming_report.txt + - path: results/trimgalore/RAP1_UNINDUCED_REP2.fastq.gz_trimming_report.txt + - path: results/trimgalore/WT_REP1_1.fastq.gz_trimming_report.txt + - path: results/trimgalore/WT_REP1_2.fastq.gz_trimming_report.txt + - path: results/trimgalore/WT_REP2_1.fastq.gz_trimming_report.txt + - path: results/trimgalore/WT_REP2_2.fastq.gz_trimming_report.txt diff --git a/tests/test_hisat2.yml b/tests/test_hisat2.yml new file mode 100644 index 000000000..ac37e103e --- /dev/null +++ b/tests/test_hisat2.yml @@ -0,0 +1,12 @@ +# NOTE For now these tests don't support any containers beside docker +- name: Test HISAT2 with skip_qc workflow parameters + command: nextflow run main.nf -profile test,docker --aligner hisat2 --skip_qc + tags: + - hisat2 + - skip_qc + +- name: Test HISAT2 with hisat2_index false workflow parameters + command: nextflow run main.nf -profile test,docker --aligner hisat2 --hisat2_index false + tags: + - hisat2 + - "hisat2_index false" diff --git a/tests/test_salmon.yml b/tests/test_salmon.yml new file mode 100644 index 000000000..52481c344 --- /dev/null +++ b/tests/test_salmon.yml @@ -0,0 +1,16 @@ +# NOTE For now these tests don't support any containers beside docker +- name: Test Salmon with skip_qc skip_alignment workflow parameters + command: nextflow run main.nf -profile test,docker --pseudo_aligner salmon --skip_qc --skip_alignment + tags: + - salmon + - skip_qc + - skip_alignment + - "skip_qc skip_alignment" + +- name: Test Salmon with salmon_index false transcript_fasta false workflow parameters + command: nextflow run main.nf -profile test,docker --salmon_index false --transcript_fasta false + tags: + - salmon + - salmon_index + - transcript_fasta + - "salmon_index false transcript_fasta false" diff --git a/tests/test_sra_download.yml b/tests/test_sra_download.yml new file mode 100644 index 000000000..cc6bd05ef --- /dev/null +++ b/tests/test_sra_download.yml @@ -0,0 +1,5 @@ +# NOTE For now these tests don't support any containers beside docker +- name: Run pipeline to download public data + command: nextflow run main.nf -profile test_sra,docker + tags: + - sra_download diff --git a/tests/test_star_rsem.yml b/tests/test_star_rsem.yml new file mode 100644 index 000000000..c5f499f26 --- /dev/null +++ b/tests/test_star_rsem.yml @@ -0,0 +1,14 @@ +# NOTE For now these tests don't support any containers beside docker +- name: Test STAR RSEM with skip_qc workflow parameters + command: nextflow run main.nf -profile test,docker --aligner star_rsem --skip_qc + tags: + - star + - rsem + - skip_qc + +- name: Test STAR RSEM with rsem_index false workflow parameters + command: nextflow run main.nf -profile test,docker --aligner star_rsem --rsem_index false + tags: + - star + - rsem + - "rsem_index false" diff --git a/tests/test_star_salmon.yml b/tests/test_star_salmon.yml new file mode 100644 index 000000000..e040c2b09 --- /dev/null +++ b/tests/test_star_salmon.yml @@ -0,0 +1,87 @@ +# NOTE For now these tests don't support any containers beside docker +- name: Test STAR Salmon with skip_qc workflow parameters + command: nextflow run main.nf -profile test,docker --aligner star_salmon --skip_qc + tags: + - star + - salmon + - skip_qc + +- name: Test STAR Salmon with remove_ribo_rna workflow parameters + command: nextflow run main.nf -profile test,docker --aligner star_salmon --remove_ribo_rna + tags: + - star + - salmon + - remove_ribo_rna + - failing + exit_code: 1 + +- name: Test STAR Salmon with skim_trimming workflow parameters + command: nextflow run main.nf -profile test,docker --aligner star_salmon --skim_trimming + tags: + - star + - salmon + - skim_trimming + +- name: Test STAR Salmon with gtf false workflow parameters + command: nextflow run main.nf -profile test,docker --aligner star_salmon --gtf false + tags: + - star + - salmon + - "gtf false" + +- name: Test STAR Salmon with star_index false workflow parameters + command: nextflow run main.nf -profile test,docker --aligner star_salmon --star_index false + tags: + - star + - salmon + - "star_index false" + +- name: Test STAR Salmon with transcript_fasta false workflow parameters + command: nextflow run main.nf -profile test,docker --aligner star_salmon --transcript_fasta false + tags: + - star + - salmon + - "transcript_fasta" + +- name: Test STAR Salmon with min_mapped_reads 90 workflow parameters + command: nextflow run main.nf -profile test,docker --aligner star_salmon --min_mapped_reads 90 + tags: + - star + - salmon + - "min_mapped_reads 90" + +- name: Test STAR Salmon with with_umi workflow parameters + command: nextflow run main.nf -profile test,docker --aligner star_salmon --with_umi + tags: + - star + - salmon + - with_umi + +- name: Test STAR Salmon with with_umi skip_trimming workflow parameters + command: nextflow run main.nf -profile test,docker --aligner star_salmon --with_umi --skip_trimming + tags: + - star + - salmon + - with_umi + - skip_trimming + +- name: Test STAR Salmon with bam_csi_index workflow parameters + command: nextflow run main.nf -profile test,docker --aligner star_salmon --bam_csi_index + tags: + - star + - salmon + - "bam_csi_index" + +- name: Test STAR Salmon with save_align_intermeds save_reference workflow parameters + command: nextflow run main.nf -profile test,docker --aligner star_salmon --save_align_intermeds --save_reference + tags: + - star + - salmon + - "save_reference" + +- name: Test STAR Salmon with featurecounts_group_type false workflow parameters + command: nextflow run main.nf -profile test,docker --aligner star_salmon --featurecounts_group_type false + tags: + - star + - salmon + - "featurecounts_group_type" From c0ec02f280bf016fddc23b4e0ee24afafa82aa10 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 10 Oct 2022 11:53:00 +0200 Subject: [PATCH 03/49] ci: Add a few tags --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c98a7af01..c01d50e90 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -117,6 +117,9 @@ jobs: NXF_VER: - "21.10.3" test: + - "hisat2" + - "salmon" + - "skip_qc" - "skip_alignment" - "skip_qc skip_alignment" profile: ["docker"] From 8a5f904e79ed146aaa82027bb4273783a29b7257 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 10 Oct 2022 12:16:43 +0200 Subject: [PATCH 04/49] ci: Reduce test runs --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c01d50e90..976c44a95 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,7 @@ jobs: runs-on: ubuntu-latest strategy: # HACK Remove after DSL2 rewrite is done - fail-fast: false + fail-fast: true matrix: NXF_VER: - "21.10.3" @@ -112,7 +112,7 @@ jobs: runs-on: ubuntu-latest strategy: # HACK Remove after DSL2 rewrite is done - fail-fast: false + fail-fast: true matrix: NXF_VER: - "21.10.3" From c175aef05ccdcf96496a2aa12671abbbae4492d4 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 10 Oct 2022 12:29:18 +0200 Subject: [PATCH 05/49] test: Add --outdir --- tests/test_default.yml | 2 +- tests/test_hisat2.yml | 4 ++-- tests/test_salmon.yml | 4 ++-- tests/test_sra_download.yml | 2 +- tests/test_star_rsem.yml | 4 ++-- tests/test_star_salmon.yml | 24 ++++++++++++------------ 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/tests/test_default.yml b/tests/test_default.yml index 5aa4b708c..4e324b445 100644 --- a/tests/test_default.yml +++ b/tests/test_default.yml @@ -1,6 +1,6 @@ # NOTE For now these tests don't support any containers beside docker - name: Run default pipeline with test data - command: nextflow run main.nf -profile test,docker + command: nextflow run main.nf -profile test,docker --outdir results tags: - default files: diff --git a/tests/test_hisat2.yml b/tests/test_hisat2.yml index ac37e103e..23ad18f45 100644 --- a/tests/test_hisat2.yml +++ b/tests/test_hisat2.yml @@ -1,12 +1,12 @@ # NOTE For now these tests don't support any containers beside docker - name: Test HISAT2 with skip_qc workflow parameters - command: nextflow run main.nf -profile test,docker --aligner hisat2 --skip_qc + command: nextflow run main.nf -profile test,docker --aligner hisat2 --skip_qc --outdir results tags: - hisat2 - skip_qc - name: Test HISAT2 with hisat2_index false workflow parameters - command: nextflow run main.nf -profile test,docker --aligner hisat2 --hisat2_index false + command: nextflow run main.nf -profile test,docker --aligner hisat2 --hisat2_index false --outdir results tags: - hisat2 - "hisat2_index false" diff --git a/tests/test_salmon.yml b/tests/test_salmon.yml index 52481c344..bfd2ae5c1 100644 --- a/tests/test_salmon.yml +++ b/tests/test_salmon.yml @@ -1,6 +1,6 @@ # NOTE For now these tests don't support any containers beside docker - name: Test Salmon with skip_qc skip_alignment workflow parameters - command: nextflow run main.nf -profile test,docker --pseudo_aligner salmon --skip_qc --skip_alignment + command: nextflow run main.nf -profile test,docker --pseudo_aligner salmon --skip_qc --skip_alignment --outdir results tags: - salmon - skip_qc @@ -8,7 +8,7 @@ - "skip_qc skip_alignment" - name: Test Salmon with salmon_index false transcript_fasta false workflow parameters - command: nextflow run main.nf -profile test,docker --salmon_index false --transcript_fasta false + command: nextflow run main.nf -profile test,docker --salmon_index false --transcript_fasta false --outdir results tags: - salmon - salmon_index diff --git a/tests/test_sra_download.yml b/tests/test_sra_download.yml index cc6bd05ef..c31b7075c 100644 --- a/tests/test_sra_download.yml +++ b/tests/test_sra_download.yml @@ -1,5 +1,5 @@ # NOTE For now these tests don't support any containers beside docker - name: Run pipeline to download public data - command: nextflow run main.nf -profile test_sra,docker + command: nextflow run main.nf -profile test_sra,docker --outdir results tags: - sra_download diff --git a/tests/test_star_rsem.yml b/tests/test_star_rsem.yml index c5f499f26..cb397de08 100644 --- a/tests/test_star_rsem.yml +++ b/tests/test_star_rsem.yml @@ -1,13 +1,13 @@ # NOTE For now these tests don't support any containers beside docker - name: Test STAR RSEM with skip_qc workflow parameters - command: nextflow run main.nf -profile test,docker --aligner star_rsem --skip_qc + command: nextflow run main.nf -profile test,docker --aligner star_rsem --skip_qc --outdir results tags: - star - rsem - skip_qc - name: Test STAR RSEM with rsem_index false workflow parameters - command: nextflow run main.nf -profile test,docker --aligner star_rsem --rsem_index false + command: nextflow run main.nf -profile test,docker --aligner star_rsem --rsem_index false --outdir results tags: - star - rsem diff --git a/tests/test_star_salmon.yml b/tests/test_star_salmon.yml index e040c2b09..6d244fbc8 100644 --- a/tests/test_star_salmon.yml +++ b/tests/test_star_salmon.yml @@ -1,13 +1,13 @@ # NOTE For now these tests don't support any containers beside docker - name: Test STAR Salmon with skip_qc workflow parameters - command: nextflow run main.nf -profile test,docker --aligner star_salmon --skip_qc + command: nextflow run main.nf -profile test,docker --aligner star_salmon --skip_qc --outdir results tags: - star - salmon - skip_qc - name: Test STAR Salmon with remove_ribo_rna workflow parameters - command: nextflow run main.nf -profile test,docker --aligner star_salmon --remove_ribo_rna + command: nextflow run main.nf -profile test,docker --aligner star_salmon --remove_ribo_rna --outdir results tags: - star - salmon @@ -16,49 +16,49 @@ exit_code: 1 - name: Test STAR Salmon with skim_trimming workflow parameters - command: nextflow run main.nf -profile test,docker --aligner star_salmon --skim_trimming + command: nextflow run main.nf -profile test,docker --aligner star_salmon --skim_trimming --outdir results tags: - star - salmon - skim_trimming - name: Test STAR Salmon with gtf false workflow parameters - command: nextflow run main.nf -profile test,docker --aligner star_salmon --gtf false + command: nextflow run main.nf -profile test,docker --aligner star_salmon --gtf false --outdir results tags: - star - salmon - "gtf false" - name: Test STAR Salmon with star_index false workflow parameters - command: nextflow run main.nf -profile test,docker --aligner star_salmon --star_index false + command: nextflow run main.nf -profile test,docker --aligner star_salmon --star_index false --outdir results tags: - star - salmon - "star_index false" - name: Test STAR Salmon with transcript_fasta false workflow parameters - command: nextflow run main.nf -profile test,docker --aligner star_salmon --transcript_fasta false + command: nextflow run main.nf -profile test,docker --aligner star_salmon --transcript_fasta false --outdir results tags: - star - salmon - "transcript_fasta" - name: Test STAR Salmon with min_mapped_reads 90 workflow parameters - command: nextflow run main.nf -profile test,docker --aligner star_salmon --min_mapped_reads 90 + command: nextflow run main.nf -profile test,docker --aligner star_salmon --min_mapped_reads 90 --outdir results tags: - star - salmon - "min_mapped_reads 90" - name: Test STAR Salmon with with_umi workflow parameters - command: nextflow run main.nf -profile test,docker --aligner star_salmon --with_umi + command: nextflow run main.nf -profile test,docker --aligner star_salmon --with_umi --outdir results tags: - star - salmon - with_umi - name: Test STAR Salmon with with_umi skip_trimming workflow parameters - command: nextflow run main.nf -profile test,docker --aligner star_salmon --with_umi --skip_trimming + command: nextflow run main.nf -profile test,docker --aligner star_salmon --with_umi --skip_trimming --outdir results tags: - star - salmon @@ -66,21 +66,21 @@ - skip_trimming - name: Test STAR Salmon with bam_csi_index workflow parameters - command: nextflow run main.nf -profile test,docker --aligner star_salmon --bam_csi_index + command: nextflow run main.nf -profile test,docker --aligner star_salmon --bam_csi_index --outdir results tags: - star - salmon - "bam_csi_index" - name: Test STAR Salmon with save_align_intermeds save_reference workflow parameters - command: nextflow run main.nf -profile test,docker --aligner star_salmon --save_align_intermeds --save_reference + command: nextflow run main.nf -profile test,docker --aligner star_salmon --save_align_intermeds --save_reference --outdir results tags: - star - salmon - "save_reference" - name: Test STAR Salmon with featurecounts_group_type false workflow parameters - command: nextflow run main.nf -profile test,docker --aligner star_salmon --featurecounts_group_type false + command: nextflow run main.nf -profile test,docker --aligner star_salmon --featurecounts_group_type false --outdir results tags: - star - salmon From 263719c4b4e541288d54f060dd1e6f1990c097b8 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 10 Oct 2022 12:50:25 +0200 Subject: [PATCH 06/49] test: Update default md5sums --- tests/test_default.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/test_default.yml b/tests/test_default.yml index 4e324b445..54aa6451d 100644 --- a/tests/test_default.yml +++ b/tests/test_default.yml @@ -38,25 +38,25 @@ - path: results/salmon/salmon.merged.transcript_tpm.tsv # star_salmon - - md5sum: c35b111750b6dec93b2605a4efd05ecf + - md5sum: 777fc7806490edd2ce0a7cbfc742ceaa path: results/star_salmon/RAP1_IAA_30M_REP1.markdup.sorted.bam - - md5sum: e66938e03e213321e25bbf71a59dc0f7 + - md5sum: 76998f645d101fb06bec6df1f14acd1f path: results/star_salmon/RAP1_IAA_30M_REP1.markdup.sorted.bam.bai - - md5sum: a428c6152079a33c8835d1dbe02080e1 + - md5sum: a3d5a48db015dd9a1aeab92acf19d311 path: results/star_salmon/RAP1_UNINDUCED_REP1.markdup.sorted.bam - - md5sum: 51d711d916e06fb27f24e93207b99648 + - md5sum: 81c12f756d9f2c650ccaa0442d7c1207 path: results/star_salmon/RAP1_UNINDUCED_REP1.markdup.sorted.bam.bai - - md5sum: 4f4f85c87e9212e8eb5f82accbe4a6ce + - md5sum: 7ce1704a8ac04afbb6d944c038144baf path: results/star_salmon/RAP1_UNINDUCED_REP2.markdup.sorted.bam - - md5sum: f5bf66d3fba93b98ef7326a30bc2fd0f + - md5sum: 2a0a2934001813bacb4dd36004acf487 path: results/star_salmon/RAP1_UNINDUCED_REP2.markdup.sorted.bam.bai - - md5sum: 16f89e27cbaf438883dd1708c5dfe631 + - md5sum: 9e346d843a583cbf32cb46fa92c2d5b3 path: results/star_salmon/WT_REP1.markdup.sorted.bam - - md5sum: 22e88cd06d331f84fb00d599851c0382 + - md5sum: 4959827b17abd9eb2d24c40e3fae7cae path: results/star_salmon/WT_REP1.markdup.sorted.bam.bai - - md5sum: bbe49353f4b18ba6445503c3b7a23cc5 + - md5sum: 127b085e48849f448dccde482f2c2991 path: results/star_salmon/WT_REP2.markdup.sorted.bam - - md5sum: 90df98e87262785a3776b9cf82b30457 + - md5sum: 1b93886a4c71d1fb356dc36ca135390d path: results/star_salmon/WT_REP2.markdup.sorted.bam.bai # Trimgalore From bb4d75cad0f6fcb2191bb08d00a801f23fa63888 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 10 Oct 2022 13:17:53 +0200 Subject: [PATCH 07/49] test: Remove md5sums for bams in default --- tests/test_default.yml | 30 ++++++++++-------------------- 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/tests/test_default.yml b/tests/test_default.yml index 54aa6451d..7294f7824 100644 --- a/tests/test_default.yml +++ b/tests/test_default.yml @@ -38,26 +38,16 @@ - path: results/salmon/salmon.merged.transcript_tpm.tsv # star_salmon - - md5sum: 777fc7806490edd2ce0a7cbfc742ceaa - path: results/star_salmon/RAP1_IAA_30M_REP1.markdup.sorted.bam - - md5sum: 76998f645d101fb06bec6df1f14acd1f - path: results/star_salmon/RAP1_IAA_30M_REP1.markdup.sorted.bam.bai - - md5sum: a3d5a48db015dd9a1aeab92acf19d311 - path: results/star_salmon/RAP1_UNINDUCED_REP1.markdup.sorted.bam - - md5sum: 81c12f756d9f2c650ccaa0442d7c1207 - path: results/star_salmon/RAP1_UNINDUCED_REP1.markdup.sorted.bam.bai - - md5sum: 7ce1704a8ac04afbb6d944c038144baf - path: results/star_salmon/RAP1_UNINDUCED_REP2.markdup.sorted.bam - - md5sum: 2a0a2934001813bacb4dd36004acf487 - path: results/star_salmon/RAP1_UNINDUCED_REP2.markdup.sorted.bam.bai - - md5sum: 9e346d843a583cbf32cb46fa92c2d5b3 - path: results/star_salmon/WT_REP1.markdup.sorted.bam - - md5sum: 4959827b17abd9eb2d24c40e3fae7cae - path: results/star_salmon/WT_REP1.markdup.sorted.bam.bai - - md5sum: 127b085e48849f448dccde482f2c2991 - path: results/star_salmon/WT_REP2.markdup.sorted.bam - - md5sum: 1b93886a4c71d1fb356dc36ca135390d - path: results/star_salmon/WT_REP2.markdup.sorted.bam.bai + - path: results/star_salmon/RAP1_IAA_30M_REP1.markdup.sorted.bam + - path: results/star_salmon/RAP1_IAA_30M_REP1.markdup.sorted.bam.bai + - path: results/star_salmon/RAP1_UNINDUCED_REP1.markdup.sorted.bam + - path: results/star_salmon/RAP1_UNINDUCED_REP1.markdup.sorted.bam.bai + - path: results/star_salmon/RAP1_UNINDUCED_REP2.markdup.sorted.bam + - path: results/star_salmon/RAP1_UNINDUCED_REP2.markdup.sorted.bam.bai + - path: results/star_salmon/WT_REP1.markdup.sorted.bam + - path: results/star_salmon/WT_REP1.markdup.sorted.bam.bai + - path: results/star_salmon/WT_REP2.markdup.sorted.bam + - path: results/star_salmon/WT_REP2.markdup.sorted.bam.bai # Trimgalore - path: results/trimgalore/RAP1_IAA_30M_REP1_1.fastq.gz_trimming_report.txt From 7c5bcc787a7ab7a02a23e957235b9069376d4ea7 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 10 Oct 2022 13:20:51 +0200 Subject: [PATCH 08/49] ci: Only test on docker for now --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 976c44a95..d47d8b5a6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,7 @@ jobs: - "latest-everything" test: - "default" - profile: ["docker", "singularity", "conda"] + profile: ["docker"] exclude: - NXF_VER: "latest-everything" profile: "conda" From d7b3deba657a0d5e3d425ac8596cde8b39d738b7 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 10 Oct 2022 13:51:00 +0200 Subject: [PATCH 09/49] ci: Remove broken test --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d47d8b5a6..2eb78a0c4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -121,7 +121,7 @@ jobs: - "salmon" - "skip_qc" - "skip_alignment" - - "skip_qc skip_alignment" + # FIXME - "skip_qc skip_alignment" profile: ["docker"] steps: - name: Check out pipeline code From ff99fac050792a751955c5a92c25e61a45909943 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 10 Oct 2022 15:53:33 +0200 Subject: [PATCH 10/49] chore: Update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ad658113..ac3f801b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Enhancements & fixes - Update modules to latest restructuring on nf-core/modules +- [[#883](https://github.com/nf-core/rnaseq/pull/883)] - Migrate CI tests to Pytest workflow ### Parameters From 967ed570f155eaddbed21711c3c5a61c7ee2fd74 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 10 Oct 2022 16:19:40 +0200 Subject: [PATCH 11/49] ci: Add pytest for tags --- .github/workflows/ci.yml | 109 ++++++++++++------ .../{pytest_software.yml => pytest_tags.yml} | 0 2 files changed, 72 insertions(+), 37 deletions(-) rename tests/config/{pytest_software.yml => pytest_tags.yml} (100%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2eb78a0c4..0cf1ef5a4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,7 @@ env: jobs: test: - name: Run pipeline with test data + name: Run default pipeline test # Only run on push if this is the nf-core dev branch (merged PRs) if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/rnaseq') }}" runs-on: ubuntu-latest @@ -82,7 +82,7 @@ jobs: - name: Run pipeline with tests settings uses: Wandalen/wretry.action@v1.0.11 with: - command: TMPDIR=~ PROFILE=${{ matrix.profile }} pytest --tag ${{ matrix.test }} --symlink --kwdof --git-aware --color=yes + command: TMPDIR=~ PROFILE=${{ matrix.profile }} pytest --tag default --symlink --kwdof --git-aware --color=yes attempt_limit: 3 - name: Output log on failure @@ -105,52 +105,88 @@ jobs: !/tmp/pytest_workflow_*/*/work/conda !/tmp/pytest_workflow_*/*/work/singularity - test_all: - name: Run pipeline with test data (complete) - # Only run on push if this is the nf-core dev branch (merged PRs) - if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/rnaseq') }}" + changes: + name: Check for changes runs-on: ubuntu-latest - strategy: - # HACK Remove after DSL2 rewrite is done - fail-fast: true - matrix: - NXF_VER: - - "21.10.3" - test: - - "hisat2" - - "salmon" - - "skip_qc" - - "skip_alignment" - # FIXME - "skip_qc skip_alignment" - profile: ["docker"] + outputs: + # Expose matched filters as job 'modules' output variable + tags: ${{ steps.filter.outputs.changes }} steps: - - name: Check out pipeline code - uses: actions/checkout@v2 + - uses: actions/checkout@v2 - - name: Install Nextflow - uses: nf-core/setup-nextflow@v1 + - uses: dorny/paths-filter@v2 + id: filter with: - version: "${{ matrix.NXF_VER }}" + filters: "tests/config/pytest_tags.yml" + test_changes: + name: ${{ matrix.tags }} ${{ matrix.profile }} + runs-on: ubuntu-20.04 + needs: changes + if: needs.changes.outputs.tags != '[]' + strategy: + fail-fast: false + matrix: + tags: ["${{ fromJson(needs.changes.outputs.tags) }}"] + profile: ["docker"] # TODO Add singularity/conda + steps: - name: Set up Python uses: actions/setup-python@v2 with: python-version: "3.x" - - name: Install dependencies + - uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: Install Python dependencies run: python -m pip install --upgrade pip pytest-workflow - - name: Run pipeline with tests settings - uses: Wandalen/wretry.action@v1.0.11 + - uses: actions/cache@v2 with: - command: PROFILE=${{ matrix.profile }} pytest --tag ${{ matrix.test }} --symlink --kwdof --git-aware --color=yes - attempt_limit: 3 + path: /usr/local/bin/nextflow + key: ${{ runner.os }} + restore-keys: | + ${{ runner.os }}-nextflow- + + - name: Install Nextflow + env: + CAPSULE_LOG: none + run: | + wget -qO- get.nextflow.io | bash + sudo mv nextflow /usr/local/bin/ + + - name: Set up Singularity + if: matrix.profile == 'singularity' + uses: eWaterCycle/setup-singularity@v5 + with: + singularity-version: 3.7.1 + + - name: Set up miniconda + if: matrix.profile == 'conda' + uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + channels: conda-forge,bioconda,defaults + python-version: ${{ matrix.python-version }} + + - name: Conda clean + if: matrix.profile == 'conda' + run: conda clean -a + + # Test the module + - name: Run pytest-workflow + # only use one thread for pytest-workflow to avoid race condition on conda cache. + run: TMPDIR=~ PROFILE=${{ matrix.profile }} pytest --tag ${{ matrix.tags }} --symlink --kwdof --git-aware --color=yes - name: Output log on failure if: failure() run: | sudo apt install bat > /dev/null - batcat --decorations=always --color=always /tmp/pytest_workflow_*/*/log.{out,err} + batcat --decorations=always --color=always /home/runner/pytest_workflow_*/*/log.{out,err} - name: Upload logs on failure if: failure() @@ -158,10 +194,9 @@ jobs: with: name: logs-${{ matrix.profile }} path: | - /tmp/pytest_workflow_*/*/.nextflow.log - /tmp/pytest_workflow_*/*/log.out - /tmp/pytest_workflow_*/*/log.err - /tmp/pytest_workflow_*/*/work - /tmp/pytest_workflow_*/**/.command.log - !/tmp/pytest_workflow_*/*/work/conda - !/tmp/pytest_workflow_*/*/work/singularity + /home/runner/pytest_workflow_*/*/.nextflow.log + /home/runner/pytest_workflow_*/*/log.out + /home/runner/pytest_workflow_*/*/log.err + /home/runner/pytest_workflow_*/*/work + !/home/runner/pytest_workflow_*/*/work/conda + !/home/runner/pytest_workflow_*/*/work/singularity diff --git a/tests/config/pytest_software.yml b/tests/config/pytest_tags.yml similarity index 100% rename from tests/config/pytest_software.yml rename to tests/config/pytest_tags.yml From e892f515ffbbf471915462e25f5bad489d1031f9 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 10 Oct 2022 16:20:50 +0200 Subject: [PATCH 12/49] style: Rename pipeline tests to their own dir --- tests/{ => pipeline}/test_default.yml | 0 tests/{ => pipeline}/test_hisat2.yml | 0 tests/{ => pipeline}/test_salmon.yml | 0 tests/{ => pipeline}/test_sra_download.yml | 0 tests/{ => pipeline}/test_star_rsem.yml | 0 tests/{ => pipeline}/test_star_salmon.yml | 0 6 files changed, 0 insertions(+), 0 deletions(-) rename tests/{ => pipeline}/test_default.yml (100%) rename tests/{ => pipeline}/test_hisat2.yml (100%) rename tests/{ => pipeline}/test_salmon.yml (100%) rename tests/{ => pipeline}/test_sra_download.yml (100%) rename tests/{ => pipeline}/test_star_rsem.yml (100%) rename tests/{ => pipeline}/test_star_salmon.yml (100%) diff --git a/tests/test_default.yml b/tests/pipeline/test_default.yml similarity index 100% rename from tests/test_default.yml rename to tests/pipeline/test_default.yml diff --git a/tests/test_hisat2.yml b/tests/pipeline/test_hisat2.yml similarity index 100% rename from tests/test_hisat2.yml rename to tests/pipeline/test_hisat2.yml diff --git a/tests/test_salmon.yml b/tests/pipeline/test_salmon.yml similarity index 100% rename from tests/test_salmon.yml rename to tests/pipeline/test_salmon.yml diff --git a/tests/test_sra_download.yml b/tests/pipeline/test_sra_download.yml similarity index 100% rename from tests/test_sra_download.yml rename to tests/pipeline/test_sra_download.yml diff --git a/tests/test_star_rsem.yml b/tests/pipeline/test_star_rsem.yml similarity index 100% rename from tests/test_star_rsem.yml rename to tests/pipeline/test_star_rsem.yml diff --git a/tests/test_star_salmon.yml b/tests/pipeline/test_star_salmon.yml similarity index 100% rename from tests/test_star_salmon.yml rename to tests/pipeline/test_star_salmon.yml From e1258b4f25408cb30ed14c9d85a24ddace94ce40 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 10 Oct 2022 16:23:19 +0200 Subject: [PATCH 13/49] test: Add star tag --- tests/config/pytest_tags.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/config/pytest_tags.yml b/tests/config/pytest_tags.yml index 84b3f181d..252114466 100644 --- a/tests/config/pytest_tags.yml +++ b/tests/config/pytest_tags.yml @@ -13,3 +13,8 @@ quantify_rsem: - tests/subworkflows/local/quantify_rsem/** - modules/nf-core/software/rsem/calculateexpression/** - modules/local/rsem_merge_counts.nf + +star: + - modules/nf-core/star/** + - subworkflows/local/align_star.nf + - modules/local/star_align_igenomes.nf From a5c3a9732a2463bc82e77b7d67c45b1335ec1838 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 11 Oct 2022 10:37:08 +0200 Subject: [PATCH 14/49] test: Add quantify_salmon to full workflow tests --- tests/pipeline/test_salmon.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/pipeline/test_salmon.yml b/tests/pipeline/test_salmon.yml index bfd2ae5c1..7dff542ca 100644 --- a/tests/pipeline/test_salmon.yml +++ b/tests/pipeline/test_salmon.yml @@ -3,6 +3,7 @@ command: nextflow run main.nf -profile test,docker --pseudo_aligner salmon --skip_qc --skip_alignment --outdir results tags: - salmon + - quantify_salmon - skip_qc - skip_alignment - "skip_qc skip_alignment" @@ -11,6 +12,7 @@ command: nextflow run main.nf -profile test,docker --salmon_index false --transcript_fasta false --outdir results tags: - salmon + - quantify_salmon - salmon_index - transcript_fasta - "salmon_index false transcript_fasta false" From cfaa74a87f0855b40c5e0ffa0967836265bb15b6 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 11 Oct 2022 10:45:23 +0200 Subject: [PATCH 15/49] ci: Use a seperate workflow for pytest-workflow --- .github/workflows/ci.yml | 96 ----------------------- .github/workflows/pytest-workflow.yml | 109 ++++++++++++++++++++++++++ 2 files changed, 109 insertions(+), 96 deletions(-) create mode 100644 .github/workflows/pytest-workflow.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0cf1ef5a4..5fef6be8b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -104,99 +104,3 @@ jobs: /tmp/pytest_workflow_*/**/.command.log !/tmp/pytest_workflow_*/*/work/conda !/tmp/pytest_workflow_*/*/work/singularity - - changes: - name: Check for changes - runs-on: ubuntu-latest - outputs: - # Expose matched filters as job 'modules' output variable - tags: ${{ steps.filter.outputs.changes }} - steps: - - uses: actions/checkout@v2 - - - uses: dorny/paths-filter@v2 - id: filter - with: - filters: "tests/config/pytest_tags.yml" - - test_changes: - name: ${{ matrix.tags }} ${{ matrix.profile }} - runs-on: ubuntu-20.04 - needs: changes - if: needs.changes.outputs.tags != '[]' - strategy: - fail-fast: false - matrix: - tags: ["${{ fromJson(needs.changes.outputs.tags) }}"] - profile: ["docker"] # TODO Add singularity/conda - steps: - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.x" - - - uses: actions/cache@v2 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip- - - - name: Install Python dependencies - run: python -m pip install --upgrade pip pytest-workflow - - - uses: actions/cache@v2 - with: - path: /usr/local/bin/nextflow - key: ${{ runner.os }} - restore-keys: | - ${{ runner.os }}-nextflow- - - - name: Install Nextflow - env: - CAPSULE_LOG: none - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ - - - name: Set up Singularity - if: matrix.profile == 'singularity' - uses: eWaterCycle/setup-singularity@v5 - with: - singularity-version: 3.7.1 - - - name: Set up miniconda - if: matrix.profile == 'conda' - uses: conda-incubator/setup-miniconda@v2 - with: - auto-update-conda: true - channels: conda-forge,bioconda,defaults - python-version: ${{ matrix.python-version }} - - - name: Conda clean - if: matrix.profile == 'conda' - run: conda clean -a - - # Test the module - - name: Run pytest-workflow - # only use one thread for pytest-workflow to avoid race condition on conda cache. - run: TMPDIR=~ PROFILE=${{ matrix.profile }} pytest --tag ${{ matrix.tags }} --symlink --kwdof --git-aware --color=yes - - - name: Output log on failure - if: failure() - run: | - sudo apt install bat > /dev/null - batcat --decorations=always --color=always /home/runner/pytest_workflow_*/*/log.{out,err} - - - name: Upload logs on failure - if: failure() - uses: actions/upload-artifact@v2 - with: - name: logs-${{ matrix.profile }} - path: | - /home/runner/pytest_workflow_*/*/.nextflow.log - /home/runner/pytest_workflow_*/*/log.out - /home/runner/pytest_workflow_*/*/log.err - /home/runner/pytest_workflow_*/*/work - !/home/runner/pytest_workflow_*/*/work/conda - !/home/runner/pytest_workflow_*/*/work/singularity diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml new file mode 100644 index 000000000..f481289f9 --- /dev/null +++ b/.github/workflows/pytest-workflow.yml @@ -0,0 +1,109 @@ +name: pytest-workflow +# This workflow runs the pipeline with the minimal test dataset to check that it completes without any syntax errors +on: + push: + branches: + - dev + pull_request: + release: + types: [published] + +env: + NXF_ANSI_LOG: false + +jobs: + changes: + name: Check for changes + runs-on: ubuntu-latest + outputs: + # Expose matched filters as job 'modules' output variable + tags: ${{ steps.filter.outputs.changes }} + steps: + - uses: actions/checkout@v2 + + - uses: dorny/paths-filter@v2 + id: filter + with: + filters: "tests/config/pytest_tags.yml" + + test_changes: + name: ${{ matrix.tags }} ${{ matrix.profile }} + runs-on: ubuntu-20.04 + needs: changes + if: needs.changes.outputs.tags != '[]' + strategy: + fail-fast: false + matrix: + tags: ["${{ fromJson(needs.changes.outputs.tags) }}"] + profile: ["docker"] # TODO Add singularity/conda + steps: + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: "3.x" + + - uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: Install Python dependencies + run: python -m pip install --upgrade pip pytest-workflow + + - uses: actions/cache@v2 + with: + path: /usr/local/bin/nextflow + key: ${{ runner.os }} + restore-keys: | + ${{ runner.os }}-nextflow- + + - name: Install Nextflow + env: + CAPSULE_LOG: none + run: | + wget -qO- get.nextflow.io | bash + sudo mv nextflow /usr/local/bin/ + + - name: Set up Singularity + if: matrix.profile == 'singularity' + uses: eWaterCycle/setup-singularity@v5 + with: + singularity-version: 3.7.1 + + - name: Set up miniconda + if: matrix.profile == 'conda' + uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + channels: conda-forge,bioconda,defaults + python-version: ${{ matrix.python-version }} + + - name: Conda clean + if: matrix.profile == 'conda' + run: conda clean -a + + # Test the module + - name: Run pytest-workflow + # only use one thread for pytest-workflow to avoid race condition on conda cache. + run: TMPDIR=~ PROFILE=${{ matrix.profile }} pytest --tag ${{ matrix.tags }} --symlink --kwdof --git-aware --color=yes + + - name: Output log on failure + if: failure() + run: | + sudo apt install bat > /dev/null + batcat --decorations=always --color=always /home/runner/pytest_workflow_*/*/log.{out,err} + + - name: Upload logs on failure + if: failure() + uses: actions/upload-artifact@v2 + with: + name: logs-${{ matrix.profile }} + path: | + /home/runner/pytest_workflow_*/*/.nextflow.log + /home/runner/pytest_workflow_*/*/log.out + /home/runner/pytest_workflow_*/*/log.err + /home/runner/pytest_workflow_*/*/work + !/home/runner/pytest_workflow_*/*/work/conda + !/home/runner/pytest_workflow_*/*/work/singularity From 96e275c135152f53687c0863d8f75c41b3d44fef Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 11 Oct 2022 10:46:50 +0200 Subject: [PATCH 16/49] ci: Add code checkout --- .github/workflows/pytest-workflow.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml index f481289f9..d79ed45de 100644 --- a/.github/workflows/pytest-workflow.yml +++ b/.github/workflows/pytest-workflow.yml @@ -37,6 +37,9 @@ jobs: tags: ["${{ fromJson(needs.changes.outputs.tags) }}"] profile: ["docker"] # TODO Add singularity/conda steps: + - name: Check out pipeline code + uses: actions/checkout@v2 + - name: Set up Python uses: actions/setup-python@v2 with: From d63d6cbcd5e2de20ce355ff35035318d96f31a44 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 11 Oct 2022 12:15:11 +0200 Subject: [PATCH 17/49] test: Clean up and confirm they're up to date with current CI --- tests/pipeline/test_hisat2.yml | 2 +- tests/pipeline/test_salmon.yml | 6 ++---- tests/pipeline/test_sra_download.yml | 5 ----- tests/pipeline/test_star_rsem.yml | 2 +- tests/pipeline/test_star_salmon.yml | 32 +++++++++++++++------------- 5 files changed, 21 insertions(+), 26 deletions(-) delete mode 100644 tests/pipeline/test_sra_download.yml diff --git a/tests/pipeline/test_hisat2.yml b/tests/pipeline/test_hisat2.yml index 23ad18f45..dada2ff31 100644 --- a/tests/pipeline/test_hisat2.yml +++ b/tests/pipeline/test_hisat2.yml @@ -9,4 +9,4 @@ command: nextflow run main.nf -profile test,docker --aligner hisat2 --hisat2_index false --outdir results tags: - hisat2 - - "hisat2_index false" + - hisat2_index-false diff --git a/tests/pipeline/test_salmon.yml b/tests/pipeline/test_salmon.yml index 7dff542ca..ffc0082da 100644 --- a/tests/pipeline/test_salmon.yml +++ b/tests/pipeline/test_salmon.yml @@ -6,13 +6,11 @@ - quantify_salmon - skip_qc - skip_alignment - - "skip_qc skip_alignment" - name: Test Salmon with salmon_index false transcript_fasta false workflow parameters command: nextflow run main.nf -profile test,docker --salmon_index false --transcript_fasta false --outdir results tags: - salmon - quantify_salmon - - salmon_index - - transcript_fasta - - "salmon_index false transcript_fasta false" + - salmon_index-false + - transcript_fasta-false diff --git a/tests/pipeline/test_sra_download.yml b/tests/pipeline/test_sra_download.yml deleted file mode 100644 index c31b7075c..000000000 --- a/tests/pipeline/test_sra_download.yml +++ /dev/null @@ -1,5 +0,0 @@ -# NOTE For now these tests don't support any containers beside docker -- name: Run pipeline to download public data - command: nextflow run main.nf -profile test_sra,docker --outdir results - tags: - - sra_download diff --git a/tests/pipeline/test_star_rsem.yml b/tests/pipeline/test_star_rsem.yml index cb397de08..554f75ba3 100644 --- a/tests/pipeline/test_star_rsem.yml +++ b/tests/pipeline/test_star_rsem.yml @@ -11,4 +11,4 @@ tags: - star - rsem - - "rsem_index false" + - rsem_index-false diff --git a/tests/pipeline/test_star_salmon.yml b/tests/pipeline/test_star_salmon.yml index 6d244fbc8..fd762e03b 100644 --- a/tests/pipeline/test_star_salmon.yml +++ b/tests/pipeline/test_star_salmon.yml @@ -6,15 +6,6 @@ - salmon - skip_qc -- name: Test STAR Salmon with remove_ribo_rna workflow parameters - command: nextflow run main.nf -profile test,docker --aligner star_salmon --remove_ribo_rna --outdir results - tags: - - star - - salmon - - remove_ribo_rna - - failing - exit_code: 1 - - name: Test STAR Salmon with skim_trimming workflow parameters command: nextflow run main.nf -profile test,docker --aligner star_salmon --skim_trimming --outdir results tags: @@ -27,21 +18,21 @@ tags: - star - salmon - - "gtf false" + - gtf-false - name: Test STAR Salmon with star_index false workflow parameters command: nextflow run main.nf -profile test,docker --aligner star_salmon --star_index false --outdir results tags: - star - salmon - - "star_index false" + - star_index-false - name: Test STAR Salmon with transcript_fasta false workflow parameters command: nextflow run main.nf -profile test,docker --aligner star_salmon --transcript_fasta false --outdir results tags: - star - salmon - - "transcript_fasta" + - transcript_fasta-false - name: Test STAR Salmon with min_mapped_reads 90 workflow parameters command: nextflow run main.nf -profile test,docker --aligner star_salmon --min_mapped_reads 90 --outdir results @@ -65,23 +56,34 @@ - with_umi - skip_trimming +- name: Test STAR Salmon with remove_ribo_rna and skip_qualimap workflow parameters + command: nextflow run main.nf -profile test,docker --aligner star_salmon --remove_ribo_rna --skip_qualimap --outdir results + tags: + - star + - salmon + - remove_ribo_rna + - skip_qualimap + - failing + exit_code: 1 + - name: Test STAR Salmon with bam_csi_index workflow parameters command: nextflow run main.nf -profile test,docker --aligner star_salmon --bam_csi_index --outdir results tags: - star - salmon - - "bam_csi_index" + - bam_csi_index - name: Test STAR Salmon with save_align_intermeds save_reference workflow parameters command: nextflow run main.nf -profile test,docker --aligner star_salmon --save_align_intermeds --save_reference --outdir results tags: - star - salmon - - "save_reference" + - save_align_intermeds + - save_reference - name: Test STAR Salmon with featurecounts_group_type false workflow parameters command: nextflow run main.nf -profile test,docker --aligner star_salmon --featurecounts_group_type false --outdir results tags: - star - salmon - - "featurecounts_group_type" + - featurecounts_group_type-false From 9d55816e901d707417c42f7fb8cc37d2241f1461 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 11 Oct 2022 12:15:33 +0200 Subject: [PATCH 18/49] test: Add tag paths --- tests/config/pytest_tags.yml | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/tests/config/pytest_tags.yml b/tests/config/pytest_tags.yml index 252114466..9f44fd774 100644 --- a/tests/config/pytest_tags.yml +++ b/tests/config/pytest_tags.yml @@ -7,6 +7,11 @@ quantify_salmon: - modules/local/salmon_merge_counts.nf - modules/local/salmon_summarizedexperiment.nf +hisat2: + - modules/nf-core/hisat2/** + - subworkflows/nf-core/align_hisat2.nf + - tests/pipeline/test_hisat2.yml + quantify_rsem: - subworkflows/local/quantify_rsem.nf - subworkflows/nf-core/bam_sort_samtools.nf @@ -14,7 +19,32 @@ quantify_rsem: - modules/nf-core/software/rsem/calculateexpression/** - modules/local/rsem_merge_counts.nf +salmon: + - bin/salmon_summarizedexperiment.r + - bin/salmon_tx2gene.py + - bin/salmon_tximport.r + - modules/local/salmon_summarizedexperiment.nf + - modules/local/salmon_tx2gene.nf + - modules/local/salmon_tximport.nf + - modules/nf-core/salmon/** + - subworkflows/local/quantify_salmon.nf + - tests/pipeline/test_salmon.yml + - tests/pipeline/test_star_salmon.yml + # - tests/subworkflows/local/quantify_salmon/ + star: + - modules/local/star_align_igenomes.nf + - modules/local/star_genomegenerate_igenomes.nf - modules/nf-core/star/** - subworkflows/local/align_star.nf - - modules/local/star_align_igenomes.nf + - tests/pipeline/test_star_rsem.yml + - tests/pipeline/test_star_salmon.yml + +rsem: + - bin/prepare-for-rsem.py + - modules/local/rsem_merge_counts.nf + - modules/local/umitools_prepareforrsem.nf + - modules/nf-core/rsem/** + - subworkflows/local/quantify_rsem.nf + - tests/pipeline/test_star_rsem.yml + # - tests/subworkflows/local/quantify_rsem/ From acecd8eb8ae6bd80b50fbf626788396e2cfd41c4 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 11 Oct 2022 12:46:23 +0200 Subject: [PATCH 19/49] ci: Add a release CI that runs all of the tests on release --- .github/workflows/release.yml | 124 ++++++++++++++++++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000..8ef8d3af9 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,124 @@ +name: nf-core CI +# This workflow runs the pipeline with the minimal test dataset to check that it completes without any syntax errors +on: + push: + branches: + - dev + pull_request: + release: + types: [published] + +env: + NXF_ANSI_LOG: false + +jobs: + test: + name: ${{ matrix.test }} (${{ matrix.profile }}-${{ matrix.NXF_VER }}) + # Only run on push if this is the nf-core dev branch (merged PRs) + if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/rnaseq') }}" + runs-on: ubuntu-latest + strategy: + # HACK Remove after DSL2 rewrite is done + fail-fast: true + matrix: + NXF_VER: + - "21.10.3" + - "latest-everything" + test: + - "Run default pipeline with test data" + - "Test STAR Salmon with skip_qc workflow parameters" + - "Test STAR Salmon with skim_trimming workflow parameters" + - "Test STAR Salmon with gtf false workflow parameters" + - "Test STAR Salmon with star_index false workflow parameters" + - "Test STAR Salmon with transcript_fasta false workflow parameters" + - "Test STAR Salmon with min_mapped_reads 90 workflow parameters" + - "Test STAR Salmon with with_umi workflow parameters" + - "Test STAR Salmon with with_umi skip_trimming workflow parameters" + - "Test STAR Salmon with remove_ribo_rna and skip_qualimap workflow parameters" + - "Test STAR Salmon with bam_csi_index workflow parameters" + - "Test STAR Salmon with save_align_intermeds save_reference workflow parameters" + - "Test STAR Salmon with featurecounts_group_type false workflow parameters" + - "Test STAR RSEM with skip_qc workflow parameters" + - "Test STAR RSEM with rsem_index false workflow parameters" + - "Test HISAT2 with skip_qc workflow parameters" + - "Test HISAT2 with hisat2_index false workflow parameters" + - "Test Salmon with skip_qc skip_alignment workflow parameters" + - "Test Salmon with salmon_index false transcript_fasta false workflow parameters" + profile: ["docker", "singularity", "conda"] + exclude: + - NXF_VER: "latest-everything" + profile: "conda" + steps: + - name: Check out pipeline code + uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: "3.x" + + - uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: Install Python dependencies + run: python -m pip install --upgrade pip pytest-workflow + + - uses: actions/cache@v2 + with: + path: /usr/local/bin/nextflow + key: ${{ runner.os }} + restore-keys: | + ${{ runner.os }}-nextflow- + + - name: Install Nextflow + uses: nf-core/setup-nextflow@v1 + with: + version: "${{ matrix.NXF_VER }}" + + - name: Set up Singularity + if: matrix.profile == 'singularity' + uses: eWaterCycle/setup-singularity@v5 + with: + singularity-version: 3.7.1 + + - name: Set up miniconda + if: matrix.profile == 'conda' + uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + channels: conda-forge,bioconda,defaults + python-version: ${{ matrix.python-version }} + + - name: Conda clean + if: matrix.profile == 'conda' + run: conda clean -a + + - name: Run pipeline with tests settings + uses: Wandalen/wretry.action@v1.0.11 + with: + command: TMPDIR=~ PROFILE=${{ matrix.profile }} pytest --tag release --symlink --kwdof --git-aware --color=yes + attempt_limit: 3 + + - name: Output log on failure + if: failure() + run: | + sudo apt install bat > /dev/null + batcat --decorations=always --color=always /tmp/pytest_workflow_*/*/log.{out,err} + + - name: Upload logs on failure + if: failure() + uses: actions/upload-artifact@v2 + with: + name: logs-${{ matrix.profile }} + path: | + /tmp/pytest_workflow_*/*/.nextflow.log + /tmp/pytest_workflow_*/*/log.out + /tmp/pytest_workflow_*/*/log.err + /tmp/pytest_workflow_*/*/work + /tmp/pytest_workflow_*/**/.command.log + !/tmp/pytest_workflow_*/*/work/conda + !/tmp/pytest_workflow_*/*/work/singularity From 08d15603daf71e4a6f99599c689970138cf85844 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 11 Oct 2022 12:52:00 +0200 Subject: [PATCH 20/49] ci: Add in profile discovery Co-authored-by: maxulysse --- conf/test.config | 30 +++++++++++++++++++++++++++++ tests/pipeline/test_default.yml | 3 +-- tests/pipeline/test_hisat2.yml | 5 ++--- tests/pipeline/test_salmon.yml | 5 ++--- tests/pipeline/test_star_rsem.yml | 5 ++--- tests/pipeline/test_star_salmon.yml | 25 ++++++++++++------------ 6 files changed, 49 insertions(+), 24 deletions(-) diff --git a/conf/test.config b/conf/test.config index 187977e7d..25c6746d5 100644 --- a/conf/test.config +++ b/conf/test.config @@ -46,3 +46,33 @@ process { ext.args2 = "--genomeSAindexNbases 7" } } + +// Enable container engines/virtualisation envs for CI testing +// only works when specified with the profile ENV +// otherwise tests can be done with the regular provided profiles +if (System.getenv('PROFILE')) { + if ("$PROFILE" == "conda") { + conda.createTimeout = "120 min" + params.enable_conda = true + charliecloud.enabled = false + docker.enabled = false + podman.enabled = false + shifter.enabled = false + singularity.enabled = false + } else if ("$PROFILE" == "docker") { + docker.enabled = true + docker.userEmulation = { params.use_gatk_spark ? false : true }.call() + charliecloud.enabled = false + podman.enabled = false + shifter.enabled = false + singularity.enabled = false + } else if ("$PROFILE" == "singularity") { + params.singularity_pull_docker_container = false + singularity.autoMounts = true + singularity.enabled = true + charliecloud.enabled = false + docker.enabled = false + podman.enabled = false + shifter.enabled = false + } +} diff --git a/tests/pipeline/test_default.yml b/tests/pipeline/test_default.yml index 7294f7824..590908b58 100644 --- a/tests/pipeline/test_default.yml +++ b/tests/pipeline/test_default.yml @@ -1,6 +1,5 @@ -# NOTE For now these tests don't support any containers beside docker - name: Run default pipeline with test data - command: nextflow run main.nf -profile test,docker --outdir results + command: nextflow run main.nf -profile test --outdir results tags: - default files: diff --git a/tests/pipeline/test_hisat2.yml b/tests/pipeline/test_hisat2.yml index dada2ff31..70921af90 100644 --- a/tests/pipeline/test_hisat2.yml +++ b/tests/pipeline/test_hisat2.yml @@ -1,12 +1,11 @@ -# NOTE For now these tests don't support any containers beside docker - name: Test HISAT2 with skip_qc workflow parameters - command: nextflow run main.nf -profile test,docker --aligner hisat2 --skip_qc --outdir results + command: nextflow run main.nf -profile test --aligner hisat2 --skip_qc --outdir results tags: - hisat2 - skip_qc - name: Test HISAT2 with hisat2_index false workflow parameters - command: nextflow run main.nf -profile test,docker --aligner hisat2 --hisat2_index false --outdir results + command: nextflow run main.nf -profile test --aligner hisat2 --hisat2_index false --outdir results tags: - hisat2 - hisat2_index-false diff --git a/tests/pipeline/test_salmon.yml b/tests/pipeline/test_salmon.yml index ffc0082da..0ed1f4c45 100644 --- a/tests/pipeline/test_salmon.yml +++ b/tests/pipeline/test_salmon.yml @@ -1,6 +1,5 @@ -# NOTE For now these tests don't support any containers beside docker - name: Test Salmon with skip_qc skip_alignment workflow parameters - command: nextflow run main.nf -profile test,docker --pseudo_aligner salmon --skip_qc --skip_alignment --outdir results + command: nextflow run main.nf -profile test --pseudo_aligner salmon --skip_qc --skip_alignment --outdir results tags: - salmon - quantify_salmon @@ -8,7 +7,7 @@ - skip_alignment - name: Test Salmon with salmon_index false transcript_fasta false workflow parameters - command: nextflow run main.nf -profile test,docker --salmon_index false --transcript_fasta false --outdir results + command: nextflow run main.nf -profile test --salmon_index false --transcript_fasta false --outdir results tags: - salmon - quantify_salmon diff --git a/tests/pipeline/test_star_rsem.yml b/tests/pipeline/test_star_rsem.yml index 554f75ba3..f8cd921bc 100644 --- a/tests/pipeline/test_star_rsem.yml +++ b/tests/pipeline/test_star_rsem.yml @@ -1,13 +1,12 @@ -# NOTE For now these tests don't support any containers beside docker - name: Test STAR RSEM with skip_qc workflow parameters - command: nextflow run main.nf -profile test,docker --aligner star_rsem --skip_qc --outdir results + command: nextflow run main.nf -profile test --aligner star_rsem --skip_qc --outdir results tags: - star - rsem - skip_qc - name: Test STAR RSEM with rsem_index false workflow parameters - command: nextflow run main.nf -profile test,docker --aligner star_rsem --rsem_index false --outdir results + command: nextflow run main.nf -profile test --aligner star_rsem --rsem_index false --outdir results tags: - star - rsem diff --git a/tests/pipeline/test_star_salmon.yml b/tests/pipeline/test_star_salmon.yml index fd762e03b..ca11d2856 100644 --- a/tests/pipeline/test_star_salmon.yml +++ b/tests/pipeline/test_star_salmon.yml @@ -1,55 +1,54 @@ -# NOTE For now these tests don't support any containers beside docker - name: Test STAR Salmon with skip_qc workflow parameters - command: nextflow run main.nf -profile test,docker --aligner star_salmon --skip_qc --outdir results + command: nextflow run main.nf -profile test --aligner star_salmon --skip_qc --outdir results tags: - star - salmon - skip_qc - name: Test STAR Salmon with skim_trimming workflow parameters - command: nextflow run main.nf -profile test,docker --aligner star_salmon --skim_trimming --outdir results + command: nextflow run main.nf -profile test --aligner star_salmon --skim_trimming --outdir results tags: - star - salmon - skim_trimming - name: Test STAR Salmon with gtf false workflow parameters - command: nextflow run main.nf -profile test,docker --aligner star_salmon --gtf false --outdir results + command: nextflow run main.nf -profile test --aligner star_salmon --gtf false --outdir results tags: - star - salmon - gtf-false - name: Test STAR Salmon with star_index false workflow parameters - command: nextflow run main.nf -profile test,docker --aligner star_salmon --star_index false --outdir results + command: nextflow run main.nf -profile test --aligner star_salmon --star_index false --outdir results tags: - star - salmon - star_index-false - name: Test STAR Salmon with transcript_fasta false workflow parameters - command: nextflow run main.nf -profile test,docker --aligner star_salmon --transcript_fasta false --outdir results + command: nextflow run main.nf -profile test --aligner star_salmon --transcript_fasta false --outdir results tags: - star - salmon - transcript_fasta-false - name: Test STAR Salmon with min_mapped_reads 90 workflow parameters - command: nextflow run main.nf -profile test,docker --aligner star_salmon --min_mapped_reads 90 --outdir results + command: nextflow run main.nf -profile test --aligner star_salmon --min_mapped_reads 90 --outdir results tags: - star - salmon - "min_mapped_reads 90" - name: Test STAR Salmon with with_umi workflow parameters - command: nextflow run main.nf -profile test,docker --aligner star_salmon --with_umi --outdir results + command: nextflow run main.nf -profile test --aligner star_salmon --with_umi --outdir results tags: - star - salmon - with_umi - name: Test STAR Salmon with with_umi skip_trimming workflow parameters - command: nextflow run main.nf -profile test,docker --aligner star_salmon --with_umi --skip_trimming --outdir results + command: nextflow run main.nf -profile test --aligner star_salmon --with_umi --skip_trimming --outdir results tags: - star - salmon @@ -57,7 +56,7 @@ - skip_trimming - name: Test STAR Salmon with remove_ribo_rna and skip_qualimap workflow parameters - command: nextflow run main.nf -profile test,docker --aligner star_salmon --remove_ribo_rna --skip_qualimap --outdir results + command: nextflow run main.nf -profile test --aligner star_salmon --remove_ribo_rna --skip_qualimap --outdir results tags: - star - salmon @@ -67,14 +66,14 @@ exit_code: 1 - name: Test STAR Salmon with bam_csi_index workflow parameters - command: nextflow run main.nf -profile test,docker --aligner star_salmon --bam_csi_index --outdir results + command: nextflow run main.nf -profile test --aligner star_salmon --bam_csi_index --outdir results tags: - star - salmon - bam_csi_index - name: Test STAR Salmon with save_align_intermeds save_reference workflow parameters - command: nextflow run main.nf -profile test,docker --aligner star_salmon --save_align_intermeds --save_reference --outdir results + command: nextflow run main.nf -profile test --aligner star_salmon --save_align_intermeds --save_reference --outdir results tags: - star - salmon @@ -82,7 +81,7 @@ - save_reference - name: Test STAR Salmon with featurecounts_group_type false workflow parameters - command: nextflow run main.nf -profile test,docker --aligner star_salmon --featurecounts_group_type false --outdir results + command: nextflow run main.nf -profile test --aligner star_salmon --featurecounts_group_type false --outdir results tags: - star - salmon From 7431d513f6216b3629386d832c878cd59eda4abb Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 11 Oct 2022 13:00:58 +0200 Subject: [PATCH 21/49] ci: Remove release fail-fast --- .github/workflows/release.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8ef8d3af9..6e2092465 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -18,8 +18,7 @@ jobs: if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/rnaseq') }}" runs-on: ubuntu-latest strategy: - # HACK Remove after DSL2 rewrite is done - fail-fast: true + fail-fast: false matrix: NXF_VER: - "21.10.3" From 987720a7ba322afa4ceb98db8733195513485a9f Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 11 Oct 2022 13:47:21 +0200 Subject: [PATCH 22/49] ci: Fix docker profile --- conf/test.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/test.config b/conf/test.config index 25c6746d5..2c2f6f0c5 100644 --- a/conf/test.config +++ b/conf/test.config @@ -61,7 +61,7 @@ if (System.getenv('PROFILE')) { singularity.enabled = false } else if ("$PROFILE" == "docker") { docker.enabled = true - docker.userEmulation = { params.use_gatk_spark ? false : true }.call() + docker.userEmulation = true charliecloud.enabled = false podman.enabled = false shifter.enabled = false From 03650ba42f8b5b89b0e555ddaf76ecc5a64a0075 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 11 Oct 2022 13:57:44 +0200 Subject: [PATCH 23/49] ci: Remove retries This takes too long --- .github/workflows/ci.yml | 5 +---- .github/workflows/release.yml | 5 +---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5fef6be8b..15d9adfd8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -80,10 +80,7 @@ jobs: run: conda clean -a - name: Run pipeline with tests settings - uses: Wandalen/wretry.action@v1.0.11 - with: - command: TMPDIR=~ PROFILE=${{ matrix.profile }} pytest --tag default --symlink --kwdof --git-aware --color=yes - attempt_limit: 3 + run: TMPDIR=~ PROFILE=${{ matrix.profile }} pytest --tag default --symlink --kwdof --git-aware --color=yes - name: Output log on failure if: failure() diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6e2092465..aa7e36183 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -97,10 +97,7 @@ jobs: run: conda clean -a - name: Run pipeline with tests settings - uses: Wandalen/wretry.action@v1.0.11 - with: - command: TMPDIR=~ PROFILE=${{ matrix.profile }} pytest --tag release --symlink --kwdof --git-aware --color=yes - attempt_limit: 3 + run: TMPDIR=~ PROFILE=${{ matrix.profile }} pytest --tag ${{ matrix.test }} --symlink --kwdof --git-aware --color=yes - name: Output log on failure if: failure() From adbae6a2f149b9453186669ef154301315c5fa77 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 11 Oct 2022 15:23:40 +0200 Subject: [PATCH 24/49] ci: Add quotes Co-authored-by: awgymer --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index aa7e36183..c12b53332 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -97,7 +97,7 @@ jobs: run: conda clean -a - name: Run pipeline with tests settings - run: TMPDIR=~ PROFILE=${{ matrix.profile }} pytest --tag ${{ matrix.test }} --symlink --kwdof --git-aware --color=yes + run: TMPDIR=~ PROFILE=${{ matrix.profile }} pytest --tag "${{ matrix.test }}" --symlink --kwdof --git-aware --color=yes - name: Output log on failure if: failure() From 4c6be708aa7d3dfd0b10c9ef7fb6641001f5328d Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 11 Oct 2022 15:57:14 +0200 Subject: [PATCH 25/49] ci: tmp => home Co-authored-by: awgymer --- .github/workflows/ci.yml | 15 +++++++-------- .github/workflows/release.yml | 15 +++++++-------- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 15d9adfd8..0f3b0a23f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,7 +86,7 @@ jobs: if: failure() run: | sudo apt install bat > /dev/null - batcat --decorations=always --color=always /tmp/pytest_workflow_*/*/log.{out,err} + batcat --decorations=always --color=always /home/runner/pytest_workflow_*/*/log.{out,err} - name: Upload logs on failure if: failure() @@ -94,10 +94,9 @@ jobs: with: name: logs-${{ matrix.profile }} path: | - /tmp/pytest_workflow_*/*/.nextflow.log - /tmp/pytest_workflow_*/*/log.out - /tmp/pytest_workflow_*/*/log.err - /tmp/pytest_workflow_*/*/work - /tmp/pytest_workflow_*/**/.command.log - !/tmp/pytest_workflow_*/*/work/conda - !/tmp/pytest_workflow_*/*/work/singularity + /home/runner/pytest_workflow_*/*/.nextflow.log + /home/runner/pytest_workflow_*/*/log.out + /home/runner/pytest_workflow_*/*/log.err + /home/runner/pytest_workflow_*/*/work + !/home/runner/pytest_workflow_*/*/work/conda + !/home/runner/pytest_workflow_*/*/work/singularity diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c12b53332..e9a1f2d01 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -103,7 +103,7 @@ jobs: if: failure() run: | sudo apt install bat > /dev/null - batcat --decorations=always --color=always /tmp/pytest_workflow_*/*/log.{out,err} + batcat --decorations=always --color=always /home/runner/pytest_workflow_*/*/log.{out,err} - name: Upload logs on failure if: failure() @@ -111,10 +111,9 @@ jobs: with: name: logs-${{ matrix.profile }} path: | - /tmp/pytest_workflow_*/*/.nextflow.log - /tmp/pytest_workflow_*/*/log.out - /tmp/pytest_workflow_*/*/log.err - /tmp/pytest_workflow_*/*/work - /tmp/pytest_workflow_*/**/.command.log - !/tmp/pytest_workflow_*/*/work/conda - !/tmp/pytest_workflow_*/*/work/singularity + /home/runner/pytest_workflow_*/*/.nextflow.log + /home/runner/pytest_workflow_*/*/log.out + /home/runner/pytest_workflow_*/*/log.err + /home/runner/pytest_workflow_*/*/work + !/home/runner/pytest_workflow_*/*/work/conda + !/home/runner/pytest_workflow_*/*/work/singularity From 1579a1ac0cffe0e7290f4841983dcc7f949207db Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 11 Oct 2022 15:58:13 +0200 Subject: [PATCH 26/49] ci: apt => apt-get Co-authored-by: awgymer --- .github/workflows/ci.yml | 2 +- .github/workflows/pytest-workflow.yml | 2 +- .github/workflows/release.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0f3b0a23f..3598c8f01 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -85,7 +85,7 @@ jobs: - name: Output log on failure if: failure() run: | - sudo apt install bat > /dev/null + sudo apt-get install bat > /dev/null batcat --decorations=always --color=always /home/runner/pytest_workflow_*/*/log.{out,err} - name: Upload logs on failure diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml index d79ed45de..08bbe87a6 100644 --- a/.github/workflows/pytest-workflow.yml +++ b/.github/workflows/pytest-workflow.yml @@ -95,7 +95,7 @@ jobs: - name: Output log on failure if: failure() run: | - sudo apt install bat > /dev/null + sudo apt-get install bat > /dev/null batcat --decorations=always --color=always /home/runner/pytest_workflow_*/*/log.{out,err} - name: Upload logs on failure diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e9a1f2d01..b4311cf08 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -102,7 +102,7 @@ jobs: - name: Output log on failure if: failure() run: | - sudo apt install bat > /dev/null + sudo apt-get install bat > /dev/null batcat --decorations=always --color=always /home/runner/pytest_workflow_*/*/log.{out,err} - name: Upload logs on failure From 8ff17cc4b17686350b6d641dd65f7b1cd56049e6 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 11 Oct 2022 16:49:44 +0200 Subject: [PATCH 27/49] ci: Redo triggers --- .github/workflows/ci.yml | 3 +-- .github/workflows/pytest-workflow.yml | 5 +---- .github/workflows/release.yml | 7 ++----- 3 files changed, 4 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3598c8f01..82ecebfe0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,10 +1,9 @@ name: nf-core CI # This workflow runs the pipeline with the minimal test dataset to check that it completes without any syntax errors on: - push: + pull_request: branches: - dev - pull_request: release: types: [published] diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml index 08bbe87a6..06a115917 100644 --- a/.github/workflows/pytest-workflow.yml +++ b/.github/workflows/pytest-workflow.yml @@ -1,12 +1,9 @@ name: pytest-workflow # This workflow runs the pipeline with the minimal test dataset to check that it completes without any syntax errors on: - push: + pull_request: branches: - dev - pull_request: - release: - types: [published] env: NXF_ANSI_LOG: false diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b4311cf08..4b48b57fa 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,12 +1,9 @@ -name: nf-core CI +name: Run all pytest workflow jobs # This workflow runs the pipeline with the minimal test dataset to check that it completes without any syntax errors on: - push: - branches: - - dev - pull_request: release: types: [published] + workflow_dispatch: env: NXF_ANSI_LOG: false From 7e3d95557ae21036e2f38ace1e65ab75fb944829 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 11 Oct 2022 16:53:29 +0200 Subject: [PATCH 28/49] ci: Remove fail-fast which is true by default --- .github/workflows/ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 82ecebfe0..ca002b61a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,8 +17,6 @@ jobs: if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/rnaseq') }}" runs-on: ubuntu-latest strategy: - # HACK Remove after DSL2 rewrite is done - fail-fast: true matrix: NXF_VER: - "21.10.3" From 40643e974ecaedbae7d6902e0cd8310a8587688c Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Wed, 12 Oct 2022 01:10:01 +0200 Subject: [PATCH 29/49] style: Ignore actions ci triggering in linting --- .nf-core.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.nf-core.yml b/.nf-core.yml index 40bcac74d..ca3842aeb 100644 --- a/.nf-core.yml +++ b/.nf-core.yml @@ -4,3 +4,4 @@ lint: - assets/email_template.html - assets/email_template.txt - lib/NfcoreTemplate.groovy + actions_ci: False From 55cf7f2f7700e65bcdb169994d090998c67c9b76 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Wed, 12 Oct 2022 10:17:42 +0200 Subject: [PATCH 30/49] test: Mark STAR Salmon with remove_ribo_rna as passing --- tests/pipeline/test_star_salmon.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/pipeline/test_star_salmon.yml b/tests/pipeline/test_star_salmon.yml index ca11d2856..c0658da6d 100644 --- a/tests/pipeline/test_star_salmon.yml +++ b/tests/pipeline/test_star_salmon.yml @@ -62,8 +62,6 @@ - salmon - remove_ribo_rna - skip_qualimap - - failing - exit_code: 1 - name: Test STAR Salmon with bam_csi_index workflow parameters command: nextflow run main.nf -profile test --aligner star_salmon --bam_csi_index --outdir results From 92733f89a0383fa9caa29880c8f7a16a3b68630d Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sat, 5 Nov 2022 09:16:59 -0500 Subject: [PATCH 31/49] build: Run nf-test init --- .gitignore | 1 + nf-test.config | 8 ++++++++ 2 files changed, 9 insertions(+) create mode 100644 nf-test.config diff --git a/.gitignore b/.gitignore index 5124c9ac7..089a40797 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ results/ testing/ testing* *.pyc +.nf-test diff --git a/nf-test.config b/nf-test.config new file mode 100644 index 000000000..703b5a4ec --- /dev/null +++ b/nf-test.config @@ -0,0 +1,8 @@ +config { + + testsDir "tests" + workDir ".nf-test" + configFile "conf/test.config" + // profile "docker" + +} From 83f0436e6e272e965146bffacb031ab90ac53da9 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Fri, 4 Nov 2022 15:03:28 -0500 Subject: [PATCH 32/49] test(nf-test): Add initial hisat2 tests --- tests/pipeline/hisat2/main.nf.test | 38 ++++++++++++++++++++++++++++++ tests/pipeline/test_hisat2.yml | 11 --------- 2 files changed, 38 insertions(+), 11 deletions(-) create mode 100644 tests/pipeline/hisat2/main.nf.test delete mode 100644 tests/pipeline/test_hisat2.yml diff --git a/tests/pipeline/hisat2/main.nf.test b/tests/pipeline/hisat2/main.nf.test new file mode 100644 index 000000000..109944dd8 --- /dev/null +++ b/tests/pipeline/hisat2/main.nf.test @@ -0,0 +1,38 @@ +nextflow_pipeline { + + name "hisat2" + script "main.nf" + + test("with skip_qc workflow parameters") { + when { + params { + aligner = "hisat2" + skip_qc = true + outdir = "$outputDir" + } + } + + then { + assert workflow.success + assert workflow.trace.tasks().size() == 120 + // FIXME assert snapshot(workflow).match() + } + } + + test("with hisat2_index false workflow parameters") { + when { + params { + aligner = "hisat2" + hisat2_index = false + outdir = "$outputDir" + } + } + + then { + assert workflow.success + assert workflow.trace.tasks().size() == 186 + // FIXME assert snapshot(workflow).match() + } + } + +} diff --git a/tests/pipeline/test_hisat2.yml b/tests/pipeline/test_hisat2.yml deleted file mode 100644 index 70921af90..000000000 --- a/tests/pipeline/test_hisat2.yml +++ /dev/null @@ -1,11 +0,0 @@ -- name: Test HISAT2 with skip_qc workflow parameters - command: nextflow run main.nf -profile test --aligner hisat2 --skip_qc --outdir results - tags: - - hisat2 - - skip_qc - -- name: Test HISAT2 with hisat2_index false workflow parameters - command: nextflow run main.nf -profile test --aligner hisat2 --hisat2_index false --outdir results - tags: - - hisat2 - - hisat2_index-false From 02a30af2c4cabb64676bb1e7f05f096e9761f36e Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 6 Nov 2022 21:18:55 -0600 Subject: [PATCH 33/49] test(nf-test): Move salmon tests --- tests/pipeline/salmon/main.nf.test | 40 ++++++++++++++++++++++++++++++ tests/pipeline/test_salmon.yml | 15 ----------- 2 files changed, 40 insertions(+), 15 deletions(-) create mode 100644 tests/pipeline/salmon/main.nf.test delete mode 100644 tests/pipeline/test_salmon.yml diff --git a/tests/pipeline/salmon/main.nf.test b/tests/pipeline/salmon/main.nf.test new file mode 100644 index 000000000..700d219f6 --- /dev/null +++ b/tests/pipeline/salmon/main.nf.test @@ -0,0 +1,40 @@ +nextflow_pipeline { + + name "Salmon" + script "main.nf" + + test("with skip_qc skip_alignment workflow parameters") { + when { + params { + pseudo_aligner = "salmon" + skip_qc = true + skip_alignment = true + outdir = "$outputDir" + } + } + + then { + assert workflow.success + assert workflow.trace.tasks().size() == 33 + // FIXME assert snapshot(workflow).match() + } + } + + test("with salmon_index false transcript_fasta false workflow parameters") { + when { + params { + pseudo_aligner = "salmon" + salmon_index = false + transcript_fasta = false + outdir = "$outputDir" + } + } + + then { + assert workflow.success + assert workflow.trace.tasks().size() == 199 + // FIXME assert snapshot(workflow).match() + } + } + +} diff --git a/tests/pipeline/test_salmon.yml b/tests/pipeline/test_salmon.yml deleted file mode 100644 index 0ed1f4c45..000000000 --- a/tests/pipeline/test_salmon.yml +++ /dev/null @@ -1,15 +0,0 @@ -- name: Test Salmon with skip_qc skip_alignment workflow parameters - command: nextflow run main.nf -profile test --pseudo_aligner salmon --skip_qc --skip_alignment --outdir results - tags: - - salmon - - quantify_salmon - - skip_qc - - skip_alignment - -- name: Test Salmon with salmon_index false transcript_fasta false workflow parameters - command: nextflow run main.nf -profile test --salmon_index false --transcript_fasta false --outdir results - tags: - - salmon - - quantify_salmon - - salmon_index-false - - transcript_fasta-false From 8481a298afd8cd0843e0511b91e363f14dae7666 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 6 Nov 2022 21:35:03 -0600 Subject: [PATCH 34/49] test(nf-test): Move star_rsem tests over --- tests/pipeline/star_rsem/main.nf.test | 39 +++++++++++++++++++++++++++ tests/pipeline/test_star_rsem.yml | 13 --------- 2 files changed, 39 insertions(+), 13 deletions(-) create mode 100644 tests/pipeline/star_rsem/main.nf.test delete mode 100644 tests/pipeline/test_star_rsem.yml diff --git a/tests/pipeline/star_rsem/main.nf.test b/tests/pipeline/star_rsem/main.nf.test new file mode 100644 index 000000000..ce00fa8e7 --- /dev/null +++ b/tests/pipeline/star_rsem/main.nf.test @@ -0,0 +1,39 @@ +nextflow_pipeline { + + name "STAR RSEM" + script "main.nf" + + test("with skip_qc workflow parameters") { + when { + params { + aligner = "star_rsem" + skip_qc = true + outdir = "$outputDir" + } + } + + then { + assert workflow.success + assert workflow.trace.tasks().size() == 120 + // FIXME assert snapshot(workflow).match() + } + } + + test("with rsem_index false workflow parameters") { + when { + params { + pseudo_aligner = "salmon" + aligner = "star_rsem" + rsem_index = false + outdir = "$outputDir" + } + } + + then { + assert workflow.success + assert workflow.trace.tasks().size() == 187 + // FIXME assert snapshot(workflow).match() + } + } + +} diff --git a/tests/pipeline/test_star_rsem.yml b/tests/pipeline/test_star_rsem.yml deleted file mode 100644 index f8cd921bc..000000000 --- a/tests/pipeline/test_star_rsem.yml +++ /dev/null @@ -1,13 +0,0 @@ -- name: Test STAR RSEM with skip_qc workflow parameters - command: nextflow run main.nf -profile test --aligner star_rsem --skip_qc --outdir results - tags: - - star - - rsem - - skip_qc - -- name: Test STAR RSEM with rsem_index false workflow parameters - command: nextflow run main.nf -profile test --aligner star_rsem --rsem_index false --outdir results - tags: - - star - - rsem - - rsem_index-false From f47a07287f6a9cb857d4f7db4a7083e183f70018 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 6 Nov 2022 23:00:39 -0600 Subject: [PATCH 35/49] test(nf-test): Move star_salmon over --- tests/pipeline/star_salmon/main.nf.test | 187 ++++++++++++++++++++++++ tests/pipeline/test_star_salmon.yml | 86 ----------- 2 files changed, 187 insertions(+), 86 deletions(-) create mode 100644 tests/pipeline/star_salmon/main.nf.test delete mode 100644 tests/pipeline/test_star_salmon.yml diff --git a/tests/pipeline/star_salmon/main.nf.test b/tests/pipeline/star_salmon/main.nf.test new file mode 100644 index 000000000..cf1bee7a2 --- /dev/null +++ b/tests/pipeline/star_salmon/main.nf.test @@ -0,0 +1,187 @@ +nextflow_pipeline { + + name "STAR Salmon" + script "main.nf" + + test("with skip_qc workflow parameters") { + when { + params { + aligner = "star_salmon" + skip_qc = true + outdir = "$outputDir" + } + } + + then { + assert workflow.success + assert workflow.trace.tasks().size() == 130 + // FIXME assert snapshot(workflow).match() + } + } + + test("with skim_trimming workflow parameters") { + when { + params { + aligner = "star_salmon" + skim_trimming = true + outdir = "$outputDir" + } + } + + then { + assert workflow.success + assert workflow.trace.tasks().size() == 197 + // FIXME assert snapshot(workflow).match() + } + } + + test("with gtf false workflow parameters") { + when { + params { + aligner = "star_salmon" + gtf = false + outdir = "$outputDir" + } + } + + then { + assert workflow.success + assert workflow.trace.tasks().size() == 198 + // FIXME assert snapshot(workflow).match() + } + } + + test("with star_index false workflow parameters") { + when { + params { + aligner = "star_salmon" + star_index = false + outdir = "$outputDir" + } + } + + then { + assert workflow.success + assert workflow.trace.tasks().size() == 197 + // FIXME assert snapshot(workflow).match() + } + } + + test("with transcript_fasta false workflow parameters") { + when { + params { + aligner = "star_salmon" + transcript_fasta = false + outdir = "$outputDir" + } + } + + then { + assert workflow.success + assert workflow.trace.tasks().size() == 199 + // FIXME assert snapshot(workflow).match() + } + } + + test("with min_mapped_reads 90 workflow parameters") { + when { + params { + aligner = "star_salmon" + min_mapped_reads = 152 + outdir = "$outputDir" + } + } + + then { + assert workflow.success + assert workflow.trace.tasks().size() == 186 + // FIXME assert snapshot(workflow).match() + } + } + + test("with with_umi workflow parameters") { + when { + params { + aligner = "star_salmon" + with_umi = true + outdir = "$outputDir" + } + } + + then { + // FIXME ValueError: Must supply --bc-pattern for single-end + assert workflow.failed + // assert workflow.trace.tasks().size() == 186 + // FIXME assert snapshot(workflow).match() + } + } + + test("with with_umi skip_trimming workflow parameters") { + when { + params { + aligner = "star_salmon" + with_umi = true + skip_trimming = true + outdir = "$outputDir" + } + } + + then { + // FIXME ValueError: Must supply --bc-pattern for single-end + assert workflow.failed + assert workflow.trace.tasks().size() == 186 + // FIXME assert snapshot(workflow).match() + } + } + + test("with remove_ribo_rna and skip_qualimap workflow parameters") { + when { + params { + aligner = "star_salmon" + remove_ribo_rna = true + skip_qualimap = true + outdir = "$outputDir" + } + } + + then { + assert workflow.success + assert workflow.trace.tasks().size() == 197 + // FIXME assert snapshot(workflow).match() + } + } + + test("with save_align_intermeds save_reference workflow parameters") { + when { + params { + aligner = "star_salmon" + save_align_intermeds = true + save_reference = true + outdir = "$outputDir" + } + } + + then { + assert workflow.success + assert workflow.trace.tasks().size() == 197 + // FIXME assert snapshot(workflow).match() + } + } + + test("with featurecounts_group_type false workflow parameters") { + when { + params { + aligner = "star_salmon" + featurecounts_group_type = false + outdir = "$outputDir" + } + } + + then { + assert workflow.success + assert workflow.trace.tasks().size() == 187 + // FIXME assert snapshot(workflow).match() + } + } + +} diff --git a/tests/pipeline/test_star_salmon.yml b/tests/pipeline/test_star_salmon.yml deleted file mode 100644 index c0658da6d..000000000 --- a/tests/pipeline/test_star_salmon.yml +++ /dev/null @@ -1,86 +0,0 @@ -- name: Test STAR Salmon with skip_qc workflow parameters - command: nextflow run main.nf -profile test --aligner star_salmon --skip_qc --outdir results - tags: - - star - - salmon - - skip_qc - -- name: Test STAR Salmon with skim_trimming workflow parameters - command: nextflow run main.nf -profile test --aligner star_salmon --skim_trimming --outdir results - tags: - - star - - salmon - - skim_trimming - -- name: Test STAR Salmon with gtf false workflow parameters - command: nextflow run main.nf -profile test --aligner star_salmon --gtf false --outdir results - tags: - - star - - salmon - - gtf-false - -- name: Test STAR Salmon with star_index false workflow parameters - command: nextflow run main.nf -profile test --aligner star_salmon --star_index false --outdir results - tags: - - star - - salmon - - star_index-false - -- name: Test STAR Salmon with transcript_fasta false workflow parameters - command: nextflow run main.nf -profile test --aligner star_salmon --transcript_fasta false --outdir results - tags: - - star - - salmon - - transcript_fasta-false - -- name: Test STAR Salmon with min_mapped_reads 90 workflow parameters - command: nextflow run main.nf -profile test --aligner star_salmon --min_mapped_reads 90 --outdir results - tags: - - star - - salmon - - "min_mapped_reads 90" - -- name: Test STAR Salmon with with_umi workflow parameters - command: nextflow run main.nf -profile test --aligner star_salmon --with_umi --outdir results - tags: - - star - - salmon - - with_umi - -- name: Test STAR Salmon with with_umi skip_trimming workflow parameters - command: nextflow run main.nf -profile test --aligner star_salmon --with_umi --skip_trimming --outdir results - tags: - - star - - salmon - - with_umi - - skip_trimming - -- name: Test STAR Salmon with remove_ribo_rna and skip_qualimap workflow parameters - command: nextflow run main.nf -profile test --aligner star_salmon --remove_ribo_rna --skip_qualimap --outdir results - tags: - - star - - salmon - - remove_ribo_rna - - skip_qualimap - -- name: Test STAR Salmon with bam_csi_index workflow parameters - command: nextflow run main.nf -profile test --aligner star_salmon --bam_csi_index --outdir results - tags: - - star - - salmon - - bam_csi_index - -- name: Test STAR Salmon with save_align_intermeds save_reference workflow parameters - command: nextflow run main.nf -profile test --aligner star_salmon --save_align_intermeds --save_reference --outdir results - tags: - - star - - salmon - - save_align_intermeds - - save_reference - -- name: Test STAR Salmon with featurecounts_group_type false workflow parameters - command: nextflow run main.nf -profile test --aligner star_salmon --featurecounts_group_type false --outdir results - tags: - - star - - salmon - - featurecounts_group_type-false From db27489b058783fa6aba27f2b195004dc484e070 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 7 Nov 2022 09:06:57 -0600 Subject: [PATCH 36/49] test(nf-test): Move default tests over --- tests/pipeline/default/main.nf.test | 71 ++++++++++++++++++++++++ tests/pipeline/default/main.nf.test.snap | 17 ++++++ tests/pipeline/test_default.yml | 59 -------------------- 3 files changed, 88 insertions(+), 59 deletions(-) create mode 100644 tests/pipeline/default/main.nf.test create mode 100644 tests/pipeline/default/main.nf.test.snap delete mode 100644 tests/pipeline/test_default.yml diff --git a/tests/pipeline/default/main.nf.test b/tests/pipeline/default/main.nf.test new file mode 100644 index 000000000..d089be895 --- /dev/null +++ b/tests/pipeline/default/main.nf.test @@ -0,0 +1,71 @@ +nextflow_pipeline { + + name "default" + script "main.nf" + + test("default test config") { + when { + params { + outdir = "$outputDir" + } + } + + then { + assert workflow.success + assert workflow.trace.tasks().size() == 197 + + assert snapshot( + path("$outputDir/salmon/salmon.merged.gene_counts_length_scaled.rds"), + path("$outputDir/salmon/salmon.merged.gene_counts_length_scaled.tsv"), + path("$outputDir/salmon/salmon.merged.gene_counts.rds"), + path("$outputDir/salmon/salmon.merged.gene_counts_scaled.rds"), + path("$outputDir/salmon/salmon.merged.gene_counts_scaled.tsv"), + path("$outputDir/salmon/salmon.merged.gene_counts.tsv"), + path("$outputDir/salmon/salmon.merged.gene_tpm.tsv"), + path("$outputDir/salmon/salmon.merged.transcript_counts.rds"), + path("$outputDir/salmon/salmon.merged.transcript_counts.tsv"), + path("$outputDir/salmon/salmon.merged.transcript_tpm.tsv") + ).match() + + assert new File("$outputDir/fastqc/RAP1_IAA_30M_REP1_1_fastqc.html").exists() + assert new File("$outputDir/fastqc/RAP1_IAA_30M_REP1_1_fastqc.zip").exists() + assert new File("$outputDir/fastqc/RAP1_IAA_30M_REP1_2_fastqc.html").exists() + assert new File("$outputDir/fastqc/RAP1_IAA_30M_REP1_2_fastqc.zip").exists() + assert new File("$outputDir/fastqc/RAP1_UNINDUCED_REP1_fastqc.html").exists() + assert new File("$outputDir/fastqc/RAP1_UNINDUCED_REP1_fastqc.zip").exists() + assert new File("$outputDir/fastqc/RAP1_UNINDUCED_REP2_fastqc.html").exists() + assert new File("$outputDir/fastqc/RAP1_UNINDUCED_REP2_fastqc.zip").exists() + assert new File("$outputDir/fastqc/WT_REP1_1_fastqc.html").exists() + assert new File("$outputDir/fastqc/WT_REP1_1_fastqc.zip").exists() + assert new File("$outputDir/fastqc/WT_REP1_2_fastqc.html").exists() + assert new File("$outputDir/fastqc/WT_REP1_2_fastqc.zip").exists() + assert new File("$outputDir/fastqc/WT_REP2_1_fastqc.html").exists() + assert new File("$outputDir/fastqc/WT_REP2_1_fastqc.zip").exists() + assert new File("$outputDir/fastqc/WT_REP2_2_fastqc.html").exists() + assert new File("$outputDir/fastqc/WT_REP2_2_fastqc.zip").exists() + + assert new File("$outputDir/multiqc/star_salmon/multiqc_report.html").exists() + + assert new File("$outputDir/star_salmon/RAP1_IAA_30M_REP1.markdup.sorted.bam").exists() + assert new File("$outputDir/star_salmon/RAP1_IAA_30M_REP1.markdup.sorted.bam.bai").exists() + assert new File("$outputDir/star_salmon/RAP1_UNINDUCED_REP1.markdup.sorted.bam").exists() + assert new File("$outputDir/star_salmon/RAP1_UNINDUCED_REP1.markdup.sorted.bam.bai").exists() + assert new File("$outputDir/star_salmon/RAP1_UNINDUCED_REP2.markdup.sorted.bam").exists() + assert new File("$outputDir/star_salmon/RAP1_UNINDUCED_REP2.markdup.sorted.bam.bai").exists() + assert new File("$outputDir/star_salmon/WT_REP1.markdup.sorted.bam").exists() + assert new File("$outputDir/star_salmon/WT_REP1.markdup.sorted.bam.bai").exists() + assert new File("$outputDir/star_salmon/WT_REP2.markdup.sorted.bam").exists() + assert new File("$outputDir/star_salmon/WT_REP2.markdup.sorted.bam.bai").exists() + + assert new File("$outputDir/trimgalore/RAP1_IAA_30M_REP1_1.fastq.gz_trimming_report.txt").exists() + assert new File("$outputDir/trimgalore/RAP1_IAA_30M_REP1_2.fastq.gz_trimming_report.txt").exists() + assert new File("$outputDir/trimgalore/RAP1_UNINDUCED_REP1.fastq.gz_trimming_report.txt").exists() + assert new File("$outputDir/trimgalore/RAP1_UNINDUCED_REP2.fastq.gz_trimming_report.txt").exists() + assert new File("$outputDir/trimgalore/WT_REP1_1.fastq.gz_trimming_report.txt").exists() + assert new File("$outputDir/trimgalore/WT_REP1_2.fastq.gz_trimming_report.txt").exists() + assert new File("$outputDir/trimgalore/WT_REP2_1.fastq.gz_trimming_report.txt").exists() + assert new File("$outputDir/trimgalore/WT_REP2_2.fastq.gz_trimming_report.txt").exists() + } + } + +} diff --git a/tests/pipeline/default/main.nf.test.snap b/tests/pipeline/default/main.nf.test.snap new file mode 100644 index 000000000..a84f0ef37 --- /dev/null +++ b/tests/pipeline/default/main.nf.test.snap @@ -0,0 +1,17 @@ +{ + "default test config": { + "content": [ + "salmon.merged.gene_counts_length_scaled.rds:md5,f8003e4bb35be88be899b101278b649a", + "salmon.merged.gene_counts_length_scaled.tsv:md5,156aa4f17a9bca0c02f6ced658d1441d", + "salmon.merged.gene_counts.rds:md5,5cd138a017b614b87332ab49cbe52cbf", + "salmon.merged.gene_counts_scaled.rds:md5,db353b0aa735f82fa191fc60488bea57", + "salmon.merged.gene_counts_scaled.tsv:md5,22da1c1e76dd4e015e19b450feeb3843", + "salmon.merged.gene_counts.tsv:md5,d8a058897b775f3f09eb71ee4b62f606", + "salmon.merged.gene_tpm.tsv:md5,db3651b0cf0583493c578f9b24c2f1c8", + "salmon.merged.transcript_counts.rds:md5,45574d7d5a63a4f690d532b0247dc29a", + "salmon.merged.transcript_counts.tsv:md5,05fe3760f6dbdde83e358a443f91528b", + "salmon.merged.transcript_tpm.tsv:md5,8b071524cbdff06a2d1cf0900608af18" + ], + "timestamp": "2022-11-07T15:06:34+0000" + } +} \ No newline at end of file diff --git a/tests/pipeline/test_default.yml b/tests/pipeline/test_default.yml deleted file mode 100644 index 590908b58..000000000 --- a/tests/pipeline/test_default.yml +++ /dev/null @@ -1,59 +0,0 @@ -- name: Run default pipeline with test data - command: nextflow run main.nf -profile test --outdir results - tags: - - default - files: - # fastqc - - path: results/fastqc/RAP1_IAA_30M_REP1_1_fastqc.html - - path: results/fastqc/RAP1_IAA_30M_REP1_1_fastqc.zip - - path: results/fastqc/RAP1_IAA_30M_REP1_2_fastqc.html - - path: results/fastqc/RAP1_IAA_30M_REP1_2_fastqc.zip - - path: results/fastqc/RAP1_UNINDUCED_REP1_fastqc.html - - path: results/fastqc/RAP1_UNINDUCED_REP1_fastqc.zip - - path: results/fastqc/RAP1_UNINDUCED_REP2_fastqc.html - - path: results/fastqc/RAP1_UNINDUCED_REP2_fastqc.zip - - path: results/fastqc/WT_REP1_1_fastqc.html - - path: results/fastqc/WT_REP1_1_fastqc.zip - - path: results/fastqc/WT_REP1_2_fastqc.html - - path: results/fastqc/WT_REP1_2_fastqc.zip - - path: results/fastqc/WT_REP2_1_fastqc.html - - path: results/fastqc/WT_REP2_1_fastqc.zip - - path: results/fastqc/WT_REP2_2_fastqc.html - - path: results/fastqc/WT_REP2_2_fastqc.zip - - # multiqc - - path: results/multiqc/star_salmon/multiqc_report.html - - # salmon - - path: results/salmon/salmon.merged.gene_counts_length_scaled.rds - - path: results/salmon/salmon.merged.gene_counts_length_scaled.tsv - - path: results/salmon/salmon.merged.gene_counts.rds - - path: results/salmon/salmon.merged.gene_counts_scaled.rds - - path: results/salmon/salmon.merged.gene_counts_scaled.tsv - - path: results/salmon/salmon.merged.gene_counts.tsv - - path: results/salmon/salmon.merged.gene_tpm.tsv - - path: results/salmon/salmon.merged.transcript_counts.rds - - path: results/salmon/salmon.merged.transcript_counts.tsv - - path: results/salmon/salmon.merged.transcript_tpm.tsv - - # star_salmon - - path: results/star_salmon/RAP1_IAA_30M_REP1.markdup.sorted.bam - - path: results/star_salmon/RAP1_IAA_30M_REP1.markdup.sorted.bam.bai - - path: results/star_salmon/RAP1_UNINDUCED_REP1.markdup.sorted.bam - - path: results/star_salmon/RAP1_UNINDUCED_REP1.markdup.sorted.bam.bai - - path: results/star_salmon/RAP1_UNINDUCED_REP2.markdup.sorted.bam - - path: results/star_salmon/RAP1_UNINDUCED_REP2.markdup.sorted.bam.bai - - path: results/star_salmon/WT_REP1.markdup.sorted.bam - - path: results/star_salmon/WT_REP1.markdup.sorted.bam.bai - - path: results/star_salmon/WT_REP2.markdup.sorted.bam - - path: results/star_salmon/WT_REP2.markdup.sorted.bam.bai - - # Trimgalore - - path: results/trimgalore/RAP1_IAA_30M_REP1_1.fastq.gz_trimming_report.txt - - path: results/trimgalore/RAP1_IAA_30M_REP1_2.fastq.gz_trimming_report.txt - - path: results/trimgalore/RAP1_UNINDUCED_REP1.fastq.gz_trimming_report.txt - - path: results/trimgalore/RAP1_UNINDUCED_REP2.fastq.gz_trimming_report.txt - - path: results/trimgalore/WT_REP1_1.fastq.gz_trimming_report.txt - - path: results/trimgalore/WT_REP1_2.fastq.gz_trimming_report.txt - - path: results/trimgalore/WT_REP2_1.fastq.gz_trimming_report.txt - - path: results/trimgalore/WT_REP2_2.fastq.gz_trimming_report.txt From 778f0588231412fea267cb8762dcf9949c92d4d0 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 7 Nov 2022 09:55:25 -0600 Subject: [PATCH 37/49] ci: Copy ci from methylseq --- .github/workflows/ci.yml | 46 ++++++++++------------------------------ 1 file changed, 11 insertions(+), 35 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ca002b61a..546fbabd8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ env: jobs: test: - name: Run default pipeline test + name: Run ${{ matrix.profile }}-${{ matrix.NXF_VER }} pipeline test # Only run on push if this is the nf-core dev branch (merged PRs) if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/rnaseq') }}" runs-on: ubuntu-latest @@ -31,21 +31,6 @@ jobs: - name: Check out pipeline code uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.x" - - - uses: actions/cache@v2 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip- - - - name: Install Python dependencies - run: python -m pip install --upgrade pip pytest-workflow - - uses: actions/cache@v2 with: path: /usr/local/bin/nextflow @@ -58,6 +43,11 @@ jobs: with: version: "${{ matrix.NXF_VER }}" + - name: Install nf-test + run: | + wget -qO- https://code.askimed.com/install/nf-test | bash + sudo mv nf-test /usr/local/bin/ + - name: Set up Singularity if: matrix.profile == 'singularity' uses: eWaterCycle/setup-singularity@v5 @@ -76,24 +66,10 @@ jobs: if: matrix.profile == 'conda' run: conda clean -a - - name: Run pipeline with tests settings - run: TMPDIR=~ PROFILE=${{ matrix.profile }} pytest --tag default --symlink --kwdof --git-aware --color=yes - - - name: Output log on failure - if: failure() - run: | - sudo apt-get install bat > /dev/null - batcat --decorations=always --color=always /home/runner/pytest_workflow_*/*/log.{out,err} + - name: Run nf-test + run: nf-test test --profile=${{ matrix.profile }} tests/pipeline/default/*.nf.test --tap=test.tap - - name: Upload logs on failure - if: failure() - uses: actions/upload-artifact@v2 + - uses: pcolby/tap-summary@v1 with: - name: logs-${{ matrix.profile }} - path: | - /home/runner/pytest_workflow_*/*/.nextflow.log - /home/runner/pytest_workflow_*/*/log.out - /home/runner/pytest_workflow_*/*/log.err - /home/runner/pytest_workflow_*/*/work - !/home/runner/pytest_workflow_*/*/work/conda - !/home/runner/pytest_workflow_*/*/work/singularity + path: >- + test.tap From 24c0a2f8ab57a41722ca88f3308ed1175d464288 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 7 Nov 2022 10:02:18 -0600 Subject: [PATCH 38/49] ci: Add pipeline, subworkflows, and modules test based on changes --- .../{pytest-workflow.yml => ci-changes.yml} | 55 +++++-------------- tests/config/{pytest_tags.yml => tags.yml} | 42 +++++++------- 2 files changed, 35 insertions(+), 62 deletions(-) rename .github/workflows/{pytest-workflow.yml => ci-changes.yml} (52%) rename tests/config/{pytest_tags.yml => tags.yml} (50%) diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/ci-changes.yml similarity index 52% rename from .github/workflows/pytest-workflow.yml rename to .github/workflows/ci-changes.yml index 06a115917..a21c7a74d 100644 --- a/.github/workflows/pytest-workflow.yml +++ b/.github/workflows/ci-changes.yml @@ -1,4 +1,4 @@ -name: pytest-workflow +name: nf-test on changes # This workflow runs the pipeline with the minimal test dataset to check that it completes without any syntax errors on: pull_request: @@ -21,10 +21,10 @@ jobs: - uses: dorny/paths-filter@v2 id: filter with: - filters: "tests/config/pytest_tags.yml" + filters: "tests/config/tags.yml" test_changes: - name: ${{ matrix.tags }} ${{ matrix.profile }} + name: ${{ matrix.tags }} ${{ matrix.component }} ${{ matrix.profile }} runs-on: ubuntu-20.04 needs: changes if: needs.changes.outputs.tags != '[]' @@ -33,28 +33,14 @@ jobs: matrix: tags: ["${{ fromJson(needs.changes.outputs.tags) }}"] profile: ["docker"] # TODO Add singularity/conda + component: ["pipeline", "subworkflows", "modules"] steps: - name: Check out pipeline code uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.x" - - uses: actions/cache@v2 with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip- - - - name: Install Python dependencies - run: python -m pip install --upgrade pip pytest-workflow - - - uses: actions/cache@v2 - with: - path: /usr/local/bin/nextflow + path: /usr/local/bin/ key: ${{ runner.os }} restore-keys: | ${{ runner.os }}-nextflow- @@ -66,6 +52,11 @@ jobs: wget -qO- get.nextflow.io | bash sudo mv nextflow /usr/local/bin/ + - name: Install nf-test + run: | + wget -qO- https://code.askimed.com/install/nf-test | bash + sudo mv nf-test /usr/local/bin/ + - name: Set up Singularity if: matrix.profile == 'singularity' uses: eWaterCycle/setup-singularity@v5 @@ -84,26 +75,10 @@ jobs: if: matrix.profile == 'conda' run: conda clean -a - # Test the module - - name: Run pytest-workflow - # only use one thread for pytest-workflow to avoid race condition on conda cache. - run: TMPDIR=~ PROFILE=${{ matrix.profile }} pytest --tag ${{ matrix.tags }} --symlink --kwdof --git-aware --color=yes - - - name: Output log on failure - if: failure() - run: | - sudo apt-get install bat > /dev/null - batcat --decorations=always --color=always /home/runner/pytest_workflow_*/*/log.{out,err} + - name: Run nf-test + run: nf-test test --profile=${{ matrix.profile }} tests/${{ matrix.component }}/${{ matrix.tags }}/*.nf.test --tap=test.tap - - name: Upload logs on failure - if: failure() - uses: actions/upload-artifact@v2 + - uses: pcolby/tap-summary@v1 with: - name: logs-${{ matrix.profile }} - path: | - /home/runner/pytest_workflow_*/*/.nextflow.log - /home/runner/pytest_workflow_*/*/log.out - /home/runner/pytest_workflow_*/*/log.err - /home/runner/pytest_workflow_*/*/work - !/home/runner/pytest_workflow_*/*/work/conda - !/home/runner/pytest_workflow_*/*/work/singularity + path: >- + test.tap diff --git a/tests/config/pytest_tags.yml b/tests/config/tags.yml similarity index 50% rename from tests/config/pytest_tags.yml rename to tests/config/tags.yml index 9f44fd774..0ff8e8ab9 100644 --- a/tests/config/pytest_tags.yml +++ b/tests/config/tags.yml @@ -1,23 +1,23 @@ -quantify_salmon: - - subworkflows/local/quantify_salmon.nf - - tests/subworkflows/local/quantify_salmon/** - - modules/nf-core/software/salmon/quant/** - - modules/local/salmon_tx2gene.nf - - modules/local/salmon_tximport.nf - - modules/local/salmon_merge_counts.nf - - modules/local/salmon_summarizedexperiment.nf +# quantify_salmon: +# - subworkflows/local/quantify_salmon.nf +# - tests/subworkflows/local/quantify_salmon/** +# - modules/nf-core/software/salmon/quant/** +# - modules/local/salmon_tx2gene.nf +# - modules/local/salmon_tximport.nf +# - modules/local/salmon_merge_counts.nf +# - modules/local/salmon_summarizedexperiment.nf hisat2: - modules/nf-core/hisat2/** - subworkflows/nf-core/align_hisat2.nf - - tests/pipeline/test_hisat2.yml + - tests/pipeline/hisat2/** -quantify_rsem: - - subworkflows/local/quantify_rsem.nf - - subworkflows/nf-core/bam_sort_samtools.nf - - tests/subworkflows/local/quantify_rsem/** - - modules/nf-core/software/rsem/calculateexpression/** - - modules/local/rsem_merge_counts.nf +# quantify_rsem: +# - subworkflows/local/quantify_rsem.nf +# - subworkflows/nf-core/bam_sort_samtools.nf +# - tests/subworkflows/local/quantify_rsem/** +# - modules/nf-core/software/rsem/calculateexpression/** +# - modules/local/rsem_merge_counts.nf salmon: - bin/salmon_summarizedexperiment.r @@ -28,23 +28,21 @@ salmon: - modules/local/salmon_tximport.nf - modules/nf-core/salmon/** - subworkflows/local/quantify_salmon.nf - - tests/pipeline/test_salmon.yml - - tests/pipeline/test_star_salmon.yml + - tests/pipeline/salmon/** # - tests/subworkflows/local/quantify_salmon/ -star: +star_salmon: - modules/local/star_align_igenomes.nf - modules/local/star_genomegenerate_igenomes.nf - modules/nf-core/star/** - subworkflows/local/align_star.nf - - tests/pipeline/test_star_rsem.yml - - tests/pipeline/test_star_salmon.yml + - tests/pipeline/star_salmon/** -rsem: +star_rsem: - bin/prepare-for-rsem.py - modules/local/rsem_merge_counts.nf - modules/local/umitools_prepareforrsem.nf - modules/nf-core/rsem/** - subworkflows/local/quantify_rsem.nf - - tests/pipeline/test_star_rsem.yml + - tests/pipeline/star_rsem/** # - tests/subworkflows/local/quantify_rsem/ From 188f1e5d4046a47a9e4ce7e12c60f6c7d8851543 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 7 Nov 2022 10:28:27 -0600 Subject: [PATCH 39/49] ci: Comment out subworkflows and modules for now --- .github/workflows/ci-changes.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-changes.yml b/.github/workflows/ci-changes.yml index a21c7a74d..056b73c21 100644 --- a/.github/workflows/ci-changes.yml +++ b/.github/workflows/ci-changes.yml @@ -33,7 +33,7 @@ jobs: matrix: tags: ["${{ fromJson(needs.changes.outputs.tags) }}"] profile: ["docker"] # TODO Add singularity/conda - component: ["pipeline", "subworkflows", "modules"] + component: ["pipeline"] # TODO , "subworkflows", "modules"] steps: - name: Check out pipeline code uses: actions/checkout@v2 From f64a6e3f205280550250e75bf39d927a74a13726 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 7 Nov 2022 10:33:15 -0600 Subject: [PATCH 40/49] ci: tap => junitxml --- .github/workflows/ci-changes.yml | 9 +++++---- .github/workflows/ci.yml | 9 +++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci-changes.yml b/.github/workflows/ci-changes.yml index 056b73c21..398f7d312 100644 --- a/.github/workflows/ci-changes.yml +++ b/.github/workflows/ci-changes.yml @@ -76,9 +76,10 @@ jobs: run: conda clean -a - name: Run nf-test - run: nf-test test --profile=${{ matrix.profile }} tests/${{ matrix.component }}/${{ matrix.tags }}/*.nf.test --tap=test.tap + run: nf-test test --profile=${{ matrix.profile }} tests/${{ matrix.component }}/${{ matrix.tags }}/*.nf.test --junitxml=${{ matrix.tags }}.xml - - uses: pcolby/tap-summary@v1 + - name: Publish Test Report + uses: mikepenz/action-junit-report@v3 + if: always() # always run even if the previous step fails with: - path: >- - test.tap + report_paths: "${{ matrix.tags }}.xml" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 546fbabd8..1c4fdb8fc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,9 +67,10 @@ jobs: run: conda clean -a - name: Run nf-test - run: nf-test test --profile=${{ matrix.profile }} tests/pipeline/default/*.nf.test --tap=test.tap + run: nf-test test --profile=${{ matrix.profile }} tests/pipeline/default/*.nf.test --junitxml=default.xml - - uses: pcolby/tap-summary@v1 + - name: Publish Test Report + uses: mikepenz/action-junit-report@v3 + if: always() # always run even if the previous step fails with: - path: >- - test.tap + report_paths: "default.xml" From bbcef6f3a1af033572b584966455ea05e6b3d6b1 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 7 Nov 2022 11:00:58 -0600 Subject: [PATCH 41/49] test(default): Remove snapshot These files can't be md5summed --- tests/pipeline/default/main.nf.test | 22 ++++++++++------------ tests/pipeline/default/main.nf.test.snap | 17 ----------------- 2 files changed, 10 insertions(+), 29 deletions(-) delete mode 100644 tests/pipeline/default/main.nf.test.snap diff --git a/tests/pipeline/default/main.nf.test b/tests/pipeline/default/main.nf.test index d089be895..1d01b4abd 100644 --- a/tests/pipeline/default/main.nf.test +++ b/tests/pipeline/default/main.nf.test @@ -14,18 +14,16 @@ nextflow_pipeline { assert workflow.success assert workflow.trace.tasks().size() == 197 - assert snapshot( - path("$outputDir/salmon/salmon.merged.gene_counts_length_scaled.rds"), - path("$outputDir/salmon/salmon.merged.gene_counts_length_scaled.tsv"), - path("$outputDir/salmon/salmon.merged.gene_counts.rds"), - path("$outputDir/salmon/salmon.merged.gene_counts_scaled.rds"), - path("$outputDir/salmon/salmon.merged.gene_counts_scaled.tsv"), - path("$outputDir/salmon/salmon.merged.gene_counts.tsv"), - path("$outputDir/salmon/salmon.merged.gene_tpm.tsv"), - path("$outputDir/salmon/salmon.merged.transcript_counts.rds"), - path("$outputDir/salmon/salmon.merged.transcript_counts.tsv"), - path("$outputDir/salmon/salmon.merged.transcript_tpm.tsv") - ).match() + assert new File("$outputDir/salmon/salmon.merged.gene_counts_length_scaled.rds") + assert new File("$outputDir/salmon/salmon.merged.gene_counts_length_scaled.tsv") + assert new File("$outputDir/salmon/salmon.merged.gene_counts.rds") + assert new File("$outputDir/salmon/salmon.merged.gene_counts_scaled.rds") + assert new File("$outputDir/salmon/salmon.merged.gene_counts_scaled.tsv") + assert new File("$outputDir/salmon/salmon.merged.gene_counts.tsv") + assert new File("$outputDir/salmon/salmon.merged.gene_tpm.tsv") + assert new File("$outputDir/salmon/salmon.merged.transcript_counts.rds") + assert new File("$outputDir/salmon/salmon.merged.transcript_counts.tsv") + assert new File("$outputDir/salmon/salmon.merged.transcript_tpm.tsv") assert new File("$outputDir/fastqc/RAP1_IAA_30M_REP1_1_fastqc.html").exists() assert new File("$outputDir/fastqc/RAP1_IAA_30M_REP1_1_fastqc.zip").exists() diff --git a/tests/pipeline/default/main.nf.test.snap b/tests/pipeline/default/main.nf.test.snap deleted file mode 100644 index a84f0ef37..000000000 --- a/tests/pipeline/default/main.nf.test.snap +++ /dev/null @@ -1,17 +0,0 @@ -{ - "default test config": { - "content": [ - "salmon.merged.gene_counts_length_scaled.rds:md5,f8003e4bb35be88be899b101278b649a", - "salmon.merged.gene_counts_length_scaled.tsv:md5,156aa4f17a9bca0c02f6ced658d1441d", - "salmon.merged.gene_counts.rds:md5,5cd138a017b614b87332ab49cbe52cbf", - "salmon.merged.gene_counts_scaled.rds:md5,db353b0aa735f82fa191fc60488bea57", - "salmon.merged.gene_counts_scaled.tsv:md5,22da1c1e76dd4e015e19b450feeb3843", - "salmon.merged.gene_counts.tsv:md5,d8a058897b775f3f09eb71ee4b62f606", - "salmon.merged.gene_tpm.tsv:md5,db3651b0cf0583493c578f9b24c2f1c8", - "salmon.merged.transcript_counts.rds:md5,45574d7d5a63a4f690d532b0247dc29a", - "salmon.merged.transcript_counts.tsv:md5,05fe3760f6dbdde83e358a443f91528b", - "salmon.merged.transcript_tpm.tsv:md5,8b071524cbdff06a2d1cf0900608af18" - ], - "timestamp": "2022-11-07T15:06:34+0000" - } -} \ No newline at end of file From a11181f7ff816493863b1c8243bcb783cc394cab Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 7 Nov 2022 11:36:39 -0600 Subject: [PATCH 42/49] ci: Remove release for now --- .github/workflows/release.yml | 116 ---------------------------------- 1 file changed, 116 deletions(-) delete mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index 4b48b57fa..000000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,116 +0,0 @@ -name: Run all pytest workflow jobs -# This workflow runs the pipeline with the minimal test dataset to check that it completes without any syntax errors -on: - release: - types: [published] - workflow_dispatch: - -env: - NXF_ANSI_LOG: false - -jobs: - test: - name: ${{ matrix.test }} (${{ matrix.profile }}-${{ matrix.NXF_VER }}) - # Only run on push if this is the nf-core dev branch (merged PRs) - if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/rnaseq') }}" - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - NXF_VER: - - "21.10.3" - - "latest-everything" - test: - - "Run default pipeline with test data" - - "Test STAR Salmon with skip_qc workflow parameters" - - "Test STAR Salmon with skim_trimming workflow parameters" - - "Test STAR Salmon with gtf false workflow parameters" - - "Test STAR Salmon with star_index false workflow parameters" - - "Test STAR Salmon with transcript_fasta false workflow parameters" - - "Test STAR Salmon with min_mapped_reads 90 workflow parameters" - - "Test STAR Salmon with with_umi workflow parameters" - - "Test STAR Salmon with with_umi skip_trimming workflow parameters" - - "Test STAR Salmon with remove_ribo_rna and skip_qualimap workflow parameters" - - "Test STAR Salmon with bam_csi_index workflow parameters" - - "Test STAR Salmon with save_align_intermeds save_reference workflow parameters" - - "Test STAR Salmon with featurecounts_group_type false workflow parameters" - - "Test STAR RSEM with skip_qc workflow parameters" - - "Test STAR RSEM with rsem_index false workflow parameters" - - "Test HISAT2 with skip_qc workflow parameters" - - "Test HISAT2 with hisat2_index false workflow parameters" - - "Test Salmon with skip_qc skip_alignment workflow parameters" - - "Test Salmon with salmon_index false transcript_fasta false workflow parameters" - profile: ["docker", "singularity", "conda"] - exclude: - - NXF_VER: "latest-everything" - profile: "conda" - steps: - - name: Check out pipeline code - uses: actions/checkout@v2 - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.x" - - - uses: actions/cache@v2 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip- - - - name: Install Python dependencies - run: python -m pip install --upgrade pip pytest-workflow - - - uses: actions/cache@v2 - with: - path: /usr/local/bin/nextflow - key: ${{ runner.os }} - restore-keys: | - ${{ runner.os }}-nextflow- - - - name: Install Nextflow - uses: nf-core/setup-nextflow@v1 - with: - version: "${{ matrix.NXF_VER }}" - - - name: Set up Singularity - if: matrix.profile == 'singularity' - uses: eWaterCycle/setup-singularity@v5 - with: - singularity-version: 3.7.1 - - - name: Set up miniconda - if: matrix.profile == 'conda' - uses: conda-incubator/setup-miniconda@v2 - with: - auto-update-conda: true - channels: conda-forge,bioconda,defaults - python-version: ${{ matrix.python-version }} - - - name: Conda clean - if: matrix.profile == 'conda' - run: conda clean -a - - - name: Run pipeline with tests settings - run: TMPDIR=~ PROFILE=${{ matrix.profile }} pytest --tag "${{ matrix.test }}" --symlink --kwdof --git-aware --color=yes - - - name: Output log on failure - if: failure() - run: | - sudo apt-get install bat > /dev/null - batcat --decorations=always --color=always /home/runner/pytest_workflow_*/*/log.{out,err} - - - name: Upload logs on failure - if: failure() - uses: actions/upload-artifact@v2 - with: - name: logs-${{ matrix.profile }} - path: | - /home/runner/pytest_workflow_*/*/.nextflow.log - /home/runner/pytest_workflow_*/*/log.out - /home/runner/pytest_workflow_*/*/log.err - /home/runner/pytest_workflow_*/*/work - !/home/runner/pytest_workflow_*/*/work/conda - !/home/runner/pytest_workflow_*/*/work/singularity From ea8215ef641602f1b555a5e557c81c6530179b1d Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 7 Nov 2022 11:39:06 -0600 Subject: [PATCH 43/49] chore: Update CHANGELOG --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac3f801b9..d5375a7c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Enhancements & fixes - Update modules to latest restructuring on nf-core/modules -- [[#883](https://github.com/nf-core/rnaseq/pull/883)] - Migrate CI tests to Pytest workflow +- [[#883](https://github.com/nf-core/rnaseq/pull/883)] - Migrate CI tests to nf-test ### Parameters From 4b16f3859c759587d3db6485449382e618bb32b5 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 7 Nov 2022 11:39:41 -0600 Subject: [PATCH 44/49] chore: Remove PROFILE hack as nf-test has first-class support for profiles --- conf/test.config | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/conf/test.config b/conf/test.config index 2c2f6f0c5..187977e7d 100644 --- a/conf/test.config +++ b/conf/test.config @@ -46,33 +46,3 @@ process { ext.args2 = "--genomeSAindexNbases 7" } } - -// Enable container engines/virtualisation envs for CI testing -// only works when specified with the profile ENV -// otherwise tests can be done with the regular provided profiles -if (System.getenv('PROFILE')) { - if ("$PROFILE" == "conda") { - conda.createTimeout = "120 min" - params.enable_conda = true - charliecloud.enabled = false - docker.enabled = false - podman.enabled = false - shifter.enabled = false - singularity.enabled = false - } else if ("$PROFILE" == "docker") { - docker.enabled = true - docker.userEmulation = true - charliecloud.enabled = false - podman.enabled = false - shifter.enabled = false - singularity.enabled = false - } else if ("$PROFILE" == "singularity") { - params.singularity_pull_docker_container = false - singularity.autoMounts = true - singularity.enabled = true - charliecloud.enabled = false - docker.enabled = false - podman.enabled = false - shifter.enabled = false - } -} From 81963db4246f82316ef9f53baa9471d9870c241b Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 7 Nov 2022 12:18:34 -0600 Subject: [PATCH 45/49] test(nf-test): Move quantify_rsem tests over --- .../subworkflows/local/quantify_rsem/main.nf | 25 ------------------- .../local/quantify_rsem/main.nf.test | 25 +++++++++++++++++++ .../subworkflows/local/quantify_rsem/test.yml | 7 ------ 3 files changed, 25 insertions(+), 32 deletions(-) delete mode 100644 tests/subworkflows/local/quantify_rsem/main.nf create mode 100644 tests/subworkflows/local/quantify_rsem/main.nf.test delete mode 100644 tests/subworkflows/local/quantify_rsem/test.yml diff --git a/tests/subworkflows/local/quantify_rsem/main.nf b/tests/subworkflows/local/quantify_rsem/main.nf deleted file mode 100644 index 6da37f6e2..000000000 --- a/tests/subworkflows/local/quantify_rsem/main.nf +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env nextflow - -nextflow.enable.dsl = 2 - -def modules = params.modules.clone() - -include { QUANTIFY_RSEM } from '../../../../subworkflows/local/quantify_rsem' addParams( - calculateexpression_options: rsem_calculateexpression_options, - samtools_sort_options: samtools_sort_genome_options, - samtools_index_options: samtools_index_genome_options, - samtools_stats_options: samtools_index_genome_options, - merge_counts_options: modules['rsem_merge_counts'] -) - -workflow test_quantify_rsem { - input = [ [ id:'test', single_end:true ], - file(params.test_data['sarscov2']['illumina']['bam']['test_single_end_bam'], checkIfExists: true)] - - rsem_index = file('https://github.com/nf-core/test-datasets/raw/rnaseq/reference/rsem.tar.gz', checkIfExists: true) - - QUANTIFY_RSEM ( - input, - rsem_index - ) -} diff --git a/tests/subworkflows/local/quantify_rsem/main.nf.test b/tests/subworkflows/local/quantify_rsem/main.nf.test new file mode 100644 index 000000000..c8005aeef --- /dev/null +++ b/tests/subworkflows/local/quantify_rsem/main.nf.test @@ -0,0 +1,25 @@ +nextflow_workflow { + + name "Test quantify_rsem subworkflow" + script "subworkflows/local/quantify_rsem.nf" + workflow "QUANTIFY_RSEM" + + test("Basic test") { + when { + workflow { + """ + input[0] = [ [ id:'test', single_end:true ], + file('https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/sarscov2/illumina/bam/test.single_end.sorted.bam', checkIfExists: true)] + + input[1] = file('https://github.com/nf-core/test-datasets/raw/rnaseq/reference/rsem.tar.gz', checkIfExists: true) + """ + } + } + + then { + // FIXME + assert workflow.failed + } + } + +} diff --git a/tests/subworkflows/local/quantify_rsem/test.yml b/tests/subworkflows/local/quantify_rsem/test.yml deleted file mode 100644 index 7ec7f3128..000000000 --- a/tests/subworkflows/local/quantify_rsem/test.yml +++ /dev/null @@ -1,7 +0,0 @@ -- name: subworkflow quantify_rsem - command: nextflow run ./tests/subworkflows/local/quantify_rsem/ -entry test_quantify_rsem -c tests/config/nextflow.config - tags: - - quantify_rsem - - failing - exit_code: 1 - # TODO files: From 5d4bf8b5066960242c161a2dc459106ce9f5cc77 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 7 Nov 2022 12:34:55 -0600 Subject: [PATCH 46/49] test(nf-test): Migrate quantify_salmon --- .../local/quantify_salmon/main.nf | 25 ---------------- .../local/quantify_salmon/main.nf.test | 29 +++++++++++++++++++ 2 files changed, 29 insertions(+), 25 deletions(-) delete mode 100644 tests/subworkflows/local/quantify_salmon/main.nf create mode 100644 tests/subworkflows/local/quantify_salmon/main.nf.test diff --git a/tests/subworkflows/local/quantify_salmon/main.nf b/tests/subworkflows/local/quantify_salmon/main.nf deleted file mode 100644 index cbc0be131..000000000 --- a/tests/subworkflows/local/quantify_salmon/main.nf +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env nextflow - -nextflow.enable.dsl = 2 - -def modules = params.modules.clone() - -include { SALMON_INDEX } from '../../../../modules/nf-core/software/salmon/index/main.nf' addParams( options: [:] ) -include { QUANTIFY_SALMON } from '../../../../subworkflows/local/quantify_salmon' addParams( - genome_options: '', - tximport_options: modules['salmon_tximport'], - salmon_quant_options: modules['salmon_quant'], - merge_counts_options: modules['salmon_merge_counts'] -) - -workflow test_quantify_salmon { - input = [ [ id:'test', single_end:true ], // meta map - file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true) - ] - genome_fasta = file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true) - transcript_fasta = file(params.test_data['sarscov2']['genome']['transcriptome_fasta'], checkIfExists: true) - gtf = file(params.test_data['sarscov2']['genome']['genome_gtf'], checkIfExists: true) - - SALMON_INDEX ( genome_fasta, transcript_fasta ) - QUANTIFY_SALMON ( input, SALMON_INDEX.out.index, transcript_fasta, gtf, false ) -} diff --git a/tests/subworkflows/local/quantify_salmon/main.nf.test b/tests/subworkflows/local/quantify_salmon/main.nf.test new file mode 100644 index 000000000..7791e826b --- /dev/null +++ b/tests/subworkflows/local/quantify_salmon/main.nf.test @@ -0,0 +1,29 @@ +nextflow_workflow { + + name "Test quantify_salmon subworkflow" + script "subworkflows/local/quantify_salmon.nf" + workflow "QUANTIFY_SALMON" + + test("Basic test") { + when { + workflow { + """ + input[0] = [ [ id:'test', single_end: false ], + file(params.test_data['homo_sapiens']['illumina']['test_paired_end_sorted_bam'], checkIfExists: true) + ] + input[1] = file(params.test_data['homo_sapiens']['genome']['index_salmon'], checkIfExists: true) + input[2] = file(params.test_data['homo_sapiens']['genome']['transcriptome_fasta'], checkIfExists: true) + input[3] = file(params.test_data['homo_sapiens']['genome']['genome_gtf'], checkIfExists: true) + input[4] = true + input[5] = '' + """ + } + } + + then { + // FIXME salmon index doesn't exist + assert workflow.failed + } + } + +} From 00ed9b1ed7e2cbcb6c35b7234c9b4a251ee69db4 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 7 Nov 2022 16:45:41 -0600 Subject: [PATCH 47/49] test(nf-test): Remove local in subworkflows Redundant as all the nf-core subworkflows should be tested in nf-core/modules --- tests/subworkflows/{local => }/quantify_rsem/main.nf.test | 0 tests/subworkflows/{local => }/quantify_salmon/main.nf.test | 0 tests/subworkflows/{local => }/quantify_salmon/test.yml | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename tests/subworkflows/{local => }/quantify_rsem/main.nf.test (100%) rename tests/subworkflows/{local => }/quantify_salmon/main.nf.test (100%) rename tests/subworkflows/{local => }/quantify_salmon/test.yml (100%) diff --git a/tests/subworkflows/local/quantify_rsem/main.nf.test b/tests/subworkflows/quantify_rsem/main.nf.test similarity index 100% rename from tests/subworkflows/local/quantify_rsem/main.nf.test rename to tests/subworkflows/quantify_rsem/main.nf.test diff --git a/tests/subworkflows/local/quantify_salmon/main.nf.test b/tests/subworkflows/quantify_salmon/main.nf.test similarity index 100% rename from tests/subworkflows/local/quantify_salmon/main.nf.test rename to tests/subworkflows/quantify_salmon/main.nf.test diff --git a/tests/subworkflows/local/quantify_salmon/test.yml b/tests/subworkflows/quantify_salmon/test.yml similarity index 100% rename from tests/subworkflows/local/quantify_salmon/test.yml rename to tests/subworkflows/quantify_salmon/test.yml From 08a4caae449db6195ca711aeea96821175b20851 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 7 Nov 2022 16:51:28 -0600 Subject: [PATCH 48/49] test(subworkflows): Add input_check test --- tests/subworkflows/input_check/main.nf.test | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 tests/subworkflows/input_check/main.nf.test diff --git a/tests/subworkflows/input_check/main.nf.test b/tests/subworkflows/input_check/main.nf.test new file mode 100644 index 000000000..d468baa2c --- /dev/null +++ b/tests/subworkflows/input_check/main.nf.test @@ -0,0 +1,21 @@ +nextflow_workflow { + + name "input_check subworkflow" + script "subworkflows/local/input_check.nf" + workflow "INPUT_CHECK" + + test("Should run without failure") { + when { + workflow { + """ + input[0] = file('https://raw.githubusercontent.com/nf-core/test-datasets/rnaseq/samplesheet/v3.4/samplesheet_test.csv', checkIfExists: true) + """ + } + } + + then { + assert workflow.success + } + } + +} From 05df41c2dd1af06c512b60d63210f3cec45db88c Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 7 Nov 2022 16:55:30 -0600 Subject: [PATCH 49/49] test(modules): Add basic samplesheet_check --- tests/modules/samplesheet_check/main.nf.test | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 tests/modules/samplesheet_check/main.nf.test diff --git a/tests/modules/samplesheet_check/main.nf.test b/tests/modules/samplesheet_check/main.nf.test new file mode 100644 index 000000000..4e5ef97fa --- /dev/null +++ b/tests/modules/samplesheet_check/main.nf.test @@ -0,0 +1,19 @@ +nextflow_process { + name "samplesheet_check" + script "modules/local/samplesheet_check.nf" + process "SAMPLESHEET_CHECK" + + test("Should run without failure") { + when { + process { + """ + input[0] = file('https://raw.githubusercontent.com/nf-core/test-datasets/rnaseq/samplesheet/v3.4/samplesheet_test.csv', checkIfExists: true) + """ + } + } + + then { + assert process.success + } + } +}