Merge pull request #243 from drpatelh/fixes

Fixes for PR #238

.github/workflows/ci.yml

@@ -13,7 +13,9 @@ on:
 
 env:
   NXF_ANSI_LOG: false
-  NFTEST_VER: "0.8.1"
+  NFTEST_VER: "0.8.3"
+  NFT_DIFF: "icdiff"
+  NFT_DIFF_ARGS: "-N --cols 200 -L expected -L observed -t"
 
 concurrency:
   group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}"
@@ -89,9 +91,13 @@ jobs:
           wget -qO- https://code.askimed.com/install/nf-test | bash
           sudo mv nf-test /usr/local/bin/
 
+      - name: Install icdiff for nf-test diff viewing
+        run: |
+          sudo apt install icdiff
+
       - name: Run nf-test
         run: |
-          nf-test test --tag ${{ matrix.tags }} --profile "test,${{ matrix.profile }}" --junitxml=test.xml
+          nf-test test --verbose --tag ${{ matrix.tags }} --profile "test,${{ matrix.profile }}" --junitxml=test.xml
 
       - name: Output log on failure
         if: failure()

.nf-core.yml

@@ -17,3 +17,6 @@ lint:
     - .gitignore
     - assets/sendmail_template.txt
     - lib/NfcoreTemplate.groovy
+    - .github/CONTRIBUTING.md
+    - .github/PULL_REQUEST_TEMPLATE.md
+    - .github/workflows/linting.yml

CHANGELOG.md

@@ -10,14 +10,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 Special thanks to the following for their contributions to the release:
 
 - [Alexandru Mizeranschi](https://github.com/nicolae06)
+- [Alexander Blaessle](https://github.com/alexblaessle)
 - [Sebastian Uhrig](https://github.com/suhrig)
 
 Thank you to everyone else that has contributed by reporting bugs, enhancements or in any other way, shape or form.
 
 ### Enhancements & fixes
 
+- [PR #238](https://github.com/nf-core/fetchngs/pull/238) - Resolved bug when prefetching large studies ([#236](https://github.com/nf-core/fetchngs/issues/236))
 - [PR #241](https://github.com/nf-core/fetchngs/pull/241) - Download of big files time out and don't resume ([#169](https://github.com/nf-core/fetchngs/issues/169))
 - [PR #242](https://github.com/nf-core/fetchngs/pull/242) - Template update for nf-core/tools v2.11
+- [PR #243](https://github.com/nf-core/fetchngs/pull/243) - Fixes for [PR #238](https://github.com/nf-core/fetchngs/pull/238)
 
 ### Software dependencies
 

bin/multiqc_mappings_config.py

@@ -7,6 +7,7 @@
     config = "sample_names_rename_buttons:\n"
     config += "\n".join(["  - " + x.strip('"') for x in header])
     config += "sample_names_rename:\n"
+    rename = []
     for line in fin:
-        config += f"  - [{', '.join(line.strip().split(','))}]\n"
-    fout.write(config)
+        rename.append(f"  - [{', '.join(line.strip().split(','))}]")
+    fout.write(config + "\n".join(sorted(rename)) + "\n")

conf/test.config

@@ -20,7 +20,7 @@ params {
     max_time   = '6.h'
 
     // Input data
-    input  = '${projectDir}/tests/sra_ids_test.csv'
+    input = "https://raw.githubusercontent.com/nf-core/test-datasets/fetchngs/sra_ids_test.csv"
 
-    validationSchemaIgnoreParams = 'test_data_base,merge_samplesheet_ids,fastq_ftp_ids,test_data'
+    validationSchemaIgnoreParams = 'test_data_base,fastq_ftp_ids,test_data'
 }

conf/test_data.config

@@ -2,19 +2,7 @@ params {
     // Base directory for test data
     test_data_base = "https://raw.githubusercontent.com/nf-core/test-datasets/fetchngs"
 
-    merge_samplesheet_ids = [ "DRX024467_DRR026872", "SRX11047067_SRR14709033", "SRX9504942_SRR13055517", "DRX026011_DRR028935", "SRX17709227_SRR21711856", "SRX9504942_SRR13055518", "ERX1188904_ERR1109373", "SRX17709228_SRR21711855", "SRX9504942_SRR13055519", "ERX1234253_ERR1160846", "SRX6725035_SRR9984183", "SRX9504942_SRR13055520", "SRX10940790_SRR14593545", "SRX9315476_SRR12848126", "SRX9626017_SRR13191702" ]
-
-    def merge_samplesheet_url  = "${params.test_data_base}/modules/local/sra_merge_samplesheet/samplesheets/"
-    def merge_mappings_url     = "${params.test_data_base}/modules/local/sra_merge_samplesheet/mappings/"
-    def merge_samplesheet_urls = []
-    def merge_mappings_urls    = []
-
-    merge_samplesheet_ids.each { id ->
-        merge_samplesheet_urls += "${merge_samplesheet_url}${id}.samplesheet.csv"
-        merge_mappings_urls    += "${merge_mappings_url}/${id}.mappings.csv"
-    }
-
-    fastq_ftp_ids      = ["SRR13191702"]
+    fastq_ftp_ids = ["SRR13191702"]
     def fastq_ftp_url  = "ftp.sra.ebi.ac.uk/vol1/fastq/SRR131/002/SRR13191702/"
     def fastq_ftp_urls = []
 
@@ -26,22 +14,20 @@ params {
     test_data {
         'sarscov2'{
             'illumina' {
-                SRR11140744_tar_gz                  = "${params.test_data_base}/data/genomics/sarscov2/illumina/sra/SRR11140744.tar.gz"
-                SRR13255544_tar_gz                  = "${params.test_data_base}/data/genomics/sarscov2/illumina/sra/SRR13255544.tar.gz"
+                SRR11140744_tar_gz = "${params.test_data_base}/data/genomics/sarscov2/illumina/sra/SRR11140744.tar.gz"
+                SRR13255544_tar_gz = "${params.test_data_base}/data/genomics/sarscov2/illumina/sra/SRR13255544.tar.gz"
             }
         }
         'generic' {
             'config' {
-                ncbi_user_settings                  = "${params.test_data_base}/data/generic/config/ncbi_user_settings.mkfg"
+                ncbi_user_settings = "${params.test_data_base}/data/generic/config/ncbi_user_settings.mkfg"
             }
         }
         'modules_local' {
-                multiqc_mappings_config             = "${params.test_data_base}/modules/local/multiqc_mappings_config/SRX9626017_SRR13191702.mappings.csv"
-                sra_merge_samplesheet_samplesheets  = merge_samplesheet_urls
-                sra_merge_samplesheet_mappings      = merge_mappings_urls
-                sra_to_samplesheet                  = "${params.test_data_base}/modules/local/sra_to_samplesheet/SRX9626017_SRR13191702.mappings.csv"
-                sra_fastq_ftp                       = fastq_ftp_urls
-                sra_runinfo_to_ftp                  = "${params.test_data_base}/modules/local/sra_runinfo_to_ftp/SRR13191702.runinfo.tsv"
+                multiqc_mappings_config = "${params.test_data_base}/modules/local/multiqc_mappings_config/SRX9626017_SRR13191702.mappings.csv"
+                sra_to_samplesheet      = "${params.test_data_base}/modules/local/sra_to_samplesheet/SRX9626017_SRR13191702.mappings.csv"
+                sra_fastq_ftp           = fastq_ftp_urls
+                sra_runinfo_to_ftp      = "${params.test_data_base}/modules/local/sra_runinfo_to_ftp/SRR13191702.runinfo.tsv"
         }
     }
 }

nextflow.config

@@ -114,7 +114,7 @@ profiles {
         shifter.enabled        = false
         charliecloud.enabled   = false
         apptainer.enabled      = false
-        runOptions             = '-u $(id -u):$(id -g)'
+        docker.runOptions      = '-u $(id -u):$(id -g)'
     }
     arm {
         docker.runOptions = '-u $(id -u):$(id -g) --platform=linux/amd64'

nf-test.config

@@ -1,16 +1,10 @@
 config {
-    // location for all nf-tests
+    // Location of nf-tests
     testsDir "."
 
-    // nf-test directory including temporary files for each test
-    workDir "/tmp"
+    // nf-test directory used to create temporary files for each test
+    workDir "nf-test"
 
-    // location of library folder that is added automatically to the classpath
-    libDir "lib/"
-
-    // location of an optional nextflow.config file specific for executing tests
-    configFile "nextflow.config"
-
-    // run all test with the defined docker profile from the main nextflow.config
-    profile ""
+    // Location of an optional nextflow.config file specific for executing pipeline tests
+    configFile "tests/nextflow.config"
 }

tests/main.nf.test

@@ -2,27 +2,20 @@ nextflow_pipeline {
 
     name "Test pipeline"
     script "../main.nf"
+    config "../conf/test.config"
     tag "pipeline"
     tag "pipeline_fetchngs"
 
     test("Run with profile test") {
 
         when {
             params {
-                outdir     = "results"
-                max_cpus   = 2
-                max_memory = '6.GB'
-                max_time   = '6.h'
-                input      = "$projectDir/tests/sra_ids_test.csv"
-                validationSchemaIgnoreParams = 'test_data_base,merge_samplesheet_ids,fastq_ftp_ids,test_data'
+                outdir = "$outputDir"
             }
         }
 
         then {
-            assertAll(
-                { assert workflow.success }
-            )
+            assert workflow.success
         }
     }
-
 }

workflows/sra/main.nf

@@ -61,7 +61,6 @@ workflow SRA {
         .unique()
         .set { ch_sra_metadata }
 
-    fastq_files = Channel.empty()
     if (!params.skip_fastq_download) {
 
         ch_sra_metadata
@@ -93,8 +92,10 @@ workflow SRA {
         ch_versions = ch_versions.mix(FASTQ_DOWNLOAD_PREFETCH_FASTERQDUMP_SRATOOLS.out.versions.first())
 
         // Isolate FASTQ channel which will be added to emit block
-        fastq_files
-            .mix(SRA_FASTQ_FTP.out.fastq, FASTQ_DOWNLOAD_PREFETCH_FASTERQDUMP_SRATOOLS.out.reads)
+        SRA_FASTQ_FTP
+            .out
+            .fastq
+            .mix(FASTQ_DOWNLOAD_PREFETCH_FASTERQDUMP_SRATOOLS.out.reads)
             .map {
                 meta, fastq ->
                     def reads = fastq instanceof List ? fastq.flatten() : [ fastq ]
@@ -118,10 +119,24 @@ workflow SRA {
         params.sample_mapping_fields
     )
 
-
-    // Combine all sample sheet/mapping paths into two files
-    ch_samplesheet=SRA_TO_SAMPLESHEET.out.samplesheet.map{it[1]}.collectFile(name:'samplesheets.csv', newLine: true, keepHeader: true, storeDir:params.outdir)
-    ch_mappings=SRA_TO_SAMPLESHEET.out.samplesheet.map{it[1]}.collectFile(name:'id_mappings.csv', newLine: true, keepHeader: true, storeDir:params.outdir)
+    // Merge samplesheets and mapping files across all samples
+    SRA_TO_SAMPLESHEET
+        .out
+        .samplesheet
+        .map { it[1] }
+        .collectFile(name:'tmp_samplesheet.csv', newLine: true, keepHeader: true, sort: { it.baseName })
+        .map { it.text.tokenize('\n').join('\n') }
+        .collectFile(name:'samplesheet.csv', storeDir: "${params.outdir}/samplesheet")
+        .set { ch_samplesheet }
+
+    SRA_TO_SAMPLESHEET
+        .out
+        .mappings
+        .map { it[1] }
+        .collectFile(name:'tmp_id_mappings.csv', newLine: true, keepHeader: true, sort: { it.baseName })
+        .map { it.text.tokenize('\n').join('\n') }
+        .collectFile(name:'id_mappings.csv', storeDir: "${params.outdir}/samplesheet")
+        .set { ch_mappings }
 
     //
     // MODULE: Create a MutiQC config file with sample name mappings
@@ -136,10 +151,10 @@ workflow SRA {
     }
 
     emit:
-    fastq           = fastq_files
     samplesheet     = ch_samplesheet
     mappings        = ch_mappings
     sample_mappings = ch_sample_mappings_yml
+    sra_metadata    = ch_sra_metadata
     versions        = ch_versions.unique()
 }
 

workflows/sra/tests/main.nf.test

@@ -13,19 +13,32 @@ nextflow_workflow {
         when {
             workflow {
                 """
-                input[0] = Channel.from('ERR1160846', 'GSE214215', 'SRR12848126')
+                input[0] = Channel.from("DRX026011", "ERX1234253", "SRX6725035")
                 """
             }
             params {
-                outdir = "results"
+                outdir = "$outputDir"
             }
         }
 
         then {
-            assertAll(
-                { assert workflow.success },
-                { assert snapshot(workflow.out).match() }
-            )
+            assert workflow.success
+
+            with(workflow.out.samplesheet) {
+                assert path(get(0)).readLines().size() == 4
+                assert path(get(0)).readLines()*.split(',').collect { it[0] } == ['"sample"', '"DRX026011"', '"ERX1234253"', '"SRX6725035"']
+                assert path(get(0)).text.contains('Illumina HiSeq 2500')
+            }
+
+            with(workflow.out.mappings) {
+                assert path(get(0)).readLines().size() == 4
+                assert path(get(0)).readLines()*.split(',').collect { it[0] } == ['"sample"', '"DRX026011"', '"ERX1234253"', '"SRX6725035"']
+                assert path(get(0)).text.contains('Illumina HiSeq 2500')
+            }
+
+            with(workflow.out.sample_mappings) {
+                assert path(get(0)[0]).md5 == "1ac06bb95b503703430e74660bbdd768"
+            }
         }
     }
 }

workflows/sra/tests/sra_custom_ena_metadata_fields.nf.test

@@ -13,22 +13,35 @@ nextflow_workflow {
         when {
             workflow {
                 """
-                input[0] = Channel.from('ERR1160846', 'GSE214215', 'SRR12848126')
+                input[0] = Channel.from("DRX026011", "ERX1234253", "SRX6725035")
                 """
             }
             params {
-                outdir = "results"
+                outdir = "$outputDir"
                 nf_core_pipeline = "rnaseq"
                 ena_metadata_fields = "run_accession,experiment_accession,library_layout,fastq_ftp,fastq_md5"
                 sample_mapping_fields = "run_accession,library_layout"
             }
         }
 
         then {
-            assertAll(
-                { assert workflow.success },
-                { assert snapshot(workflow.out).match() }
-            )
+            assert workflow.success
+
+            with(workflow.out.samplesheet) {
+                assert path(get(0)).readLines().size() == 4
+                assert path(get(0)).readLines()*.split(',').collect { it[0] } == ['"sample"', '"DRX026011"', '"ERX1234253"', '"SRX6725035"']
+                assert path(get(0)).text.contains('SINGLE')
+            }
+
+            with(workflow.out.mappings) {
+                assert path(get(0)).readLines().size() == 4
+                assert path(get(0)).readLines()*.split(',').collect { it[0] } == ['"sample"', '"DRX026011"', '"ERX1234253"', '"SRX6725035"']
+                assert path(get(0)).text.contains('SINGLE')
+            }
+
+            with(workflow.out.sample_mappings) {
+                assert path(get(0)[0]).md5 == "3b70bc9658eab4ba2f4ec98cb749ac9d"
+            }
         }
     }
 }