Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Delete lib directory and replace with utils_* subworkflows #1197

Merged
merged 9 commits into from
Jan 22, 2024
Merged

Delete lib directory and replace with utils_* subworkflows #1197

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
cbdfbc6
Move main workflow into it's own directory
drpatelh Jan 20, 2024
91c5742
Remove nfcore_external_java_deps.jar
drpatelh Jan 20, 2024
c85a4cb
Install utils subworkflows from nf-core modules
drpatelh Jan 20, 2024
15924c4
Delete lib directory and replace with utils_* subworkflows
drpatelh Jan 20, 2024
433b455
Fix linting
drpatelh Jan 20, 2024
b0fc8cb
Add prepare_tool_indices logic to rnaseq workflow
drpatelh Jan 20, 2024
2c53607
Move rrna-db-defaults.txt to workflow assets
drpatelh Jan 21, 2024
82f0eea
Replace local MultiQC module with one from nf-core/modules
drpatelh Jan 21, 2024
3734375
Remove test_cache profile
drpatelh Jan 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions .devcontainer/devcontainer.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,11 @@
"python.linting.flake8Path": "/opt/conda/bin/flake8",
"python.linting.pycodestylePath": "/opt/conda/bin/pycodestyle",
"python.linting.pydocstylePath": "/opt/conda/bin/pydocstyle",
"python.linting.pylintPath": "/opt/conda/bin/pylint"
"python.linting.pylintPath": "/opt/conda/bin/pylint",
},

// Add the IDs of extensions you want installed when the container is created.
"extensions": ["ms-python.python", "ms-python.vscode-pylance", "nf-core.nf-core-extensionpack"]
}
}
"extensions": ["ms-python.python", "ms-python.vscode-pylance", "nf-core.nf-core-extensionpack"],
},
},
}
155 changes: 5 additions & 150 deletions .github/workflows/ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,38 +35,9 @@ jobs:
with:
version: "${{ matrix.NXF_VER }}"

- name: Hash Github Workspace
id: hash_workspace
run: |
echo "digest=$(echo RNA_3.10.1_${{ github.workspace }} | md5sum | cut -c 1-25)" >> $GITHUB_OUTPUT

- name: Cache test data
id: cache-testdata
uses: actions/cache@v3
with:
path: test-datasets/
key: ${{ steps.hash_workspace.outputs.digest }}

- name: Check out test data
if: steps.cache-testdata.outputs.cache-hit != 'true'
uses: actions/checkout@v3
with:
repository: nf-core/test-datasets
ref: rnaseq3
path: test-datasets/

- name: Replace remote paths in samplesheets
run: |
for f in ${{ github.workspace }}/test-datasets/samplesheet/v3.10/*.csv; do
sed -i "s=https://raw.githubusercontent.com/nf-core/test-datasets/rnaseq/=${{ github.workspace }}/test-datasets/=g" $f
echo "========== $f ============"
cat $f
echo "========================================"
done;

- name: Run pipeline with test data
run: |
nextflow run ${GITHUB_WORKSPACE} -profile test_cache,docker --outdir ./results --test_data_base ${{ github.workspace }}/test-datasets/
nextflow run ${GITHUB_WORKSPACE} -profile test,docker --outdir ./results

star_salmon:
name: Test STAR Salmon with workflow parameters
Expand All @@ -92,43 +63,14 @@ jobs:
- name: Check out pipeline code
uses: actions/checkout@v2

- name: Hash Github Workspace
id: hash_workspace
run: |
echo "digest=$(echo RNA_3.10.1_${{ github.workspace }} | md5sum | cut -c 1-25)" >> $GITHUB_OUTPUT

- name: Cache test data
id: cache-testdata
uses: actions/cache@v3
with:
path: test-datasets/
key: ${{ steps.hash_workspace.outputs.digest }}

- name: Check out test data
if: steps.cache-testdata.outputs.cache-hit != 'true'
uses: actions/checkout@v3
with:
repository: nf-core/test-datasets
ref: rnaseq3
path: test-datasets/

- name: Replace remote paths in samplesheets
run: |
for f in ${{ github.workspace }}/test-datasets/samplesheet/v3.10/*.csv; do
sed -i "s=https://raw.githubusercontent.com/nf-core/test-datasets/rnaseq/=${{ github.workspace }}/test-datasets/=g" $f
echo "========== $f ============"
cat $f
echo "========================================"
done;

- name: Install Nextflow
run: |
wget -qO- get.nextflow.io | bash
sudo mv nextflow /usr/local/bin/

- name: Run pipeline with STAR and various parameters
run: |
nextflow run ${GITHUB_WORKSPACE} -profile test_cache,docker --aligner star_salmon ${{ matrix.parameters }} --outdir ./results --test_data_base ${{ github.workspace }}/test-datasets/
nextflow run ${GITHUB_WORKSPACE} -profile test,docker --aligner star_salmon ${{ matrix.parameters }} --outdir ./results

star_rsem:
name: Test STAR RSEM with workflow parameters
Expand All @@ -143,43 +85,14 @@ jobs:
- name: Check out pipeline code
uses: actions/checkout@v2

- name: Hash Github Workspace
id: hash_workspace
run: |
echo "digest=$(echo RNA_3.10.1_${{ github.workspace }} | md5sum | cut -c 1-25)" >> $GITHUB_OUTPUT

- name: Cache test data
id: cache-testdata
uses: actions/cache@v3
with:
path: test-datasets/
key: ${{ steps.hash_workspace.outputs.digest }}

- name: Check out test data
if: steps.cache-testdata.outputs.cache-hit != 'true'
uses: actions/checkout@v3
with:
repository: nf-core/test-datasets
ref: rnaseq3
path: test-datasets/

- name: Replace remote paths in samplesheets
run: |
for f in ${{ github.workspace }}/test-datasets/samplesheet/v3.10/*.csv; do
sed -i "s=https://raw.githubusercontent.com/nf-core/test-datasets/rnaseq/=${{ github.workspace }}/test-datasets/=g" $f
echo "========== $f ============"
cat $f
echo "========================================"
done;

- name: Install Nextflow
run: |
wget -qO- get.nextflow.io | bash
sudo mv nextflow /usr/local/bin/

- name: Run pipeline with RSEM STAR and various parameters
run: |
nextflow run ${GITHUB_WORKSPACE} -profile test_cache,docker --aligner star_rsem ${{ matrix.parameters }} --outdir ./results --test_data_base ${{ github.workspace }}/test-datasets/
nextflow run ${GITHUB_WORKSPACE} -profile test,docker --aligner star_rsem ${{ matrix.parameters }} --outdir ./results

hisat2:
name: Test HISAT2 with workflow parameters
Expand All @@ -194,43 +107,14 @@ jobs:
- name: Check out pipeline code
uses: actions/checkout@v2

- name: Hash Github Workspace
id: hash_workspace
run: |
echo "digest=$(echo RNA_3.10.1_${{ github.workspace }} | md5sum | cut -c 1-25)" >> $GITHUB_OUTPUT

- name: Cache test data
id: cache-testdata
uses: actions/cache@v3
with:
path: test-datasets/
key: ${{ steps.hash_workspace.outputs.digest }}

- name: Check out test data
if: steps.cache-testdata.outputs.cache-hit != 'true'
uses: actions/checkout@v3
with:
repository: nf-core/test-datasets
ref: rnaseq3
path: test-datasets/

- name: Replace remote paths in samplesheets
run: |
for f in ${{ github.workspace }}/test-datasets/samplesheet/v3.10/*.csv; do
sed -i "s=https://raw.githubusercontent.com/nf-core/test-datasets/rnaseq/=${{ github.workspace }}/test-datasets/=g" $f
echo "========== $f ============"
cat $f
echo "========================================"
done;

- name: Install Nextflow
run: |
wget -qO- get.nextflow.io | bash
sudo mv nextflow /usr/local/bin/

- name: Run pipeline with HISAT2 and various parameters
run: |
nextflow run ${GITHUB_WORKSPACE} -profile test_cache,docker --aligner hisat2 ${{ matrix.parameters }} --outdir ./results --test_data_base ${{ github.workspace }}/test-datasets/
nextflow run ${GITHUB_WORKSPACE} -profile test,docker --aligner hisat2 ${{ matrix.parameters }} --outdir ./results

pseudo:
name: Test Pseudoaligners with workflow parameters
Expand All @@ -249,40 +133,11 @@ jobs:
- name: Check out pipeline code
uses: actions/checkout@v2

- name: Hash Github Workspace
id: hash_workspace
run: |
echo "digest=$(echo RNA_3.10.1_${{ github.workspace }} | md5sum | cut -c 1-25)" >> $GITHUB_OUTPUT

- name: Cache test data
id: cache-testdata
uses: actions/cache@v3
with:
path: test-datasets/
key: ${{ steps.hash_workspace.outputs.digest }}

- name: Check out test data
if: steps.cache-testdata.outputs.cache-hit != 'true'
uses: actions/checkout@v3
with:
repository: nf-core/test-datasets
ref: rnaseq3
path: test-datasets/

- name: Replace remote paths in samplesheets
run: |
for f in ${{ github.workspace }}/test-datasets/samplesheet/v3.10/*.csv; do
sed -i "s=https://raw.githubusercontent.com/nf-core/test-datasets/rnaseq/=${{ github.workspace }}/test-datasets/=g" $f
echo "========== $f ============"
cat $f
echo "========================================"
done;

- name: Install Nextflow
run: |
wget -qO- get.nextflow.io | bash
sudo mv nextflow /usr/local/bin/

- name: Run pipeline with Salmon or Kallisto and various parameters
run: |
nextflow run ${GITHUB_WORKSPACE} -profile test_cache,docker ${{ matrix.parameters }} --outdir ./results --test_data_base ${{ github.workspace }}/test-datasets/
nextflow run ${GITHUB_WORKSPACE} -profile test,docker ${{ matrix.parameters }} --outdir ./results
6 changes: 4 additions & 2 deletions .nf-core.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ lint:
files_unchanged:
- assets/email_template.html
- assets/email_template.txt
files_exist:
- lib/nfcore_external_java_deps.jar
- lib/NfcoreTemplate.groovy
- pyproject.toml
multiqc_config: false
- lib/Utils.groovy
- lib/WorkflowMain.groovy
1 change: 0 additions & 1 deletion assets/dummy_file.txt
View file
Open in desktop

This file was deleted.

2 changes: 1 addition & 1 deletion assets/email_template.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
<div style="font-family: Helvetica, Arial, sans-serif; padding: 30px; max-width: 800px; margin: 0 auto">
<img src="cid:nfcorepipelinelogo" />

<h1>nf-core/rnaseq v${version}</h1>
<h1>nf-core/rnaseq ${version}</h1>
<h2>Run Name: $runName</h2>

<% if (!success){ out << """
Expand Down
2 changes: 1 addition & 1 deletion assets/email_template.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
|\\ | |__ __ / ` / \\ |__) |__ } {
| \\| | \\__, \\__/ | \\ |___ \\`-._,-`-,
`._,._,'
nf-core/rnaseq v${version}
nf-core/rnaseq ${version}
----------------------------------------------------
Run Name: $runName

Expand Down
3 changes: 0 additions & 3 deletions conf/base.config
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,4 @@ process {
errorStrategy = 'retry'
maxRetries = 2
}
withName:CUSTOM_DUMPSOFTWAREVERSIONS {
cache = false
}
}
26 changes: 9 additions & 17 deletions conf/modules.config
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,14 +22,6 @@ process {
mode: params.publish_dir_mode,
saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
]

withName: 'CUSTOM_DUMPSOFTWAREVERSIONS' {
publishDir = [
path: { "${params.outdir}/pipeline_info" },
mode: params.publish_dir_mode,
pattern: '*_versions.yml'
]
}
}

//
Expand Down Expand Up @@ -148,7 +140,7 @@ process {

if (!params.skip_bbsplit && params.bbsplit_fasta_list) {
process {
withName: '.*:PREPARE_GENOME:BBMAP_BBSPLIT' {
withName: 'PREPARE_GENOME:BBMAP_BBSPLIT' {
ext.args = 'build=1'
publishDir = [
path: { params.save_reference ? "${params.outdir}/genome/index" : params.outdir },
Expand Down Expand Up @@ -348,7 +340,7 @@ if (params.remove_ribo_rna) {

if (!params.skip_alignment) {
process {
withName: 'NFCORE_RNASEQ:RNASEQ:.*:BAM_SORT_STATS_SAMTOOLS:BAM_STATS_SAMTOOLS:.*' {
withName: 'NFCORE_RNASEQ:.*:BAM_SORT_STATS_SAMTOOLS:BAM_STATS_SAMTOOLS:.*' {
ext.prefix = { "${meta.id}.sorted.bam" }
publishDir = [
path: { "${params.outdir}/${params.aligner}/samtools_stats" },
Expand All @@ -357,7 +349,7 @@ if (!params.skip_alignment) {
]
}

withName: 'NFCORE_RNASEQ:RNASEQ:.*:BAM_SORT_STATS_SAMTOOLS:SAMTOOLS_SORT' {
withName: 'NFCORE_RNASEQ:.*:BAM_SORT_STATS_SAMTOOLS:SAMTOOLS_SORT' {
ext.prefix = { "${meta.id}.sorted" }
publishDir = [
path: { ( ['star_salmon','hisat2'].contains(params.aligner) &&
Expand All @@ -371,7 +363,7 @@ if (!params.skip_alignment) {
]
}

withName: 'NFCORE_RNASEQ:RNASEQ:.*:BAM_SORT_STATS_SAMTOOLS:SAMTOOLS_INDEX' {
withName: 'NFCORE_RNASEQ:.*:BAM_SORT_STATS_SAMTOOLS:SAMTOOLS_INDEX' {
ext.args = { params.bam_csi_index ? '-c' : '' }
publishDir = [
path: { ( ['star_salmon','hisat2'].contains(params.aligner) &&
Expand Down Expand Up @@ -611,7 +603,7 @@ if (!params.skip_alignment && params.aligner == 'star_salmon') {

if (params.with_umi) {
process {
withName: 'NFCORE_RNASEQ:RNASEQ:SAMTOOLS_SORT' {
withName: 'NFCORE_RNASEQ:SAMTOOLS_SORT' {
ext.args = '-n'
ext.prefix = { "${meta.id}.umi_dedup.transcriptome" }
publishDir = [
Expand All @@ -622,7 +614,7 @@ if (!params.skip_alignment && params.aligner == 'star_salmon') {
]
}

withName: 'NFCORE_RNASEQ:RNASEQ:UMITOOLS_PREPAREFORSALMON' {
withName: 'NFCORE_RNASEQ:UMITOOLS_PREPAREFORSALMON' {
ext.prefix = { "${meta.id}.umi_dedup.transcriptome.filtered" }
publishDir = [
[
Expand All @@ -639,7 +631,7 @@ if (!params.skip_alignment && params.aligner == 'star_salmon') {
]
}

withName: 'NFCORE_RNASEQ:RNASEQ:BAM_SORT_STATS_SAMTOOLS:SAMTOOLS_SORT' {
withName: 'NFCORE_RNASEQ:BAM_SORT_STATS_SAMTOOLS:SAMTOOLS_SORT' {
ext.prefix = { "${meta.id}.transcriptome.sorted" }
publishDir = [
path: { params.save_align_intermeds || params.save_umi_intermeds ? "${params.outdir}/${params.aligner}" : params.outdir },
Expand All @@ -649,7 +641,7 @@ if (!params.skip_alignment && params.aligner == 'star_salmon') {
]
}

withName: 'NFCORE_RNASEQ:RNASEQ:BAM_SORT_STATS_SAMTOOLS:SAMTOOLS_INDEX' {
withName: 'NFCORE_RNASEQ:BAM_SORT_STATS_SAMTOOLS:SAMTOOLS_INDEX' {
publishDir = [
path: { params.save_align_intermeds || params.save_umi_intermeds ? "${params.outdir}/${params.aligner}" : params.outdir },
mode: params.publish_dir_mode,
Expand All @@ -658,7 +650,7 @@ if (!params.skip_alignment && params.aligner == 'star_salmon') {
]
}

withName: 'NFCORE_RNASEQ:RNASEQ:BAM_SORT_STATS_SAMTOOLS:BAM_STATS_SAMTOOLS:.*' {
withName: 'NFCORE_RNASEQ:BAM_SORT_STATS_SAMTOOLS:BAM_STATS_SAMTOOLS:.*' {
ext.prefix = { "${meta.id}.transcriptome.sorted.bam" }
publishDir = [
path: { params.save_align_intermeds || params.save_umi_intermeds ? "${params.outdir}/${params.aligner}/samtools_stats" : params.outdir },
Expand Down
Loading
Loading