Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions data/templates/0.mapping_minimap2_db.sbatch
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,11 @@
#SBATCH -n {{nprocs}}
#SBATCH --time {{wall_time_limit}}
#SBATCH --mem {{mem_in_gb}}G
#SBATCH -o {{output}}/step-1/logs/%x-%A_%a.out
#SBATCH -e {{output}}/step-1/logs/%x-%A_%a.out
#SBATCH -o {{output}}/step-0/logs/%x-%A_%a.out
#SBATCH -e {{output}}/step-0/logs/%x-%A_%a.out
#SBATCH --array {{array_params}}

source ~/.bashrc
conda activate {{conda_environment}}


Expand All @@ -21,4 +22,4 @@ filename=`echo $input | awk '{print $2}'`

fn=`basename ${filename}`

minimap2 -x map-hifi -t {{nprocs}} -a --secondary=no --MD --eqx -o {{output}}/step-0/${sample_name}.sam /projects/wol/qiyun/wol2/databases/minimap2/WoLr2.mmi ${filename}
minimap2 -x map-hifi -t {{nprocs}} -a --secondary=no --MD --eqx /projects/wol/qiyun/wol2/databases/minimap2/WoLr2.mmi ${filename} | samtools sort -@ {{nprocs}} -o {{output}}/step-1/${sample_name}_WoLr2.sorted.sam
2 changes: 1 addition & 1 deletion data/templates/1.hifiasm-meta_new.sbatch
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
#SBATCH -o {{output}}/step-1/logs/%x-%A_%a.out
#SBATCH -e {{output}}/step-1/logs/%x-%A_%a.out
#SBATCH --array {{array_params}}

source ~/.bashrc
conda activate {{conda_environment}}

cd {{output}}/step-1
Expand Down
1 change: 1 addition & 0 deletions data/templates/2.get-circular-genomes.sbatch
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
#SBATCH -e {{output}}/step-2/logs/%x-%A_%a.out
#SBATCH --array {{array_params}}

source ~/.bashrc

conda activate {{conda_environment}}

Expand Down
2 changes: 1 addition & 1 deletion data/templates/3.minimap2_assembly.sbatch
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
#SBATCH -e {{output}}/step-3/logs/%x-%A_%a.err
#SBATCH --array {{array_params}}


source ~/.bashrc
conda activate {{conda_environment}}
cd {{output}}

Expand Down
3 changes: 2 additions & 1 deletion data/templates/4.metawrap_binning_new.sbatch
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
#SBATCH -e {{output}}/step-4/logs/%x-%A_%a.out
#SBATCH --array {{array_params}}

source ~/.bashrc
conda activate {{conda_environment}}

cd {{output}}
Expand All @@ -26,5 +27,5 @@ cp {{output}}/step-3/${sample_name}_binning/${sample_name}.sorted.bam {{output}}
rm {{output}}/step-3/${sample_name}_binning/${sample_name}.sorted.bam

ln -s ${filename} {{output}}/step-4/${sample_name}_binning/work_files/${sample_name}.fastq
metawrap binning -a {{output}}/step-2/${sample_name}.noLCG.fa -o {{output}}/step-4/${sample_name}_binning \
metawrap binning -a {{output}}/step-2/${sample_name}_noLCG.fa -o {{output}}/step-4/${sample_name}_binning \
-t {{nprocs}} -m 100 -l 16000 --single-end --metabat2 --maxbin2 --concoct --universal {{output}}/step-4/${sample_name}_binning/work_files/${sample_name}.fastq
4 changes: 3 additions & 1 deletion data/templates/5.DAS_Tools_prepare_batch3_test.sbatch
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@
#SBATCH -e {{output}}/step-5/logs/%x-%A_%a.out
#SBATCH --array {{array_params}}

source ~/.bashrc

conda activate {{conda_environment}}
cd {{output}}

Expand All @@ -33,4 +35,4 @@ Fasta_to_Contig2Bin.sh -i ./concoct_bins -e fa > ${sample_name}.concoct.tsv
Fasta_to_Contig2Bin.sh -i ./maxbin2_bins -e fa > ${sample_name}.maxbin2.tsv
Fasta_to_Contig2Bin.sh -i ./metabat2_bins -e fa > ${sample_name}.metabat2.tsv

DAS_Tool --bins=${sample_name}.concoct.tsv,${sample_name}.maxbin2.tsv,${sample_name}.metabat2.tsv --contigs={{output}}/step-2/${sample_name}.noLCG.fa --outputbasename={{output}}/${folder}/${sample_name}/${sample_name} --labels=CONCOCT,MaxBin,MetaBAT --threads={{nprocs}} --search_engine=diamond --dbDirectory=${DAS_db} --write_bins
DAS_Tool --bins=${sample_name}.concoct.tsv,${sample_name}.maxbin2.tsv,${sample_name}.metabat2.tsv --contigs={{output}}/step-2/${sample_name}_noLCG.fa --outputbasename={{output}}/${folder}/${sample_name}/${sample_name} --labels=CONCOCT,MaxBin,MetaBAT --threads={{nprocs}} --search_engine=diamond --dbDirectory=${DAS_db} --write_bins
1 change: 1 addition & 0 deletions data/templates/6.MAG_rename.sbatch
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
#SBATCH -e {{output}}/step-6/logs/%x-%A_%a.out
#SBATCH --array {{array_params}}

source ~/.bashrc
conda activate {{conda_environment}}
cd {{output}}

Expand Down
3 changes: 3 additions & 0 deletions data/templates/7.checkm_batch.sbatch
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,9 @@
#SBATCH -e {{output}}/step-7/logs/%x-%A_%a.out
#SBATCH --array {{array_params}}


source ~/.bashrc

conda activate {{conda_environment}}

cd {{output}}
Expand Down
61 changes: 41 additions & 20 deletions qp_pacbio/tests/test_pacbio.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,37 +6,54 @@
# The full license is in the file LICENSE, distributed with this software.
# -----------------------------------------------------------------------------

from unittest import main
from qiita_client import ArtifactInfo
from qiita_client.testing import PluginTestCase
from json import dumps
from os import remove
from os.path import exists, isdir, join
from shutil import copyfile, rmtree
from tempfile import mkdtemp
from shutil import rmtree, copyfile
from json import dumps
from unittest import main

from qiita_client import ArtifactInfo
from qiita_client.testing import PluginTestCase

from qp_pacbio import plugin
from qp_pacbio.qp_pacbio import pacbio_processing

# Keep these in sync with your generator defaults
CONDA_ENV = "qp_pacbio_2025.9"
STEP1_NPROCS = 16
STEP1_WALL = 1000
STEP1_MEM_GB = 300 # generator uses 300G for step-1
NODE_COUNT = 1
PARTITION = "qiita"

# Exact expected Step-1 script matching your template.
# Escape ${...} -> ${{...}} and awk braces -> '{{print $1}}' etc.
STEP_1_EXP = (
"#!/bin/bash\n"
"#SBATCH -J s1-my-job-id\n"
"#SBATCH -N 1\n"
"#SBATCH -n 32\n"
"#SBATCH --time 1000\n"
"#SBATCH --mem 300G\n"
"#SBATCH -o {out_dir}/step-1/logs/%x-%A.out\n"
"#SBATCH -e {out_dir}/step-1/logs/%x-%A.err\n"
"#SBATCH --array 1:2%16\n"
"\n"
"conda activate qp_pacbio_2025.9\n"
"#SBATCH -J s1-{job_id}\n"
f"#SBATCH -p {PARTITION}\n"
f"#SBATCH -N {NODE_COUNT}\n"
f"#SBATCH -n {STEP1_NPROCS}\n"
f"#SBATCH --time {STEP1_WALL}\n"
f"#SBATCH --mem {STEP1_MEM_GB}G\n"
"#SBATCH -o {out_dir}/step-1/logs/%x-%A_%a.out\n"
"#SBATCH -e {out_dir}/step-1/logs/%x-%A_%a.out\n"
"#SBATCH --array 1-{njobs}%16\n"
"source ~/.bashrc\n"
f"conda activate {CONDA_ENV}\n"
"\n"
"cd {out_dir}/step-1\n"
"step=${{SLURM_ARRAY_TASK_ID}}\n"
"input=$(head -n $step {out_dir}/file_list.txt | tail -n 1)\n"
"fn=`basename ${{input}}`\n"
"hifiasm_meta -t 60 -o {out_dir}/step-1/${{fn}} ${{input}}"
"input=$(head -n $step {out_dir}/sample_list.txt | tail -n 1)\n"
"\n"
"sample_name=`echo $input | awk '{{print $1}}'`\n"
"filename=`echo $input | awk '{{print $2}}'`\n"
"\n"
"fn=`basename ${{filename}}`\n"
"\n"
f"hifiasm_meta -t {STEP1_NPROCS} -o "
"{out_dir}/step-1/${{sample_name}} ${{filename}}"
)


Expand Down Expand Up @@ -110,14 +127,18 @@ def test_pacbio_processing(self):
with open(f"{out_dir}/sample_list.txt", "r") as f:
obs_lines = f.readlines()
self.assertEqual(2, len(obs_lines))

njobs = len(obs_lines)
# testing step-1
with open(f"{out_dir}/step-1/step-1.slurm", "r") as f:
# removing \n
obs_lines = [ln.replace("\n", "") for ln in f.readlines()]

self.assertCountEqual(
STEP_1_EXP.format(out_dir=out_dir).split("\n"),
STEP_1_EXP.format(
out_dir=out_dir,
job_id=job_id,
njobs=njobs,
).split("\n"),
obs_lines,
)

Expand Down
Loading