Skip to content

Commit 986c534

Browse files
jianshu93jianshu93
authored
Test (#6)
* format * step 4 error * step 5 error * test with flexibility * test with flexibility * test with flexibility * test with flexibility * test with flexibility * test with flexibility * test with flexibility * test with flexibility * test with flexibility * test with flexibility * test with flexibility * test with flexibility * test with flexibility * test with flexibility * update test * update test * update test * update test * update test * update test * update test
1 parent 565ec78 commit 986c534

9 files changed

+57
-27
lines changed

‎data/templates/0.mapping_minimap2_db.sbatch‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,11 @@
55
#SBATCH -n {{nprocs}}
66
#SBATCH --time {{wall_time_limit}}
77
#SBATCH --mem {{mem_in_gb}}G
8-
#SBATCH -o {{output}}/step-1/logs/%x-%A_%a.out
9-
#SBATCH -e {{output}}/step-1/logs/%x-%A_%a.out
8+
#SBATCH -o {{output}}/step-0/logs/%x-%A_%a.out
9+
#SBATCH -e {{output}}/step-0/logs/%x-%A_%a.out
1010
#SBATCH --array {{array_params}}
1111

12+
source ~/.bashrc
1213
conda activate {{conda_environment}}
1314

1415

@@ -21,4 +22,4 @@ filename=`echo $input | awk '{print $2}'`
2122

2223
fn=`basename ${filename}`
2324

24-
minimap2 -x map-hifi -t {{nprocs}} -a --secondary=no --MD --eqx -o {{output}}/step-0/${sample_name}.sam /projects/wol/qiyun/wol2/databases/minimap2/WoLr2.mmi ${filename}
25+
minimap2 -x map-hifi -t {{nprocs}} -a --secondary=no --MD --eqx /projects/wol/qiyun/wol2/databases/minimap2/WoLr2.mmi ${filename} | samtools sort -@ {{nprocs}} -o {{output}}/step-1/${sample_name}_WoLr2.sorted.sam

‎data/templates/1.hifiasm-meta_new.sbatch‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
#SBATCH -o {{output}}/step-1/logs/%x-%A_%a.out
99
#SBATCH -e {{output}}/step-1/logs/%x-%A_%a.out
1010
#SBATCH --array {{array_params}}
11-
11+
source ~/.bashrc
1212
conda activate {{conda_environment}}
1313

1414
cd {{output}}/step-1

‎data/templates/2.get-circular-genomes.sbatch‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
#SBATCH -e {{output}}/step-2/logs/%x-%A_%a.out
1010
#SBATCH --array {{array_params}}
1111

12+
source ~/.bashrc
1213

1314
conda activate {{conda_environment}}
1415

‎data/templates/3.minimap2_assembly.sbatch‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
#SBATCH -e {{output}}/step-3/logs/%x-%A_%a.err
1010
#SBATCH --array {{array_params}}
1111

12-
12+
source ~/.bashrc
1313
conda activate {{conda_environment}}
1414
cd {{output}}
1515

‎data/templates/4.metawrap_binning_new.sbatch‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
#SBATCH -e {{output}}/step-4/logs/%x-%A_%a.out
1010
#SBATCH --array {{array_params}}
1111

12+
source ~/.bashrc
1213
conda activate {{conda_environment}}
1314

1415
cd {{output}}
@@ -26,5 +27,5 @@ cp {{output}}/step-3/${sample_name}_binning/${sample_name}.sorted.bam {{output}}
2627
rm {{output}}/step-3/${sample_name}_binning/${sample_name}.sorted.bam
2728

2829
ln -s ${filename} {{output}}/step-4/${sample_name}_binning/work_files/${sample_name}.fastq
29-
metawrap binning -a {{output}}/step-2/${sample_name}.noLCG.fa -o {{output}}/step-4/${sample_name}_binning \
30+
metawrap binning -a {{output}}/step-2/${sample_name}_noLCG.fa -o {{output}}/step-4/${sample_name}_binning \
3031
-t {{nprocs}} -m 100 -l 16000 --single-end --metabat2 --maxbin2 --concoct --universal {{output}}/step-4/${sample_name}_binning/work_files/${sample_name}.fastq

‎data/templates/5.DAS_Tools_prepare_batch3_test.sbatch‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@
99
#SBATCH -e {{output}}/step-5/logs/%x-%A_%a.out
1010
#SBATCH --array {{array_params}}
1111

12+
source ~/.bashrc
13+
1214
conda activate {{conda_environment}}
1315
cd {{output}}
1416

@@ -33,4 +35,4 @@ Fasta_to_Contig2Bin.sh -i ./concoct_bins -e fa > ${sample_name}.concoct.tsv
3335
Fasta_to_Contig2Bin.sh -i ./maxbin2_bins -e fa > ${sample_name}.maxbin2.tsv
3436
Fasta_to_Contig2Bin.sh -i ./metabat2_bins -e fa > ${sample_name}.metabat2.tsv
3537

36-
DAS_Tool --bins=${sample_name}.concoct.tsv,${sample_name}.maxbin2.tsv,${sample_name}.metabat2.tsv --contigs={{output}}/step-2/${sample_name}.noLCG.fa --outputbasename={{output}}/${folder}/${sample_name}/${sample_name} --labels=CONCOCT,MaxBin,MetaBAT --threads={{nprocs}} --search_engine=diamond --dbDirectory=${DAS_db} --write_bins
38+
DAS_Tool --bins=${sample_name}.concoct.tsv,${sample_name}.maxbin2.tsv,${sample_name}.metabat2.tsv --contigs={{output}}/step-2/${sample_name}_noLCG.fa --outputbasename={{output}}/${folder}/${sample_name}/${sample_name} --labels=CONCOCT,MaxBin,MetaBAT --threads={{nprocs}} --search_engine=diamond --dbDirectory=${DAS_db} --write_bins

‎data/templates/6.MAG_rename.sbatch‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
#SBATCH -e {{output}}/step-6/logs/%x-%A_%a.out
1010
#SBATCH --array {{array_params}}
1111

12+
source ~/.bashrc
1213
conda activate {{conda_environment}}
1314
cd {{output}}
1415

‎data/templates/7.checkm_batch.sbatch‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,9 @@
99
#SBATCH -e {{output}}/step-7/logs/%x-%A_%a.out
1010
#SBATCH --array {{array_params}}
1111

12+
13+
source ~/.bashrc
14+
1215
conda activate {{conda_environment}}
1316

1417
cd {{output}}

‎qp_pacbio/tests/test_pacbio.py‎

Lines changed: 41 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -6,37 +6,54 @@
66
# The full license is in the file LICENSE, distributed with this software.
77
# -----------------------------------------------------------------------------
88

9-
from unittest import main
10-
from qiita_client import ArtifactInfo
11-
from qiita_client.testing import PluginTestCase
9+
from json import dumps
1210
from os import remove
1311
from os.path import exists, isdir, join
12+
from shutil import copyfile, rmtree
1413
from tempfile import mkdtemp
15-
from shutil import rmtree, copyfile
16-
from json import dumps
14+
from unittest import main
15+
16+
from qiita_client import ArtifactInfo
17+
from qiita_client.testing import PluginTestCase
1718

1819
from qp_pacbio import plugin
1920
from qp_pacbio.qp_pacbio import pacbio_processing
2021

22+
# Keep these in sync with your generator defaults
23+
CONDA_ENV = "qp_pacbio_2025.9"
24+
STEP1_NPROCS = 16
25+
STEP1_WALL = 1000
26+
STEP1_MEM_GB = 300 # generator uses 300G for step-1
27+
NODE_COUNT = 1
28+
PARTITION = "qiita"
2129

30+
# Exact expected Step-1 script matching your template.
31+
# Escape ${...} -> ${{...}} and awk braces -> '{{print $1}}' etc.
2232
STEP_1_EXP = (
2333
"#!/bin/bash\n"
24-
"#SBATCH -J s1-my-job-id\n"
25-
"#SBATCH -N 1\n"
26-
"#SBATCH -n 32\n"
27-
"#SBATCH --time 1000\n"
28-
"#SBATCH --mem 300G\n"
29-
"#SBATCH -o {out_dir}/step-1/logs/%x-%A.out\n"
30-
"#SBATCH -e {out_dir}/step-1/logs/%x-%A.err\n"
31-
"#SBATCH --array 1:2%16\n"
32-
"\n"
33-
"conda activate qp_pacbio_2025.9\n"
34+
"#SBATCH -J s1-{job_id}\n"
35+
f"#SBATCH -p {PARTITION}\n"
36+
f"#SBATCH -N {NODE_COUNT}\n"
37+
f"#SBATCH -n {STEP1_NPROCS}\n"
38+
f"#SBATCH --time {STEP1_WALL}\n"
39+
f"#SBATCH --mem {STEP1_MEM_GB}G\n"
40+
"#SBATCH -o {out_dir}/step-1/logs/%x-%A_%a.out\n"
41+
"#SBATCH -e {out_dir}/step-1/logs/%x-%A_%a.out\n"
42+
"#SBATCH --array 1-{njobs}%16\n"
43+
"source ~/.bashrc\n"
44+
f"conda activate {CONDA_ENV}\n"
3445
"\n"
3546
"cd {out_dir}/step-1\n"
3647
"step=${{SLURM_ARRAY_TASK_ID}}\n"
37-
"input=$(head -n $step {out_dir}/file_list.txt | tail -n 1)\n"
38-
"fn=`basename ${{input}}`\n"
39-
"hifiasm_meta -t 60 -o {out_dir}/step-1/${{fn}} ${{input}}"
48+
"input=$(head -n $step {out_dir}/sample_list.txt | tail -n 1)\n"
49+
"\n"
50+
"sample_name=`echo $input | awk '{{print $1}}'`\n"
51+
"filename=`echo $input | awk '{{print $2}}'`\n"
52+
"\n"
53+
"fn=`basename ${{filename}}`\n"
54+
"\n"
55+
f"hifiasm_meta -t {STEP1_NPROCS} -o "
56+
"{out_dir}/step-1/${{sample_name}} ${{filename}}"
4057
)
4158

4259

@@ -110,14 +127,18 @@ def test_pacbio_processing(self):
110127
with open(f"{out_dir}/sample_list.txt", "r") as f:
111128
obs_lines = f.readlines()
112129
self.assertEqual(2, len(obs_lines))
113-
130+
njobs = len(obs_lines)
114131
# testing step-1
115132
with open(f"{out_dir}/step-1/step-1.slurm", "r") as f:
116133
# removing \n
117134
obs_lines = [ln.replace("\n", "") for ln in f.readlines()]
118135

119136
self.assertCountEqual(
120-
STEP_1_EXP.format(out_dir=out_dir).split("\n"),
137+
STEP_1_EXP.format(
138+
out_dir=out_dir,
139+
job_id=job_id,
140+
njobs=njobs,
141+
).split("\n"),
121142
obs_lines,
122143
)
123144

0 commit comments

Comments
 (0)