Move QC outputs to the bids folder

config/config.yml

@@ -140,6 +140,13 @@ report:
     slice_start: 0
     slice_step: 50
     colour_map: viridis
+  resources:
+    readme_md: resources/qc/README.md
+    vol_viewer_dir: resources/qc/volViewer
+    ff_html: resources/qc/ff_html_temp.html
+    report_html: resources/qc/qc_report_temp.html
+    subject_html: resources/qc/subject_html_temp.html
+    ws_html: resources/qc/ws_html_temp.html
 
 
 containers:

qc/README.md → resources/qc/README.md

@@ -2,18 +2,13 @@
 
 ### 1: Run the workflow with personalized report generation configurations
 
-### 2: Navigate to qc directory:
-
-```bash
-cd ./qc
-```
-### 3: Create a Python web server:
+### 2: Navigate to qc directory, then create a Python web server:
 
 ```bash
 python -m http.server
 ```
-### 4: Open the link generated in a browser and open qc_report.html
+### 3: Open the link generated in a browser and open `qc_report.html`
 
 
 ### Note:
- Can view the whole slice images and the flatfield corrections without running the web server. However, to view the volume rendered brain, the web server is required.
+ Can view the whole slice images and the flatfield corrections without running the web server. However, to view the volume rendered brain, the web server is required.

workflow/Snakefile

@@ -33,6 +33,7 @@ rule all:
     input:
         get_all_targets(),
         get_bids_toplevel_targets(),
+        get_qc_targets(),
     localrule: True
 
 

workflow/rules/common.smk

@@ -29,6 +29,20 @@ def expand_bids(expand_kwargs, **bids_kwargs):
         return files
 
 
+def remote_file(filename):
+    if is_remote(filename):
+        return storage(str(filename))
+    else:
+        return filename
+
+
+def remote_directory(dirname):
+    if is_remote(dirname):
+        return storage(directory(str(dirname)))
+    else:
+        return directory(dirname)
+
+
 def directory_bids(root, *args, **kwargs):
     """Similar to expand_bids, this replacement function
     is needed to ensure storage() comes after directory() tags"""
@@ -107,18 +121,25 @@ def get_all_targets():
                 ),
             )
         )
-        if config["report"]["create_report"]:
-            targets.extend(
-                expand(
-                    "qc/sub-{subject}_sample-{sample}_acq-{acq}/subject.html",
-                    subject=datasets.loc[i, "subject"],
-                    sample=datasets.loc[i, "sample"],
-                    acq=datasets.loc[i, "acq"],
-                )
-            )
     return targets
 
 
+def get_all_subj_html(wildcards):
+    htmls = []
+
+    for i in range(len(datasets)):
+
+        html = "{root}/qc/sub-{subject}_sample-{sample}_acq-{acq}/subject.html".format(
+            root=root,
+            subject=datasets.loc[i, "subject"],
+            sample=datasets.loc[i, "sample"],
+            acq=datasets.loc[i, "acq"],
+        )
+        htmls.append(remote_file(html))
+
+    return htmls
+
+
 def get_bids_toplevel_targets():
     targets = []
     targets.append(bids_toplevel(root, "README.md"))
@@ -129,6 +150,14 @@ def get_bids_toplevel_targets():
     return targets
 
 
+def get_qc_targets():
+    targets = []
+    if config["report"]["create_report"]:
+        targets.append(remote_file(Path(root) / "qc" / "qc_report.html"))
+        targets.append(remote_file(Path(root) / "qc" / "README.md"))
+    return targets
+
+
 def dataset_is_remote(wildcards):
     return is_remote_gcs(Path(get_dataset_path(wildcards)))
 

workflow/rules/qc.smk

@@ -1,3 +1,16 @@
+
+rule setup_qc_dir:
+    "Copies QC resources to the output bids folder"
+    input:
+        readme_md=config["report"]["resources"]["readme_md"],
+    output:
+        readme_md=remote_file(Path(root) / "qc" / "README.md"),
+    log:
+        "logs/setup_qc_dir_log.txt",
+    shell:
+        "cp {input.readme_md} {output.readme_md}"
+
+
 rule generate_flatfield_qc:
     "Generates an html file for comparing before and after flatfield correction"
     input:
@@ -19,17 +32,30 @@ rule generate_flatfield_qc:
             desc="flatcorr",
             suffix="SPIM.zarr",
         ),
+        ff_html=config["report"]["resources"]["ff_html"],
     params:
         ff_s_start=config["report"]["flatfield_corrected"]["slice_start"],
         ff_s_step=config["report"]["flatfield_corrected"]["slice_step"],
         ff_cmap=config["report"]["flatfield_corrected"]["colour_map"],
     output:
-        html="qc/sub-{subject}_sample-{sample}_acq-{acq}/flatfieldqc.html",
-        corr_images_dir=directory(
-            "qc/sub-{subject}_sample-{sample}_acq-{acq}/images/corr"
+        html=remote_file(
+            Path(root)
+            / "qc"
+            / "sub-{subject}_sample-{sample}_acq-{acq}/flatfieldqc.html"
         ),
-        uncorr_images_dir=directory(
-            "qc/sub-{subject}_sample-{sample}_acq-{acq}/images/uncorr"
+        corr_images_dir=remote_directory(
+            Path(root)
+            / "qc"
+            / "sub-{subject}_sample-{sample}_acq-{acq}"
+            / "images"
+            / "corr"
+        ),
+        uncorr_images_dir=remote_directory(
+            Path(root)
+            / "qc"
+            / "sub-{subject}_sample-{sample}_acq-{acq}"
+            / "images"
+            / "uncorr"
         ),
     log:
         bids(
@@ -49,15 +75,27 @@ rule generate_whole_slice_qc:
     input:
         **get_storage_creds(),
         ome=get_input_ome_zarr_to_nii(),
+        ws_html=config["report"]["resources"]["ws_html"],
     params:
         ws_s_start=config["report"]["whole_slice_viewer"]["slice_start"],
         ws_s_step=config["report"]["whole_slice_viewer"]["slice_step"],
         ws_cmap=config["report"]["whole_slice_viewer"]["colour_map"],
         uri=get_output_ome_zarr_uri(),
         storage_provider_settings=workflow.storage_provider_settings,
     output:
-        html="qc/sub-{subject}_sample-{sample}_acq-{acq}/whole_slice_qc.html",
-        images_dir=directory("qc/sub-{subject}_sample-{sample}_acq-{acq}/images/whole"),
+        html=remote_file(
+            Path(root)
+            / "qc"
+            / "sub-{subject}_sample-{sample}_acq-{acq}"
+            / "whole_slice_qc.html"
+        ),
+        images_dir=remote_directory(
+            Path(root)
+            / "qc"
+            / "sub-{subject}_sample-{sample}_acq-{acq}"
+            / "images"
+            / "whole"
+        ),
     log:
         bids(
             root="logs",
@@ -76,14 +114,23 @@ rule generate_volume_qc:
     input:
         **get_storage_creds(),
         ome=get_input_ome_zarr_to_nii(),
+        vol_viewer_dir=config["report"]["resources"]["vol_viewer_dir"],
     params:
         uri=get_output_ome_zarr_uri(),
         storage_provider_settings=workflow.storage_provider_settings,
     output:
-        resources=directory(
-            "qc/sub-{subject}_sample-{sample}_acq-{acq}/volume_resources"
+        resources=remote_directory(
+            Path(root)
+            / "qc"
+            / "sub-{subject}_sample-{sample}_acq-{acq}"
+            / "volume_resources"
+        ),
+        html=remote_file(
+            Path(root)
+            / "qc"
+            / "sub-{subject}_sample-{sample}_acq-{acq}"
+            / "volume_qc.html"
         ),
-        html="qc/sub-{subject}_sample-{sample}_acq-{acq}/volume_qc.html",
     log:
         bids(
             root="logs",
@@ -100,11 +147,17 @@ rule generate_volume_qc:
 rule generate_subject_qc:
     "Generates html files to access all the subjects qc reports in one place"
     input:
-        ws_html="qc/sub-{subject}_sample-{sample}_acq-{acq}/whole_slice_qc.html",
-        ff_html="qc/sub-{subject}_sample-{sample}_acq-{acq}/flatfieldqc.html",
-        vol_html="qc/sub-{subject}_sample-{sample}_acq-{acq}/volume_qc.html",
+        subject_html=config["report"]["resources"]["subject_html"],
+        ws_html=rules.generate_whole_slice_qc.output.html,
+        ff_html=rules.generate_flatfield_qc.output.html,
+        vol_html=rules.generate_volume_qc.output.html,
     output:
-        sub_html="qc/sub-{subject}_sample-{sample}_acq-{acq}/subject.html",
+        sub_html=remote_file(
+            Path(root)
+            / "qc"
+            / "sub-{subject}_sample-{sample}_acq-{acq}"
+            / "subject.html"
+        ),
     log:
         bids(
             root="logs",
@@ -116,3 +169,21 @@ rule generate_subject_qc:
         ),
     script:
         "../scripts/generate_subject_qc.py"
+
+
+rule generate_aggregate_qc:
+    input:
+        report_html=config["report"]["resources"]["report_html"],
+        subj_htmls=get_all_subj_html,
+    params:
+        datasets=datasets,
+    output:
+        total_html=remote_file(Path(root) / "qc" / "qc_report.html"),
+    log:
+        bids(
+            root="logs",
+            datatype="generate_aggregate_qc",
+            suffix="log.txt",
+        ),
+    script:
+        "../scripts/generate_aggregate_qc.py"

workflow/scripts/generate_flatfield_qc.py

@@ -9,7 +9,7 @@
 # load the html template from jinja
 file_loader = FileSystemLoader(".")
 env = Environment(loader=file_loader)
-template = env.get_template("qc/resources/ff_html_temp.html")
+template = env.get_template(snakemake.input.ff_html)
 
 # User set configurations
 ff_s_start=snakemake.params.ff_s_start

workflow/scripts/generate_subject_qc.py

@@ -1,10 +1,11 @@
 from jinja2 import Environment, FileSystemLoader
 import os.path as path
+from pathlib import Path
 
 # load jinja template
 file_loader = FileSystemLoader(".")
 env = Environment(loader=file_loader)
-template = env.get_template("qc/resources/subject_html_temp.html")
+template = env.get_template(snakemake.input.subject_html)
 
 # input html files
 ws_html = snakemake.input.ws_html
@@ -13,36 +14,21 @@
 
 # output html files
 sub_html = snakemake.output.sub_html
-total_html = "qc/qc_report.html"
 
 # Wildcards
 subject = snakemake.wildcards.subject
 sample = snakemake.wildcards.sample
 acq = snakemake.wildcards.acq
 
 # Get relative path to the subjects QC htmls
-ws_rel_path = path.relpath(path.dirname(sub_html), path.dirname(ws_html))+"/"+path.basename(ws_html)
-ff_rel_path = path.relpath(path.dirname(sub_html), path.dirname(ff_html))+"/"+path.basename(ff_html)
-vol_rel_path = path.relpath(path.dirname(sub_html), path.dirname(vol_html))+ "/" +path.basename(vol_html)
+
+ws_rel_path = Path(ws_html).relative_to(Path(sub_html).parent)
+ff_rel_path = Path(ff_html).relative_to(Path(sub_html).parent)
+vol_rel_path = Path(vol_html).relative_to(Path(sub_html).parent)
 
 # Fill in jinja template for subject html and write it out
 output = template.render(back_link="../qc_report.html",subject=subject,sample=sample,acq=acq,
                          ffhtml=ff_rel_path,wshtml=ws_rel_path, volhtml=vol_rel_path)
 with open(sub_html, 'w') as f:
     f.write(output)
 
-# Create line to add link to subject into final qc report combining all subjects
-sub_link = f'\n\t\t<a href="./{sub_html.split("/")[1]}/subject.html">{subject}-{sample}-{acq}</a><br>'
-
-# if not first sample just add the one link
-if(path.exists(total_html)):
-    with open(total_html,'a') as f:
-        f.write(sub_link)
-
-# if it is the first sample write out the template
-else:
-    template = env.get_template("qc/resources/qc_report_temp.html")
-    output = template.render()
-    output+=sub_link
-    with open(total_html, 'w') as f:
-        f.write(output)

workflow/scripts/generate_volume_qc.py

@@ -10,16 +10,16 @@
 import zarr
 
 # directory containing the volume rendering files
-resource_dir = Path(snakemake.output.resources)
+resource_dir = snakemake.output.resources
 # where html file should be written
 html_dest = snakemake.output.html
 
 # inputted ome-zarr path
 ome_data = snakemake.input.ome
 
 # move volume renderer into the subjects directory
-copy_tree("qc/resources/volViewer", str(resource_dir))
-shutil.move(resource_dir / "volRender.html", html_dest)
+copy_tree(snakemake.input.vol_viewer_dir, resource_dir)
+shutil.move(Path(resource_dir) / "volRender.html", html_dest)
 
 uri = snakemake.params.uri
 if is_remote(uri):
@@ -43,7 +43,7 @@
 ds_z = ds_z.downsample(along_z=downsample_factor)
 
 # Write it to a JSON for js script to read
-with open(resource_dir / "volumeData.json", 'w') as f:
+with open(Path(resource_dir) / "volumeData.json", 'w') as f:
     json_data = json.dumps(ds_z.darr.compute().tolist())
     f.write(json_data)
 

workflow/scripts/generate_whole_slice_qc.py

@@ -12,7 +12,7 @@
 # load jinja html template
 file_loader = FileSystemLoader(".")
 env = Environment(loader=file_loader)
-template = env.get_template("qc/resources/ws_html_temp.html")
+template = env.get_template(snakemake.input.ws_html)
 
 # user set configurations
 ws_s_step=snakemake.params.ws_s_step