Perform quality control with coSMicQC with UMAP analyses

0.5.quality_control/run.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+# Runs processing related to step 5.quality_control
+set -e
+
+# run umap analysis via python script based on notebook
+# use a subshell to keep existing code intact
+(
+cd ./0.quality_control || exit
+conda run -n jump_sc python ./0.jump_umap_analysis_with_cosmicqc.py
+)

0.download_data/2.download_images.py

@@ -157,10 +157,13 @@ def download_jump_cpg000_images_from_s3(
     return "data/images"
 
 
+# -
+
+
 # +
 # reference the file without reading it entirely
 target_file = pq.ParquetFile(
-    "../0.download_data/data/plates/BR00117006/BR00117006.parquet"
+    (parquet_path := "../0.download_data/data/plates/BR00117006/BR00117006.parquet")
 )
 
 # target image names
@@ -243,3 +246,18 @@ def download_jump_cpg000_images_from_s3(
         indent=4,
     ),
 )
+
+# read the full data
+df_full = pd.read_parquet(parquet_path, columns=target_flattened_columns)
+df_full
+
+# find the s3 paths for the full dataset
+df_full, s3_columns = add_jump_cpg0000_s3_paths(
+    df=df_full, image_column_groups=target_image_column_groups
+)
+df_full
+
+# download the images for the dataset
+image_path = download_jump_cpg000_images_from_s3(
+    df=df_full, s3_columns=s3_columns, data_path="data/images"
+)

environment.yml

@@ -18,6 +18,16 @@ dependencies:
   - conda-forge::requests
   - conda-forge::seaborn
   - conda-forge::scikit-posthocs
+  # used for cosmicqc quality control visualizations
+  - conda-forge::hvplot
+  - conda-forge::plotly
+  - conda-forge::umap-learn
+  - conda-forge::ipywidgets
+  - conda-forge::fsspec
+  - conda-forge::s3fs
+  - conda-forge::datashader
+  - conda-forge::selenium
+  - conda-forge::dask
   # used for managing jupyter environments
   - conda-forge::notebook
   - conda-forge::jupyterlab
@@ -34,7 +44,8 @@ dependencies:
   # used for formatting code within .ipynb files
   - conda-forge::jupyterlab_code_formatter
   - pip:
-      - git+https://github.com/cytomining/pycytominer.git@8e3c28d3b81efd2c241d4c792edfefaa46698115
+      - pycytominer
       - lmoments3
       - cytotable==0.0.10
       - cloudpathlib
+      - cosmicqc

justfile

@@ -63,3 +63,10 @@ default:
 
     # run step 0.download_data
     source 0.download_data/run.sh
+
+# run step 0.5.quality_control
+@run-step-0-5:
+    #!{{hashbang}}
+
+    # run step 0.5.quality_control
+    source 0.5.quality_control/run.sh