From 90b6dec360b4a8eb00667626ba994d4823d98398 Mon Sep 17 00:00:00 2001 From: jenna-tomkinson Date: Tue, 14 Mar 2023 15:45:26 -0600 Subject: [PATCH 1/5] add IC module and run to save images (ignored) --- .gitignore | 6 +- 1.cellprofiler_ic/NF1_illum.cppipe | 217 ++++++++++++++++++++++++++++ 1.cellprofiler_ic/README.md | 15 ++ 1.cellprofiler_ic/nf1_ic.ipynb | 167 +++++++++++++++++++++ 1.cellprofiler_ic/nf1_ic.sh | 6 + 1.cellprofiler_ic/scripts/nf1_ic.py | 57 ++++++++ nf1_cellpainting_env.yml | 5 +- utils/cp_utils.py | 71 +++++++++ 8 files changed, 541 insertions(+), 3 deletions(-) create mode 100644 1.cellprofiler_ic/NF1_illum.cppipe create mode 100644 1.cellprofiler_ic/README.md create mode 100644 1.cellprofiler_ic/nf1_ic.ipynb create mode 100644 1.cellprofiler_ic/nf1_ic.sh create mode 100644 1.cellprofiler_ic/scripts/nf1_ic.py create mode 100644 utils/cp_utils.py diff --git a/.gitignore b/.gitignore index 5ea04aa..b71be73 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,10 @@ -# ignore all plates as they are downloaded from figshare in instructions, including the metadata folder created +# ignore all plates as they are downloaded from figshare in instructions 0.download_data/Plate_1 0.download_data/Plate_2 # ignore pycache __pycache__ + +# ignore corrected plates images (like in download data +1.cellprofiler_ic/Corrected_Plate_1 +1.cellprofiler_ic/Corrected_Plate_2 diff --git a/1.cellprofiler_ic/NF1_illum.cppipe b/1.cellprofiler_ic/NF1_illum.cppipe new file mode 100644 index 0000000..2fef5f2 --- /dev/null +++ b/1.cellprofiler_ic/NF1_illum.cppipe @@ -0,0 +1,217 @@ +CellProfiler Pipeline: http://www.cellprofiler.org +Version:5 +DateRevision:424 +GitHash: +ModuleCount:11 +HasImagePlaneDetails:False + +Images:[module_num:1|svn_version:'Unknown'|variable_revision_number:2|show_window:False|notes:['To begin creating your project, use the Images module to compile a list of files and/or folders that you want to analyze. You can also specify a set of rules to include only the desired files in your selected folders.']|batch_state:array([], dtype=uint8)|enabled:True|wants_pause:False] + : + Filter images?:Images only + Select the rule criteria:and (extension does isimage) (directory doesnot containregexp "[\\\\/]\\.") + +Metadata:[module_num:2|svn_version:'Unknown'|variable_revision_number:6|show_window:False|notes:['The Metadata module optionally allows you to extract information describing your images (i.e, metadata) which will be stored along with your measurements. This information can be contained in the file name and/or location, or in an external file.']|batch_state:array([], dtype=uint8)|enabled:True|wants_pause:False] + Extract metadata?:No + Metadata data type:Text + Metadata types:{} + Extraction method count:1 + Metadata extraction method:Extract from file/folder names + Metadata source:File name + Regular expression to extract from file name:^(?P.*)_(?P[A-P][0-9]{2})_s(?P[0-9])_w(?P[0-9]) + Regular expression to extract from folder name:(?P[0-9]{4}_[0-9]{2}_[0-9]{2})$ + Extract metadata from:All images + Select the filtering criteria:and (file does contain "") + Metadata file location:Elsewhere...| + Match file and image metadata:[] + Use case insensitive matching?:No + Metadata file name:None + Does cached metadata exist?:No + +NamesAndTypes:[module_num:3|svn_version:'Unknown'|variable_revision_number:8|show_window:False|notes:['The NamesAndTypes module allows you to assign a meaningful name to each image by which other modules will refer to it.']|batch_state:array([], dtype=uint8)|enabled:True|wants_pause:False] + Assign a name to:Images matching rules + Select the image type:Grayscale image + Name to assign these images:DNA + Match metadata:[] + Image set matching method:Order + Set intensity range from:Image metadata + Assignments count:3 + Single images count:0 + Maximum intensity:255.0 + Process as 3D?:No + Relative pixel spacing in X:1.0 + Relative pixel spacing in Y:1.0 + Relative pixel spacing in Z:1.0 + Select the rule criteria:and (file does contain "DAPI") + Name to assign these images:OrigDAPI + Name to assign these objects:Cell + Select the image type:Grayscale image + Set intensity range from:Image metadata + Maximum intensity:255.0 + Select the rule criteria:and (file does contain "GFP") + Name to assign these images:OrigGFP + Name to assign these objects:Nucleus + Select the image type:Grayscale image + Set intensity range from:Image metadata + Maximum intensity:255.0 + Select the rule criteria:and (file does contain "RFP") + Name to assign these images:OrigRFP + Name to assign these objects:Cytoplasm + Select the image type:Grayscale image + Set intensity range from:Image metadata + Maximum intensity:255.0 + +Groups:[module_num:4|svn_version:'Unknown'|variable_revision_number:2|show_window:False|notes:['The Groups module optionally allows you to split your list of images into image subsets (groups) which will be processed independently of each other. Examples of groupings include screening batches, microtiter plates, time-lapse movies, etc.']|batch_state:array([], dtype=uint8)|enabled:True|wants_pause:False] + Do you want to group your images?:No + grouping metadata count:1 + Metadata category:None + +CorrectIlluminationCalculate:[module_num:5|svn_version:'Unknown'|variable_revision_number:2|show_window:False|notes:['Create an IC function for each image and we use the Background (due to majority of the images have less objects taking up the images) + Fit Polynomial smoothing method as this seems to work well at this time.']|batch_state:array([], dtype=uint8)|enabled:True|wants_pause:False] + Select the input image:OrigDAPI + Name the output image:IllumDAPI + Select how the illumination function is calculated:Background + Dilate objects in the final averaged image?:No + Dilation radius:1 + Block size:50 + Rescale the illumination function?:Yes + Calculate function for each image individually, or based on all images?:Each + Smoothing method:Fit Polynomial + Method to calculate smoothing filter size:Automatic + Approximate object diameter:80 + Smoothing filter size:200 + Retain the averaged image?:No + Name the averaged image:IllumBlueAvg + Retain the dilated image?:No + Name the dilated image:IllumBlueDilated + Automatically calculate spline parameters?:No + Background mode:dark + Number of spline points:5 + Background threshold:2.0 + Image resampling factor:2.0 + Maximum number of iterations:40 + Residual value for convergence:0.001 + +CorrectIlluminationCalculate:[module_num:6|svn_version:'Unknown'|variable_revision_number:2|show_window:False|notes:['Create an IC function for each image and we use the Background (due to majority of the images have less objects taking up the images) + Fit Polynomial smoothing method as this seems to work well at this time.']|batch_state:array([], dtype=uint8)|enabled:True|wants_pause:False] + Select the input image:OrigGFP + Name the output image:IllumGFP + Select how the illumination function is calculated:Background + Dilate objects in the final averaged image?:No + Dilation radius:1 + Block size:50 + Rescale the illumination function?:Yes + Calculate function for each image individually, or based on all images?:Each + Smoothing method:Fit Polynomial + Method to calculate smoothing filter size:Automatic + Approximate object diameter:200 + Smoothing filter size:200 + Retain the averaged image?:No + Name the averaged image:IllumBlueAvg + Retain the dilated image?:No + Name the dilated image:IllumBlueDilated + Automatically calculate spline parameters?:Yes + Background mode:auto + Number of spline points:5 + Background threshold:2.0 + Image resampling factor:2.0 + Maximum number of iterations:40 + Residual value for convergence:0.001 + +CorrectIlluminationCalculate:[module_num:7|svn_version:'Unknown'|variable_revision_number:2|show_window:False|notes:['Create an IC function for each image and we use the Background (due to majority of the images have less objects taking up the images) + Fit Polynomial smoothing method as this seems to work well at this time.']|batch_state:array([], dtype=uint8)|enabled:True|wants_pause:False] + Select the input image:OrigRFP + Name the output image:IllumRFP + Select how the illumination function is calculated:Background + Dilate objects in the final averaged image?:No + Dilation radius:1 + Block size:50 + Rescale the illumination function?:Yes + Calculate function for each image individually, or based on all images?:Each + Smoothing method:Fit Polynomial + Method to calculate smoothing filter size:Automatic + Approximate object diameter:10 + Smoothing filter size:600 + Retain the averaged image?:No + Name the averaged image:IllumBlueAvg + Retain the dilated image?:No + Name the dilated image:IllumBlueDilated + Automatically calculate spline parameters?:Yes + Background mode:auto + Number of spline points:5 + Background threshold:2.0 + Image resampling factor:2.0 + Maximum number of iterations:40 + Residual value for convergence:0.001 + +CorrectIlluminationApply:[module_num:8|svn_version:'Unknown'|variable_revision_number:5|show_window:False|notes:['Apply illumination correction functions to all channel groups']|batch_state:array([], dtype=uint8)|enabled:True|wants_pause:False] + Select the input image:OrigDAPI + Name the output image:CorrDAPI + Select the illumination function:IllumDAPI + Select how the illumination function is applied:Divide + Select the input image:OrigGFP + Name the output image:CorrGFP + Select the illumination function:IllumGFP + Select how the illumination function is applied:Divide + Select the input image:OrigRFP + Name the output image:CorrRFP + Select the illumination function:IllumRFP + Select how the illumination function is applied:Divide + Set output image values less than 0 equal to 0?:Yes + Set output image values greater than 1 equal to 1?:Yes + +SaveImages:[module_num:9|svn_version:'Unknown'|variable_revision_number:16|show_window:False|notes:['Save DAPI images as same tiff bit depth']|batch_state:array([], dtype=uint8)|enabled:True|wants_pause:False] + Select the type of image to save:Image + Select the image to save:CorrDAPI + Select method for constructing file names:From image filename + Select image name for file prefix:OrigDAPI + Enter single file name:OrigBlue + Number of digits:4 + Append a suffix to the image file name?:Yes + Text to append to the image name:_illumcorrect + Saved file format:tiff + Output file location:Default Output Folder|NF1_SchwannCell_data/CellProfiler_pipelines/IC_Output + Image bit depth:16-bit integer + Overwrite existing files without warning?:Yes + When to save:Every cycle + Record the file and path information to the saved image?:No + Create subfolders in the output folder?:No + Base image folder:Elsewhere...| + How to save the series:T (Time) + Save with lossless compression?:Yes + +SaveImages:[module_num:10|svn_version:'Unknown'|variable_revision_number:16|show_window:False|notes:['Save GFP images as same bit depth']|batch_state:array([], dtype=uint8)|enabled:True|wants_pause:False] + Select the type of image to save:Image + Select the image to save:CorrGFP + Select method for constructing file names:From image filename + Select image name for file prefix:OrigGFP + Enter single file name:OrigBlue + Number of digits:4 + Append a suffix to the image file name?:Yes + Text to append to the image name:_illumcorrect + Saved file format:tiff + Output file location:Default Output Folder|NF1_SchwannCell_data/CellProfiler_pipelines/IC_Output + Image bit depth:16-bit integer + Overwrite existing files without warning?:Yes + When to save:Every cycle + Record the file and path information to the saved image?:No + Create subfolders in the output folder?:No + Base image folder:Elsewhere...| + How to save the series:T (Time) + Save with lossless compression?:Yes + +SaveImages:[module_num:11|svn_version:'Unknown'|variable_revision_number:16|show_window:False|notes:['Save RFP images as same bit depth']|batch_state:array([], dtype=uint8)|enabled:True|wants_pause:False] + Select the type of image to save:Image + Select the image to save:CorrRFP + Select method for constructing file names:From image filename + Select image name for file prefix:OrigRFP + Enter single file name:OrigBlue + Number of digits:4 + Append a suffix to the image file name?:Yes + Text to append to the image name:_illumcorrect + Saved file format:tiff + Output file location:Default Output Folder|NF1_SchwannCell_data/CellProfiler_pipelines/IC_Output + Image bit depth:16-bit integer + Overwrite existing files without warning?:Yes + When to save:Every cycle + Record the file and path information to the saved image?:No + Create subfolders in the output folder?:No + Base image folder:Elsewhere...| + How to save the series:T (Time) + Save with lossless compression?:Yes diff --git a/1.cellprofiler_ic/README.md b/1.cellprofiler_ic/README.md new file mode 100644 index 0000000..94f2def --- /dev/null +++ b/1.cellprofiler_ic/README.md @@ -0,0 +1,15 @@ +## Perform illumination correction and save corrected images + +In this module, we perform illumination correction (IC) on the images for each plate and save the corrected images into new folders. +The images are saved as 16-bit depth, which is the same as the raw data. + +## Run the `nf1_ic` notebook + +To calculate and apply an IC function on each channel, run the [nf1_ic.ipynb](dnf1_ic.ipynb) notebook as a python script using the code block below: + +```bash +# Run this script in terminal +conda activate nf1_cellpainting_data +cd 0.cellprofiler_ic +bash nf1_ic.sh +``` diff --git a/1.cellprofiler_ic/nf1_ic.ipynb b/1.cellprofiler_ic/nf1_ic.ipynb new file mode 100644 index 0000000..e60382e --- /dev/null +++ b/1.cellprofiler_ic/nf1_ic.ipynb @@ -0,0 +1,167 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Correct illumination and save images for each plate using CellProfiler" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Import libraries" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import pathlib\n", + "\n", + "import sys\n", + "sys.path.append(\"../utils\")\n", + "import cp_utils" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Set paths for each plate" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "path_to_pipeline = pathlib.Path(\"NF1_illum.cppipe\").absolute()\n", + "\n", + "plates_info_dictionary = {\n", + " \"Plate_1\": {\n", + " \"path_to_images\": pathlib.Path(\"../0.download_data/Plate_1/\").absolute(),\n", + " \"path_to_output\": pathlib.Path(\"Corrected_Plate_1\").absolute(),\n", + " },\n", + " \"Plate_2\": {\n", + " \"path_to_images\": pathlib.Path(\"../0.download_data/Plate_2/\").absolute(),\n", + " \"path_to_output\": pathlib.Path(\"Corrected_Plate_2\").absolute(),\n", + " },\n", + "}" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Run illumination correction pipeline on each plate" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Times reported are CPU and Wall-clock times for each module\n", + "Tue Mar 14 14:13:52 2023: Image # 1, module Images # 1: CPU_time = 0.00 secs, Wall_time = 0.00 secs\n", + "Tue Mar 14 14:13:52 2023: Image # 1, module Metadata # 2: CPU_time = 0.00 secs, Wall_time = 0.00 secs\n", + "Tue Mar 14 14:13:52 2023: Image # 1, module NamesAndTypes # 3: CPU_time = 1.06 secs, Wall_time = 0.25 secs\n", + "Tue Mar 14 14:13:52 2023: Image # 1, module Groups # 4: CPU_time = 0.00 secs, Wall_time = 0.00 secs\n", + "Tue Mar 14 14:13:52 2023: Image # 1, module CorrectIlluminationCalculate # 5: CPU_time = 1.50 secs, Wall_time = 1.25 secs\n", + "Tue Mar 14 14:13:53 2023: Image # 1, module CorrectIlluminationApply # 6: CPU_time = 0.00 secs, Wall_time = 0.00 secs\n", + "Tue Mar 14 14:13:53 2023: Image # 1, module CorrectIlluminationCalculate # 7: CPU_time = 1.26 secs, Wall_time = 1.25 secs\n", + "Tue Mar 14 14:13:54 2023: Image # 1, module CorrectIlluminationApply # 8: CPU_time = 0.00 secs, Wall_time = 0.00 secs\n", + "Tue Mar 14 14:13:54 2023: Image # 1, module CorrectIlluminationCalculate # 9: CPU_time = 3.34 secs, Wall_time = 3.33 secs\n", + "Tue Mar 14 14:13:58 2023: Image # 1, module CorrectIlluminationApply # 10: CPU_time = 0.00 secs, Wall_time = 0.00 secs\n", + "Tue Mar 14 14:13:58 2023: Image # 1, module SaveImages # 11: CPU_time = 0.14 secs, Wall_time = 0.14 secs\n", + "Tue Mar 14 14:13:58 2023: Image # 1, module SaveImages # 12: CPU_time = 0.09 secs, Wall_time = 0.09 secs\n", + "Tue Mar 14 14:13:58 2023: Image # 1, module SaveImages # 13: CPU_time = 0.09 secs, Wall_time = 0.09 secs\n", + "Tue Mar 14 14:13:58 2023: Image # 2, module Images # 1: CPU_time = 0.00 secs, Wall_time = 0.00 secs\n", + "Tue Mar 14 14:13:58 2023: Image # 2, module Metadata # 2: CPU_time = 0.00 secs, Wall_time = 0.00 secs\n", + "Tue Mar 14 14:13:58 2023: Image # 2, module NamesAndTypes # 3: CPU_time = 0.63 secs, Wall_time = 0.20 secs\n", + "Tue Mar 14 14:13:58 2023: Image # 2, module Groups # 4: CPU_time = 0.00 secs, Wall_time = 0.00 secs\n", + "Tue Mar 14 14:13:58 2023: Image # 2, module CorrectIlluminationCalculate # 5: CPU_time = 1.45 secs, Wall_time = 1.24 secs\n", + "Tue Mar 14 14:13:59 2023: Image # 2, module CorrectIlluminationApply # 6: CPU_time = 0.01 secs, Wall_time = 0.00 secs\n", + "Tue Mar 14 14:13:59 2023: Image # 2, module CorrectIlluminationCalculate # 7: CPU_time = 1.28 secs, Wall_time = 1.24 secs\n", + "Tue Mar 14 14:14:01 2023: Image # 2, module CorrectIlluminationApply # 8: CPU_time = 0.00 secs, Wall_time = 0.00 secs\n", + "Tue Mar 14 14:14:01 2023: Image # 2, module CorrectIlluminationCalculate # 9: CPU_time = 3.37 secs, Wall_time = 3.37 secs\n", + "Tue Mar 14 14:14:04 2023: Image # 2, module CorrectIlluminationApply # 10: CPU_time = 0.01 secs, Wall_time = 0.00 secs\n", + "Tue Mar 14 14:14:04 2023: Image # 2, module SaveImages # 11: CPU_time = 0.13 secs, Wall_time = 0.14 secs\n", + "Tue Mar 14 14:14:04 2023: Image # 2, module SaveImages # 12: CPU_time = 0.10 secs, Wall_time = 0.10 secs\n", + "Tue Mar 14 14:14:04 2023: Image # 2, module SaveImages # 13: CPU_time = 0.11 secs, Wall_time = 0.11 secs\n", + "Tue Mar 14 14:14:04 2023: Image # 3, module Images # 1: CPU_time = 0.00 secs, Wall_time = 0.00 secs\n", + "Tue Mar 14 14:14:04 2023: Image # 3, module Metadata # 2: CPU_time = 0.00 secs, Wall_time = 0.00 secs\n", + "Tue Mar 14 14:14:04 2023: Image # 3, module NamesAndTypes # 3: CPU_time = 0.54 secs, Wall_time = 0.19 secs\n", + "Tue Mar 14 14:14:05 2023: Image # 3, module Groups # 4: CPU_time = 0.00 secs, Wall_time = 0.00 secs\n", + "Tue Mar 14 14:14:05 2023: Image # 3, module CorrectIlluminationCalculate # 5: CPU_time = 1.35 secs, Wall_time = 1.25 secs\n", + "Tue Mar 14 14:14:06 2023: Image # 3, module CorrectIlluminationApply # 6: CPU_time = 0.00 secs, Wall_time = 0.00 secs\n", + "Tue Mar 14 14:14:06 2023: Image # 3, module CorrectIlluminationCalculate # 7: CPU_time = 1.30 secs, Wall_time = 1.25 secs\n", + "Tue Mar 14 14:14:07 2023: Image # 3, module CorrectIlluminationApply # 8: CPU_time = 0.00 secs, Wall_time = 0.00 secs\n", + "Tue Mar 14 14:14:07 2023: Image # 3, module CorrectIlluminationCalculate # 9: CPU_time = 3.36 secs, Wall_time = 3.32 secs\n", + "Tue Mar 14 14:14:10 2023: Image # 3, module CorrectIlluminationApply # 10: CPU_time = 0.00 secs, Wall_time = 0.00 secs\n", + "Tue Mar 14 14:14:11 2023: Image # 3, module SaveImages # 11: CPU_time = 0.14 secs, Wall_time = 0.13 secs\n", + "Tue Mar 14 14:14:11 2023: Image # 3, module SaveImages # 12: CPU_time = 0.09 secs, Wall_time = 0.09 secs\n", + "Tue Mar 14 14:14:11 2023: Image # 3, module SaveImages # 13: CPU_time = 0.09 secs, Wall_time = 0.09 secs\n", + "Tue Mar 14 14:14:11 2023: Image # 4, module Images # 1: CPU_time = 0.00 secs, Wall_time = 0.00 secs\n", + "Tue Mar 14 14:14:11 2023: Image # 4, module Metadata # 2: CPU_time = 0.00 secs, Wall_time = 0.00 secs\n", + "Tue Mar 14 14:14:11 2023: Image # 4, module NamesAndTypes # 3: CPU_time = 0.41 secs, Wall_time = 0.16 secs\n", + "Tue Mar 14 14:14:11 2023: Image # 4, module Groups # 4: CPU_time = 0.00 secs, Wall_time = 0.00 secs\n", + "Tue Mar 14 14:14:11 2023: Image # 4, module CorrectIlluminationCalculate # 5: CPU_time = 1.50 secs, Wall_time = 1.24 secs\n", + "Tue Mar 14 14:14:12 2023: Image # 4, module CorrectIlluminationApply # 6: CPU_time = 0.00 secs, Wall_time = 0.00 secs\n" + ] + } + ], + "source": [ + "for plate in plates_info_dictionary:\n", + " # access the plate info stored in the dictionary\n", + " plate_info = plates_info_dictionary[plate]\n", + " path_to_output=plate_info[\"path_to_output\"]\n", + " path_to_images=plate_info[\"path_to_images\"]\n", + "\n", + " # run illumination correction pipeline and save images\n", + " cp_utils.run_cellprofiler(\n", + " path_to_pipeline=path_to_pipeline,\n", + " path_to_output=path_to_output,\n", + " path_to_images=path_to_images,\n", + " # these variables are turned off for illum pipeline\n", + " sqlite_name=None,\n", + " analysis_run=False,\n", + " )" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "nf1_cellpainting_data", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.15" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/1.cellprofiler_ic/nf1_ic.sh b/1.cellprofiler_ic/nf1_ic.sh new file mode 100644 index 0000000..38fc6dc --- /dev/null +++ b/1.cellprofiler_ic/nf1_ic.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +# convert the notebook into a python and run the file +jupyter nbconvert --to python \ + --FilesWriter.build_directory=scripts/ \ + --execute nf1_ic.ipynb diff --git a/1.cellprofiler_ic/scripts/nf1_ic.py b/1.cellprofiler_ic/scripts/nf1_ic.py new file mode 100644 index 0000000..c005829 --- /dev/null +++ b/1.cellprofiler_ic/scripts/nf1_ic.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python +# coding: utf-8 + +# # Correct illumination and save images for each plate using CellProfiler + +# ## Import libraries + +# In[1]: + + +import pathlib + +import sys +sys.path.append("../utils") +import cp_utils + + +# ## Set paths for each plate + +# In[2]: + + +path_to_pipeline = pathlib.Path("NF1_illum.cppipe").absolute() + +plates_info_dictionary = { + "Plate_1": { + "path_to_images": pathlib.Path("../0.download_data/Plate_1/").absolute(), + "path_to_output": pathlib.Path("Corrected_Plate_1").absolute(), + }, + "Plate_2": { + "path_to_images": pathlib.Path("../0.download_data/Plate_2/").absolute(), + "path_to_output": pathlib.Path("Corrected_Plate_2").absolute(), + }, +} + + +# ## Run illumination correction pipeline on each plate + +# In[3]: + + +for plate in plates_info_dictionary: + # access the plate info stored in the dictionary + plate_info = plates_info_dictionary[plate] + path_to_output=plate_info["path_to_output"] + path_to_images=plate_info["path_to_images"] + + # run illumination correction pipeline and save images + cp_utils.run_cellprofiler( + path_to_pipeline=path_to_pipeline, + path_to_output=path_to_output, + path_to_images=path_to_images, + # these variables are turned off for illum pipeline + sqlite_name=None, + analysis_run=False, + ) + diff --git a/nf1_cellpainting_env.yml b/nf1_cellpainting_env.yml index 0477da5..c0ed320 100644 --- a/nf1_cellpainting_env.yml +++ b/nf1_cellpainting_env.yml @@ -6,6 +6,7 @@ channels: dependencies: - conda-forge::python>=3.8 - conda-forge::ipykernel>=6.21.3 +- conda-forge::nbconvert=6.4.4 - conda-forge::pip>=23.0.1 - conda-forge::numpy>=1.24.2 - conda-forge::matplotlib>=3.7.1 @@ -15,8 +16,8 @@ dependencies: - conda-forge::scikit-learn>=1.2.2 - conda-forge::mahotas>=1.4.13 - conda-forge::gtk2>=2.24.33 -# these are strict because that is how it is on the CellProfiler wiki -- conda-forge::Jinja2=3.0.1 +# these are strict because that is how it is on the CellProfiler wiki (Jinja updated for nbconvert) +- conda-forge::Jinja2=3.0.3 - conda-forge::inflect=5.3.0 - conda-forge::wxpython=4.1.0 - conda-forge::sentry-sdk=0.18.0 diff --git a/utils/cp_utils.py b/utils/cp_utils.py new file mode 100644 index 0000000..d6bb671 --- /dev/null +++ b/utils/cp_utils.py @@ -0,0 +1,71 @@ +""" +This collection of functions runs CellProfiler and will rename the .sqlite outputed to any specified name if running an analysis pipeline. +""" + +import os +import pathlib + + +def rename_sqlite_file(sqlite_dir_path: pathlib.Path, name: str): + """Rename the .sqlite file to be {name}.sqlite as to differentiate between the files. + + Args: + sqlite_dir_path (pathlib.Path): path to CellProfiler_output directory + name (str): new name for the SQLite file + """ + try: + # CellProfiler requires a name to be set in to pipeline, so regardless of plate or method, all sqlite files name are hardcoded + sqlite_file_path = pathlib.Path(f"{sqlite_dir_path}/NF1_data.sqlite") + + new_file_name = str(sqlite_file_path).replace( + sqlite_file_path.name, f"{name}.sqlite" + ) + + # change the file name in the directory + pathlib.Path(sqlite_file_path).rename(pathlib.Path(new_file_name)) + print(f"The file is renamed to {pathlib.Path(new_file_name).name}!") + + except FileNotFoundError as e: + print( + f"The NF1_data.sqlite file is not found in directory. Either the pipeline wasn't ran properly or the file is already renamed.\n" + f"{e}" + ) + + +def run_cellprofiler( + path_to_pipeline: str, + path_to_output: str, + path_to_images: str, + sqlite_name: str = None, + analysis_run: bool = False, +): + """Run CellProfiler on data. It can be used for both a illumination correction pipeline and analysis pipeline. + + Args: + path_to_pipeline (str): path to the CellProfiler .cppipe file with the segmentation and feature measurement modules + path_to_output (str): path to the output folder + path_to_images (str): path to the images + sqlite_name (str, optional): string with name for SQLite file for an analysis pipeline (default is None) + analysis_run (bool, optional): will use functions to complete an analysis pipeline (default is False) + """ + # run CellProfiler on a plate that has not been analyzed yet + command = f"cellprofiler -c -r -p {path_to_pipeline} -o {path_to_output} -i {path_to_images}" + os.system(command) + + if analysis_run: + # runs through any files that are in the output path + if any( + files.name.startswith(sqlite_name) + for files in pathlib.Path(path_to_output).iterdir() + ): + print("This plate has already been analyzed!") + return + + # run CellProfiler on corrected images + command = f"cellprofiler -c -r -p {path_to_pipeline} -o {path_to_output} -i {path_to_images}" + os.system(command) + + # rename the outputted .sqlite file to the + rename_sqlite_file(sqlite_dir_path=pathlib.Path(path_to_output), name=sqlite_name) + + From 267c3b8af4ac1c99ed81cc6d81b02f50e1c09fd9 Mon Sep 17 00:00:00 2001 From: jenna-tomkinson Date: Tue, 14 Mar 2023 15:45:43 -0600 Subject: [PATCH 2/5] edit main README to add new module --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 05c522b..20a9135 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,8 @@ Once we discover a biomarker from these cells, we hope that our method can be us | Module | Purpose | Description | | :---- | :----- | :---------- | -| [0_download_data](0_download_data/) | Download NF1 pilot data | Download images from each plate of NF1 dataset for analysis from Figshare | +| [0_download_data](0.download_data/) | Download NF1 pilot data | Download images from each plate of NF1 dataset for analysis from Figshare | +| [1.cellprofiler_ic](1.cellprofiler_ic/) | Apply CellProfiler illumination correction (IC)| Use a CellProfiler pipeline to calculate and apply IC the images and save them | ## Main environment From d2dda2b437826d09e4d015de2a64546ef37a1925 Mon Sep 17 00:00:00 2001 From: jenna-tomkinson Date: Tue, 14 Mar 2023 15:46:22 -0600 Subject: [PATCH 3/5] add file to utils and change import statements --- 0.download_data/download_plates.ipynb | 3 ++- 0.download_data/scripts/download_plates.py | 2 +- {0.download_data => utils}/download_figshare.py | 0 3 files changed, 3 insertions(+), 2 deletions(-) rename {0.download_data => utils}/download_figshare.py (100%) diff --git a/0.download_data/download_plates.ipynb b/0.download_data/download_plates.ipynb index e61724e..acf85b2 100644 --- a/0.download_data/download_plates.ipynb +++ b/0.download_data/download_plates.ipynb @@ -23,7 +23,8 @@ "outputs": [], "source": [ "import pathlib\n", - "from download_figshare import download_figshare" + "\n", + "from utils.download_figshare import download_figshare" ] }, { diff --git a/0.download_data/scripts/download_plates.py b/0.download_data/scripts/download_plates.py index 4901dba..7e61134 100644 --- a/0.download_data/scripts/download_plates.py +++ b/0.download_data/scripts/download_plates.py @@ -9,7 +9,7 @@ import pathlib -from download_figshare import download_figshare +from utils.download_figshare import download_figshare # ## Set constant paths/variables diff --git a/0.download_data/download_figshare.py b/utils/download_figshare.py similarity index 100% rename from 0.download_data/download_figshare.py rename to utils/download_figshare.py From 86a9a39c37abc3f9e1cfd8284c08d18f0f3f9269 Mon Sep 17 00:00:00 2001 From: jenna-tomkinson Date: Tue, 14 Mar 2023 15:56:46 -0600 Subject: [PATCH 4/5] change imports to reflect utils folder --- 1.cellprofiler_ic/nf1_ic.ipynb | 4 +--- 1.cellprofiler_ic/scripts/nf1_ic.py | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/1.cellprofiler_ic/nf1_ic.ipynb b/1.cellprofiler_ic/nf1_ic.ipynb index e60382e..729c127 100644 --- a/1.cellprofiler_ic/nf1_ic.ipynb +++ b/1.cellprofiler_ic/nf1_ic.ipynb @@ -24,9 +24,7 @@ "source": [ "import pathlib\n", "\n", - "import sys\n", - "sys.path.append(\"../utils\")\n", - "import cp_utils" + "import utils.cp_utils as cp_utils" ] }, { diff --git a/1.cellprofiler_ic/scripts/nf1_ic.py b/1.cellprofiler_ic/scripts/nf1_ic.py index c005829..8ccdbda 100644 --- a/1.cellprofiler_ic/scripts/nf1_ic.py +++ b/1.cellprofiler_ic/scripts/nf1_ic.py @@ -10,9 +10,7 @@ import pathlib -import sys -sys.path.append("../utils") -import cp_utils +import utils.cp_utils as cp_utils # ## Set paths for each plate From 777def3b7b96b7a5ed851c97aec98fcfa83e18c8 Mon Sep 17 00:00:00 2001 From: jenna-tomkinson Date: Thu, 16 Mar 2023 15:12:07 -0600 Subject: [PATCH 5/5] erik and dave suggestions + add log function --- .gitignore | 3 ++ 1.cellprofiler_ic/README.md | 13 +++--- 1.cellprofiler_ic/nf1_ic.ipynb | 64 +++++------------------------ 1.cellprofiler_ic/nf1_ic.sh | 4 ++ 1.cellprofiler_ic/scripts/nf1_ic.py | 14 ++++--- README.md | 2 +- nf1_cellpainting_env.yml | 24 ++++++----- utils/cp_utils.py | 51 +++++++++++++++-------- 8 files changed, 82 insertions(+), 93 deletions(-) diff --git a/.gitignore b/.gitignore index b71be73..f86b706 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,6 @@ __pycache__ # ignore corrected plates images (like in download data 1.cellprofiler_ic/Corrected_Plate_1 1.cellprofiler_ic/Corrected_Plate_2 + +# ignore log files +1.cellprofiler_ic/logs \ No newline at end of file diff --git a/1.cellprofiler_ic/README.md b/1.cellprofiler_ic/README.md index 94f2def..0504cad 100644 --- a/1.cellprofiler_ic/README.md +++ b/1.cellprofiler_ic/README.md @@ -1,15 +1,16 @@ -## Perform illumination correction and save corrected images +# Perform illumination correction and save corrected images -In this module, we perform illumination correction (IC) on the images for each plate and save the corrected images into new folders. -The images are saved as 16-bit depth, which is the same as the raw data. +In this module, we perform illumination correction (IC) on images for each plate and save the corrected images into new folders. +Images are saved as 16-bit depth, which is the same as the raw data. ## Run the `nf1_ic` notebook -To calculate and apply an IC function on each channel, run the [nf1_ic.ipynb](dnf1_ic.ipynb) notebook as a python script using the code block below: +To calculate and apply an IC function on each channel, run the [nf1_ic.ipynb](nf1_ic.ipynb) notebook as a python script using the code block below: ```bash # Run this script in terminal -conda activate nf1_cellpainting_data -cd 0.cellprofiler_ic +# move to the 1.cellprofiler_ic directory to acess the `sh` script +cd 1.cellprofiler_ic +# run the notebook as a python script bash nf1_ic.sh ``` diff --git a/1.cellprofiler_ic/nf1_ic.ipynb b/1.cellprofiler_ic/nf1_ic.ipynb index 729c127..76ecc77 100644 --- a/1.cellprofiler_ic/nf1_ic.ipynb +++ b/1.cellprofiler_ic/nf1_ic.ipynb @@ -24,7 +24,9 @@ "source": [ "import pathlib\n", "\n", - "import utils.cp_utils as cp_utils" + "import sys\n", + "sys.path.append(\"../\")\n", + "from utils import cp_utils" ] }, { @@ -69,64 +71,20 @@ "metadata": {}, "outputs": [ { - "name": "stderr", + "name": "stdout", "output_type": "stream", "text": [ - "Times reported are CPU and Wall-clock times for each module\n", - "Tue Mar 14 14:13:52 2023: Image # 1, module Images # 1: CPU_time = 0.00 secs, Wall_time = 0.00 secs\n", - "Tue Mar 14 14:13:52 2023: Image # 1, module Metadata # 2: CPU_time = 0.00 secs, Wall_time = 0.00 secs\n", - "Tue Mar 14 14:13:52 2023: Image # 1, module NamesAndTypes # 3: CPU_time = 1.06 secs, Wall_time = 0.25 secs\n", - "Tue Mar 14 14:13:52 2023: Image # 1, module Groups # 4: CPU_time = 0.00 secs, Wall_time = 0.00 secs\n", - "Tue Mar 14 14:13:52 2023: Image # 1, module CorrectIlluminationCalculate # 5: CPU_time = 1.50 secs, Wall_time = 1.25 secs\n", - "Tue Mar 14 14:13:53 2023: Image # 1, module CorrectIlluminationApply # 6: CPU_time = 0.00 secs, Wall_time = 0.00 secs\n", - "Tue Mar 14 14:13:53 2023: Image # 1, module CorrectIlluminationCalculate # 7: CPU_time = 1.26 secs, Wall_time = 1.25 secs\n", - "Tue Mar 14 14:13:54 2023: Image # 1, module CorrectIlluminationApply # 8: CPU_time = 0.00 secs, Wall_time = 0.00 secs\n", - "Tue Mar 14 14:13:54 2023: Image # 1, module CorrectIlluminationCalculate # 9: CPU_time = 3.34 secs, Wall_time = 3.33 secs\n", - "Tue Mar 14 14:13:58 2023: Image # 1, module CorrectIlluminationApply # 10: CPU_time = 0.00 secs, Wall_time = 0.00 secs\n", - "Tue Mar 14 14:13:58 2023: Image # 1, module SaveImages # 11: CPU_time = 0.14 secs, Wall_time = 0.14 secs\n", - "Tue Mar 14 14:13:58 2023: Image # 1, module SaveImages # 12: CPU_time = 0.09 secs, Wall_time = 0.09 secs\n", - "Tue Mar 14 14:13:58 2023: Image # 1, module SaveImages # 13: CPU_time = 0.09 secs, Wall_time = 0.09 secs\n", - "Tue Mar 14 14:13:58 2023: Image # 2, module Images # 1: CPU_time = 0.00 secs, Wall_time = 0.00 secs\n", - "Tue Mar 14 14:13:58 2023: Image # 2, module Metadata # 2: CPU_time = 0.00 secs, Wall_time = 0.00 secs\n", - "Tue Mar 14 14:13:58 2023: Image # 2, module NamesAndTypes # 3: CPU_time = 0.63 secs, Wall_time = 0.20 secs\n", - "Tue Mar 14 14:13:58 2023: Image # 2, module Groups # 4: CPU_time = 0.00 secs, Wall_time = 0.00 secs\n", - "Tue Mar 14 14:13:58 2023: Image # 2, module CorrectIlluminationCalculate # 5: CPU_time = 1.45 secs, Wall_time = 1.24 secs\n", - "Tue Mar 14 14:13:59 2023: Image # 2, module CorrectIlluminationApply # 6: CPU_time = 0.01 secs, Wall_time = 0.00 secs\n", - "Tue Mar 14 14:13:59 2023: Image # 2, module CorrectIlluminationCalculate # 7: CPU_time = 1.28 secs, Wall_time = 1.24 secs\n", - "Tue Mar 14 14:14:01 2023: Image # 2, module CorrectIlluminationApply # 8: CPU_time = 0.00 secs, Wall_time = 0.00 secs\n", - "Tue Mar 14 14:14:01 2023: Image # 2, module CorrectIlluminationCalculate # 9: CPU_time = 3.37 secs, Wall_time = 3.37 secs\n", - "Tue Mar 14 14:14:04 2023: Image # 2, module CorrectIlluminationApply # 10: CPU_time = 0.01 secs, Wall_time = 0.00 secs\n", - "Tue Mar 14 14:14:04 2023: Image # 2, module SaveImages # 11: CPU_time = 0.13 secs, Wall_time = 0.14 secs\n", - "Tue Mar 14 14:14:04 2023: Image # 2, module SaveImages # 12: CPU_time = 0.10 secs, Wall_time = 0.10 secs\n", - "Tue Mar 14 14:14:04 2023: Image # 2, module SaveImages # 13: CPU_time = 0.11 secs, Wall_time = 0.11 secs\n", - "Tue Mar 14 14:14:04 2023: Image # 3, module Images # 1: CPU_time = 0.00 secs, Wall_time = 0.00 secs\n", - "Tue Mar 14 14:14:04 2023: Image # 3, module Metadata # 2: CPU_time = 0.00 secs, Wall_time = 0.00 secs\n", - "Tue Mar 14 14:14:04 2023: Image # 3, module NamesAndTypes # 3: CPU_time = 0.54 secs, Wall_time = 0.19 secs\n", - "Tue Mar 14 14:14:05 2023: Image # 3, module Groups # 4: CPU_time = 0.00 secs, Wall_time = 0.00 secs\n", - "Tue Mar 14 14:14:05 2023: Image # 3, module CorrectIlluminationCalculate # 5: CPU_time = 1.35 secs, Wall_time = 1.25 secs\n", - "Tue Mar 14 14:14:06 2023: Image # 3, module CorrectIlluminationApply # 6: CPU_time = 0.00 secs, Wall_time = 0.00 secs\n", - "Tue Mar 14 14:14:06 2023: Image # 3, module CorrectIlluminationCalculate # 7: CPU_time = 1.30 secs, Wall_time = 1.25 secs\n", - "Tue Mar 14 14:14:07 2023: Image # 3, module CorrectIlluminationApply # 8: CPU_time = 0.00 secs, Wall_time = 0.00 secs\n", - "Tue Mar 14 14:14:07 2023: Image # 3, module CorrectIlluminationCalculate # 9: CPU_time = 3.36 secs, Wall_time = 3.32 secs\n", - "Tue Mar 14 14:14:10 2023: Image # 3, module CorrectIlluminationApply # 10: CPU_time = 0.00 secs, Wall_time = 0.00 secs\n", - "Tue Mar 14 14:14:11 2023: Image # 3, module SaveImages # 11: CPU_time = 0.14 secs, Wall_time = 0.13 secs\n", - "Tue Mar 14 14:14:11 2023: Image # 3, module SaveImages # 12: CPU_time = 0.09 secs, Wall_time = 0.09 secs\n", - "Tue Mar 14 14:14:11 2023: Image # 3, module SaveImages # 13: CPU_time = 0.09 secs, Wall_time = 0.09 secs\n", - "Tue Mar 14 14:14:11 2023: Image # 4, module Images # 1: CPU_time = 0.00 secs, Wall_time = 0.00 secs\n", - "Tue Mar 14 14:14:11 2023: Image # 4, module Metadata # 2: CPU_time = 0.00 secs, Wall_time = 0.00 secs\n", - "Tue Mar 14 14:14:11 2023: Image # 4, module NamesAndTypes # 3: CPU_time = 0.41 secs, Wall_time = 0.16 secs\n", - "Tue Mar 14 14:14:11 2023: Image # 4, module Groups # 4: CPU_time = 0.00 secs, Wall_time = 0.00 secs\n", - "Tue Mar 14 14:14:11 2023: Image # 4, module CorrectIlluminationCalculate # 5: CPU_time = 1.50 secs, Wall_time = 1.24 secs\n", - "Tue Mar 14 14:14:12 2023: Image # 4, module CorrectIlluminationApply # 6: CPU_time = 0.00 secs, Wall_time = 0.00 secs\n" + "Correcting Plate_1\n", + "Correcting Plate_2\n" ] } ], "source": [ - "for plate in plates_info_dictionary:\n", - " # access the plate info stored in the dictionary\n", - " plate_info = plates_info_dictionary[plate]\n", - " path_to_output=plate_info[\"path_to_output\"]\n", - " path_to_images=plate_info[\"path_to_images\"]\n", + "# run through each plate with each set of paths based on dictionary\n", + "for plate, info in plates_info_dictionary.items():\n", + " path_to_output = info[\"path_to_output\"]\n", + " path_to_images = info[\"path_to_images\"]\n", + " print(f\"Correcting {plate}\")\n", "\n", " # run illumination correction pipeline and save images\n", " cp_utils.run_cellprofiler(\n", diff --git a/1.cellprofiler_ic/nf1_ic.sh b/1.cellprofiler_ic/nf1_ic.sh index 38fc6dc..5a49488 100644 --- a/1.cellprofiler_ic/nf1_ic.sh +++ b/1.cellprofiler_ic/nf1_ic.sh @@ -1,5 +1,9 @@ #!/bin/bash +# activate the main conda environment +eval "$(conda shell.bash hook)" +conda activate nf1_cellpainting_data + # convert the notebook into a python and run the file jupyter nbconvert --to python \ --FilesWriter.build_directory=scripts/ \ diff --git a/1.cellprofiler_ic/scripts/nf1_ic.py b/1.cellprofiler_ic/scripts/nf1_ic.py index 8ccdbda..04713cb 100644 --- a/1.cellprofiler_ic/scripts/nf1_ic.py +++ b/1.cellprofiler_ic/scripts/nf1_ic.py @@ -10,7 +10,9 @@ import pathlib -import utils.cp_utils as cp_utils +import sys +sys.path.append("../") +from utils import cp_utils # ## Set paths for each plate @@ -37,11 +39,11 @@ # In[3]: -for plate in plates_info_dictionary: - # access the plate info stored in the dictionary - plate_info = plates_info_dictionary[plate] - path_to_output=plate_info["path_to_output"] - path_to_images=plate_info["path_to_images"] +# run through each plate with each set of paths based on dictionary +for plate, info in plates_info_dictionary.items(): + path_to_output = info["path_to_output"] + path_to_images = info["path_to_images"] + print(f"Correcting {plate}") # run illumination correction pipeline and save images cp_utils.run_cellprofiler( diff --git a/README.md b/README.md index 20a9135..63880e5 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ Once we discover a biomarker from these cells, we hope that our method can be us | Module | Purpose | Description | | :---- | :----- | :---------- | -| [0_download_data](0.download_data/) | Download NF1 pilot data | Download images from each plate of NF1 dataset for analysis from Figshare | +| [0.download_data](0.download_data/) | Download NF1 pilot data | Download images from each plate of NF1 dataset for analysis from Figshare | | [1.cellprofiler_ic](1.cellprofiler_ic/) | Apply CellProfiler illumination correction (IC)| Use a CellProfiler pipeline to calculate and apply IC the images and save them | ## Main environment diff --git a/nf1_cellpainting_env.yml b/nf1_cellpainting_env.yml index c0ed320..361a3cc 100644 --- a/nf1_cellpainting_env.yml +++ b/nf1_cellpainting_env.yml @@ -4,18 +4,20 @@ channels: - conda-forge dependencies: -- conda-forge::python>=3.8 -- conda-forge::ipykernel>=6.21.3 +# this is restricted to Python 3.8 at this time due to other conflicts +- conda-forge::python=3.8 +- conda-forge::ipykernel - conda-forge::nbconvert=6.4.4 -- conda-forge::pip>=23.0.1 -- conda-forge::numpy>=1.24.2 -- conda-forge::matplotlib>=3.7.1 -- conda-forge::pandas>=1.5.3 -- conda-forge::mysqlclient>=2.0.3 -- conda-forge::openjdk>=11.0.9.1 -- conda-forge::scikit-learn>=1.2.2 -- conda-forge::mahotas>=1.4.13 -- conda-forge::gtk2>=2.24.33 +- conda-forge::pip +- conda-forge::numpy +- conda-forge::matplotlib +- conda-forge::pandas +- conda-forge::mysqlclient +- conda-forge::openjdk +- conda-forge::scikit-learn +- conda-forge::mahotas +- conda-forge::gtk2 +- conda-forge::typing-extensions # these are strict because that is how it is on the CellProfiler wiki (Jinja updated for nbconvert) - conda-forge::Jinja2=3.0.3 - conda-forge::inflect=5.3.0 diff --git a/utils/cp_utils.py b/utils/cp_utils.py index d6bb671..8aabf42 100644 --- a/utils/cp_utils.py +++ b/utils/cp_utils.py @@ -1,17 +1,26 @@ """ -This collection of functions runs CellProfiler and will rename the .sqlite outputed to any specified name if running an analysis pipeline. +These collection of functions runs CellProfiler and renames the .sqlite outputs to any specified name if +running an analysis pipeline. """ -import os +# must use the annotations import as CellProfiler is restricted to Python 3.8 at this time so Optional +# by itself only works in Python 3.10 +from __future__ import annotations +from typing import Optional +import subprocess import pathlib def rename_sqlite_file(sqlite_dir_path: pathlib.Path, name: str): - """Rename the .sqlite file to be {name}.sqlite as to differentiate between the files. + """Rename the .sqlite file into {name}.sqlite as to differentiate between different files. Args: sqlite_dir_path (pathlib.Path): path to CellProfiler_output directory name (str): new name for the SQLite file + + Raises: + FileNotFoundError: This error will occur if you do not have a SQLite file with the hard coded file name in the specified directory. + This means that this function can not find the right file to rename, so it raises an error. """ try: # CellProfiler requires a name to be set in to pipeline, so regardless of plate or method, all sqlite files name are hardcoded @@ -36,21 +45,28 @@ def run_cellprofiler( path_to_pipeline: str, path_to_output: str, path_to_images: str, - sqlite_name: str = None, - analysis_run: bool = False, + sqlite_name: Optional[None | str] = None, + analysis_run: Optional[False | bool] = False, ): - """Run CellProfiler on data. It can be used for both a illumination correction pipeline and analysis pipeline. + """Run CellProfiler on data. It can be used for either a illumination correction pipeline (default) and analysis pipeline (when + parameter is set to True). Args: path_to_pipeline (str): path to the CellProfiler .cppipe file with the segmentation and feature measurement modules path_to_output (str): path to the output folder path_to_images (str): path to the images - sqlite_name (str, optional): string with name for SQLite file for an analysis pipeline (default is None) + sqlite_name (str, optional): string with name for SQLite file for an analysis pipeline if you plan on running + multiple sets of images (e.g., per plate) so that the outputs will have different names (default is None) analysis_run (bool, optional): will use functions to complete an analysis pipeline (default is False) + """ - # run CellProfiler on a plate that has not been analyzed yet - command = f"cellprofiler -c -r -p {path_to_pipeline} -o {path_to_output} -i {path_to_images}" - os.system(command) + print(f"Starting CellProfiler run on {pathlib.Path(path_to_images).name}") + # A log file is created for each plate or data set name (based on folder name with images) + with open(pathlib.Path(f"logs/cellprofiler_output_{pathlib.Path(path_to_images).name}.log"), "w") as cellprofiler_output_file: + # run CellProfiler for a illumination correction pipeline + command = f"cellprofiler -c -r -p {path_to_pipeline} -o {path_to_output} -i {path_to_images}" + subprocess.run(command, shell=True, stdout=cellprofiler_output_file, stderr=cellprofiler_output_file) + cellprofiler_output_file.close() if analysis_run: # runs through any files that are in the output path @@ -62,10 +78,13 @@ def run_cellprofiler( return # run CellProfiler on corrected images - command = f"cellprofiler -c -r -p {path_to_pipeline} -o {path_to_output} -i {path_to_images}" - os.system(command) - - # rename the outputted .sqlite file to the - rename_sqlite_file(sqlite_dir_path=pathlib.Path(path_to_output), name=sqlite_name) - + with open("cellprofiler_output_analysis.log", "w") as cellprofiler_output_file: + command = f"cellprofiler -c -r -p {path_to_pipeline} -o {path_to_output} -i {path_to_images}" + subprocess.run(command, shell=True, capture_output=True) + cellprofiler_output_file.close() + if sqlite_name: + # rename the outputted .sqlite file as a specified name (used when running multiple plates with same CP pipeline) + rename_sqlite_file( + sqlite_dir_path=pathlib.Path(path_to_output), name=sqlite_name + )