From e8813ae1ca383a59109e4f4717a8f38bfc524eb2 Mon Sep 17 00:00:00 2001 From: Anish Date: Wed, 23 Aug 2023 10:29:49 -0400 Subject: [PATCH 1/3] Update copy test data command to mkdir if it does not exist Add Dockerfile for bioformats2raw --- em_workflows/utils/bioformats2raw/Dockerfile | 36 ++++++++++++++++++++ em_workflows/utils/bioformats2raw/README.md | 13 +++++++ test/copy_test_data.sh | 6 ++-- 3 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 em_workflows/utils/bioformats2raw/Dockerfile create mode 100644 em_workflows/utils/bioformats2raw/README.md diff --git a/em_workflows/utils/bioformats2raw/Dockerfile b/em_workflows/utils/bioformats2raw/Dockerfile new file mode 100644 index 00000000..21686bf5 --- /dev/null +++ b/em_workflows/utils/bioformats2raw/Dockerfile @@ -0,0 +1,36 @@ +ARG BUILD_IMAGE=ubuntu:20.04 + +FROM ${BUILD_IMAGE} + +RUN DEBIAN_FRONTEND=noninteractive \ + apt-get update && \ + apt-get install -y python3 unzip && \ + apt-get install -y ca-certificates-java && \ + apt-get install -y openjdk-8-jdk && \ + apt-get install -y libblosc1 && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/*; + +# Fix certificate issues +RUN apt-get update && \ + apt-get clean && \ + update-ca-certificates -f; + +# Setup JAVA_HOME -- useful for docker commandline +ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/ + +RUN export JAVA_HOME + +RUN mkdir /app + +ENV BIOFORMATS2RAW_VERSION 0.7.0 + +COPY bioformats2raw-${BIOFORMATS2RAW_VERSION}.zip ./app + +WORKDIR /app + +RUN unzip bioformats2raw-${BIOFORMATS2RAW_VERSION}.zip + +ENV PATH="/app/bioformats2raw-${BIOFORMATS2RAW_VERSION}/bin:${PATH}" + +ENTRYPOINT ["/bin/bash"] diff --git a/em_workflows/utils/bioformats2raw/README.md b/em_workflows/utils/bioformats2raw/README.md new file mode 100644 index 00000000..55b06c9f --- /dev/null +++ b/em_workflows/utils/bioformats2raw/README.md @@ -0,0 +1,13 @@ +This Dockerfile was built for mac users in mind who has issues running `bioformats2raw` command on their machine. + +In order to build this image, you need to download one of the releases. +[0.7.0 Release](https://github.com/glencoesoftware/bioformats2raw/releases/tag/v0.7.0) +Place the `zip` file in the same directory as the Dockerfile to build it. + +The Dockerfile is using the version: _0.7.0_ currently and can be updated as per necessary. + +In order to build the docker image, use `--platform linux/amd64` option. + +```bash +$ docker build . -t hedwig_bioformats --platform linux/amd64 +``` diff --git a/test/copy_test_data.sh b/test/copy_test_data.sh index 3d44ffdb..6b0a584c 100755 --- a/test/copy_test_data.sh +++ b/test/copy_test_data.sh @@ -7,10 +7,12 @@ # - cd to top-level "image_portal_workflows" dir of your repo # test_brt - one 970MB file -rsync -ave ssh ai-rmlsbatch1.niaid.nih.gov:/mnt/ai-fas12/RMLEMHedwigQA/test/input_files/brt_inputs/Projects/2013-1220-dA30_5-BSC-1_10.mrc test/input_files/brt_inputs/ +mkdir -p test/input_files/brt_inputs +rsync --progress --stats -ave ssh ai-rmlsbatch1.niaid.nih.gov:/mnt/ai-fas12/RMLEMHedwigQA/test/input_files/brt_inputs/Projects/2013-1220-dA30_5-BSC-1_10.mrc test/input_files/brt_inputs/ # test_sem - directory of smallish files, but 1.1GB total -rsync -ave ssh ai-rmlsbatch1.niaid.nih.gov:/mnt/ai-fas12/RMLEMHedwigDev/Projects/BCBB_TEST/test/input_files/sem_inputs/ test/input_files/sem_inputs/ +mkdir -p test/input_files/sem_inputs +rsync --progress --stats -ave ssh ai-rmlsbatch1.niaid.nih.gov:/mnt/ai-fas12/RMLEMHedwigDev/Projects/BCBB_TEST/test/input_files/sem_inputs/ test/input_files/sem_inputs/ # test_dm - These were in the repo as of March 2023, but uncomment to copy #rsync -ave ssh ai-rmlsbatch1.niaid.nih.gov:/mnt/ai-fas12/RMLEMHedwigQA/test/input_files/dm_inputs/Projects/Lab/PI/ test/input_files/dm_inputs/ From 5a04b237785e47e0a6c674090ec31ddc538a9543 Mon Sep 17 00:00:00 2001 From: Anish Date: Thu, 24 Aug 2023 14:59:43 -0400 Subject: [PATCH 2/3] Dockerfile for Hedwig workflow --- .gitignore | 1 + Dockerfile | 51 ++++++++++++++++++++ README.rst | 13 +++++ em_workflows/utils/bioformats2raw/Dockerfile | 34 ++++++------- em_workflows/utils/bioformats2raw/README.md | 13 ----- pytest.ini | 1 + test/test_utils.py | 3 +- 7 files changed, 83 insertions(+), 33 deletions(-) create mode 100644 Dockerfile delete mode 100644 em_workflows/utils/bioformats2raw/README.md diff --git a/.gitignore b/.gitignore index 22ae1d53..04635233 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ __pycache__/ *.py[cod] *$py.class +*.DS_store # C extensions *.so diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..fdf7222d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,51 @@ +ARG BUILD_IMAGE=ubuntu:20.04 + +FROM ${BUILD_IMAGE} + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt update -y && \ + apt install -y python3 unzip wget git python3-pip && \ + apt install -y ca-certificates-java && \ + apt install -y openjdk-8-jdk && \ + apt install -y libblosc1 && \ + apt install -y ffmpeg && \ + apt install -y graphicsmagick && \ + apt clean && \ + rm -rf /var/lib/apt/lists/*; + +RUN ln -fs /usr/bin/python3.8 /usr/bin/python + +# Setup JAVA_HOME -- useful for docker commandline +ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/ + +RUN export JAVA_HOME + +# setup IMOD +# Install libGL ? +ARG IMOD=imod_4.11.24_RHEL7-64_CUDA10.1.sh + +RUN wget https://bio3d.colorado.edu/imod/AMD64-RHEL5/${IMOD} && \ + sh ${IMOD} -yes && \ + rm -f ${IMOD} + +ENV IMOD_DIR=/usr/local/IMOD +ENV PATH=$IMOD_DIR/bin:$PATH + +ARG BIOFORMATS2RAW_VERSION=0.7.0 + +WORKDIR /opt/bin +# setup bioformats2raw +RUN wget https://github.com/glencoesoftware/bioformats2raw/releases/download/v${BIOFORMATS2RAW_VERSION}/bioformats2raw-${BIOFORMATS2RAW_VERSION}.zip +RUN unzip bioformats2raw-${BIOFORMATS2RAW_VERSION}.zip && \ + rm bioformats2raw-${BIOFORMATS2RAW_VERSION}.zip + +ENV PATH="/opt/bin/bioformats2raw-${BIOFORMATS2RAW_VERSION}/bin:${PATH}" + +COPY requirements.txt . +RUN pip install -r requirements.txt +RUN rm requirements.txt + +WORKDIR /image_portal_workflows + +ENTRYPOINT ["/bin/bash"] diff --git a/README.rst b/README.rst index 51d9939e..67417e59 100644 --- a/README.rst +++ b/README.rst @@ -13,3 +13,16 @@ Build/Test: Test Coverage: .. image:: ../../coverage.svg + +#### Docker + +The Dockerfile is currently using,\ +- bioformats2raw version: 0.7.0. +- imod version: 4.11.24 + +In order to build the docker image, use `--platform linux/amd64` option. +Explanation can be found [here](https://teams.microsoft.com/l/entity/com.microsoft.teamspace.tab.wiki/tab::5f55363b-bb53-4e5b-9564-8bed5289fdd5?context=%7B%22subEntityId%22%3A%22%7B%5C%22pageId%5C%22%3A15%2C%5C%22sectionId%5C%22%3A17%2C%5C%22origin%5C%22%3A2%7D%22%2C%22channelId%22%3A%2219%3A869be6677ee54848bc13f2066d847cc0%40thread.skype%22%7D&tenantId=14b77578-9773-42d5-8507-251ca2dc2b06) + +```bash +$ docker build . -t hedwig_pipelines --platform linux/amd64 +``` diff --git a/em_workflows/utils/bioformats2raw/Dockerfile b/em_workflows/utils/bioformats2raw/Dockerfile index 21686bf5..6de10327 100644 --- a/em_workflows/utils/bioformats2raw/Dockerfile +++ b/em_workflows/utils/bioformats2raw/Dockerfile @@ -2,35 +2,31 @@ ARG BUILD_IMAGE=ubuntu:20.04 FROM ${BUILD_IMAGE} -RUN DEBIAN_FRONTEND=noninteractive \ - apt-get update && \ - apt-get install -y python3 unzip && \ - apt-get install -y ca-certificates-java && \ - apt-get install -y openjdk-8-jdk && \ - apt-get install -y libblosc1 && \ - apt-get clean && \ +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt update -y && \ + apt install -y unzip wget && \ + apt install -y ca-certificates-java && \ + apt install -y openjdk-8-jdk && \ + apt install -y libblosc1 && \ + apt clean && \ rm -rf /var/lib/apt/lists/*; -# Fix certificate issues -RUN apt-get update && \ - apt-get clean && \ - update-ca-certificates -f; - # Setup JAVA_HOME -- useful for docker commandline ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/ RUN export JAVA_HOME -RUN mkdir /app - -ENV BIOFORMATS2RAW_VERSION 0.7.0 +ARG BIOFORMATS2RAW_VERSION=0.7.0 -COPY bioformats2raw-${BIOFORMATS2RAW_VERSION}.zip ./app +WORKDIR /opt/bin -WORKDIR /app +# setup bioformats2raw +RUN wget https://github.com/glencoesoftware/bioformats2raw/releases/download/v${BIOFORMATS2RAW_VERSION}/bioformats2raw-${BIOFORMATS2RAW_VERSION}.zip -RUN unzip bioformats2raw-${BIOFORMATS2RAW_VERSION}.zip +RUN unzip bioformats2raw-${BIOFORMATS2RAW_VERSION}.zip && \ + rm bioformats2raw-${BIOFORMATS2RAW_VERSION}.zip -ENV PATH="/app/bioformats2raw-${BIOFORMATS2RAW_VERSION}/bin:${PATH}" +ENV PATH="/opt/bin/bioformats2raw-${BIOFORMATS2RAW_VERSION}/bin:${PATH}" ENTRYPOINT ["/bin/bash"] diff --git a/em_workflows/utils/bioformats2raw/README.md b/em_workflows/utils/bioformats2raw/README.md deleted file mode 100644 index 55b06c9f..00000000 --- a/em_workflows/utils/bioformats2raw/README.md +++ /dev/null @@ -1,13 +0,0 @@ -This Dockerfile was built for mac users in mind who has issues running `bioformats2raw` command on their machine. - -In order to build this image, you need to download one of the releases. -[0.7.0 Release](https://github.com/glencoesoftware/bioformats2raw/releases/tag/v0.7.0) -Place the `zip` file in the same directory as the Dockerfile to build it. - -The Dockerfile is using the version: _0.7.0_ currently and can be updated as per necessary. - -In order to build the docker image, use `--platform linux/amd64` option. - -```bash -$ docker build . -t hedwig_bioformats --platform linux/amd64 -``` diff --git a/pytest.ini b/pytest.ini index 18f96bb4..ddab6ce6 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,6 +1,7 @@ [pytest] env = HEDWIG_ENV=dev + USER=test addopts = --strict-markers markers = diff --git a/test/test_utils.py b/test/test_utils.py index 493ce0f4..9a217b61 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -67,7 +67,7 @@ def test_mount_config(mock_nfs_mount): assert os.path.exists(Config.brt_binary) assert os.path.exists(Config.dm2mrc_loc) assert os.path.exists(Config.clip_loc) - assert os.path.exists(Config.convert_loc) + # assert os.path.exists(Config.convert_loc) # uses gm instead (graphicsmagick) assert os.path.exists(Config.header_loc) assert os.path.exists(Config.mrc2tif_loc) assert os.path.exists(Config.newstack_loc) @@ -237,6 +237,7 @@ def test_mrc_to_movie(mock_nfs_mount): input_dir = "test/input_files/sem_inputs/Projects/mrc_movie_test" input_path = Path(os.path.join(proj_dir, input_dir)) # FIXME input directory `sem_inputs` in test/input_files is missing + print(input_path) assert os.path.exists(input_path) # FIXME adjusted.mrc is missing image_path = Path(os.path.join(proj_dir, input_dir, "adjusted.mrc")) From 6a9cd8c6fa532a4dee7f026f34953a5f2099c948 Mon Sep 17 00:00:00 2001 From: Anish Date: Fri, 25 Aug 2023 12:46:07 -0400 Subject: [PATCH 3/3] Update development.rst --- README.rst | 13 ------------- docs/source/development.rst | 35 ++++++++++++++++++++++++++++++++++- test/copy_test_data.sh | 8 ++++---- test/test_utils.py | 1 - 4 files changed, 38 insertions(+), 19 deletions(-) diff --git a/README.rst b/README.rst index 67417e59..51d9939e 100644 --- a/README.rst +++ b/README.rst @@ -13,16 +13,3 @@ Build/Test: Test Coverage: .. image:: ../../coverage.svg - -#### Docker - -The Dockerfile is currently using,\ -- bioformats2raw version: 0.7.0. -- imod version: 4.11.24 - -In order to build the docker image, use `--platform linux/amd64` option. -Explanation can be found [here](https://teams.microsoft.com/l/entity/com.microsoft.teamspace.tab.wiki/tab::5f55363b-bb53-4e5b-9564-8bed5289fdd5?context=%7B%22subEntityId%22%3A%22%7B%5C%22pageId%5C%22%3A15%2C%5C%22sectionId%5C%22%3A17%2C%5C%22origin%5C%22%3A2%7D%22%2C%22channelId%22%3A%2219%3A869be6677ee54848bc13f2066d847cc0%40thread.skype%22%7D&tenantId=14b77578-9773-42d5-8507-251ca2dc2b06) - -```bash -$ docker build . -t hedwig_pipelines --platform linux/amd64 -``` diff --git a/docs/source/development.rst b/docs/source/development.rst index bd731224..10a905b2 100644 --- a/docs/source/development.rst +++ b/docs/source/development.rst @@ -50,15 +50,21 @@ pre-commit format compliance at commit time. It is configured via a top-level ``.pre-commit-config.yaml`` file that runs ``black``, ``Flake8`` and other checks. +Docker +====== + +`Docker `_ is a platform that provides OS-level virtualization so that softwares are developed and packaged in containers. This helps consistent development in different environments. + ******* Testing ******* -There are currently four ``pytest`` files in the `test` directory: +All ``pytest`` files reside in the `test` directory: - ``test_brt``: end-to-end test of batchruntomo pipeline - ``test_dm``: 2D end-to-end pipeline test - ``test_sem``: end-to-end test of FIBSEM pipeline +- ``test_lrg_2d``: Large 2d pipeline test - ``test_utils``: unit tests of utils/utils.py module There is test data for `test_dm` in the Git repo, but not for the others. These files need to be @@ -125,6 +131,33 @@ not have binaries built for the M1. Need to install using an x86_64 version of P are issues installing `biofomats2raw` as there is no OpenCV package for arm64 chip. There currently is no fix or workaround for this issue. +Docker +====== + +You can also choose to use Docker for local development and testing. + +In order to build the docker image, use `--platform linux/amd64` option. +Explanation can be found `here `_ + +The basic usage for testing would look like below. The command assumes that you are running the container from the project directory where the main Dockerfile is located. + +.. code-block:: sh + + # To build the image + docker build . -t hedwig_pipelines --platform linux/amd64 + + +In order to boot up a container with pipeline image, you can run the command below.\ +Note that we are setting a USER environment variable here. This is because `class Config` requires this environment variable set. + +.. code-block:: sh + + # To run the container of that image + docker run -v "$(pwd):/image_portal_workflows" -e USER=root -it --rm hedwig_pipelines:latest + + +Once you are in the container, you can run the commands you want to. For example: `pytest`. + HPC Set up ========== diff --git a/test/copy_test_data.sh b/test/copy_test_data.sh index 6b0a584c..c9b15893 100755 --- a/test/copy_test_data.sh +++ b/test/copy_test_data.sh @@ -7,12 +7,12 @@ # - cd to top-level "image_portal_workflows" dir of your repo # test_brt - one 970MB file -mkdir -p test/input_files/brt_inputs -rsync --progress --stats -ave ssh ai-rmlsbatch1.niaid.nih.gov:/mnt/ai-fas12/RMLEMHedwigQA/test/input_files/brt_inputs/Projects/2013-1220-dA30_5-BSC-1_10.mrc test/input_files/brt_inputs/ +mkdir -p test/input_files/brt_inputs/Projects +rsync --progress --stats -ave ssh ai-rmlsbatch1.niaid.nih.gov:/mnt/ai-fas12/RMLEMHedwigQA/test/input_files/brt_inputs/Projects/2013-1220-dA30_5-BSC-1_10.mrc test/input_files/brt_inputs/Projects # test_sem - directory of smallish files, but 1.1GB total -mkdir -p test/input_files/sem_inputs -rsync --progress --stats -ave ssh ai-rmlsbatch1.niaid.nih.gov:/mnt/ai-fas12/RMLEMHedwigDev/Projects/BCBB_TEST/test/input_files/sem_inputs/ test/input_files/sem_inputs/ +mkdir -p test/input_files/sem_inputs/Projects +rsync --progress --stats -ave ssh ai-rmlsbatch1.niaid.nih.gov:/mnt/ai-fas12/RMLEMHedwigDev/Projects/BCBB_TEST/test/input_files/sem_inputs/1/ test/input_files/sem_inputs/Projects/ # test_dm - These were in the repo as of March 2023, but uncomment to copy #rsync -ave ssh ai-rmlsbatch1.niaid.nih.gov:/mnt/ai-fas12/RMLEMHedwigQA/test/input_files/dm_inputs/Projects/Lab/PI/ test/input_files/dm_inputs/ diff --git a/test/test_utils.py b/test/test_utils.py index 9a217b61..b5dfefd3 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -237,7 +237,6 @@ def test_mrc_to_movie(mock_nfs_mount): input_dir = "test/input_files/sem_inputs/Projects/mrc_movie_test" input_path = Path(os.path.join(proj_dir, input_dir)) # FIXME input directory `sem_inputs` in test/input_files is missing - print(input_path) assert os.path.exists(input_path) # FIXME adjusted.mrc is missing image_path = Path(os.path.join(proj_dir, input_dir, "adjusted.mrc"))