Merge pull request #284 from niaid/feature/containers

Adds Dockerfile which can be used to generate a test environment

.gitignore

@@ -2,6 +2,7 @@
 __pycache__/
 *.py[cod]
 *$py.class
+*.DS_store
 
 # C extensions
 *.so

Dockerfile

@@ -0,0 +1,51 @@
+ARG BUILD_IMAGE=ubuntu:20.04
+
+FROM ${BUILD_IMAGE}
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt update -y && \
+  apt install -y python3 unzip wget git python3-pip && \
+  apt install -y ca-certificates-java && \
+  apt install -y openjdk-8-jdk && \
+  apt install -y libblosc1 && \
+  apt install -y ffmpeg && \
+  apt install -y graphicsmagick && \
+  apt clean && \
+  rm -rf /var/lib/apt/lists/*;
+
+RUN ln -fs /usr/bin/python3.8 /usr/bin/python
+
+# Setup JAVA_HOME -- useful for docker commandline
+ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/
+
+RUN export JAVA_HOME
+
+# setup IMOD
+# Install libGL ?
+ARG IMOD=imod_4.11.24_RHEL7-64_CUDA10.1.sh
+
+RUN wget https://bio3d.colorado.edu/imod/AMD64-RHEL5/${IMOD} && \
+    sh ${IMOD} -yes && \
+    rm -f ${IMOD}
+
+ENV IMOD_DIR=/usr/local/IMOD
+ENV PATH=$IMOD_DIR/bin:$PATH
+
+ARG BIOFORMATS2RAW_VERSION=0.7.0
+
+WORKDIR /opt/bin
+# setup bioformats2raw
+RUN wget https://github.com/glencoesoftware/bioformats2raw/releases/download/v${BIOFORMATS2RAW_VERSION}/bioformats2raw-${BIOFORMATS2RAW_VERSION}.zip
+RUN unzip bioformats2raw-${BIOFORMATS2RAW_VERSION}.zip && \
+    rm bioformats2raw-${BIOFORMATS2RAW_VERSION}.zip
+
+ENV PATH="/opt/bin/bioformats2raw-${BIOFORMATS2RAW_VERSION}/bin:${PATH}"
+
+COPY requirements.txt .
+RUN pip install -r requirements.txt
+RUN rm requirements.txt
+
+WORKDIR /image_portal_workflows
+
+ENTRYPOINT ["/bin/bash"]

docs/source/development.rst

@@ -50,15 +50,21 @@ pre-commit
 format compliance at commit time. It is configured via a top-level ``.pre-commit-config.yaml`` file that runs ``black``,
 ``Flake8`` and other checks.
 
+Docker
+======
+
+`Docker <https://www.docker.com/products/docker-desktop/>`_ is a platform that provides OS-level virtualization so that softwares are developed and packaged in containers. This helps consistent development in different environments.
+
 *******
 Testing
 *******
 
-There are currently four ``pytest`` files in the `test` directory:
+All ``pytest`` files reside in the `test` directory:
 
 - ``test_brt``: end-to-end test of batchruntomo pipeline
 - ``test_dm``: 2D end-to-end pipeline test
 - ``test_sem``: end-to-end test of FIBSEM pipeline
+- ``test_lrg_2d``: Large 2d pipeline test
 - ``test_utils``: unit tests of utils/utils.py module
 
 There is test data for `test_dm` in the Git repo, but not for the others. These files need to be
@@ -125,6 +131,33 @@ not have binaries built for the M1. Need to install using an x86_64 version of P
 are issues installing `biofomats2raw` as there is no OpenCV package for arm64 chip. There currently
 is no fix or workaround for this issue.
 
+Docker
+======
+
+You can also choose to use Docker for local development and testing.
+
+In order to build the docker image, use `--platform linux/amd64` option.
+Explanation can be found `here <https://teams.microsoft.com/l/entity/com.microsoft.teamspace.tab.wiki/tab::5f55363b-bb53-4e5b-9564-8bed5289fdd5?context=%7B%22subEntityId%22%3A%22%7B%5C%22pageId%5C%22%3A15%2C%5C%22sectionId%5C%22%3A17%2C%5C%22origin%5C%22%3A2%7D%22%2C%22channelId%22%3A%2219%3A869be6677ee54848bc13f2066d847cc0%40thread.skype%22%7D&tenantId=14b77578-9773-42d5-8507-251ca2dc2b06>`_
+
+The basic usage for testing would look like below. The command assumes that you are running the container from the project directory where the main Dockerfile is located.
+
+.. code-block:: sh
+
+   # To build the image
+   docker build . -t hedwig_pipelines --platform linux/amd64
+
+
+In order to boot up a container with pipeline image, you can run the command below.\
+Note that we are setting a USER environment variable here. This is because `class Config` requires this environment variable set.
+
+.. code-block:: sh
+
+   # To run the container of that image
+   docker run -v "$(pwd):/image_portal_workflows" -e USER=root -it --rm hedwig_pipelines:latest
+
+
+Once you are in the container, you can run the commands you want to. For example: `pytest`.
+
 HPC Set up
 ==========
 

em_workflows/utils/bioformats2raw/Dockerfile

@@ -0,0 +1,32 @@
+ARG BUILD_IMAGE=ubuntu:20.04
+
+FROM ${BUILD_IMAGE}
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt update -y && \
+  apt install -y unzip wget && \
+  apt install -y ca-certificates-java && \
+  apt install -y openjdk-8-jdk && \
+  apt install -y libblosc1 && \
+  apt clean && \
+  rm -rf /var/lib/apt/lists/*;
+
+# Setup JAVA_HOME -- useful for docker commandline
+ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/
+
+RUN export JAVA_HOME
+
+ARG BIOFORMATS2RAW_VERSION=0.7.0
+
+WORKDIR /opt/bin
+
+# setup bioformats2raw
+RUN wget https://github.com/glencoesoftware/bioformats2raw/releases/download/v${BIOFORMATS2RAW_VERSION}/bioformats2raw-${BIOFORMATS2RAW_VERSION}.zip
+
+RUN unzip bioformats2raw-${BIOFORMATS2RAW_VERSION}.zip && \
+    rm bioformats2raw-${BIOFORMATS2RAW_VERSION}.zip
+
+ENV PATH="/opt/bin/bioformats2raw-${BIOFORMATS2RAW_VERSION}/bin:${PATH}"
+
+ENTRYPOINT ["/bin/bash"]

pytest.ini

@@ -1,6 +1,7 @@
 [pytest]
 env =
     HEDWIG_ENV=dev
+    USER=test
 
 addopts = --strict-markers
 markers =

test/copy_test_data.sh

@@ -7,10 +7,12 @@
 #  - cd to top-level "image_portal_workflows" dir of your repo
 
 # test_brt - one 970MB file
-rsync -ave ssh ai-rmlsbatch1.niaid.nih.gov:/mnt/ai-fas12/RMLEMHedwigQA/test/input_files/brt_inputs/Projects/2013-1220-dA30_5-BSC-1_10.mrc test/input_files/brt_inputs/
+mkdir -p test/input_files/brt_inputs/Projects
+rsync --progress --stats -ave ssh ai-rmlsbatch1.niaid.nih.gov:/mnt/ai-fas12/RMLEMHedwigQA/test/input_files/brt_inputs/Projects/2013-1220-dA30_5-BSC-1_10.mrc test/input_files/brt_inputs/Projects
 
 # test_sem - directory of smallish files, but 1.1GB total
-rsync -ave ssh ai-rmlsbatch1.niaid.nih.gov:/mnt/ai-fas12/RMLEMHedwigDev/Projects/BCBB_TEST/test/input_files/sem_inputs/ test/input_files/sem_inputs/
+mkdir -p test/input_files/sem_inputs/Projects
+rsync --progress --stats -ave ssh ai-rmlsbatch1.niaid.nih.gov:/mnt/ai-fas12/RMLEMHedwigDev/Projects/BCBB_TEST/test/input_files/sem_inputs/1/ test/input_files/sem_inputs/Projects/
 
 # test_dm - These were in the repo as of March 2023, but uncomment to copy
 #rsync -ave ssh ai-rmlsbatch1.niaid.nih.gov:/mnt/ai-fas12/RMLEMHedwigQA/test/input_files/dm_inputs/Projects/Lab/PI/  test/input_files/dm_inputs/

test/test_utils.py

@@ -67,7 +67,7 @@ def test_mount_config(mock_nfs_mount):
     assert os.path.exists(Config.brt_binary)
     assert os.path.exists(Config.dm2mrc_loc)
     assert os.path.exists(Config.clip_loc)
-    assert os.path.exists(Config.convert_loc)
+    # assert os.path.exists(Config.convert_loc) # uses gm instead (graphicsmagick)
     assert os.path.exists(Config.header_loc)
     assert os.path.exists(Config.mrc2tif_loc)
     assert os.path.exists(Config.newstack_loc)