reverting to py3.7.3

- previous commits included bumping to py39. - going back to py3.7.3 as bumping created segfault in mpi version - added info on build readme
OpenChemistry · Jan 14, 2023 · eefba22 · eefba22
1 parent 0781a87
commit eefba22
Show file tree

Hide file tree

Showing 27 changed files with 152 additions and 308 deletions.
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -4,15 +4,21 @@ on:
   push:
     branches:
       - "master"
-      - "update-dockerfile-staging"
 
 jobs:
-  # Builds base image for stempy (no cuda) if Dockerfile.base has changed
+  # Builds base image for stempy of various verions if Dockerfile.base has changed
   build-base:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        version: [stempy, stempy-mpi, stempy-mpi-old, stempy-mpi-mpich4, stempy-mpi-mpich4-apt-jammy]
+        include:
+          - version: stempy
+            tag-suffix: -conda-jammy
+          - version: stempy-mpi
+            tag-suffix: -conda
+          - version: stempy-mpi
+            tag-suffix: -conda-jammy
+
     steps:
       -
         name: Checkout
@@ -25,7 +31,7 @@ jobs:
         uses: tj-actions/changed-files@v35
         with:
           files: |
-            docker/${{ matrix.version }}/Dockerfile.base
+            docker/${{ matrix.version }}${{ matrix.tag-suffix}}/Dockerfile.base
       -
         name: Login to Docker Hub
         uses: docker/login-action@v2
@@ -40,30 +46,30 @@ jobs:
         uses: docker/build-push-action@v3
         if: ${{ contains(github.event.head_commit.message, 'trigger-ci') || steps.changed-dockerfile-base.outputs.any_changed == 'true'}}
         with:
-          context: ./docker/${{ matrix.version }}
-          file: ./docker/${{ matrix.version }}/Dockerfile.base
+          context: ./docker/${{ matrix.version }}${{ matrix.tag-suffix}}
+          file: ./docker/${{ matrix.version }}${{ matrix.tag-suffix}}/Dockerfile.base
           push: true
-          tags: ${{ secrets.DOCKERHUB_USERNAME }}/${{ matrix.version }}-base:latest
+          tags: ${{ secrets.DOCKERHUB_USERNAME }}/${{ matrix.version }}-base:latest${{ matrix.tag-suffix }}
           cache-to: type=gha
           cache-from: type=gha,mode=max
-          # build-args: |
-          #             PYTHON_VERSION=${{ secrets.JFROG_CREDS_USR }}
-          #             JFROG_PWD=${{ secrets.JFROG_CREDS_PSW }}
 
 # Builds stempy every time after build-base is done
-  Build-stempy:
+  build-stempy:
     runs-on: ubuntu-latest
     needs: [build-base]
     strategy:
       matrix:
         include:
           - version: stempy
-          - version: stempy-mpi
+            tag-suffix: -conda-jammy
           - version: stempy
+            tag-suffix: -conda-jammy
             notebook: -ipykernel
-          - version: stempy-mpi-old
-          - version: stempy-mpi-mpich4
-          - version: stempy-mpi-mpich4-apt-jammy
+          - version: stempy-mpi
+            tag-suffix: -conda
+          - version: stempy-mpi
+            tag-suffix: -conda-jammy
+
       fail-fast: false
     steps:
       -
@@ -84,7 +90,56 @@ jobs:
         name: Build/push ${{ matrix.version }}${{matrix.notebook}}
         uses: docker/build-push-action@v3
         with:
-          context: ./docker/${{ matrix.version }}
-          file: ./docker/${{ matrix.version }}/Dockerfile.stempy${{matrix.notebook}}
+          context: ./docker/${{ matrix.version }}${{ matrix.tag-suffix}}
+          file: ./docker/${{ matrix.version }}${{ matrix.tag-suffix}}/Dockerfile.stempy${{matrix.notebook}}
+          push: true
+          tags: ${{ secrets.DOCKERHUB_USERNAME }}/${{ matrix.version }}${{matrix.notebook}}:latest${{ matrix.tag-suffix}}
+
+  # Seperate job for building original version
+  build-stempy-old:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - version: stempy
+            MPI: OFF
+          - version: stempy-mpi
+            MPI: ON
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      -
+        name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - # Building base versions using original dockerfile
+        name: Build/push ${{ matrix.version }}
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          file: ./docker/Dockerfile
+          push: true
+          tags: ${{ secrets.DOCKERHUB_USERNAME }}/${{ matrix.version }}:latest
+          build-args: |
+                      MPI=${{ matrix.MPI }}
+          cache-to: type=gha
+          cache-from: type=gha, mode=max
+
+      - # Building ipykernel
+        name: Build/push ${{ matrix.version }}-ipykernel
+        if: ${{ matrix.version == 'stempy' }}
+        uses: docker/build-push-action@v3
+        with:
+          context: ./docker
+          file: ./docker/Dockerfile.ipykernel
           push: true
-          tags: ${{ secrets.DOCKERHUB_USERNAME }}/${{ matrix.version }}${{matrix.notebook}}:latest
+          tags: ${{ secrets.DOCKERHUB_USERNAME }}/${{ matrix.version }}-ipykernel:latest
diff --git a/docker/BUILD.md b/docker/BUILD.md
@@ -9,8 +9,8 @@
     shifterimg pull openchemistry/stempy
    N.B. I have noticed that it takes sometime for shifter to realize that a new image is available, so sometimes you have to wait say 10 minutes after pushing to dockerhub, not sure why!
 
-# Information on GitHub Actions
+# GitHub Actions
 
-The GitHub Actions docker workflow (.github/workflows/docker.yml) will automatically build the images in docker/stempy and docker/stempy-mpi. The base image will be built only when "trigger-ci" is in the head commit or if Dockerfile.base changes in the commit. This is to save build time.
+The GitHub Actions docker workflow (.github/workflows/docker.yml) will automatically build the images in docker/stempy-conda-jammy, docker/stempy-mpi-conda, and stempy-mpi-conda-jammy. These images run with conda's python. The base image (`Docker.base`) will only be built if "trigger-ci" is in the head commit or if it changes in the commit. The workflow will check on these things, and then proceed to building the images for stempy. This significantly improves build time - we shouldn't have to build mpich every time. 
 
-These images contain a conda environment. 
+The original versions are also built with GHA in a separate job in the same docker GHA workflow.
diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -1,6 +1,6 @@
 FROM ubuntu:18.04
 
-ARG MPI=OFF
+ARG MPI=MPI
 
 # Install deps
 RUN apt-get update && \
@@ -69,22 +69,22 @@ RUN mkdir -p /build/vtk-m && \
   make -j4
 
 # Build stempy
-COPY . /source/stempy
-
-RUN mkdir -p /build/stempy && \
-  cd /build/stempy && \
+RUN cd /source                                                                              && \
+  git clone --recursive https://github.com/OpenChemistry/stempy.git                         && \
+  mkdir -p /build/stempy                                                                    && \
+  cd /build/stempy                                                                          && \
   cmake -DCMAKE_BUILD_TYPE:STRING=Release \
-  -Dstempy_ENABLE_VTKm:BOOL=ON \
-  -DVTKm_DIR:PATH=/build/vtk-m/lib/cmake/vtkm-1.5 \
+  -Dstempy_ENABLE_VTKm:BOOL=OFF \
   -Dstempy_ENABLE_MPI:BOOL=${MPI} \
-  /source/stempy . && \
-  make -j4
+  /source/stempy .                                                                          && \
+  make -j 16                                                             
 
 # Install stempy
 RUN pip3 install -r /source/stempy/requirements.txt && \
   cp -r -L /build/stempy/lib/stempy /usr/local/lib/python3.7/site-packages && \
-  pip3 install matplotlib click imageio ncempy
+  pip3 install matplotlib click imageio ncempy                                              && \
+  rm -rf /source/stempy        
 
 RUN rm -rf /build
 
-RUN /sbin/ldconfig
+RUN /sbin/ldconfig
diff --git a/docker/conda/environment.yml b/docker/conda/environment.yml
diff --git a/docker/stempy/Dockerfile.base → docker/stempy-conda-jammy/Dockerfile.base b/docker/stempy/Dockerfile.base → docker/stempy-conda-jammy/Dockerfile.base
@@ -1,7 +1,7 @@
 FROM ubuntu:jammy as base
 
 ENV DEBIAN_FRONTEND noninteractive
-ENV installer=Miniconda3-py38_22.11.1-1-Linux-x86_64.sh
+ENV installer=Miniconda3-py39_22.11.1-1-Linux-x86_64.sh
 ENV PATH=/opt/miniconda3/bin:$PATH
 
 WORKDIR /source

diff --git a/docker/stempy/Dockerfile.stempy → docker/stempy-conda-jammy/Dockerfile.stempy b/docker/stempy/Dockerfile.stempy → docker/stempy-conda-jammy/Dockerfile.stempy
@@ -1,24 +1,25 @@
-FROM samwelborn/stempy-base:latest AS stempy
+FROM openchemistry/stempy-base:latest-conda-jammy AS stempy
 
 ENV PATH=/opt/miniconda3/bin:$PATH
-WORKDIR /source
-ARG PYTHON_VERSION=3.8
+ARG PYTHON_VERSION=3.9
 COPY ./conda/** ./
+WORKDIR /source
 
-# Build stempy and add mpi4py and h5py to conda env
+# Build stempy
 RUN mamba env update -n base -f /source/environment_before.yml                              && \
   cd /source                                                                                && \
-  git clone --recursive -b update-dockerfile-staging https://github.com/swelborn/stempy.git && \
+  git clone --recursive https://github.com/OpenChemistry/stempy.git                         && \
   mkdir -p /build/stempy                                                                    && \
   cd /build/stempy                                                                          && \
-  cmake -DCMAKE_BUILD_TYPE:STRING=RelWithDebInfo \
+  cmake -DCMAKE_BUILD_TYPE:STRING=Release \
   -Dstempy_ENABLE_VTKm:BOOL=OFF \
   -Dstempy_ENABLE_MPI:BOOL=OFF \
   /source/stempy .                                                                          && \
   make -j 16                                                                                && \
   cp -r -L /build/stempy/lib/stempy \                                                   
   /opt/miniconda3/lib/python${PYTHON_VERSION}/site-packages                                 && \
   rm -rf /source/stempy                                                                     && \
-  conda env update -n base -f /source/environment_after.yml --prune
+  mamba env update -n base -f /source/environment_after.yml                                 && \
+  conda clean -a -y -q
 
 RUN /sbin/ldconfig
diff --git a/docker/stempy/Dockerfile.stempy-ipykernel → ...y-conda-jammy/Dockerfile.stempy-ipykernel b/docker/stempy/Dockerfile.stempy-ipykernel → ...y-conda-jammy/Dockerfile.stempy-ipykernel
@@ -1,24 +1,25 @@
-FROM samwelborn/stempy-base:latest AS stempy
+FROM openchemistry/stempy-base:latest-conda-jammy AS stempy
 
 ENV PATH=/opt/miniconda3/bin:$PATH
-WORKDIR /source
-ARG PYTHON_VERSION=3.8
+ARG PYTHON_VERSION=3.9
 COPY ./conda/** ./
+WORKDIR /source
 
-# Build stempy and add mpi4py and h5py to conda env
+# Build stempy
 RUN mamba env update -n base -f /source/environment_before.yml                              && \
   cd /source                                                                                && \
-  git clone --recursive -b update-dockerfile-staging https://github.com/swelborn/stempy.git && \
+  git clone --recursive https://github.com/OpenChemistry/stempy.git                         && \
   mkdir -p /build/stempy                                                                    && \
   cd /build/stempy                                                                          && \
-  cmake -DCMAKE_BUILD_TYPE:STRING=RelWithDebInfo \
+  cmake -DCMAKE_BUILD_TYPE:STRING=Release \
   -Dstempy_ENABLE_VTKm:BOOL=OFF \
   -Dstempy_ENABLE_MPI:BOOL=OFF \
   /source/stempy .                                                                          && \
   make -j 16                                                                                && \
   cp -r -L /build/stempy/lib/stempy \                                                   
   /opt/miniconda3/lib/python${PYTHON_VERSION}/site-packages                                 && \
   rm -rf /source/stempy                                                                     && \
-  conda env update -n base -f /source/environment_ipykernel.yml --prune
+  mamba env update -n base -f /source/environment_ipykernel.yml                             && \
+  conda clean -a -y -q
 
 RUN /sbin/ldconfig
diff --git a/docker/stempy/conda/environment_after.yml → ...y-conda-jammy/conda/environment_after.yml b/docker/stempy/conda/environment_after.yml → ...y-conda-jammy/conda/environment_after.yml
@@ -1,6 +1,7 @@
 channels:
   - conda-forge
 dependencies:
+  - python=3.9
   - pip
   - numpy=1.23
   - deprecation

diff --git a/...h4-apt-jammy/conda/environment_before.yml → ...-conda-jammy/conda/environment_before.yml b/...h4-apt-jammy/conda/environment_before.yml → ...-conda-jammy/conda/environment_before.yml
@@ -3,5 +3,6 @@ channels:
 dependencies:
   - python=3.9
   - pip
+  - numpy=1.23
   - pip:
     - cmake
diff --git a/...er/stempy/conda/environment_ipykernel.yml → ...nda-jammy/conda/environment_ipykernel.yml b/...er/stempy/conda/environment_ipykernel.yml → ...nda-jammy/conda/environment_ipykernel.yml
@@ -1,14 +1,14 @@
 channels:
   - conda-forge
 dependencies:
-  - python=3.8
+  - python=3.9
   - pip
   - numpy=1.23
   - deprecation
-  - h5py=3.6
   - click 
   - imageio 
   - ncempy
+  - h5py=3.6
   - ipykernel=6.4.2
   - ipympl=0.8.6
   - matplotlib=3.4.3
diff --git a/...empy-mpi-mpich4-apt-jammy/Dockerfile.base → ...er/stempy-mpi-conda-jammy/Dockerfile.base b/...empy-mpi-mpich4-apt-jammy/Dockerfile.base → ...er/stempy-mpi-conda-jammy/Dockerfile.base
@@ -1,8 +1,6 @@
 FROM ubuntu:jammy as base
 
 ENV DEBIAN_FRONTEND noninteractive
-ENV installer=Miniconda3-py39_22.11.1-1-Linux-x86_64.sh
-ENV PATH=/opt/miniconda3/bin:$PATH
 
 WORKDIR /source
 
@@ -34,9 +32,17 @@ RUN apt-get update                                      && \
     rm -rf /var/lib/apt/lists/*                         && \
     mkdir /build/                                       
 
+# Install conda
+# Update conda and update --all have to happen to stay specific to python 3.7.3
+# otherwise does not work
+ENV installer=Miniconda3-py37_4.12.0-Linux-x86_64.sh
+ENV PATH=/opt/miniconda3/bin:$PATH
 RUN wget https://repo.anaconda.com/miniconda/$installer && \
     /bin/bash $installer -b -p /opt/miniconda3          && \
     rm -rf $installer                                   && \
-    conda install mamba -c conda-forge -y
+    conda install python=${PYTHON_VERSION}              && \
+    conda update conda -y                               && \
+    conda update --all -y                               && \
+    conda install mamba -c conda-forge -y     
 
 RUN /sbin/ldconfig
diff --git a/...py-mpi-mpich4-apt-jammy/Dockerfile.stempy → .../stempy-mpi-conda-jammy/Dockerfile.stempy b/...py-mpi-mpich4-apt-jammy/Dockerfile.stempy → .../stempy-mpi-conda-jammy/Dockerfile.stempy
@@ -1,26 +1,27 @@
-FROM samwelborn/stempy-mpi-mpich4-apt-jammy-base:latest AS stempy
+FROM openchemistry/stempy-mpi-base:latest-conda-jammy AS stempy
 
-WORKDIR /source
-ARG PYTHON_VERSION=3.9
 ENV PATH=/opt/miniconda3/bin:$PATH
-
+ARG PYTHON_VERSION=3.7
 COPY ./conda/** ./
+WORKDIR /source
 
 # Build stempy
-RUN cd /source && mamba env update -n base -f /source/environment_before.yml                && \
+RUN mamba env update -n base -f /source/environment_before.yml                              && \
+  cd /source                                                                                && \
   git clone --recursive https://github.com/OpenChemistry/stempy.git                         && \
   mkdir -p /build/stempy                                                                    && \
   cd /build/stempy                                                                          && \
-  cmake -DCMAKE_BUILD_TYPE:STRING=RelWithDebInfo \
+  cmake -DCMAKE_BUILD_TYPE:STRING=Release \
   -Dstempy_ENABLE_VTKm:BOOL=OFF \
   -Dstempy_ENABLE_MPI:BOOL=ON \
   /source/stempy .                                                                          && \
   make -j 16                                                                                && \
   cp -r -L /build/stempy/lib/stempy \
-  /opt/miniconda3/lib/python${PYTHON_VERSION}/site-packages && \
+  /opt/miniconda3/lib/python${PYTHON_VERSION}/site-packages                                 && \
   rm -rf /source/stempy                                                                     && \
-  conda env update -n base -f /source/environment_after.yml                              && \
+  mamba env update -n base -f /source/environment_after.yml                                 && \
   conda clean -a -y -q
 
 RUN /sbin/ldconfig
 
+
diff --git a/...tempy-mpi-old/conda/environment_after.yml → ...i-conda-jammy/conda/environment_after.yml b/...tempy-mpi-old/conda/environment_after.yml → ...i-conda-jammy/conda/environment_after.yml
@@ -10,4 +10,4 @@ dependencies:
   - ncempy
   - h5py=2.10
   - pip:
-    - mpi4py==3.0.1
+    - mpi4py==3.1.4
diff --git a/...empy-mpi-old/conda/environment_before.yml → ...-conda-jammy/conda/environment_before.yml b/...empy-mpi-old/conda/environment_before.yml → ...-conda-jammy/conda/environment_before.yml