From 4156805a5e7bf88c6a2875bec23fb512cb6f2557 Mon Sep 17 00:00:00 2001 From: Jonathan Shimwell Date: Thu, 9 Jun 2022 16:34:34 +0100 Subject: [PATCH 1/5] multi layer docker draft --- CI/Dockerfile | 99 ++++++++++++++++++++++++++++------- CI/Dockerfile_0_base | 43 --------------- CI/Dockerfile_1_external_deps | 21 -------- CI/Dockerfile_1_housekeeping | 8 --- CI/Dockerfile_2_hdf5 | 17 ------ CI/Dockerfile_3_moab | 16 ------ 6 files changed, 80 insertions(+), 124 deletions(-) delete mode 100644 CI/Dockerfile_0_base delete mode 100644 CI/Dockerfile_1_external_deps delete mode 100644 CI/Dockerfile_1_housekeeping delete mode 100644 CI/Dockerfile_2_hdf5 delete mode 100644 CI/Dockerfile_3_moab diff --git a/CI/Dockerfile b/CI/Dockerfile index 3f23d0141d..81332c7233 100644 --- a/CI/Dockerfile +++ b/CI/Dockerfile @@ -1,32 +1,93 @@ ARG UBUNTU_VERSION=18.04 -FROM ubuntu:${UBUNTU_VERSION} +FROM ubuntu:${UBUNTU_VERSION} as base # Use bash as the default shell SHELL ["/bin/bash", "-c"] +# Ubuntu Setup +ENV TZ=America/Chicago +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone # Update core packages -ARG UBUNTU_VERSION=18.04 RUN apt-get -y update; \ - apt-get -y install autoconf clang cmake g++ gcc gfortran git libblas-dev \ - libhdf5-dev liblapack-dev libpython2.7-dev libtool libeigen3-dev\ - python-numpy python-pip python-setuptools wget; \ - if [ "${UBUNTU_VERSION}" = "18.04" ]; then \ - pip install sphinx; \ - apt-get -y install astyle; \ - fi; \ - pip install cython + apt-get -y install autoconf \ + clang \ + cmake \ + g++ \ + gcc \ + gfortran \ + libhdf5-dev \ + libtool \ + libeigen3-dev\ + python3-numpy \ + python3 \ + python3-pip \ + python3-setuptools \ + python3-dev \ + libpython3-dev \ + wget \ + software-properties-common; \ + add-apt-repository ppa:git-core/ppa; \ + apt-get update; \ + apt-get install -y git; \ + update-alternatives --install /usr/bin/python python /usr/bin/python3 10; \ + update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 10; \ + pip install cython; + # Copy scripts to docker image RUN mkdir -p /root/etc/ COPY CI/ /root/etc/CI +#TODO move sh file contents into this Dockerfile ENV docker_env=/root/etc/CI/env.sh -# Build HDF5 then Geant4 then MOAB 5.1.0 -RUN COMPILER=gcc HDF5_VERSION=1.10.4 /root/etc/CI/docker/build_hdf5.sh && \ - COMPILER=clang HDF5_VERSION=1.10.4 /root/etc/CI/docker/build_hdf5.sh && \ - COMPILER=gcc /root/etc/CI/docker/build_geant4.sh && \ - COMPILER=clang /root/etc/CI/docker/build_geant4.sh && \ - COMPILER=gcc HDF5_VERSION=system MOAB_VERSION=5.1.0 /root/etc/CI/docker/build_moab.sh && \ - COMPILER=clang HDF5_VERSION=system MOAB_VERSION=5.1.0 /root/etc/CI/docker/build_moab.sh && \ - COMPILER=gcc HDF5_VERSION=1.10.4 MOAB_VERSION=5.1.0 /root/etc/CI/docker/build_moab.sh && \ - COMPILER=clang HDF5_VERSION=1.10.4 MOAB_VERSION=5.1.0 /root/etc/CI/docker/build_moab.sh +ENV build_dir=/root/build_dir +ENV install_dir=/root/opt + + +FROM base as external_deps + +#setting the COMPILER variable +ARG COMPILER=gcc +ENV COMPILER=${COMPILER} + +# Set Geant4 env variable +ENV geant4_build_dir=${build_dir}/geant4 +ENV geant4_install_dir=${install_dir}/geant4 + +# Build Geant4 +#TODO move sh file contents into this Dockerfile +RUN /root/etc/CI/docker/build_geant4.sh + +ENV double_down_build_dir=${build_dir}/double-down/ +ENV double_down_install_dir=${install_dir}/double-down/ + +# Build Embree +#TODO move sh file contents into this Dockerfile +RUN /root/etc/CI/docker/build_embree.sh + +FROM external_deps as hdf5 + +# Set HDF5 env variable +ENV hdf5_build_dir=${build_dir}/hdf5 +ENV hdf5_install_dir=${install_dir}/hdf5 + +# Build HDF5 +# HDF5 argument possible value: 1.10.4 or system +ARG HDF5=1.10.4 +ENV HDF5_VERSION=${HDF5} +#TODO move sh file contents into this Dockerfile +RUN /root/etc/CI/docker/build_hdf5.sh + +from hdf5 as moab + +# Set MOAB env variable +ENV moab_build_dir=${build_dir}/moab +ENV moab_install_dir=${install_dir}/moab + +ARG MOAB=5.3.0 +ENV MOAB_VERSION ${MOAB} +#TODO move sh file contents into this Dockerfile +RUN if [ "${MOAB_VERSION}" != "master" ] && [ "${MOAB_VERSION}" != "develop" ]; then \ + /root/etc/CI/docker/build_moab.sh; \ + fi; + diff --git a/CI/Dockerfile_0_base b/CI/Dockerfile_0_base deleted file mode 100644 index bc18da2f5a..0000000000 --- a/CI/Dockerfile_0_base +++ /dev/null @@ -1,43 +0,0 @@ -ARG UBUNTU_VERSION=18.04 -FROM ubuntu:${UBUNTU_VERSION} - -# Use bash as the default shell -SHELL ["/bin/bash", "-c"] -# Ubuntu Setup -ENV TZ=America/Chicago -RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone - -# Update core packages -RUN apt-get -y update; \ - apt-get -y install autoconf \ - clang \ - cmake \ - g++ \ - gcc \ - gfortran \ - libhdf5-dev \ - libtool \ - libeigen3-dev\ - python3-numpy \ - python3 \ - python3-pip \ - python3-setuptools \ - python3-dev \ - libpython3-dev \ - wget \ - software-properties-common; \ - add-apt-repository ppa:git-core/ppa; \ - apt-get update; \ - apt-get install -y git; \ - update-alternatives --install /usr/bin/python python /usr/bin/python3 10; \ - update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 10; \ - pip install cython; - - -# Copy scripts to docker image -RUN mkdir -p /root/etc/ -COPY CI/ /root/etc/CI -ENV docker_env=/root/etc/CI/env.sh - -ENV build_dir=/root/build_dir -ENV install_dir=/root/opt diff --git a/CI/Dockerfile_1_external_deps b/CI/Dockerfile_1_external_deps deleted file mode 100644 index 9d0be56cb1..0000000000 --- a/CI/Dockerfile_1_external_deps +++ /dev/null @@ -1,21 +0,0 @@ -ARG UBUNTU_VERSION=18.04ARG -ARG OWNER=svalinn -ARG TAG=latest -FROM ghcr.io/${OWNER}/dagmc-ci-ubuntu-${UBUNTU_VERSION}:$TAG - -#setting the COMPILER variable -ARG COMPILER=gcc -ENV COMPILER=${COMPILER} - -# Set Geant4 env variable -ENV geant4_build_dir=${build_dir}/geant4 -ENV geant4_install_dir=${install_dir}/geant4 - -# Build Geant4 -RUN /root/etc/CI/docker/build_geant4.sh - -ENV double_down_build_dir=${build_dir}/double-down/ -ENV double_down_install_dir=${install_dir}/double-down/ - -# Build Embree -RUN /root/etc/CI/docker/build_embree.sh diff --git a/CI/Dockerfile_1_housekeeping b/CI/Dockerfile_1_housekeeping deleted file mode 100644 index de398652ed..0000000000 --- a/CI/Dockerfile_1_housekeeping +++ /dev/null @@ -1,8 +0,0 @@ -ARG UBUNTU_VERSION=18.04 -ARG OWNER=svalinn -ARG TAG=latest -FROM ghcr.io/${OWNER}/dagmc-ci-ubuntu-${UBUNTU_VERSION}:$TAG - -RUN pip install sphinx; \ - apt-get -y install clang-format - diff --git a/CI/Dockerfile_2_hdf5 b/CI/Dockerfile_2_hdf5 deleted file mode 100644 index cf82e573fa..0000000000 --- a/CI/Dockerfile_2_hdf5 +++ /dev/null @@ -1,17 +0,0 @@ -ARG UBUNTU_VERSION=18.04 -ARG COMPILER=gcc -ARG OWNER=svalinn -ARG TAG=latest -FROM ghcr.io/${OWNER}/dagmc-ci-ubuntu-${UBUNTU_VERSION}-${COMPILER}-ext:$TAG - - - -# Set HDF5 env variable -ENV hdf5_build_dir=${build_dir}/hdf5 -ENV hdf5_install_dir=${install_dir}/hdf5 - -# Build HDF5 -# HDF5 argument possible value: 1.10.4 or system -ARG HDF5=1.10.4 -ENV HDF5_VERSION=${HDF5} -RUN /root/etc/CI/docker/build_hdf5.sh diff --git a/CI/Dockerfile_3_moab b/CI/Dockerfile_3_moab deleted file mode 100644 index 57427b0625..0000000000 --- a/CI/Dockerfile_3_moab +++ /dev/null @@ -1,16 +0,0 @@ -ARG UBUNTU_VERSION=18.04 -ARG COMPILER=gcc -ARG HDF5=1.10.4 -ARG OWNER=svalinn -ARG TAG=latest -FROM ghcr.io/${OWNER}/dagmc-ci-ubuntu-${UBUNTU_VERSION}-${COMPILER}-ext-hdf5_${HDF5}:$TAG - -# Set MOAB env variable -ENV moab_build_dir=${build_dir}/moab -ENV moab_install_dir=${install_dir}/moab - -ARG MOAB=5.3.0 -ENV MOAB_VERSION ${MOAB} -RUN if [ "${MOAB_VERSION}" != "master" ] && [ "${MOAB_VERSION}" != "develop" ]; then \ - /root/etc/CI/docker/build_moab.sh; \ - fi; From 4edd378695efac9bb907b2f5d872eed1274123b0 Mon Sep 17 00:00:00 2001 From: Jonathan Shimwell Date: Thu, 9 Jun 2022 16:36:57 +0100 Subject: [PATCH 2/5] upper case --- CI/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CI/Dockerfile b/CI/Dockerfile index 81332c7233..6fae909793 100644 --- a/CI/Dockerfile +++ b/CI/Dockerfile @@ -78,7 +78,7 @@ ENV HDF5_VERSION=${HDF5} #TODO move sh file contents into this Dockerfile RUN /root/etc/CI/docker/build_hdf5.sh -from hdf5 as moab +FROM hdf5 as moab # Set MOAB env variable ENV moab_build_dir=${build_dir}/moab From 65a5810e638119abb2e0fd565bd349d0f4895cfa Mon Sep 17 00:00:00 2001 From: Jonathan Shimwell Date: Thu, 7 Jul 2022 15:42:43 +0100 Subject: [PATCH 3/5] updated scripts to work with single dockerfile --- .github/workflows/docker_publish.yml | 12 ++++++++---- CI/Dockerfile | 6 +++--- CI/Dockerfile_1_housekeeping | 7 +++++++ CI/update_docker.sh | 14 +++++++++----- 4 files changed, 27 insertions(+), 12 deletions(-) create mode 100644 CI/Dockerfile_1_housekeeping diff --git a/.github/workflows/docker_publish.yml b/.github/workflows/docker_publish.yml index 573b0e6c09..ae4097a6a9 100644 --- a/.github/workflows/docker_publish.yml +++ b/.github/workflows/docker_publish.yml @@ -37,7 +37,8 @@ jobs: - name: Build and Export Dockerfile_0_base uses: docker/build-push-action@v2 with: - file: CI/Dockerfile_0_base + file: CI/Dockerfile + target: base push: true context: . build-args: | @@ -116,7 +117,8 @@ jobs: - name: Build and push Dockerfile_1_external_deps uses: docker/build-push-action@v2 with: - file: CI/Dockerfile_1_external_deps + file: CI/Dockerfile + target: external_deps context: . push: false build-args: | @@ -129,7 +131,8 @@ jobs: - name: Build and export Dockerfile_2_hdf5 uses: docker/build-push-action@v2 with: - file: CI/Dockerfile_2_hdf5 + file: CI/Dockerfile + target: hdf5 context: . push: true build-args: | @@ -184,7 +187,8 @@ jobs: - name: Build and push Dockerfile_3_moab uses: docker/build-push-action@v2 with: - file: CI/Dockerfile_3_moab + file: CI/Dockerfile + target: moab context: . push: true build-args: | diff --git a/CI/Dockerfile b/CI/Dockerfile index 6fae909793..2f25be054b 100644 --- a/CI/Dockerfile +++ b/CI/Dockerfile @@ -1,5 +1,5 @@ ARG UBUNTU_VERSION=18.04 -FROM ubuntu:${UBUNTU_VERSION} as base +FROM ubuntu:${UBUNTU_VERSION} AS base # Use bash as the default shell SHELL ["/bin/bash", "-c"] @@ -65,7 +65,7 @@ ENV double_down_install_dir=${install_dir}/double-down/ #TODO move sh file contents into this Dockerfile RUN /root/etc/CI/docker/build_embree.sh -FROM external_deps as hdf5 +FROM external_deps AS hdf5 # Set HDF5 env variable ENV hdf5_build_dir=${build_dir}/hdf5 @@ -78,7 +78,7 @@ ENV HDF5_VERSION=${HDF5} #TODO move sh file contents into this Dockerfile RUN /root/etc/CI/docker/build_hdf5.sh -FROM hdf5 as moab +FROM hdf5 AS moab # Set MOAB env variable ENV moab_build_dir=${build_dir}/moab diff --git a/CI/Dockerfile_1_housekeeping b/CI/Dockerfile_1_housekeeping new file mode 100644 index 0000000000..43065975c4 --- /dev/null +++ b/CI/Dockerfile_1_housekeeping @@ -0,0 +1,7 @@ +ARG UBUNTU_VERSION=18.04 +ARG OWNER=svalinn +ARG TAG=latest +FROM ghcr.io/${OWNER}/dagmc-ci-ubuntu-${UBUNTU_VERSION}:$TAG + +RUN pip install sphinx; \ + apt-get -y install clang-format diff --git a/CI/update_docker.sh b/CI/update_docker.sh index 1126eb7956..be9825912d 100755 --- a/CI/update_docker.sh +++ b/CI/update_docker.sh @@ -22,20 +22,23 @@ moab_versions="5.3.0 develop master" for ubuntu_version in ${ubuntu_versions}; do image_name="svalinn/dagmc-ci-ubuntu-${ubuntu_version}" docker build -t ${image_name} --build-arg UBUNTU_VERSION=${ubuntu_version} \ - -f CI/Dockerfile_0_base . + -f CI/Dockerfile \ + --target base . if [ "$PUSH" = true ]; then docker push ${image_name}; fi for compiler in $compilers; do image_name="svalinn/dagmc-ci-ubuntu-${ubuntu_version}-${compiler}-ext" docker build -t ${image_name} --build-arg UBUNTU_VERSION=${ubuntu_version} \ --build-arg COMPILER=${compiler} \ - -f CI/Dockerfile_1_external_deps . + -f CI/Dockerfile \ + --target external_deps . if [ "$PUSH" = true ]; then docker push ${image_name}; fi for hdf5 in $hdf5_versions; do image_name="svalinn/dagmc-ci-ubuntu-${ubuntu_version}-${compiler}-ext-hdf5_${hdf5}" docker build -t ${image_name} --build-arg UBUNTU_VERSION=${ubuntu_version} \ --build-arg COMPILER=${compiler} \ --build-arg HDF5=${hdf5} \ - -f CI/Dockerfile_2_hdf5 . + -f CI/Dockerfile \ + --target hdf5 . if [ "$PUSH" = true ]; then docker push ${image_name}; fi for moab in $moab_versions; do image_name="svalinn/dagmc-ci-ubuntu-${ubuntu_version}-${compiler}-ext-hdf5_${hdf5}-moab_${moab}" @@ -43,7 +46,8 @@ for ubuntu_version in ${ubuntu_versions}; do --build-arg COMPILER=${compiler} \ --build-arg HDF5=${hdf5} \ --build-arg MOAB=${moab} \ - -f CI/Dockerfile_3_moab . + -f CI/Dockerfile \ + --target moab . if [ "$PUSH" = true ]; then docker push ${image_name}; fi done done @@ -51,5 +55,5 @@ for ubuntu_version in ${ubuntu_versions}; do done # Building image for houe keeping -docker build -t svalinn/dagmc-ci-ubuntu-18.04-housekeeping -f CI/Dockerfile_1_housekeeping . +docker build -t svalinn/dagmc-ci-ubuntu-18.04-docs -f CI/docs.Dockerfile . if [ "$PUSH" = true ]; then docker push svalinn/dagmc-ci-ubuntu-18.04-housekeeping; fi From b55976b4618cccc166fcf0bdc77ac062dd358c05 Mon Sep 17 00:00:00 2001 From: Jonathan Shimwell Date: Thu, 7 Jul 2022 15:45:44 +0100 Subject: [PATCH 4/5] [skip ci] using housekeeping name for docs image --- CI/update_docker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CI/update_docker.sh b/CI/update_docker.sh index be9825912d..1558886dbf 100755 --- a/CI/update_docker.sh +++ b/CI/update_docker.sh @@ -55,5 +55,5 @@ for ubuntu_version in ${ubuntu_versions}; do done # Building image for houe keeping -docker build -t svalinn/dagmc-ci-ubuntu-18.04-docs -f CI/docs.Dockerfile . +docker build -t svalinn/dagmc-ci-ubuntu-18.04-housekeeping -f CI/Dockerfile_1_housekeeping . if [ "$PUSH" = true ]; then docker push svalinn/dagmc-ci-ubuntu-18.04-housekeeping; fi From 0f4e64da25aca12eb95b06cd89e07be21a34fbf3 Mon Sep 17 00:00:00 2001 From: Jonathan Shimwell Date: Thu, 7 Jul 2022 15:49:42 +0100 Subject: [PATCH 5/5] added entry for 813 --- doc/CHANGELOG.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/CHANGELOG.rst b/doc/CHANGELOG.rst index 3b404dce8d..0cbcc6a5e1 100644 --- a/doc/CHANGELOG.rst +++ b/doc/CHANGELOG.rst @@ -7,6 +7,11 @@ DAGMC Changelog Next version ==================== +**Changed:** + + * Using multi stage Dockerfile to reduce the number of Dockerfile (#813) + + v3.2.2 ====================