From de93ef4486bfc20474499784e91fc44833211b18 Mon Sep 17 00:00:00 2001 From: Landung 'Don' Setiawan Date: Tue, 13 Feb 2024 09:49:10 -0800 Subject: [PATCH 1/2] refactor: Update how the codebase is structured --- .github/workflows/build.yml | 39 ++++++------ base/Dockerfile => Dockerfile | 18 +++--- ...conda-linux-64.lock => conda-linux-64.lock | 0 base/env.yaml => env.yaml | 0 nvidia/Dockerfile | 60 ------------------- 5 files changed, 33 insertions(+), 84 deletions(-) rename base/Dockerfile => Dockerfile (72%) rename base/conda-linux-64.lock => conda-linux-64.lock (100%) rename base/env.yaml => env.yaml (100%) delete mode 100644 nvidia/Dockerfile diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 038dfcd..543c9fd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,6 +1,7 @@ # Any commit to master branch re-builds images, re-runs tests, and pushes SHA tags to DockerHub name: Build on: + pull_request: push: branches: - main @@ -14,6 +15,7 @@ env: DOCKER_ORG: ${{ github.repository_owner }} GITHUB_SHA: ${{ github.sha }} GITHUB_REF: ${{ github.ref }} + PROJECT_NAME: caustics jobs: build-images: @@ -24,17 +26,12 @@ jobs: strategy: fail-fast: false matrix: - IMAGE: [base] + CAUSTICS_VERSION: ["dev", "0.7.0"] + CUDA_VERSION: ["11.8.0"] steps: - name: Checkout Repository uses: actions/checkout@v3 - - name: Get date tag - id: get_date - run: | - DATE_TAG="$( date -u '+%Y.%m.%d' )" - echo "date_tag=$DATE_TAG" >> $GITHUB_OUTPUT - - name: Get registry and org id: registry_org run: | @@ -54,11 +51,9 @@ jobs: id: meta uses: docker/metadata-action@v4 with: - images: ${{ steps.registry_org.outputs.image_base }}/${{ matrix.IMAGE }} + images: ${{ steps.registry_org.outputs.image_base }}/${{ env.PROJECT_NAME }} tags: | - # set latest tag for default branch - type=raw,value=latest - type=raw,value=${{ steps.get_date.outputs.date_tag }} + type=raw,value=${{ matrix.CAUSTICS_VERSION }}-cuda-${{ matrix.CUDA_VERSION }} - name: Log in to registry uses: docker/login-action@v2 @@ -69,16 +64,26 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - - - name: Build and push Docker image - uses: docker/build-push-action@v4 + + - name: Build and export to Docker + uses: docker/build-push-action@v5 with: - context: ${{ matrix.IMAGE }} + context: . + load: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - push: true - name: Inspect Image run: | - docker run ${{ steps.registry_org.outputs.image_base }}/${{ matrix.IMAGE }}:latest micromamba list + docker run ${{ steps.registry_org.outputs.image_base }}/${{ env.PROJECT_NAME }}:${{ matrix.CAUSTICS_VERSION }}-cuda-${{ matrix.CUDA_VERSION }} micromamba list docker images ls + + - name: Build and push Docker image + uses: docker/build-push-action@v4 + if: ${{ github.ref == 'refs/heads/main' }} + with: + context: . + platforms: linux/amd64,linux/arm64 + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + push: true diff --git a/base/Dockerfile b/Dockerfile similarity index 72% rename from base/Dockerfile rename to Dockerfile index 9670441..b42428c 100644 --- a/base/Dockerfile +++ b/Dockerfile @@ -1,6 +1,7 @@ -FROM mambaorg/micromamba:focal-cuda-11.8.0 +ARG CUDA_VERSION=11.8.0 +FROM mambaorg/micromamba:focal-cuda-${CUDA_VERSION} -ARG CAUSTICS_BRANCH=dev +ARG CAUSTICS_VERSION=0.7.0 # Tell apt-get to not block installs by asking for interactive human input ENV DEBIAN_FRONTEND=noninteractive \ @@ -39,8 +40,11 @@ RUN micromamba install --name base --yes --file /tmp/conda-linux-64.lock \ ARG MAMBA_DOCKERFILE_ACTIVATE=1 # Install caustics from a branch for now -RUN echo "Caustics branch: ${CAUSTICS_BRANCH}" -ENV CAUSTICS_REPO="git+https://github.com/Ciela-Institute/caustics.git@${CAUSTICS_BRANCH}" - -RUN echo "Installing caustics" \ - && pip install --no-cache ${CAUSTICS_REPO} +ENV CAUSTICS_REPO="git+https://github.com/Ciela-Institute/caustics.git@${CAUSTICS_VERSION}" +RUN echo "Installing caustics ..." \ + ; if [ "${CAUSTICS_VERSION}" == "dev" ]; then \ + echo "Installing from github branch: ${CAUSTICS_VERSION}" \ + && pip install --no-cache ${CAUSTICS_REPO} \ + ; else echo "Installing from production distribution version: ${CAUSTICS_VERSION}" ; \ + pip install caustics==${CAUSTICS_VERSION} \ + ; fi diff --git a/base/conda-linux-64.lock b/conda-linux-64.lock similarity index 100% rename from base/conda-linux-64.lock rename to conda-linux-64.lock diff --git a/base/env.yaml b/env.yaml similarity index 100% rename from base/env.yaml rename to env.yaml diff --git a/nvidia/Dockerfile b/nvidia/Dockerfile deleted file mode 100644 index 11f3a76..0000000 --- a/nvidia/Dockerfile +++ /dev/null @@ -1,60 +0,0 @@ -ARG RELEASE_VERSION=23.11 -FROM nvcr.io/nvidia/pytorch:${RELEASE_VERSION}-py3 - -ARG CAUSTICS_BRANCH=main - -# Setup environment to match variables set by repo2docker as much as possible -# Tell apt-get to not block installs by asking for interactive human input -ENV DEBIAN_FRONTEND=noninteractive \ - # Set username, uid and gid (same as uid) of non-root user the container will be run as - NB_USER=jovyan \ - NB_UID=1000 \ - # Use /bin/bash as shell, not the default /bin/sh (arrow keys, etc don't work then) - SHELL=/bin/bash \ - # Setup locale to be UTF-8, avoiding gnarly hard to debug encoding errors - LANG=C.UTF-8 \ - LC_ALL=C.UTF-8 - -# All env vars that reference other env vars need to be in their own ENV block -# Path to the python environment where the jupyter notebook packages are installed -# Home directory of our non-root user -ENV HOME=/home/${NB_USER} - - -RUN echo "Creating ${NB_USER} user..." \ - # Create a group for the user to be part of, with gid same as uid - && groupadd --gid ${NB_UID} ${NB_USER} \ - # Create non-root user, with given gid, uid and create $HOME - && useradd --create-home --gid ${NB_UID} --no-log-init --uid ${NB_UID} ${NB_USER} \ - # Make sure that /home/${NB_USER} is owned by non-root user, so we can install things there - && chown -R ${NB_USER}:${NB_USER} ${HOME} \ - && chown -R ${NB_USER}:${NB_USER} /workspace - -# Install basic apt packages -RUN echo "Installing Apt-get packages..." \ - && apt-get update --fix-missing > /dev/null \ - && apt-get install -y apt-utils \ - git \ - wget \ - zip \ - tzdata \ - python3-venv \ - graphviz \ - > /dev/null \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - -# Add TZ configuration - https://github.com/PrefectHQ/prefect/issues/3061 -ENV TZ UTC -# ======================== - -# Install caustics from a branch for now -RUN echo "Caustics branch: ${CAUSTICS_BRANCH}" -ENV CAUSTICS_REPO="git+https://github.com/Ciela-Institute/caustics.git@${CAUSTICS_BRANCH}" - -RUN echo "Installing caustics" \ - && pip install --no-cache ${CAUSTICS_REPO} - -# Set to non-root user and user's home dir for working directory -USER ${NB_USER} -WORKDIR ${HOME} From f1b4bf6e9960010b85a07f612f20d1c66e0d68b8 Mon Sep 17 00:00:00 2001 From: Landung 'Don' Setiawan Date: Tue, 13 Feb 2024 09:53:07 -0800 Subject: [PATCH 2/2] build: Add build-args to docker build actions --- .github/workflows/build.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 543c9fd..d75b8d3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -72,6 +72,9 @@ jobs: load: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + build-args: | + CAUSTICS_VERSION=${{ matrix.CAUSTICS_VERSION }} + CUDA_VERSION=${{ matrix.CUDA_VERSION }} - name: Inspect Image run: | @@ -87,3 +90,6 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} push: true + build-args: | + CAUSTICS_VERSION=${{ matrix.CAUSTICS_VERSION }} + CUDA_VERSION=${{ matrix.CUDA_VERSION }}