feat(base-image): Move conda to llb and cache it

.github/workflows/CI.yml

@@ -64,16 +64,9 @@ jobs:
       matrix:
         os: [ubuntu-latest]
     runs-on: ${{ matrix.os }}
-    env:
-      GOPATH: ${{ github.workspace }}/go
-    defaults:
-      run:
-        working-directory: ${{ env.GOPATH }}/src/github.com/tensorchord/envd
     steps:
       - name: Check out code
         uses: actions/checkout@v2
-        with:
-          path: ${{ env.GOPATH }}/src/github.com/tensorchord/envd
       - name: Setup Go
         uses: actions/setup-go@v2
         with:
@@ -97,23 +90,16 @@ jobs:
         uses: actions/upload-artifact@v3
         with:
           name: coverage-out
-          path: ${{ env.GOPATH }}/src/github.com/tensorchord/envd/coverage.out
+          path: coverage.out
   e2e:
     name: e2e
     strategy:
       matrix:
         os: [ubuntu-latest]
     runs-on: ${{ matrix.os }}
-    env:
-      GOPATH: ${{ github.workspace }}/go
-    defaults:
-      run:
-        working-directory: ${{ env.GOPATH }}/src/github.com/tensorchord/envd
     steps:
       - name: Check out code
         uses: actions/checkout@v2
-        with:
-          path: ${{ env.GOPATH }}/src/github.com/tensorchord/envd
       - name: Setup Go
         uses: actions/setup-go@v2
         with:
@@ -127,29 +113,26 @@ jobs:
             ${{ runner.OS }}-build-${{ env.cache-name }}-
             ${{ runner.OS }}-build-
             ${{ runner.OS }}-
+      - uses: actions-ecosystem/action-get-latest-tag@v1
+        id: get-latest-tag
       - name: e2e test
         run: make e2e-test
+        env:
+          GIT_LATEST_TAG: ${{ steps.get-latest-tag.outputs.tag }}
       - name: Upload coverage report
         uses: actions/upload-artifact@v3
         with:
           name: e2e-coverage-out
-          path: ${{ env.GOPATH }}/src/github.com/tensorchord/envd/e2e-coverage.out
+          path: e2e-coverage.out
   build:
     name: build
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest]
     runs-on: ${{ matrix.os }}
-    env:
-      GOPATH: ${{ github.workspace }}/go
-    defaults:
-      run:
-        working-directory: ${{ env.GOPATH }}/src/github.com/tensorchord/envd
     steps:
       - name: Check out code
         uses: actions/checkout@v2
-        with:
-          path: ${{ env.GOPATH }}/src/github.com/tensorchord/envd
       - name: Setup Go
         uses: actions/setup-go@v2
         with:
@@ -171,17 +154,10 @@ jobs:
     needs:
       - test
       - e2e
-    env:
-      GOPATH: ${{ github.workspace }}/go
     runs-on: ubuntu-latest
-    defaults:
-      run:
-        working-directory: ${{ env.GOPATH }}/src/github.com/tensorchord/envd
     steps:
       - name: Check out code
         uses: actions/checkout@v2
-        with:
-          path: ${{ env.GOPATH }}/src/github.com/tensorchord/envd
       - name: Setup Go
         uses: actions/setup-go@v2
         with:
@@ -194,12 +170,12 @@ jobs:
         uses: actions/download-artifact@v3
         with:
           name: coverage-out
-          path: ${{ env.GOPATH }}/src/github.com/tensorchord/envd
+          path: coverage.out
       - name: Get coverage report
         uses: actions/download-artifact@v3
         with:
           name: e2e-coverage-out
-          path: ${{ env.GOPATH }}/src/github.com/tensorchord/envd
+          path: e2e-coverage.out
       # - name: Send coverage
       #   env:
       #     COVERALLS_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/release.yml

@@ -36,7 +36,7 @@ jobs:
           version: latest
           args: release --rm-dist
         env:
-          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}    
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - name: upload gobin
         uses: actions/upload-artifact@v3
         with:
@@ -55,13 +55,13 @@ jobs:
     strategy:
       matrix:
         os: [macos-10.15, ubuntu-20.04]
-    steps:    
+    steps:
     - uses: actions/checkout@v3
     - name: Get gobin
       uses: actions/download-artifact@v3
       with:
         name: gobin_${{ github.event.release.tag_name }}
-        path: dist/    
+        path: dist/
     - name: Configure linux build environment
       if: runner.os == 'Linux'
       run: |
@@ -78,12 +78,12 @@ jobs:
       uses: pypa/cibuildwheel@v2.8.1
       env:
         CIBW_ARCHS: auto64
-    - name: Build source distribution    
+    - name: Build source distribution
       if: runner.os == 'Linux' # Only release source under linux to avoid conflict
       run: |
         python3 setup.py sdist
         mv dist/*.tar.gz wheelhouse/
-    - name: Upload to PyPI    
+    - name: Upload to PyPI
       env:
         TWINE_USERNAME: __token__
         TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
@@ -93,7 +93,8 @@ jobs:
         python -m twine upload wheelhouse/*
   image_publish:
     name: Build & push images
-    if: github.repository == 'tensorchord/envd'
+    # only trigger on main repo when tag starts with v
+    if: github.repository == 'tensorchord/envd' && startsWith(github.ref, 'refs/tags/v')
     runs-on: ubuntu-latest
     needs: goreleaser
     steps:
@@ -115,3 +116,38 @@ jobs:
       run: |
           docker login --username "${DOCKERIO_USERNAME}" --password "${DOCKERIO_PASSWORD}"
           ./base-images/build.sh
+  cache_publish:
+    name: Build & Push the remote cache
+    # only trigger on main repo when tag starts with v
+    if: github.repository == 'tensorchord/envd' && startsWith(github.ref, 'refs/tags/v')
+    runs-on: ubuntu-latest
+    needs: goreleaser
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v2
+    - name: Cache Docker layers
+      uses: actions/cache@v3
+      id: cache
+      with:
+        path: /tmp/.buildx-cache
+        key: ${{ runner.os }}-buildx-${{ github.sha }}
+        restore-keys: |
+          ${{ runner.os }}-buildx-
+    - name: Get gobin
+      uses: actions/download-artifact@v3
+      with:
+        name: gobin_${{ github.event.release.tag_name }}
+        path: dist/
+    - name: Configure linux build environment
+      if: runner.os == 'Linux'
+      run: |
+        mv dist/envd_linux_amd64_v1/envd /usr/local/bin/envd
+        chmod +x /usr/local/bin/envd
+    - name: Build and push
+      env:
+        DOCKERIO_USERNAME: ${{ secrets.DOCKERIO_USERNAME }}
+        DOCKERIO_PASSWORD: ${{ secrets.DOCKERIO_PASSWORD }}
+      run: |
+          docker login --username "${DOCKERIO_USERNAME}" --password "${DOCKERIO_PASSWORD}"
+          ./base-images/remote-cache/build-and-push-remote-cache.sh

.goreleaser.yaml

@@ -18,6 +18,7 @@ builds:
       - -X github.com/tensorchord/envd/pkg/version.buildDate={{ .Date }}
       - -X github.com/tensorchord/envd/pkg/version.gitCommit={{ .Commit }}
       - -X github.com/tensorchord/envd/pkg/version.gitTreeState=clean
+      - -X github.com/tensorchord/envd/pkg/version.gitTag={{ .Tag }}
   - env:
       - CGO_ENABLED=0
     goos:

Makefile

@@ -74,6 +74,7 @@ GIT_COMMIT=$(shell git rev-parse HEAD)
 GIT_TAG=$(shell if [ -z "`git status --porcelain`" ]; then git describe --exact-match --tags HEAD 2>/dev/null; fi)
 GIT_TREE_STATE=$(shell if [ -z "`git status --porcelain`" ]; then echo "clean" ; else echo "dirty"; fi)
 GITSHA ?= $(shell git rev-parse --short HEAD)
+GIT_LATEST_TAG ?= $(shell git describe --tags --abbrev=0)
 
 # Track code version with Docker Label.
 DOCKER_LABELS ?= git-describe="$(shell date -u +v%Y%m%d)-$(shell git describe --tags --always --dirty)"
@@ -95,11 +96,20 @@ export GOFLAGS ?= -count=1
 #
 
 # All targets.
-.PHONY: help lint test build dev container push addlicense debug debug-local build-local generate clean test-local addlicense-install mockgen-install pypi-build
+.PHONY: help lint test build dev container push addlicense debug debug-local build-local generate clean test-local addlicense-install mockgen-install pypi-build base-image
 
-.DEFAULT_GOAL:=build
+.DEFAULT_GOAL:=build-local
 
-build: build-local  ## Build the release version of envd
+build-release:
+	@for target in $(TARGETS); do                                                      \
+	  CGO_ENABLED=$(CGO_ENABLED) go build -trimpath -v -o $(OUTPUT_DIR)/$${target}     \
+	    -ldflags "-s -w -X $(ROOT)/pkg/version.version=$(VERSION) \
+		-X $(ROOT)/pkg/version.buildDate=$(BUILD_DATE) \
+		-X $(ROOT)/pkg/version.gitCommit=$(GIT_COMMIT) \
+		-X $(ROOT)/pkg/version.gitTreeState=$(GIT_TREE_STATE)                     \
+		-X $(ROOT)/pkg/version.gitTag=$(GIT_TAG)" \
+	    $(CMD_DIR)/$${target};                                                         \
+	done
 
 help:  ## Display this help
 	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z0-9_-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
@@ -122,7 +132,12 @@ addlicense-install:
 build-local:
 	@for target in $(TARGETS); do                                                      \
 	  CGO_ENABLED=$(CGO_ENABLED) go build -trimpath -v -o $(OUTPUT_DIR)/$${target}     \
-	    -ldflags "-s -w -X $(ROOT)/pkg/version.version=$(VERSION) -X $(ROOT)/pkg/version.buildDate=$(BUILD_DATE) -X $(ROOT)/pkg/version.gitCommit=$(GIT_COMMIT) -X $(ROOT)/pkg/version.gitTreeState=$(GIT_TREE_STATE)"                     \
+	    -ldflags "-s -w -X $(ROOT)/pkg/version.version=$(VERSION) \
+		-X $(ROOT)/pkg/version.buildDate=$(BUILD_DATE) \
+		-X $(ROOT)/pkg/version.gitCommit=$(GIT_COMMIT) \
+		-X $(ROOT)/pkg/version.gitTreeState=$(GIT_TREE_STATE)                     \
+		-X $(ROOT)/pkg/version.gitTag=$(GIT_LATEST_TAG) \
+		-X $(ROOT)/pkg/version.developmentFlag=true" \
 	    $(CMD_DIR)/$${target};                                                         \
 	done
 
@@ -156,8 +171,14 @@ test: generate  ## Run the tests
 	@go test -race -coverpkg=./pkg/... -coverprofile=coverage.out $(shell go list ./... | grep -v e2e)
 	@go tool cover -func coverage.out | tail -n 1 | awk '{ print "Total coverage: " $$3 }'
 
-e2e-test: generate
-	@go test -race -coverpkg=./pkg/app -coverprofile=e2e-coverage.out ./e2e
+e2e-test:
+	@go test -ldflags "-s -w -X $(ROOT)/pkg/version.version=$(VERSION) \
+		-X $(ROOT)/pkg/version.buildDate=$(BUILD_DATE) \
+		-X $(ROOT)/pkg/version.gitCommit=$(GIT_COMMIT) \
+		-X $(ROOT)/pkg/version.gitTreeState=$(GIT_TREE_STATE)                     \
+		-X $(ROOT)/pkg/version.gitTag="$(shell git describe --tags --abbrev=0)" \
+		-X $(ROOT)/pkg/version.developmentFlag=true" \
+		-race -v -coverpkg=./pkg/app -coverprofile=e2e-coverage.out ./e2e
 
 clean:  ## Clean the outputs and artifacts
 	@-rm -vrf ${OUTPUT_DIR}

base-images/build.sh

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+set -euo pipefail
+
 ROOT_DIR=`dirname $0`
 
 GIT_TAG_VERSION=$(git describe --tags --abbrev=0 | sed -r 's/[v]+//g') # remove v from version
@@ -26,15 +28,15 @@ docker buildx build \
     --build-arg HTTP_PROXY=${HTTP_PROXY} \
     --build-arg HTTPS_PROXY=${HTTPS_PROXY} \
     --pull --push --platform linux/x86_64,linux/arm64 \
-    -t ${DOCKER_HUB_ORG}/python:${PYTHON_VERSION}-${ENVD_OS} \
+    -t ${DOCKER_HUB_ORG}/python:${PYTHON_VERSION}-${ENVD_OS}-envd-v${ENVD_VERSION} \
     -f python${PYTHON_VERSION}-${ENVD_OS}.Dockerfile .
 docker buildx build --build-arg IMAGE_NAME=docker.io/nvidia/cuda \
     --build-arg ENVD_VERSION=${ENVD_VERSION} \
     --build-arg ENVD_SSH_IMAGE=ghcr.io/tensorchord/envd-ssh-from-scratch \
     --build-arg HTTP_PROXY=${HTTP_PROXY} \
     --build-arg HTTPS_PROXY=${HTTPS_PROXY} \
     --pull --push --platform linux/x86_64,linux/arm64 \
-    -t ${DOCKER_HUB_ORG}/python:${PYTHON_VERSION}-${ENVD_OS}-cuda11.6-cudnn8 \
+    -t ${DOCKER_HUB_ORG}/python:${PYTHON_VERSION}-${ENVD_OS}-cuda11.6-cudnn8-envd-v${ENVD_VERSION} \
     -f python${PYTHON_VERSION}-${ENVD_OS}-cuda11.6.Dockerfile .
 
 # TODO(gaocegege): Support linux/arm64
@@ -43,15 +45,15 @@ docker buildx build \
     --build-arg ENVD_SSH_IMAGE=ghcr.io/tensorchord/envd-ssh-from-scratch \
     --build-arg HTTP_PROXY=${HTTP_PROXY} \
     --build-arg HTTPS_PROXY=${HTTPS_PROXY} \
-    -t ${DOCKER_HUB_ORG}/r-base:${RLANG_VERSION} \
+    -t ${DOCKER_HUB_ORG}/r-base:${RLANG_VERSION}-envd-v${ENVD_VERSION} \
     --pull --push --platform linux/x86_64 \
     -f r${RLANG_VERSION}.Dockerfile .
 docker buildx build \
     --build-arg ENVD_VERSION=${ENVD_VERSION} \
     --build-arg ENVD_SSH_IMAGE=ghcr.io/tensorchord/envd-ssh-from-scratch \
     --build-arg HTTP_PROXY=${HTTP_PROXY} \
     --build-arg HTTPS_PROXY=${HTTPS_PROXY} \
-    -t ${DOCKER_HUB_ORG}/julia:${JULIA_VERSION}-${ENVD_OS} \
+    -t ${DOCKER_HUB_ORG}/julia:${JULIA_VERSION}-${ENVD_OS}-envd-v${ENVD_VERSION} \
     --pull --push --platform linux/x86_64,linux/arm64 \
     -f julia${JULIA_VERSION}-${ENVD_OS}.Dockerfile .
 cd - > /dev/null

base-images/python3.9-ubuntu20.04-cuda11.6.Dockerfile

@@ -88,35 +88,4 @@ RUN apt-mark hold ${NV_LIBCUBLAS_DEV_PACKAGE_NAME} ${NV_LIBNCCL_DEV_PACKAGE_NAME
 
 ENV LIBRARY_PATH /usr/local/cuda/lib64/stubs
 
-# Leave these args here to better use the Docker build cache
-ARG CONDA_VERSION=py39_4.11.0
-
-RUN set -x && \
-    UNAME_M="$(uname -m)" && \
-    if [ "${UNAME_M}" = "x86_64" ]; then \
-        MINICONDA_URL="https://repo.anaconda.com/miniconda/Miniconda3-${CONDA_VERSION}-Linux-x86_64.sh"; \
-        SHA256SUM="4ee9c3aa53329cd7a63b49877c0babb49b19b7e5af29807b793a76bdb1d362b4"; \
-    elif [ "${UNAME_M}" = "s390x" ]; then \
-        MINICONDA_URL="https://repo.anaconda.com/miniconda/Miniconda3-${CONDA_VERSION}-Linux-s390x.sh"; \
-        SHA256SUM="e5e5e89cdcef9332fe632cd25d318cf71f681eef029a24495c713b18e66a8018"; \
-    elif [ "${UNAME_M}" = "aarch64" ]; then \
-        MINICONDA_URL="https://repo.anaconda.com/miniconda/Miniconda3-${CONDA_VERSION}-Linux-aarch64.sh"; \
-        SHA256SUM="00c7127a8a8d3f4b9c2ab3391c661239d5b9a88eafe895fd0f3f2a8d9c0f4556"; \
-    elif [ "${UNAME_M}" = "ppc64le" ]; then \
-        MINICONDA_URL="https://repo.anaconda.com/miniconda/Miniconda3-${CONDA_VERSION}-Linux-ppc64le.sh"; \
-        SHA256SUM="8ee1f8d17ef7c8cb08a85f7d858b1cb55866c06fcf7545b98c3b82e4d0277e66"; \
-    fi && \
-    wget "${MINICONDA_URL}" -O miniconda.sh -q && \
-    echo "${SHA256SUM} miniconda.sh" > shasum && \
-    if [ "${CONDA_VERSION}" != "latest" ]; then sha256sum --check --status shasum; fi && \
-    mkdir -p /opt && \
-    sh miniconda.sh -b -p /opt/conda && \
-    rm miniconda.sh shasum && \
-    ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \
-    echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \
-    echo "conda activate base" >> ~/.bashrc && \
-    find /opt/conda/ -follow -type f -name '*.a' -delete && \
-    find /opt/conda/ -follow -type f -name '*.js.map' -delete && \
-    /opt/conda/bin/conda clean -afy
-
 COPY --from=envd /usr/bin/envd-ssh /var/envd/bin/envd-ssh

base-images/python3.9-ubuntu20.04.Dockerfile

@@ -32,35 +32,4 @@ RUN apt-get update && \
     # prompt
     && curl --proto '=https' --tlsv1.2 -sSf https://starship.rs/install.sh | sh -s -- -y
 
-# Leave these args here to better use the Docker build cache
-ARG CONDA_VERSION=py39_4.11.0
-
-RUN set -x && \
-    UNAME_M="$(uname -m)" && \
-    if [ "${UNAME_M}" = "x86_64" ]; then \
-        MINICONDA_URL="https://repo.anaconda.com/miniconda/Miniconda3-${CONDA_VERSION}-Linux-x86_64.sh"; \
-        SHA256SUM="4ee9c3aa53329cd7a63b49877c0babb49b19b7e5af29807b793a76bdb1d362b4"; \
-    elif [ "${UNAME_M}" = "s390x" ]; then \
-        MINICONDA_URL="https://repo.anaconda.com/miniconda/Miniconda3-${CONDA_VERSION}-Linux-s390x.sh"; \
-        SHA256SUM="e5e5e89cdcef9332fe632cd25d318cf71f681eef029a24495c713b18e66a8018"; \
-    elif [ "${UNAME_M}" = "aarch64" ]; then \
-        MINICONDA_URL="https://repo.anaconda.com/miniconda/Miniconda3-${CONDA_VERSION}-Linux-aarch64.sh"; \
-        SHA256SUM="00c7127a8a8d3f4b9c2ab3391c661239d5b9a88eafe895fd0f3f2a8d9c0f4556"; \
-    elif [ "${UNAME_M}" = "ppc64le" ]; then \
-        MINICONDA_URL="https://repo.anaconda.com/miniconda/Miniconda3-${CONDA_VERSION}-Linux-ppc64le.sh"; \
-        SHA256SUM="8ee1f8d17ef7c8cb08a85f7d858b1cb55866c06fcf7545b98c3b82e4d0277e66"; \
-    fi && \
-    wget "${MINICONDA_URL}" -O miniconda.sh -q && \
-    echo "${SHA256SUM} miniconda.sh" > shasum && \
-    if [ "${CONDA_VERSION}" != "latest" ]; then sha256sum --check --status shasum; fi && \
-    mkdir -p /opt && \
-    sh miniconda.sh -b -p /opt/conda && \
-    rm miniconda.sh shasum && \
-    ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \
-    echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \
-    echo "conda activate base" >> ~/.bashrc && \
-    find /opt/conda/ -follow -type f -name '*.a' -delete && \
-    find /opt/conda/ -follow -type f -name '*.js.map' -delete && \
-    /opt/conda/bin/conda clean -afy
-
 COPY --from=envd /usr/bin/envd-ssh /var/envd/bin/envd-ssh