From b70402917edf874a0a8e630664b637fa8c22cd53 Mon Sep 17 00:00:00 2001 From: Keming Date: Wed, 28 Sep 2022 18:54:36 +0800 Subject: [PATCH] feat(ir): all in llb (#941) * change python dockerfile to llb Signed-off-by: Keming * change to docker hub Signed-off-by: Keming * copy envd-sshd Signed-off-by: Keming * add llb log Signed-off-by: Keming * delete outdated comments Signed-off-by: Keming * cuda version Signed-off-by: Keming * del python dockerfile Signed-off-by: Keming * fix test Signed-off-by: Keming Signed-off-by: Keming --- .github/workflows/release.yml | 23 +++++---- .goreleaser.yaml | 10 ++-- base-images/build.sh | 15 +----- .../python3.9-ubuntu20.04-cuda11.2.Dockerfile | 35 ------------- base-images/python3.9-ubuntu20.04.Dockerfile | 35 ------------- .../build-and-push-remote-cache.sh | 1 + base-images/remote-cache/build.envd | 4 ++ pkg/builder/builder.go | 2 +- pkg/lang/ir/compile.go | 2 +- pkg/lang/ir/interface.go | 1 + pkg/lang/ir/shell.go | 2 +- pkg/lang/ir/system.go | 50 ++++++++++++++++--- pkg/types/envd.go | 43 ++++++++++++++++ 13 files changed, 112 insertions(+), 111 deletions(-) delete mode 100644 base-images/python3.9-ubuntu20.04-cuda11.2.Dockerfile delete mode 100644 base-images/python3.9-ubuntu20.04.Dockerfile diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0f4f72c80..9ad945fac 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -26,9 +26,8 @@ jobs: - name: Docker Login uses: docker/login-action@v2 with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + username: ${{ secrets.DOCKERIO_USERNAME }} + password: ${{ secrets.DOCKERIO_TOKEN }} - name: Run GoReleaser uses: goreleaser/goreleaser-action@v3 with: @@ -109,12 +108,13 @@ jobs: key: ${{ runner.os }}-buildx-${{ github.sha }} restore-keys: | ${{ runner.os }}-buildx- + - name: Docker Login + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERIO_USERNAME }} + password: ${{ secrets.DOCKERIO_TOKEN }} - name: Docker Buildx - env: - DOCKERIO_USERNAME: ${{ secrets.DOCKERIO_USERNAME }} - DOCKERIO_PASSWORD: ${{ secrets.DOCKERIO_PASSWORD }} run: | - docker login --username "${DOCKERIO_USERNAME}" --password "${DOCKERIO_PASSWORD}" ./base-images/build.sh DOCKER_IMAGE_TAG=latest ./base-images/build.sh cache_publish: @@ -145,10 +145,11 @@ jobs: run: | mv dist/envd_linux_amd64_v1/envd /usr/local/bin/envd chmod +x /usr/local/bin/envd + - name: Docker Login + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERIO_USERNAME }} + password: ${{ secrets.DOCKERIO_TOKEN }} - name: Build and push - env: - DOCKERIO_USERNAME: ${{ secrets.DOCKERIO_USERNAME }} - DOCKERIO_PASSWORD: ${{ secrets.DOCKERIO_PASSWORD }} run: | - docker login --username "${DOCKERIO_USERNAME}" --password "${DOCKERIO_PASSWORD}" ./base-images/remote-cache/build-and-push-remote-cache.sh diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 4a6f5b56e..53efaadd5 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -69,7 +69,7 @@ changelog: - '^chore:' dockers: - image_templates: - - "ghcr.io/tensorchord/envd-sshd-from-scratch:{{ .Version }}-amd64" + - "tensorchord/envd-sshd-from-scratch:v{{ .Version }}-amd64" use: buildx dockerfile: .goreleaser/envd-sshd.Dockerfile ids: @@ -77,7 +77,7 @@ dockers: build_flag_templates: - "--platform=linux/amd64" - image_templates: - - "ghcr.io/tensorchord/envd-sshd-from-scratch:{{ .Version }}-arm64v8" + - "tensorchord/envd-sshd-from-scratch:v{{ .Version }}-arm64v8" use: buildx goarch: arm64 ids: @@ -86,10 +86,10 @@ dockers: build_flag_templates: - "--platform=linux/arm64/v8" docker_manifests: -- name_template: ghcr.io/tensorchord/envd-sshd-from-scratch:{{ .Version }} +- name_template: tensorchord/envd-sshd-from-scratch:v{{ .Version }} image_templates: - - ghcr.io/tensorchord/envd-sshd-from-scratch:{{ .Version }}-amd64 - - ghcr.io/tensorchord/envd-sshd-from-scratch:{{ .Version }}-arm64v8 + - tensorchord/envd-sshd-from-scratch:v{{ .Version }}-amd64 + - tensorchord/envd-sshd-from-scratch:v{{ .Version }}-arm64v8 # See https://github.com/tensorchord/envd/issues/908 # brews: # - name: envd diff --git a/base-images/build.sh b/base-images/build.sh index fb00594cb..503b1ccd8 100755 --- a/base-images/build.sh +++ b/base-images/build.sh @@ -16,25 +16,12 @@ RLANG_VERSION="${RLANG_VERSION:-4.2}" cd ${ROOT_DIR} # ubuntu 22.04 build require moby/buildkit version greater than 0.8.1 if ! docker buildx inspect cuda; then - docker buildx create --use --platform linux/x86_64,linux/arm64,linux/ppc64le --driver-opt image=moby/buildkit:v0.10.3 --name cuda --node cuda + docker buildx create --use --platform linux/x86_64,linux/arm64,linux/ppc64le --driver-opt image=moby/buildkit:v0.10.3 fi # https://github.com/docker/buildx/issues/495#issuecomment-754688157 docker run --rm --privileged multiarch/qemu-user-static --reset -p yes -docker buildx build \ - --build-arg ENVD_VERSION=${ENVD_VERSION} \ - --build-arg ENVD_SSH_IMAGE=ghcr.io/tensorchord/envd-sshd-from-scratch \ - --pull --push --platform linux/x86_64,linux/arm64 \ - -t ${DOCKER_HUB_ORG}/python:${PYTHON_VERSION}-${ENVD_OS}-envd-${DOCKER_IMAGE_TAG} \ - -f python${PYTHON_VERSION}-${ENVD_OS}.Dockerfile . -docker buildx build --build-arg IMAGE_NAME=docker.io/nvidia/cuda \ - --build-arg ENVD_VERSION=${ENVD_VERSION} \ - --build-arg ENVD_SSH_IMAGE=ghcr.io/tensorchord/envd-sshd-from-scratch \ - --pull --push --platform linux/x86_64,linux/arm64 \ - -t ${DOCKER_HUB_ORG}/python:${PYTHON_VERSION}-${ENVD_OS}-cuda11.2-cudnn8-envd-${DOCKER_IMAGE_TAG} \ - -f python${PYTHON_VERSION}-${ENVD_OS}-cuda11.2.Dockerfile . - # TODO(gaocegege): Support linux/arm64 docker buildx build \ --build-arg ENVD_VERSION=${ENVD_VERSION} \ diff --git a/base-images/python3.9-ubuntu20.04-cuda11.2.Dockerfile b/base-images/python3.9-ubuntu20.04-cuda11.2.Dockerfile deleted file mode 100644 index e9a1920c0..000000000 --- a/base-images/python3.9-ubuntu20.04-cuda11.2.Dockerfile +++ /dev/null @@ -1,35 +0,0 @@ -ARG IMAGE_NAME -ARG ENVD_VERSION -ARG ENVD_SSH_IMAGE -FROM ${IMAGE_NAME}:11.6.1-cudnn8-devel-ubuntu20.04 as base - -FROM ${ENVD_SSH_IMAGE}:${ENVD_VERSION} AS envd - -FROM base as base-amd64 - -FROM base as base-arm64 - -FROM base-${TARGETARCH} - -ARG TARGETARCH - -LABEL maintainer "envd-maintainers " - -ENV DEBIAN_FRONTEND noninteractive -ENV LANG C.UTF-8 -ENV LC_ALL C.UTF-8 - -RUN apt-get update && \ - apt-get install -y apt-utils && \ - apt-get install -y --no-install-recommends --no-install-suggests --fix-missing \ - bash-static libtinfo5 libncursesw5 \ - # conda dependencies - bzip2 ca-certificates libglib2.0-0 libsm6 libxext6 libxrender1 mercurial \ - procps subversion wget \ - # envd dependencies - curl openssh-client git tini sudo zsh vim \ - && rm -rf /var/lib/apt/lists/* \ - # prompt - && curl --proto '=https' --tlsv1.2 -sSf https://starship.rs/install.sh | sh -s -- -y - -COPY --from=envd /usr/bin/envd-sshd /var/envd/bin/envd-sshd diff --git a/base-images/python3.9-ubuntu20.04.Dockerfile b/base-images/python3.9-ubuntu20.04.Dockerfile deleted file mode 100644 index 46f7df522..000000000 --- a/base-images/python3.9-ubuntu20.04.Dockerfile +++ /dev/null @@ -1,35 +0,0 @@ -ARG ENVD_VERSION -ARG ENVD_SSH_IMAGE -FROM ubuntu:20.04 as base - -FROM base as base-amd64 - -FROM base as base-arm64 - -FROM ${ENVD_SSH_IMAGE}:${ENVD_VERSION} AS envd - -FROM base-${TARGETARCH} - -ARG TARGETARCH - -LABEL maintainer "envd-maintainers " - -ENV DEBIAN_FRONTEND noninteractive -ENV PATH="/usr/bin:${PATH}" -ENV LANG C.UTF-8 -ENV LC_ALL C.UTF-8 - -RUN apt-get update && \ - apt-get install -y apt-utils && \ - apt-get install -y --no-install-recommends --no-install-suggests --fix-missing \ - bash-static libtinfo5 libncursesw5 \ - # conda dependencies - bzip2 ca-certificates libglib2.0-0 libsm6 libxext6 libxrender1 mercurial \ - procps subversion wget \ - # envd dependencies - curl openssh-client git tini sudo zsh vim \ - && rm -rf /var/lib/apt/lists/* \ - # prompt - && curl --proto '=https' --tlsv1.2 -sSf https://starship.rs/install.sh | sh -s -- -y - -COPY --from=envd /usr/bin/envd-sshd /var/envd/bin/envd-sshd diff --git a/base-images/remote-cache/build-and-push-remote-cache.sh b/base-images/remote-cache/build-and-push-remote-cache.sh index 347c4110a..57abc4d1f 100755 --- a/base-images/remote-cache/build-and-push-remote-cache.sh +++ b/base-images/remote-cache/build-and-push-remote-cache.sh @@ -11,5 +11,6 @@ DOCKER_HUB_ORG="${DOCKER_HUB_ORG:-tensorchord}" cd ${ROOT_DIR} envd build --export-cache type=registry,ref=docker.io/${DOCKER_HUB_ORG}/python-cache:envd-v${ENVD_VERSION} --force +envd build -f build.envd:build_gpu --export-cache type=registry,ref=docker.io/${DOCKER_HUB_ORG}/python-cache:envd-v${ENVD_VERSION} --force cd - > /dev/null diff --git a/base-images/remote-cache/build.envd b/base-images/remote-cache/build.envd index 3b22de80e..ca3d01651 100644 --- a/base-images/remote-cache/build.envd +++ b/base-images/remote-cache/build.envd @@ -1,2 +1,6 @@ def build(): base(os="ubuntu20.04", language="python3") + +def build_gpu(): + base(os="ubuntu20.04", language="python3") + install.cuda(version="11.2", cudnn="8") diff --git a/pkg/builder/builder.go b/pkg/builder/builder.go index eac5ce47b..aafad612c 100644 --- a/pkg/builder/builder.go +++ b/pkg/builder/builder.go @@ -399,7 +399,7 @@ func (b generalBuilder) checkDepsFileUpdate(ctx context.Context, tag string, man return true, err } modifiedtime := file.ModTime().Unix() - // Only needt o use the latest modified time + // Only need to use the latest modified time if modifiedtime > latestTimestamp { latestTimestamp = modifiedtime } diff --git a/pkg/lang/ir/compile.go b/pkg/lang/ir/compile.go index 691cb49dc..09a7f14e8 100644 --- a/pkg/lang/ir/compile.go +++ b/pkg/lang/ir/compile.go @@ -86,7 +86,7 @@ func Compile(ctx context.Context, envName string, pub string) (*llb.Definition, } state, err := DefaultGraph.Compile(uid, gid) if err != nil { - return nil, errors.Wrap(err, "failed to compile") + return nil, errors.Wrap(err, "failed to compile the graph") } // TODO(gaocegege): Support multi platform. def, err := state.Marshal(ctx, llb.LinuxAmd64) diff --git a/pkg/lang/ir/interface.go b/pkg/lang/ir/interface.go index 9045f14de..bc4c629f8 100644 --- a/pkg/lang/ir/interface.go +++ b/pkg/lang/ir/interface.go @@ -112,6 +112,7 @@ func JuliaPackageServer(url string) error { func Shell(shell string) error { DefaultGraph.Shell = shell + DefaultGraph.SystemPackages = append(DefaultGraph.SystemPackages, shell) return nil } diff --git a/pkg/lang/ir/shell.go b/pkg/lang/ir/shell.go index 6938b081e..4f309b942 100644 --- a/pkg/lang/ir/shell.go +++ b/pkg/lang/ir/shell.go @@ -91,7 +91,7 @@ func (g Graph) compileZSH(root llb.State) (llb.State, error) { File(llb.Mkfile(installPath, 0644, []byte(m.InstallScript()), llb.WithUIDGID(g.uid, g.gid))) zshrc := zshStage.Run(llb.Shlex(fmt.Sprintf("bash %s", installPath)), - llb.WithCustomName("install oh-my-zsh")). + llb.WithCustomName("[internal] install oh-my-zsh")). File(llb.Mkfile(zshrcPath, 0644, []byte(m.ZSHRC()), llb.WithUIDGID(g.uid, g.gid))) return zshrc, nil diff --git a/pkg/lang/ir/system.go b/pkg/lang/ir/system.go index d64bb1e62..73a6ad1cb 100644 --- a/pkg/lang/ir/system.go +++ b/pkg/lang/ir/system.go @@ -91,14 +91,15 @@ func (g Graph) compileCopy(root llb.State) llb.State { return result } -func (g *Graph) compileCUDAPackages(org, version string) llb.State { - return llb.Image(fmt.Sprintf( - "docker.io/%s/python:3.9-%s-cuda%s-cudnn%s-envd-%s", - org, g.OS, *g.CUDA, *g.CUDNN, version)) +func (g *Graph) compileCUDAPackages(org string) llb.State { + return g.preparePythonBase(llb.Image(fmt.Sprintf( + "docker.io/%s/%s-cudnn%s-devel-%s", + org, *g.CUDA, *g.CUDNN, g.OS))) } func (g Graph) compileSystemPackages(root llb.State) llb.State { if len(g.SystemPackages) == 0 { + logrus.Debug("skip the apt since system package is not specified") return root } @@ -143,6 +144,39 @@ func (g *Graph) compileExtraSource(root llb.State) (llb.State, error) { return llb.Merge(inputs, llb.WithCustomName("[internal] build source layers")), nil } +func (g *Graph) preparePythonBase(root llb.State) llb.State { + for _, env := range types.BaseEnvironment { + root = root.AddEnv(env.Name, env.Value) + } + + // envd-sshd + sshd := root.File(llb.Copy( + llb.Image(types.EnvdSshdImage), "/usr/bin/envd-sshd", "/var/envd/bin/envd-sshd", + &llb.CopyInfo{CreateDestPath: true}), + llb.WithCustomName(fmt.Sprintf("[internal] add envd-sshd from %s", types.EnvdSshdImage))) + + // apt packages + var sb strings.Builder + sb.WriteString("apt-get update && apt-get install -y apt-utils && ") + sb.WriteString("apt-get install -y --no-install-recommends --no-install-suggests --fix-missing ") + sb.WriteString(strings.Join(types.BaseAptPackage, " ")) + sb.WriteString("&& rm -rf /var/lib/apt/lists/* ") + // shell prompt + sb.WriteString("&& curl --proto '=https' --tlsv1.2 -sSf https://starship.rs/install.sh | sh -s -- -y") + + cacheDir := "/var/cache/apt" + cacheLibDir := "/var/lib/apt" + + run := sshd.Run(llb.Shlex(fmt.Sprintf("bash -c \"%s\"", sb.String())), + llb.WithCustomName("[internal] install system packages")) + run.AddMount(cacheDir, llb.Scratch(), + llb.AsPersistentCacheDir(g.CacheID(cacheDir), llb.CacheMountShared)) + run.AddMount(cacheLibDir, llb.Scratch(), + llb.AsPersistentCacheDir(g.CacheID(cacheLibDir), llb.CacheMountShared)) + + return run.Root() +} + func (g *Graph) compileBase() (llb.State, error) { logger := logrus.WithFields(logrus.Fields{ "os": g.OS, @@ -173,16 +207,15 @@ func (g *Graph) compileBase() (llb.State, error) { g.uid = 1001 } case "python": - base = llb.Image(fmt.Sprintf( - "docker.io/%s/python:3.9-ubuntu20.04-envd-%s", org, v)) + // TODO(keming) use user input `base(os="")` + base = g.preparePythonBase(llb.Image(types.PythonBaseImage)) case "julia": base = llb.Image(fmt.Sprintf( "docker.io/%s/julia:1.8rc1-ubuntu20.04-envd-%s", org, v)) } } else { - base = g.compileCUDAPackages(org, v) + base = g.compileCUDAPackages("nvidia/cuda") } - var res llb.ExecState // Install conda first. condaStage, err := g.installConda(base) @@ -191,6 +224,7 @@ func (g *Graph) compileBase() (llb.State, error) { } // TODO(gaocegege): Refactor user to a separate stage. + var res llb.ExecState if g.uid == 0 { res = condaStage. Run(llb.Shlex(fmt.Sprintf("groupadd -g %d envd", 1001)), diff --git a/pkg/types/envd.go b/pkg/types/envd.go index 95d6ee5d1..c9f73437a 100644 --- a/pkg/types/envd.go +++ b/pkg/types/envd.go @@ -16,9 +16,12 @@ package types import ( "encoding/json" + "fmt" "github.com/docker/docker/api/types" "github.com/moby/buildkit/util/system" + + "github.com/tensorchord/envd/pkg/version" ) // DefaultPathEnvUnix is unix style list of directories to search for @@ -31,6 +34,46 @@ const DefaultPathEnvUnix = "/opt/conda/envs/envd/bin:/opt/conda/bin:/usr/local/j // ';' character . const DefaultPathEnvWindows = system.DefaultPathEnvWindows +const PythonBaseImage = "ubuntu:20.04" + +var EnvdSshdImage = fmt.Sprintf( + "tensorchord/envd-sshd-from-scratch:%s", + version.GetVersionForImageTag()) + +var BaseEnvironment = []struct { + Name string + Value string +}{ + {"DEBIAN_FRONTEND", "noninteractive"}, + {"PATH", DefaultPathEnvUnix}, + {"LANG", "C.UTF-8"}, + {"LC_ALL", "C.UTF-8"}, +} +var BaseAptPackage = []string{ + "bash-static", + "libtinfo5", + "libncursesw5", + // conda dependencies + "bzip2", + "ca-certificates", + "libglib2.0-0", + "libsm6", + "libxext6", + "libxrender1", + "mercurial", + "procps", + "subversion", + "wget", + // envd dependencies + "curl", + "openssh-client", + "git", + "tini", + "sudo", + "vim", + "zsh", +} + type EnvdImage struct { types.ImageSummary