feat: Support HTTP PROXY (#857)

feat: Add use proxy

Signed-off-by: Ce Gao <cegao@tensorchord.ai>

Signed-off-by: Ce Gao <cegao@tensorchord.ai>

base-images/build.sh

@@ -32,8 +32,8 @@ docker buildx build --build-arg IMAGE_NAME=docker.io/nvidia/cuda \
     --build-arg ENVD_VERSION=${ENVD_VERSION} \
     --build-arg ENVD_SSH_IMAGE=ghcr.io/tensorchord/envd-ssh-from-scratch \
     --pull --push --platform linux/x86_64,linux/arm64 \
-    -t ${DOCKER_HUB_ORG}/python:${PYTHON_VERSION}-${ENVD_OS}-cuda11.6-cudnn8-envd-${DOCKER_IMAGE_TAG} \
-    -f python${PYTHON_VERSION}-${ENVD_OS}-cuda11.6.Dockerfile .
+    -t ${DOCKER_HUB_ORG}/python:${PYTHON_VERSION}-${ENVD_OS}-cuda11.2-cudnn8-envd-${DOCKER_IMAGE_TAG} \
+    -f python${PYTHON_VERSION}-${ENVD_OS}-cuda11.2.Dockerfile .
 
 # TODO(gaocegege): Support linux/arm64
 docker buildx build \

base-images/python3.9-ubuntu20.04-cuda11.2.Dockerfile

@@ -0,0 +1,35 @@
+ARG IMAGE_NAME
+ARG ENVD_VERSION
+ARG ENVD_SSH_IMAGE
+FROM ${IMAGE_NAME}:11.6.1-cudnn8-devel-ubuntu20.04 as base
+
+FROM ${ENVD_SSH_IMAGE}:${ENVD_VERSION} AS envd
+
+FROM base as base-amd64
+
+FROM base as base-arm64
+
+FROM base-${TARGETARCH}
+
+ARG TARGETARCH
+
+LABEL maintainer "envd-maintainers <envd-maintainers@tensorchord.ai>"
+
+ENV DEBIAN_FRONTEND noninteractive
+ENV LANG C.UTF-8
+ENV LC_ALL C.UTF-8
+
+RUN apt-get update && \
+    apt-get install -y apt-utils && \
+    apt-get install -y --no-install-recommends --no-install-suggests --fix-missing \
+    bash-static libtinfo5 libncursesw5 \
+    # conda dependencies
+    bzip2 ca-certificates libglib2.0-0 libsm6 libxext6 libxrender1 mercurial \
+    procps subversion wget \
+    # envd dependencies
+    curl openssh-client git tini sudo zsh vim \
+    && rm -rf /var/lib/apt/lists/* \
+    # prompt
+    && curl --proto '=https' --tlsv1.2 -sSf https://starship.rs/install.sh | sh -s -- -y
+
+COPY --from=envd /usr/bin/envd-ssh /var/envd/bin/envd-ssh

pkg/app/build.go

@@ -53,6 +53,12 @@ To build and push the image to a registry:
 			Aliases: []string{"f"},
 			Value:   "build.envd:build",
 		},
+		&cli.BoolFlag{
+			Name:    "use-proxy",
+			Usage:   "Use HTTPS_PROXY/HTTP_PROXY/NO_PROXY in the build process",
+			Aliases: []string{"proxy"},
+			Value:   false,
+		},
 		&cli.PathFlag{
 			Name:    "path",
 			Usage:   "Path to the directory containing the build.envd",
@@ -173,6 +179,7 @@ func ParseBuildOpt(clicontext *cli.Context) (builder.Options, error) {
 	output := ""
 	exportCache := clicontext.String("export-cache")
 	importCache := clicontext.String("import-cache")
+	useProxy := clicontext.Bool("use-proxy")
 
 	opt := builder.Options{
 		ManifestFilePath: manifest,
@@ -185,6 +192,7 @@ func ParseBuildOpt(clicontext *cli.Context) (builder.Options, error) {
 		ProgressMode:     "auto",
 		ExportCache:      exportCache,
 		ImportCache:      importCache,
+		UseHTTPProxy:     useProxy,
 	}
 
 	debug := clicontext.Bool("debug")

pkg/app/up.go

@@ -63,6 +63,12 @@ var CommandUp = &cli.Command{
 			Aliases: []string{"f"},
 			Value:   "build.envd:build",
 		},
+		&cli.BoolFlag{
+			Name:    "use-proxy",
+			Usage:   "Use HTTPS_PROXY/HTTP_PROXY/NO_PROXY in the build process",
+			Aliases: []string{"proxy"},
+			Value:   false,
+		},
 		&cli.PathFlag{
 			Name:    "private-key",
 			Usage:   "Path to the private key",

pkg/builder/builder.go

@@ -68,6 +68,8 @@ type Options struct {
 	// ImportCache is the option to import cache.
 	// e.g. type=registry,ref=docker.io/username/image
 	ImportCache string
+	// UseHTTPProxy uses HTTPS_PROXY/HTTP_PROXY/NO_PROXY in the build process.
+	UseHTTPProxy bool
 }
 
 type generalBuilder struct {
@@ -269,6 +271,13 @@ func (b generalBuilder) build(ctx context.Context, pw progresswriter.Writer) err
 					},
 					Session: attachable,
 				}
+				if b.UseHTTPProxy {
+					solveOpt.FrontendAttrs = map[string]string{
+						"build-arg:HTTPS_PROXY": os.Getenv("HTTPS_PROXY"),
+						"build-arg:HTTP_PROXY":  os.Getenv("HTTP_PROXY"),
+						"build-arg:NO_PROXY":    os.Getenv("NO_PROXY"),
+					}
+				}
 				_, err := b.Client.Build(ctx, solveOpt, "envd", b.BuildFunc(), pw.Status())
 				if err != nil {
 					err = errors.Wrap(err, "failed to solve LLB")
@@ -304,6 +313,13 @@ func (b generalBuilder) build(ctx context.Context, pw progresswriter.Writer) err
 					},
 					Session: attachable,
 				}
+				if b.UseHTTPProxy {
+					solveOpt.FrontendAttrs = map[string]string{
+						"build-arg:HTTPS_PROXY": os.Getenv("HTTPS_PROXY"),
+						"build-arg:HTTP_PROXY":  os.Getenv("HTTP_PROXY"),
+						"build-arg:NO_PROXY":    os.Getenv("NO_PROXY"),
+					}
+				}
 				_, err := b.Client.Build(ctx, solveOpt, "envd", b.BuildFunc(), pw.Status())
 				if err != nil {
 					err = errors.Wrap(err, "failed to solve LLB")