diff --git a/deployment/Dockerfile b/deployment/Dockerfile index 5382c873..186497f6 100644 --- a/deployment/Dockerfile +++ b/deployment/Dockerfile @@ -1,5 +1,11 @@ # build-env ################################################### FROM golang:1.15.5-buster AS build-env + +RUN echo 'deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/Debian_10/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list +RUN curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/Debian_10/Release.key | apt-key add - +RUN apt-get -y update +RUN apt-get -y install skopeo + WORKDIR /app RUN apt-get update @@ -23,8 +29,9 @@ COPY pkg pkg RUN go install -ldflags="-X main.gitRef=${SOURCE_BRANCH} -X main.gitHash=${SOURCE_COMMIT}" -installsuffix cgo ./cmd/wedding ############################################################### -FROM alpine:3.12.1 -RUN apk --no-cache add ca-certificates +FROM alpine:3.12.1 AS prod +RUN apk add --no-cache ca-certificates +RUN apk add --no-cache -X http://dl-cdn.alpinelinux.org/alpine/edge/community skopeo WORKDIR /root/ -COPY --from=build-env /go/bin/wedding . +COPY --from=build-env /go/bin/wedding /usr/local/bin/wedding ENTRYPOINT [ "./wedding", "server" ] diff --git a/pkg/inspect.go b/pkg/inspect.go index c3b26d0b..be86dd09 100644 --- a/pkg/inspect.go +++ b/pkg/inspect.go @@ -5,69 +5,43 @@ import ( "fmt" "io" "log" + "math/rand" "net/http" - "strconv" - "time" "github.com/gorilla/mux" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func (s Service) inspect(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) image := fmt.Sprintf("wedding-registry:5000/images/%s", escapePort(vars["name"])) - buildScript := fmt.Sprintf(` + randomID := randStringBytes(16) + script := fmt.Sprintf(` set -euo pipefail -mkdir inspect-image -skopeo copy --quiet --retry-times 3 --src-tls-verify=false --dest-tls-verify=false docker://%s dir://inspect-image -skopeo inspect dir://inspect-image -`, image) - - pod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "wedding-inspect-", - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "skopeo", - Image: skopeoImage, - Command: []string{ - "timeout", - strconv.Itoa(int(MaxExecutionTime / time.Second)), - }, - Args: []string{ - "sh", - "-c", - buildScript, - }, - Resources: corev1.ResourceRequirements{ - Limits: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse(skopeoCPU), - corev1.ResourceMemory: resource.MustParse(skopeoMemory), - }, - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse(skopeoCPU), - corev1.ResourceMemory: resource.MustParse(skopeoMemory), - }, - }, - }, - }, - RestartPolicy: corev1.RestartPolicyNever, - }, - } +mkdir %s +skopeo copy --quiet --retry-times 3 --src-tls-verify=false docker://%s dir://%s +skopeo inspect dir://%s +rm -r %s +`, randomID, image, randomID, randomID, randomID) o := &bytes.Buffer{} - err := s.executePod(r.Context(), pod, o) + + scheduler := scheduleLocal + // scheduler = s.scheduleInKubernetes + + err := scheduler(r.Context(), o, "inspect", script, "") if err != nil { log.Printf("execute inspect: %v", err) w.WriteHeader(http.StatusNotFound) } - _, err = io.Copy(w, o) - if err != nil { - log.Printf("write inspect result: %v", err) + io.Copy(w, o) +} + +func randStringBytes(n int) string { + letterBytes := "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" + b := make([]byte, n) + for i := range b { + b[i] = letterBytes[rand.Intn(len(letterBytes))] } + return string(b) } diff --git a/pkg/kubernetes.go b/pkg/kubernetes.go index aefa4249..1b1ce5d7 100644 --- a/pkg/kubernetes.go +++ b/pkg/kubernetes.go @@ -5,14 +5,137 @@ import ( "encoding/json" "fmt" "io" + "io/ioutil" "log" "os" + "os/exec" + "path/filepath" + "strconv" "time" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +func scheduleLocal(ctx context.Context, w io.Writer, processName, script, dockerJSON string) error { + tmpHome, err := ioutil.TempDir("", "docker-secret") + if err != nil { + return fmt.Errorf("create tempdir for docker secret: %v", err) + } + defer os.RemoveAll(tmpHome) + + if dockerJSON != "" { + err = os.Mkdir(filepath.Join(tmpHome, ".docker"), os.ModePerm) + if err != nil { + return fmt.Errorf("create .docker directory for docker secret: %v", err) + } + + dockerConfigJSON := filepath.Join(tmpHome, ".docker", "config.json") + err = ioutil.WriteFile(dockerConfigJSON, []byte(dockerJSON), os.ModePerm) + if err != nil { + return fmt.Errorf("write docker secret: %v", err) + } + } + + cmd := exec.CommandContext( + ctx, + "timeout", + strconv.Itoa(int(MaxExecutionTime/time.Second)), + "bash", + "-c", + script, + ) + cmd.Stdout = w + cmd.Stderr = w + cmd.Env = append(cmd.Env, fmt.Sprintf("HOME=%s", tmpHome)) + + return cmd.Run() +} + +func (s Service) scheduleInKubernetes(ctx context.Context, w io.Writer, processName, script, dockerJSON string) error { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: fmt.Sprintf("wedding-%s-", processName), + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "skopeo", + Image: skopeoImage, + Command: []string{ + "timeout", + strconv.Itoa(int(MaxExecutionTime / time.Second)), + }, + Args: []string{ + "sh", + "-c", + script, + }, + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(skopeoCPU), + corev1.ResourceMemory: resource.MustParse(skopeoMemory), + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(skopeoCPU), + corev1.ResourceMemory: resource.MustParse(skopeoMemory), + }, + }, + }, + }, + RestartPolicy: corev1.RestartPolicyNever, + }, + } + + if dockerJSON != "" { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "wedding-docker-config-", + }, + StringData: map[string]string{ + "config.json": dockerJSON, + }, + } + + secretClient := s.kubernetesClient.CoreV1().Secrets(s.namespace) + + secret, err := secretClient.Create(ctx, secret, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("create docker.json secret: %v", err) + } + defer func() { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + err = secretClient.Delete(ctx, secret.Name, metav1.DeleteOptions{}) + if err != nil { + streamf(w, "Secret deletetion failed: %v\n", err) + log.Printf("delete secret: %v", err) + } + }() + + pod.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{ + { + MountPath: "/root/.docker", + Name: "docker-config", + }, + } + pod.Spec.Volumes = []corev1.Volume{ + { + Name: "docker-config", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: secret.Name, + }, + }, + }, + } + } + + return s.executePod(ctx, pod, w) +} + func (s Service) executePod(ctx context.Context, pod *corev1.Pod, w io.Writer) error { podClient := s.kubernetesClient.CoreV1().Pods(s.namespace) diff --git a/pkg/pull.go b/pkg/pull.go index 63bfc17e..68b845d8 100644 --- a/pkg/pull.go +++ b/pkg/pull.go @@ -1,16 +1,9 @@ package wedding import ( - "context" "fmt" "log" "net/http" - "strconv" - "time" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func (s Service) pullImage(w http.ResponseWriter, r *http.Request) { @@ -55,7 +48,6 @@ func (s Service) pullImage(w http.ResponseWriter, r *http.Request) { } from := fmt.Sprintf("%s:%s", fromImage, pullTag) - // to := fmt.Sprintf("wedding-registry:5000/images/%s", url.PathEscape(from)) to := fmt.Sprintf("wedding-registry:5000/images/%s", escapePort(from)) dockerCfg, err := xRegistryAuth(r.Header.Get("X-Registry-Auth")).toDockerConfig() @@ -66,95 +58,16 @@ func (s Service) pullImage(w http.ResponseWriter, r *http.Request) { return } - secret := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "wedding-docker-config-", - }, - StringData: map[string]string{ - "config.json": dockerCfg.mustToJSON(), - }, - } - - secretClient := s.kubernetesClient.CoreV1().Secrets(s.namespace) + script := fmt.Sprintf(`skopeo copy --retry-times 3 --dest-tls-verify=false docker://%s docker://%s`, from, to) - secret, err = secretClient.Create(r.Context(), secret, metav1.CreateOptions{}) - if err != nil { - w.WriteHeader(http.StatusInternalServerError) - streamf(w, "Secret creation failed: %v\n", err) - log.Printf("create secret: %v", err) - return - } - defer func() { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - err = secretClient.Delete(ctx, secret.Name, metav1.DeleteOptions{}) - if err != nil { - streamf(w, "Secret deletetion failed: %v\n", err) - log.Printf("delete secret: %v", err) - } - }() - - buildScript := fmt.Sprintf(` -set -euxo pipefail - -skopeo copy --retry-times 3 --dest-tls-verify=false docker://%s docker://%s -`, from, to) - - pod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "wedding-pull-", - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "skopeo", - Image: skopeoImage, - Command: []string{ - "timeout", - strconv.Itoa(int(MaxExecutionTime / time.Second)), - }, - Args: []string{ - "sh", - "-c", - buildScript, - }, - VolumeMounts: []corev1.VolumeMount{ - { - MountPath: "/root/.docker", - Name: "docker-config", - }, - }, - Resources: corev1.ResourceRequirements{ - Limits: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse(skopeoCPU), - corev1.ResourceMemory: resource.MustParse(skopeoMemory), - }, - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse(skopeoCPU), - corev1.ResourceMemory: resource.MustParse(skopeoMemory), - }, - }, - }, - }, - Volumes: []corev1.Volume{ - { - Name: "docker-config", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: secret.Name, - }, - }, - }, - }, - RestartPolicy: corev1.RestartPolicyNever, - }, - } + scheduler := scheduleLocal + // scheduler = s.scheduleInKubernetes o := &output{w: w} - err = s.executePod(r.Context(), pod, o) + err = scheduler(r.Context(), o, "pull", script, dockerCfg.mustToJSON()) if err != nil { log.Printf("execute pull: %v", err) + w.WriteHeader(http.StatusInternalServerError) o.Errorf("execute pull: %v", err) } } diff --git a/pkg/push.go b/pkg/push.go index e435b68d..ef0aace1 100644 --- a/pkg/push.go +++ b/pkg/push.go @@ -1,17 +1,11 @@ package wedding import ( - "context" "fmt" "log" "net/http" - "strconv" - "time" "github.com/gorilla/mux" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func (s Service) pushImage(w http.ResponseWriter, r *http.Request) { @@ -29,95 +23,17 @@ func (s Service) pushImage(w http.ResponseWriter, r *http.Request) { return } - secret := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "wedding-docker-config-", - }, - StringData: map[string]string{ - "config.json": dockerCfg.mustToJSON(), - }, - } - - secretClient := s.kubernetesClient.CoreV1().Secrets(s.namespace) - - secret, err = secretClient.Create(r.Context(), secret, metav1.CreateOptions{}) - if err != nil { - w.WriteHeader(http.StatusInternalServerError) - streamf(w, "Secret creation failed: %v\n", err) - log.Printf("create secret: %v", err) - return - } - defer func() { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - err = secretClient.Delete(ctx, secret.Name, metav1.DeleteOptions{}) - if err != nil { - streamf(w, "Secret deletetion failed: %v\n", err) - log.Printf("delete secret: %v", err) - } - }() - - buildScript := fmt.Sprintf(` -set -euxo pipefail + // TODO only use --dest-tls-verify=false for local registry + script := fmt.Sprintf(`skopeo copy --retry-times 3 --src-tls-verify=false --dest-tls-verify=false docker://%s docker://%s`, from, to) -skopeo copy --retry-times 3 --src-tls-verify=false --dest-tls-verify=false docker://%s docker://%s -`, from, to) - - pod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "wedding-push-", - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "skopeo", - Image: skopeoImage, - Command: []string{ - "timeout", - strconv.Itoa(int(MaxExecutionTime / time.Second)), - }, - Args: []string{ - "sh", - "-c", - buildScript, - }, - VolumeMounts: []corev1.VolumeMount{ - { - MountPath: "/root/.docker", - Name: "docker-config", - }, - }, - Resources: corev1.ResourceRequirements{ - Limits: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse(skopeoCPU), - corev1.ResourceMemory: resource.MustParse(skopeoMemory), - }, - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse(skopeoCPU), - corev1.ResourceMemory: resource.MustParse(skopeoMemory), - }, - }, - }, - }, - Volumes: []corev1.Volume{ - { - Name: "docker-config", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: secret.Name, - }, - }, - }, - }, - RestartPolicy: corev1.RestartPolicyNever, - }, - } + scheduler := scheduleLocal + // scheduler = s.scheduleInKubernetes o := &output{w: w} - err = s.executePod(r.Context(), pod, o) + err = scheduler(r.Context(), o, "push", script, dockerCfg.mustToJSON()) if err != nil { log.Printf("execute push: %v", err) + w.WriteHeader(http.StatusInternalServerError) o.Errorf("execute push: %v", err) } } diff --git a/pkg/tag.go b/pkg/tag.go index ed23bc72..95500d4b 100644 --- a/pkg/tag.go +++ b/pkg/tag.go @@ -7,14 +7,9 @@ import ( "log" "net/http" "regexp" - "strconv" "strings" - "time" "github.com/gorilla/mux" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func (s Service) tagImage(w http.ResponseWriter, r *http.Request) { @@ -37,54 +32,17 @@ func (s Service) tagImage(w http.ResponseWriter, r *http.Request) { escapePort(fmt.Sprintf("%s:%s", args.Get("repo"), tag)), ) - buildScript := fmt.Sprintf(` -set -euxo pipefail + script := fmt.Sprintf(`skopeo copy --retry-times 3 --src-tls-verify=false --dest-tls-verify=false docker://%s docker://%s`, from, to) -skopeo copy --retry-times 3 --src-tls-verify=false --dest-tls-verify=false docker://%s docker://%s -`, from, to) + scheduler := scheduleLocal + // scheduler = s.scheduleInKubernetes - pod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "wedding-tag-", - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "skopeo", - Image: skopeoImage, - Command: []string{ - "timeout", - strconv.Itoa(int(MaxExecutionTime / time.Second)), - }, - Args: []string{ - "sh", - "-c", - buildScript, - }, - Resources: corev1.ResourceRequirements{ - Limits: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse(skopeoCPU), - corev1.ResourceMemory: resource.MustParse(skopeoMemory), - }, - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse(skopeoCPU), - corev1.ResourceMemory: resource.MustParse(skopeoMemory), - }, - }, - }, - }, - RestartPolicy: corev1.RestartPolicyNever, - }, - } - - b := &bytes.Buffer{} - err := s.executePod(r.Context(), pod, b) + o := &bytes.Buffer{} + err := scheduler(r.Context(), o, "tag", script, "") if err != nil { - log.Printf("execute tagging: %v", err) + log.Printf("execute tag: %v", err) w.WriteHeader(http.StatusInternalServerError) - io.Copy(w, b) - w.Write([]byte(fmt.Sprintf("execute tagging: %v", err))) - return + io.Copy(w, o) } w.WriteHeader(http.StatusCreated)