From 45aed5085d67da4411e1a7f1c6a4b9876d3118ad Mon Sep 17 00:00:00 2001 From: Christian Weichel Date: Sun, 13 Dec 2020 23:57:38 +0000 Subject: [PATCH 1/2] [registry-facade] Implement zero-downtime handover support --- .werft/build.js | 11 +- ...ter-restricted-root-podsecuritypolicy.yaml | 1 + .../templates/registry-facade-configmap.yaml | 6 +- .../templates/registry-facade-daemonset.yaml | 29 +++- chart/values.yaml | 3 + components/registry-facade/cmd/handover.go | 65 ++++++++ components/registry-facade/cmd/run.go | 12 +- components/registry-facade/go.mod | 2 + .../registry-facade/pkg/handover/handover.go | 139 ++++++++++++++++++ .../pkg/handover/handover_test.go | 49 ++++++ .../registry-facade/pkg/registry/registry.go | 139 +++++++++++++++++- 11 files changed, 443 insertions(+), 13 deletions(-) create mode 100644 components/registry-facade/cmd/handover.go create mode 100644 components/registry-facade/pkg/handover/handover.go create mode 100644 components/registry-facade/pkg/handover/handover_test.go diff --git a/.werft/build.js b/.werft/build.js index 4c9fed70b5dd3b..1be0a0d897f4db 100644 --- a/.werft/build.js +++ b/.werft/build.js @@ -56,6 +56,7 @@ async function build(context, version) { const dynamicCPULimits = "dynamic-cpu-limits" in buildConfig; const withInstaller = "with-installer" in buildConfig || masterBuild; const noPreview = "no-preview" in buildConfig || publishRelease; + const registryFacadeHandover = "registry-facade-handover" in buildConfig; werft.log("job config", JSON.stringify({ buildConfig, version, @@ -67,6 +68,7 @@ async function build(context, version) { workspaceFeatureFlags, dynamicCPULimits, noPreview, + registryFacadeHandover, })); /** @@ -111,7 +113,7 @@ async function build(context, version) { werft.phase("deploy", "not deploying"); console.log("no-preview or publish-release is set"); } else { - await deployToDev(version, previewWithHttps, workspaceFeatureFlags, dynamicCPULimits); + await deployToDev(version, previewWithHttps, workspaceFeatureFlags, dynamicCPULimits, registryFacadeHandover); } } @@ -119,7 +121,7 @@ async function build(context, version) { /** * Deploy dev */ -async function deployToDev(version, previewWithHttps, workspaceFeatureFlags, dynamicCPULimits) { +async function deployToDev(version, previewWithHttps, workspaceFeatureFlags, dynamicCPULimits, registryFacadeHandover) { werft.phase("deploy", "deploying to dev"); const destname = version.split(".")[0]; const namespace = `staging-${destname}`; @@ -239,6 +241,11 @@ async function deployToDev(version, previewWithHttps, workspaceFeatureFlags, dyn if (dynamicCPULimits) { flags+=` -f ../.werft/values.variant.cpuLimits.yaml`; } + if (registryFacadeHandover) { + flags+=` --set components.registryFacade.handover.enabled=true`; + flags+=` --set components.registryFacade.handover.socket=/var/lib/gitpod/registry-facade-${namespace}`; + } + // const pathToVersions = `${shell.pwd().toString()}/versions.yaml`; // if (fs.existsSync(pathToVersions)) { // flags+=` -f ${pathToVersions}`; diff --git a/chart/templates/cluster-restricted-root-podsecuritypolicy.yaml b/chart/templates/cluster-restricted-root-podsecuritypolicy.yaml index 2090dbb7ffa29f..136bea949507d8 100644 --- a/chart/templates/cluster-restricted-root-podsecuritypolicy.yaml +++ b/chart/templates/cluster-restricted-root-podsecuritypolicy.yaml @@ -39,6 +39,7 @@ spec: - 'secret' - 'emptyDir' - 'persistentVolumeClaim' + - 'hostPath' hostNetwork: false hostIPC: false hostPID: false diff --git a/chart/templates/registry-facade-configmap.yaml b/chart/templates/registry-facade-configmap.yaml index d1c80e0abb0466..dbe73ea585d835 100644 --- a/chart/templates/registry-facade-configmap.yaml +++ b/chart/templates/registry-facade-configmap.yaml @@ -45,7 +45,11 @@ data: "ref": "{{ template "gitpod.comp.imageFull" (dict "root" . "gp" $.Values "comp" .Values.components.workspace.dockerUp) }}", "type": "image" } - ] + ], + "handover": { + "enabled": {{ $comp.handover.enabled }}, + "sockets": "/mnt/handover" + } }, "pprofAddr": ":6060", "prometheusAddr": ":9500" diff --git a/chart/templates/registry-facade-daemonset.yaml b/chart/templates/registry-facade-daemonset.yaml index 829ebba4ef9b5f..ad691fb397ebab 100644 --- a/chart/templates/registry-facade-daemonset.yaml +++ b/chart/templates/registry-facade-daemonset.yaml @@ -38,6 +38,21 @@ spec: spec: {{ include "gitpod.workspaceAffinity" $this | indent 6 }} serviceAccountName: registry-facade +{{- if $comp.handover.enabled }} + initContainers: + - name: handover-ownership + image: {{ template "gitpod.comp.imageFull" $this }} + command: + - "/bin/sh" + - "-c" + - "chown -R 1000:1000 /mnt/handover" + volumeMounts: + - name: handover + mountPath: "/mnt/handover" + securityContext: + privileged: false + runAsUser: 0 +{{- end }} containers: - name: registry-facade image: {{ template "gitpod.comp.imageFull" $this }} @@ -56,11 +71,15 @@ spec: {{ include "gitpod.container.defaultEnv" $this | indent 8 }} {{ include "gitpod.container.tracingEnv" $this | indent 8 }} volumeMounts: + - name: cache + mountPath: "/mnt/cache" - name: config mountPath: "/mnt/config" readOnly: true - - name: cache - mountPath: "/mnt/cache" + {{- if $comp.handover.enabled }} + - name: handover + mountPath: "/mnt/handover" + {{- end }} {{- if .Values.components.workspace.pullSecret.secretName }} - name: pull-secret mountPath: /mnt/pull-secret.json @@ -76,6 +95,12 @@ spec: - name: config configMap: name: {{ template "gitpod.comp.configMap" $this }} + {{- if $comp.handover.enabled }} + - name: handover + hostPath: + path: {{ $comp.handover.socket | quote }} + type: DirectoryOrCreate + {{- end }} {{- if .Values.components.workspace.pullSecret.secretName }} - name: pull-secret secret: diff --git a/chart/values.yaml b/chart/values.yaml index a41233220fe9d0..83f557573e9297 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -235,6 +235,9 @@ components: servicePort: 3000 svcLabels: feature: registry + handover: + enabled: false + socket: /var/lib/gitpod/registry-facade serviceType: "ClusterIP" server: diff --git a/components/registry-facade/cmd/handover.go b/components/registry-facade/cmd/handover.go new file mode 100644 index 00000000000000..d08c0f203b447f --- /dev/null +++ b/components/registry-facade/cmd/handover.go @@ -0,0 +1,65 @@ +// Copyright (c) 2020 TypeFox GmbH. All rights reserved. +// Licensed under the GNU Affero General Public License (AGPL). +// See License-AGPL.txt in the project root for license information. + +package cmd + +import ( + "context" + "os" + "os/signal" + "syscall" + "time" + + "github.com/gitpod-io/gitpod/common-go/log" + "github.com/gitpod-io/gitpod/registry-facade/pkg/registry" + "github.com/spf13/cobra" +) + +// debugHandover represents the run command +var debugHandover = &cobra.Command{ + Use: "handover ", + Short: "Attempts to get the listener socket from a registry-facade - and offers it back up for someone else", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + l, err := registry.ReceiveHandover(ctx, args[0]) + if err != nil { + return err + } + if l == nil { + log.Warn("received no listener") + return nil + } + + log.Info("handover successfull - holding listener for someone else") + + hoctx, cancelHO := context.WithCancel(context.Background()) + defer cancelHO() + + ho, err := registry.OfferHandover(hoctx, args[0], l, nil) + if err != nil { + return err + } + + log.Info("waiting for someone else to handover to - stop with Ctrl+C") + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM) + + select { + case didHO := <-ho: + if didHO { + <-ho + } + case <-sigChan: + } + + return nil + }, +} + +func init() { + rootCmd.AddCommand(debugHandover) +} diff --git a/components/registry-facade/cmd/run.go b/components/registry-facade/cmd/run.go index 2831b6d78b1bcf..854f7368bb4099 100644 --- a/components/registry-facade/cmd/run.go +++ b/components/registry-facade/cmd/run.go @@ -76,11 +76,16 @@ var runCmd = &cobra.Command{ return docker.NewResolver(resolverOpts) } + registryDoneChan := make(chan struct{}) reg, err := registry.NewRegistry(cfg.Registry, resolverProvider, prometheus.WrapRegistererWithPrefix("registry_", gpreg)) if err != nil { log.WithError(err).Fatal("cannot create registry") } - go reg.MustServe() + go func() { + defer close(registryDoneChan) + reg.MustServe() + }() + if cfg.PProfAddr != "" { go pprof.Serve(cfg.PProfAddr) } @@ -105,7 +110,10 @@ var runCmd = &cobra.Command{ log.Info("🏪 registry facade is up and running") sigChan := make(chan os.Signal, 1) signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM) - <-sigChan + select { + case <-sigChan: + case <-registryDoneChan: + } }, } diff --git a/components/registry-facade/go.mod b/components/registry-facade/go.mod index b7cd724dda26f3..cff2762238a2e9 100644 --- a/components/registry-facade/go.mod +++ b/components/registry-facade/go.mod @@ -21,6 +21,8 @@ require ( github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.1.0 github.com/spf13/cobra v0.0.5 + golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e + golang.org/x/sys v0.0.0-20201112073958-5cba982894dd golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 google.golang.org/grpc v1.34.0 gotest.tools/v3 v3.0.3 // indirect diff --git a/components/registry-facade/pkg/handover/handover.go b/components/registry-facade/pkg/handover/handover.go new file mode 100644 index 00000000000000..a04691dea08b3a --- /dev/null +++ b/components/registry-facade/pkg/handover/handover.go @@ -0,0 +1,139 @@ +package handover + +import ( + "context" + "fmt" + "net" + "os" + + "golang.org/x/sync/errgroup" + "golang.org/x/sys/unix" + "golang.org/x/xerrors" +) + +// OfferHandover opens a Unix socket on socketFN and waits for another process to ask +// for the listeners socket file descriptor. Once that happens, it closes the Unix socket and returns. +// If the context is canceled before someone asks for the listener's socket, +// this function returns context.Canceled. +func OfferHandover(ctx context.Context, socketFN string, l *net.TCPListener) error { + skt, err := net.Listen("unix", socketFN) + if err != nil { + return xerrors.Errorf("cannot create handover socket: %w", err) + } + defer skt.Close() + + done := make(chan struct{}) + eg, ctx := errgroup.WithContext(ctx) + eg.Go(func() error { + recv, err := skt.Accept() + if err != nil { + return xerrors.Errorf("cannot accept incoming handover connection: %w", err) + } + defer recv.Close() + + if ctx.Err() != nil { + return ctx.Err() + } + + listenConn := recv.(*net.UnixConn) + err = sendListener(listenConn, l) + if err != nil { + return xerrors.Errorf("cannot send listener: %w", err) + } + + defer close(done) + return nil + }) + eg.Go(func() error { + select { + case <-ctx.Done(): + return ctx.Err() + case <-done: + return nil + } + }) + return eg.Wait() +} + +// ReceiveHandover requests a net.Listener handover from a Unix socket on socketFN. +// If the context cancels before the transfer is complete, context.Canceled is returned. +func ReceiveHandover(ctx context.Context, socketFN string) (l net.Listener, err error) { + conn, err := net.Dial("unix", socketFN) + if err != nil { + return nil, err + } + defer conn.Close() + + return receiveListener(ctx, conn.(*net.UnixConn)) +} + +// sendListener sends a copy of a TCP listener's file descriptor to a Unix socket. +// Callers should close l upon successful return of this function. +// Use in conjunction with receiveListener(). +func sendListener(conn *net.UnixConn, l *net.TCPListener) error { + connf, err := conn.File() + if err != nil { + return err + } + defer connf.Close() + sktfd := int(connf.Fd()) + + lf, err := l.File() + if err != nil { + return err + } + lfd := int(lf.Fd()) + + rights := unix.UnixRights(lfd) + return unix.Sendmsg(sktfd, nil, rights, nil, 0) +} + +// receiveListener attempts to receieve a file descriptor from a Unix socket, +// and turns that fd into a net.Listener. This function makes several assumptions +// about the nature of the messages it receives: +// - there is exactly one control message, +// that contains exactly one SCM_RIGHTS message, +// that contains exaxtly one file descriptor. +// - the received file descriptor is a listening socket, s.t. we can call net.FileListener on it. +func receiveListener(ctx context.Context, conn *net.UnixConn) (l net.Listener, err error) { + buf := make([]byte, unix.CmsgSpace(4)) + + connf, err := conn.File() + if err != nil { + return nil, err + } + defer connf.Close() + connfd := int(connf.Fd()) + + recvC := make(chan error, 1) + go func() { + _, _, _, _, err := unix.Recvmsg(connfd, nil, buf, 0) + recvC <- err + }() + select { + case err = <-recvC: + case <-ctx.Done(): + err = ctx.Err() + } + if err != nil { + return nil, err + } + + msgs, err := unix.ParseSocketControlMessage(buf) + if err != nil { + return nil, err + } + if len(msgs) != 1 { + return nil, fmt.Errorf("expected a single socket control message") + } + + fds, err := unix.ParseUnixRights(&msgs[0]) + if err != nil { + return nil, err + } + if len(fds) == 0 { + return nil, fmt.Errorf("expected a single socket FD") + } + + return net.FileListener(os.NewFile(uintptr(fds[0]), "")) +} diff --git a/components/registry-facade/pkg/handover/handover_test.go b/components/registry-facade/pkg/handover/handover_test.go new file mode 100644 index 00000000000000..58c2e75f3aa8b4 --- /dev/null +++ b/components/registry-facade/pkg/handover/handover_test.go @@ -0,0 +1,49 @@ +package handover_test + +import ( + "context" + "fmt" + "net" + "os" + "path/filepath" + "testing" + "time" + + "github.com/gitpod-io/gitpod/registry-facade/pkg/handover" + "golang.org/x/sync/errgroup" +) + +func TestHandover(t *testing.T) { + socketFN := filepath.Join(os.TempDir(), fmt.Sprintf("handover-test-%d.sock", time.Now().UnixNano())) + + l, err := net.Listen("tcp", ":44444") + if err != nil { + t.Fatalf("cannot start test listener: %q", err) + } + defer l.Close() + tcpL := l.(*net.TCPListener) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + eg, ctx := errgroup.WithContext(ctx) + eg.Go(func() error { + return handover.OfferHandover(ctx, socketFN, tcpL) + }) + eg.Go(func() error { + // give the handover offer some time to start + time.Sleep(1 * time.Millisecond) + l, err := handover.ReceiveHandover(ctx, socketFN) + if err != nil { + return err + } + if l == nil { + return fmt.Errorf("l was nil") + } + l.Close() + return nil + }) + err = eg.Wait() + if err != nil { + t.Fatal(err) + } +} diff --git a/components/registry-facade/pkg/registry/registry.go b/components/registry-facade/pkg/registry/registry.go index 0c1097b9176c86..08b05edf016f02 100644 --- a/components/registry-facade/pkg/registry/registry.go +++ b/components/registry-facade/pkg/registry/registry.go @@ -10,11 +10,17 @@ import ( "crypto/x509" "fmt" "io/ioutil" + "net" "net/http" "net/http/httputil" "os" "path/filepath" "strings" + "time" + + "github.com/gitpod-io/gitpod/common-go/log" + "github.com/gitpod-io/gitpod/registry-facade/api" + "github.com/gitpod-io/gitpod/registry-facade/pkg/handover" "github.com/containerd/containerd/content" "github.com/containerd/containerd/content/local" @@ -23,8 +29,6 @@ import ( "github.com/docker/distribution/reference" "github.com/docker/distribution/registry/api/errcode" distv2 "github.com/docker/distribution/registry/api/v2" - "github.com/gitpod-io/gitpod/common-go/log" - "github.com/gitpod-io/gitpod/registry-facade/api" "github.com/gorilla/mux" grpc_opentracing "github.com/grpc-ecosystem/go-grpc-middleware/tracing/opentracing" "github.com/opentracing/opentracing-go" @@ -56,6 +60,9 @@ type Config struct { Certificate string `json:"crt"` PrivateKey string `json:"key"` } `json:"tls"` + Handover struct { + Sockets string `json:"sockets"` + } `json:"handover"` } // ResolverProvider provides new resolver @@ -71,6 +78,7 @@ type Registry struct { SpecProvider map[string]ImageSpecProvider metrics *metrics + srv *http.Server } // NewRegistry creates a new registry @@ -214,8 +222,41 @@ func (reg *Registry) Serve() error { go http.ListenAndServe(addr, mux) } + addr := fmt.Sprintf(":%d", reg.Config.Port) + var ( + l net.Listener + err error + ) + if fn := reg.Config.Handover.Sockets; fn != "" { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + l, err = ReceiveHandover(ctx, reg.Config.Handover.Sockets) + cancel() + if err != nil { + log.WithError(err).Warn("handover failed - attempting to start socket directly") + } + } + if l == nil { + // there was no handover configured or available - start our own listener + l, err = net.Listen("tcp", addr) + if err != nil { + return err + } + } + + reg.srv = &http.Server{ + Addr: addr, + Handler: mux, + } + + hoctx, cancelHO := context.WithCancel(context.Background()) + defer cancelHO() + hoc, err := OfferHandover(hoctx, reg.Config.Handover.Sockets, l, reg.srv) + if err != nil { + return err + } + if reg.Config.TLS != nil { - log.WithField("addr", fmt.Sprintf(":%d", reg.Config.Port)).Info("HTTPS server listening") + log.WithField("addr", addr).Info("HTTPS registry server listening") cert, key := reg.Config.TLS.Certificate, reg.Config.TLS.PrivateKey if tproot := os.Getenv("TELEPRESENCE_ROOT"); tproot != "" { @@ -223,11 +264,26 @@ func (reg *Registry) Serve() error { key = filepath.Join(tproot, key) } - return http.ListenAndServeTLS(fmt.Sprintf(":%d", reg.Config.Port), cert, key, mux) + return reg.srv.ServeTLS(l, cert, key) } - log.WithField("addr", fmt.Sprintf(":%d", reg.Config.Port)).Info("registry server listening") - return http.ListenAndServe(fmt.Sprintf(":%d", reg.Config.Port), mux) + srvErrChan := make(chan error, 1) + go func() { + log.WithField("addr", addr).Info("HTTP registry server listening") + srvErrChan <- reg.srv.Serve(l) + }() + + select { + case err := <-srvErrChan: + return err + case handingOver := <-hoc: + if !handingOver { + return nil + } + // we are handing over and must wait for the server to shut down + <-hoc + return nil + } } // MustServe calls serve and logs any error as Fatal @@ -238,6 +294,77 @@ func (reg *Registry) MustServe() { } } +// ReceiveHandover lists all Unix sockets in loc, finds the latest and attempts a Listener +// handover from that socket. If loc == "", this function returns nil, nil. +func ReceiveHandover(ctx context.Context, loc string) (l net.Listener, err error) { + if loc == "" { + return nil, nil + } + fs, err := ioutil.ReadDir(loc) + if err != nil { + return nil, err + } + var fn string + for _, f := range fs { + if f.Mode()*os.ModeSocket == 0 { + continue + } + if f.Name() > fn { + fn = f.Name() + } + } + if fn == "" { + return nil, nil + } + fn = filepath.Join(loc, fn) + + log.WithField("fn", fn).Debug("found handover socket - attempting listener handover") + return handover.ReceiveHandover(ctx, fn) +} + +// Shutdowner is a process that can be shut down +type Shutdowner interface { + Shutdown(context.Context) error +} + +// OfferHandover offers the registry-facade listener handover on a Unix socket +func OfferHandover(ctx context.Context, loc string, l net.Listener, s Shutdowner) (handingOver <-chan bool, err error) { + socketFN := filepath.Join(loc, fmt.Sprintf("rf-handover-%d.sock", time.Now().Unix())) + if socketFN == "" { + return nil, nil + } + tcpL, ok := l.(*net.TCPListener) + if !ok { + return nil, xerrors.Errorf("can only offer handovers for *net.TCPListener") + } + + handingOverC := make(chan bool) + go func() { + defer close(handingOverC) + + err := handover.OfferHandover(ctx, socketFN, tcpL) + if err != nil { + log.WithError(err).Error("listener handover offer failed") + return + } + + log.Warn("listener handover initiated - not accepting new connections and stopping server") + handingOverC <- true + + if s == nil { + return + } + err = s.Shutdown(ctx) + if err != nil { + log.WithError(err).Warn("error during server shutdown") + return + } + log.Warn("server shutdown complete") + }() + log.WithField("socket", socketFN).Info("offering listener handover") + return handingOverC, nil +} + func (reg *Registry) requireAuthentication(h http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fail := func() { From fae2de689bba08da8b3dd265e2ca1a2cd4696b1c Mon Sep 17 00:00:00 2001 From: Christian Weichel Date: Mon, 11 Jan 2021 14:52:56 +0000 Subject: [PATCH 2/2] [registry-facade] Run in hostNetwork to faclitate handover /werft registry-facade-handover /werft https --- .werft/build.js | 2 +- ...ter-restricted-root-podsecuritypolicy.yaml | 2 +- .../registry-facade-clusterrole.yaml | 20 ++++++ .../templates/registry-facade-configmap.yaml | 4 ++ .../templates/registry-facade-daemonset.yaml | 8 +++ .../registry-facade-podsecuritypolicy.yaml | 65 +++++++++++++++++++ .../registry-facade-rolebinding.yaml | 2 +- .../registry-facade/pkg/handover/handover.go | 1 + 8 files changed, 101 insertions(+), 3 deletions(-) create mode 100644 chart/templates/registry-facade-clusterrole.yaml create mode 100644 chart/templates/registry-facade-podsecuritypolicy.yaml diff --git a/.werft/build.js b/.werft/build.js index 1be0a0d897f4db..c0da7b6749d4a2 100644 --- a/.werft/build.js +++ b/.werft/build.js @@ -196,7 +196,7 @@ async function deployToDev(version, previewWithHttps, workspaceFeatureFlags, dyn exec(`/usr/local/bin/helm3 delete jaeger-${destname} || echo jaeger-${destname} was not installed yet`, {slice: 'predeploy cleanup'}); let objs = []; - ["ws-scheduler", "node-daemon", "cluster", "workspace", "jaeger", "jaeger-agent", "ws-sync", "ws-manager-node", "ws-daemon"].forEach(comp => + ["ws-scheduler", "node-daemon", "cluster", "workspace", "jaeger", "jaeger-agent", "ws-sync", "ws-manager-node", "ws-daemon", "registry-facade"].forEach(comp => ["ClusterRole", "ClusterRoleBinding", "PodSecurityPolicy"].forEach(kind => shell .exec(`kubectl get ${kind} -l component=${comp} --no-headers -o=custom-columns=:metadata.name | grep ${namespace}-ns`) diff --git a/chart/templates/cluster-restricted-root-podsecuritypolicy.yaml b/chart/templates/cluster-restricted-root-podsecuritypolicy.yaml index 136bea949507d8..a23e568bdf31cf 100644 --- a/chart/templates/cluster-restricted-root-podsecuritypolicy.yaml +++ b/chart/templates/cluster-restricted-root-podsecuritypolicy.yaml @@ -40,7 +40,7 @@ spec: - 'emptyDir' - 'persistentVolumeClaim' - 'hostPath' - hostNetwork: false + hostNetwork: true hostIPC: false hostPID: false hostPorts: diff --git a/chart/templates/registry-facade-clusterrole.yaml b/chart/templates/registry-facade-clusterrole.yaml new file mode 100644 index 00000000000000..0c6bf09c214fb4 --- /dev/null +++ b/chart/templates/registry-facade-clusterrole.yaml @@ -0,0 +1,20 @@ +# Copyright (c) 2020 Gitpod GmbH. All rights reserved. +# Licensed under the MIT License. See License-MIT.txt in the project root for license information. + +{{ if .Values.installPodSecurityPolicies -}} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: {{ .Release.Namespace }}-ns-registry-facade + labels: + app: {{ template "gitpod.fullname" . }} + component: cluster + kind: clusterrole + stage: {{ .Values.installation.stage }} +rules: + - apiGroups: ["policy"] + resources: ["podsecuritypolicies"] + verbs: ["use"] + resourceNames: + - {{ .Release.Namespace }}-ns-registry-facade +{{- end -}} \ No newline at end of file diff --git a/chart/templates/registry-facade-configmap.yaml b/chart/templates/registry-facade-configmap.yaml index dbe73ea585d835..faab06e55db264 100644 --- a/chart/templates/registry-facade-configmap.yaml +++ b/chart/templates/registry-facade-configmap.yaml @@ -17,7 +17,11 @@ data: { {{ if .Values.components.workspace.pullSecret.secretName -}}"dockerAuth": "/mnt/pull-secret.json",{{- end }} "registry": { + {{- if $comp.handover.enabled }} + "port": {{ $comp.ports.registry.servicePort }}, + {{- else }} "port": {{ $comp.ports.registry.containerPort }}, + {{- end }} {{- if (or .Values.certificatesSecret.secretName $comp.certificatesSecret.secretName) }} {{- if (or .Values.certificatesSecret.certManager $comp.certificatesSecret.certManager) }} "tls": { diff --git a/chart/templates/registry-facade-daemonset.yaml b/chart/templates/registry-facade-daemonset.yaml index ad691fb397ebab..815a66ee960184 100644 --- a/chart/templates/registry-facade-daemonset.yaml +++ b/chart/templates/registry-facade-daemonset.yaml @@ -53,6 +53,9 @@ spec: privileged: false runAsUser: 0 {{- end }} + {{- if $comp.handover.enabled }} + hostNetwork: true + {{- end }} containers: - name: registry-facade image: {{ template "gitpod.comp.imageFull" $this }} @@ -61,10 +64,15 @@ spec: {{ include "gitpod.container.resources" $this | indent 8 }} ports: - name: registry + {{- if $comp.handover.enabled }} + # if hostNetwork == true then containerPort == hostPort + containerPort: {{ $comp.ports.registry.servicePort }} + {{- else }} containerPort: {{ $comp.ports.registry.containerPort }} hostPort: {{ $comp.ports.registry.servicePort }} - name: metrics containerPort: 9500 + {{- end }} securityContext: privileged: false runAsUser: 1000 diff --git a/chart/templates/registry-facade-podsecuritypolicy.yaml b/chart/templates/registry-facade-podsecuritypolicy.yaml new file mode 100644 index 00000000000000..e2fe3b79820b67 --- /dev/null +++ b/chart/templates/registry-facade-podsecuritypolicy.yaml @@ -0,0 +1,65 @@ +# Copyright (c) 2020 Gitpod GmbH. All rights reserved. +# Licensed under the MIT License. See License-MIT.txt in the project root for license information. + +{{ if .Values.installPodSecurityPolicies -}} +# Taken from the examples here: +# Examples: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#example-policies +# File: https://raw.githubusercontent.com/kubernetes/website/master/content/en/examples/policy/restricted-psp.yaml +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ .Release.Namespace }}-ns-registry-facade + labels: + app: {{ template "gitpod.fullname" . }} + component: cluster + kind: podsecuritypolicy + stage: {{ .Values.installation.stage }} + annotations: + seccomp.security.alpha.kubernetes.io/allowedProfileNames: 'runtime/default' + apparmor.security.beta.kubernetes.io/allowedProfileNames: 'runtime/default' + seccomp.security.alpha.kubernetes.io/defaultProfileName: 'runtime/default' + apparmor.security.beta.kubernetes.io/defaultProfileName: 'runtime/default' +spec: + ##### + # The nginx master process (currently?) runs as root, thus we have to turn some safe things off + ##### + ### TODO root proxy + # privileged: false + # # Required to prevent escalations to root. + # allowPrivilegeEscalation: false + # # This is redundant with non-root + disallow privilege escalation, + # # but we can provide it for defense in depth. + # requiredDropCapabilities: + # - ALL + ### TODO root proxy + # Allow core volume types. + volumes: + - 'configMap' + - 'secret' + - 'emptyDir' + - 'hostPath' + hostNetwork: true + hostIPC: false + hostPID: false + hostPorts: + - min: 30000 + max: 33000 + runAsUser: + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + {{- end -}} \ No newline at end of file diff --git a/chart/templates/registry-facade-rolebinding.yaml b/chart/templates/registry-facade-rolebinding.yaml index 14b436915a40b0..52f69313b42735 100644 --- a/chart/templates/registry-facade-rolebinding.yaml +++ b/chart/templates/registry-facade-rolebinding.yaml @@ -15,5 +15,5 @@ subjects: name: registry-facade roleRef: kind: ClusterRole - name: {{ .Release.Namespace }}-ns-psp:restricted-root-user + name: {{ .Release.Namespace }}-ns-registry-facade apiGroup: rbac.authorization.k8s.io diff --git a/components/registry-facade/pkg/handover/handover.go b/components/registry-facade/pkg/handover/handover.go index a04691dea08b3a..eccdd0ada64ae2 100644 --- a/components/registry-facade/pkg/handover/handover.go +++ b/components/registry-facade/pkg/handover/handover.go @@ -84,6 +84,7 @@ func sendListener(conn *net.UnixConn, l *net.TCPListener) error { } lfd := int(lf.Fd()) + // UnixRights encodes a set of open file descriptors into a socket control message for sending to another process. rights := unix.UnixRights(lfd) return unix.Sendmsg(sktfd, nil, rights, nil, 0) }