From 13b224af410e4a66953256dedc8c7418226011f4 Mon Sep 17 00:00:00 2001 From: Manuel Alejandro de Brito Fontes Date: Fri, 14 Jan 2022 09:38:08 -0300 Subject: [PATCH] [ws-manager] Improve workspaces PodAffinity --- components/registry-facade/leeway.Dockerfile | 4 +++- components/ws-daemon/debug.Dockerfile | 2 ++ components/ws-manager/pkg/manager/create.go | 8 ++++++++ installer/pkg/common/constants.go | 3 ++- .../components/registry-facade/clusterrole.go | 18 ++++++++++++------ .../components/registry-facade/daemonset.go | 16 ++++++++++++++++ .../pkg/components/ws-daemon/daemonset.go | 18 ++++++++++++++++++ 7 files changed, 61 insertions(+), 8 deletions(-) diff --git a/components/registry-facade/leeway.Dockerfile b/components/registry-facade/leeway.Dockerfile index 1e562f8a4d0449..4390a143740942 100644 --- a/components/registry-facade/leeway.Dockerfile +++ b/components/registry-facade/leeway.Dockerfile @@ -6,7 +6,9 @@ FROM alpine:3.15 # Ensure latest packages are present, like security updates. RUN apk upgrade --no-cache \ - && apk add --no-cache ca-certificates + && apk add --no-cache ca-certificates bash + +RUN apk add --no-cache kubectl --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing RUN adduser -S -D -H -h /app -u 1000 appuser COPY components-registry-facade--app/registry-facade /app/registry-facade diff --git a/components/ws-daemon/debug.Dockerfile b/components/ws-daemon/debug.Dockerfile index a4d93262217c67..a752476bb2b13e 100644 --- a/components/ws-daemon/debug.Dockerfile +++ b/components/ws-daemon/debug.Dockerfile @@ -8,6 +8,8 @@ FROM alpine:3.15 RUN apk upgrade --no-cache \ && apk add --no-cache git bash openssh-client lz4 e2fsprogs +RUN apk add --no-cache kubectl --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing + # Add gitpod user for operations (e.g. checkout because of the post-checkout hook!) # RUN addgroup -g 33333 gitpod \ # && adduser -D -h /home/gitpod -s /bin/sh -u 33333 -G gitpod gitpod \ diff --git a/components/ws-manager/pkg/manager/create.go b/components/ws-manager/pkg/manager/create.go index a6ce50d7c176e0..02175f4afd440f 100644 --- a/components/ws-manager/pkg/manager/create.go +++ b/components/ws-manager/pkg/manager/create.go @@ -361,6 +361,14 @@ func (m *Manager) createDefiniteWorkspacePod(startContext *startWorkspaceContext Key: "gitpod.io/workload_workspace_" + workloadType, Operator: corev1.NodeSelectorOpExists, }, + { + Key: "gitpod.io/ws-daemon_ready_ns_" + m.Config.Namespace, + Operator: corev1.NodeSelectorOpExists, + }, + { + Key: "gitpod.io/registry-facade_ready_ns_" + workloadType, + Operator: corev1.NodeSelectorOpExists, + }, }, }, }, diff --git a/installer/pkg/common/constants.go b/installer/pkg/common/constants.go index 536dfab76afd99..8ac1e3d7b2e4e3 100644 --- a/installer/pkg/common/constants.go +++ b/installer/pkg/common/constants.go @@ -5,8 +5,9 @@ package common import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // This file exists to break cyclic-dependency errors diff --git a/installer/pkg/components/registry-facade/clusterrole.go b/installer/pkg/components/registry-facade/clusterrole.go index 41f698c20f5e0e..c11caf03695322 100644 --- a/installer/pkg/components/registry-facade/clusterrole.go +++ b/installer/pkg/components/registry-facade/clusterrole.go @@ -22,12 +22,18 @@ func clusterrole(ctx *common.RenderContext) ([]runtime.Object, error) { Name: fmt.Sprintf("%s-ns-%s", ctx.Namespace, Component), Labels: common.DefaultLabels(Component), }, - Rules: []rbacv1.PolicyRule{{ - APIGroups: []string{"policy"}, - Resources: []string{"podsecuritypolicies"}, - Verbs: []string{"use"}, - ResourceNames: []string{fmt.Sprintf("%s-ns-%s", ctx.Namespace, Component)}, - }}, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{"policy"}, + Resources: []string{"podsecuritypolicies"}, + Verbs: []string{"use"}, + ResourceNames: []string{fmt.Sprintf("%s-ns-%s", ctx.Namespace, Component)}, + }, { + APIGroups: []string{""}, + Resources: []string{"nodes"}, + Verbs: []string{"update"}, + }, + }, }, }, nil } diff --git a/installer/pkg/components/registry-facade/daemonset.go b/installer/pkg/components/registry-facade/daemonset.go index 79a9a20350fe21..efcb9f209b530c 100644 --- a/installer/pkg/components/registry-facade/daemonset.go +++ b/installer/pkg/components/registry-facade/daemonset.go @@ -174,6 +174,22 @@ func daemonset(ctx *common.RenderContext) ([]runtime.Object, error) { }, *common.InternalCAVolumeMount(), }, volumeMounts...), + Lifecycle: &corev1.Lifecycle{ + PostStart: &corev1.Handler{ + Exec: &corev1.ExecAction{ + Command: []string{ + "/bin/bash", "-c", `kubectl label nodes ${NODENAME} gitpod.io/registry-facade_ready_ns_${KUBE_NAMESPACE}=true`, + }, + }, + },, + PreStop: &corev1.Handler{ + Exec: &corev1.ExecAction{ + Command: []string{ + "/bin/bash", "-c", `kubectl label nodes ${NODENAME} gitpod.io/registry-facade_ready_ns_${KUBE_NAMESPACE}-`, + }, + }, + }, + }, }, *common.KubeRBACProxyContainer(ctx), diff --git a/installer/pkg/components/ws-daemon/daemonset.go b/installer/pkg/components/ws-daemon/daemonset.go index a9f2beafd1aa08..b39ea712f1f634 100644 --- a/installer/pkg/components/ws-daemon/daemonset.go +++ b/installer/pkg/components/ws-daemon/daemonset.go @@ -6,6 +6,7 @@ package wsdaemon import ( "fmt" + "github.com/gitpod-io/gitpod/installer/pkg/cluster" "github.com/gitpod-io/gitpod/installer/pkg/common" "github.com/gitpod-io/gitpod/installer/pkg/config/v1" @@ -92,6 +93,7 @@ fi SecurityContext: &corev1.SecurityContext{Privileged: pointer.Bool(true)}, }, } + if cfg.Workspace.Runtime.FSShiftMethod == config.FSShiftShiftFS { initContainers = append(initContainers, corev1.Container{ Name: "shiftfs-module-loader", @@ -279,6 +281,22 @@ fi SecurityContext: &corev1.SecurityContext{ Privileged: pointer.Bool(true), }, + Lifecycle: &corev1.Lifecycle{ + PostStart: &corev1.Handler{ + Exec: &corev1.ExecAction{ + Command: []string{ + "/bin/bash", "-c", `kubectl label nodes ${NODENAME} gitpod.io/ws-daemon_ready_ns_${KUBE_NAMESPACE}=true`, + }, + }, + }, + PreStop: &corev1.Handler{ + Exec: &corev1.ExecAction{ + Command: []string{ + "/bin/bash", "-c", `kubectl label nodes ${NODENAME} gitpod.io/ws-daemon_ready_ns_${KUBE_NAMESPACE}-`, + }, + }, + }, + }, }, *common.KubeRBACProxyContainer(ctx), },