[ws-manager] Improve workspaces PodAffinity

gitpod-io · Jan 14, 2022 · 13b224a · 13b224a
1 parent 972dfeb
commit 13b224a
Show file tree

Hide file tree

Showing 7 changed files with 61 additions and 8 deletions.
diff --git a/components/registry-facade/leeway.Dockerfile b/components/registry-facade/leeway.Dockerfile
@@ -6,7 +6,9 @@ FROM alpine:3.15
 
 # Ensure latest packages are present, like security updates.
 RUN  apk upgrade --no-cache \
-  && apk add --no-cache ca-certificates
+  && apk add --no-cache ca-certificates bash
+
+RUN apk add --no-cache kubectl --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing
 
 RUN adduser -S -D -H -h /app -u 1000 appuser
 COPY components-registry-facade--app/registry-facade /app/registry-facade

diff --git a/components/ws-daemon/debug.Dockerfile b/components/ws-daemon/debug.Dockerfile
@@ -8,6 +8,8 @@ FROM alpine:3.15
 RUN  apk upgrade --no-cache \
   && apk add --no-cache git bash openssh-client lz4 e2fsprogs
 
+RUN apk add --no-cache kubectl --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing
+
 # Add gitpod user for operations (e.g. checkout because of the post-checkout hook!)
 # RUN addgroup -g 33333 gitpod \
 #     && adduser -D -h /home/gitpod -s /bin/sh -u 33333 -G gitpod gitpod \

diff --git a/components/ws-manager/pkg/manager/create.go b/components/ws-manager/pkg/manager/create.go
@@ -361,6 +361,14 @@ func (m *Manager) createDefiniteWorkspacePod(startContext *startWorkspaceContext
 									Key:      "gitpod.io/workload_workspace_" + workloadType,
 									Operator: corev1.NodeSelectorOpExists,
 								},
+								{
+									Key:      "gitpod.io/ws-daemon_ready_ns_" + m.Config.Namespace,
+									Operator: corev1.NodeSelectorOpExists,
+								},
+								{
+									Key:      "gitpod.io/registry-facade_ready_ns_" + workloadType,
+									Operator: corev1.NodeSelectorOpExists,
+								},
 							},
 						},
 					},

diff --git a/installer/pkg/common/constants.go b/installer/pkg/common/constants.go
@@ -5,8 +5,9 @@
 package common
 
 import (
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 // This file exists to break cyclic-dependency errors

diff --git a/installer/pkg/components/registry-facade/clusterrole.go b/installer/pkg/components/registry-facade/clusterrole.go
@@ -22,12 +22,18 @@ func clusterrole(ctx *common.RenderContext) ([]runtime.Object, error) {
 				Name:   fmt.Sprintf("%s-ns-%s", ctx.Namespace, Component),
 				Labels: common.DefaultLabels(Component),
 			},
-			Rules: []rbacv1.PolicyRule{{
-				APIGroups:     []string{"policy"},
-				Resources:     []string{"podsecuritypolicies"},
-				Verbs:         []string{"use"},
-				ResourceNames: []string{fmt.Sprintf("%s-ns-%s", ctx.Namespace, Component)},
-			}},
+			Rules: []rbacv1.PolicyRule{
+				{
+					APIGroups:     []string{"policy"},
+					Resources:     []string{"podsecuritypolicies"},
+					Verbs:         []string{"use"},
+					ResourceNames: []string{fmt.Sprintf("%s-ns-%s", ctx.Namespace, Component)},
+				}, {
+					APIGroups: []string{""},
+					Resources: []string{"nodes"},
+					Verbs:     []string{"update"},
+				},
+			},
 		},
 	}, nil
 }
diff --git a/installer/pkg/components/registry-facade/daemonset.go b/installer/pkg/components/registry-facade/daemonset.go
@@ -174,6 +174,22 @@ func daemonset(ctx *common.RenderContext) ([]runtime.Object, error) {
 						},
 							*common.InternalCAVolumeMount(),
 						}, volumeMounts...),
+						Lifecycle: &corev1.Lifecycle{
+							PostStart: &corev1.Handler{
+								Exec: &corev1.ExecAction{
+									Command: []string{
+										"/bin/bash", "-c", `kubectl label nodes ${NODENAME} gitpod.io/registry-facade_ready_ns_${KUBE_NAMESPACE}=true`,
+									},
+								},
+							},,
+							PreStop: &corev1.Handler{
+								Exec: &corev1.ExecAction{
+									Command: []string{
+										"/bin/bash", "-c", `kubectl label nodes ${NODENAME} gitpod.io/registry-facade_ready_ns_${KUBE_NAMESPACE}-`,
+									},
+								},
+							},
+						},
 					},
 
 						*common.KubeRBACProxyContainer(ctx),

diff --git a/installer/pkg/components/ws-daemon/daemonset.go b/installer/pkg/components/ws-daemon/daemonset.go
@@ -6,6 +6,7 @@ package wsdaemon
 
 import (
 	"fmt"
+
 	"github.com/gitpod-io/gitpod/installer/pkg/cluster"
 	"github.com/gitpod-io/gitpod/installer/pkg/common"
 	"github.com/gitpod-io/gitpod/installer/pkg/config/v1"
@@ -92,6 +93,7 @@ fi
 			SecurityContext: &corev1.SecurityContext{Privileged: pointer.Bool(true)},
 		},
 	}
+
 	if cfg.Workspace.Runtime.FSShiftMethod == config.FSShiftShiftFS {
 		initContainers = append(initContainers, corev1.Container{
 			Name:  "shiftfs-module-loader",
@@ -279,6 +281,22 @@ fi
 				SecurityContext: &corev1.SecurityContext{
 					Privileged: pointer.Bool(true),
 				},
+				Lifecycle: &corev1.Lifecycle{
+					PostStart: &corev1.Handler{
+						Exec: &corev1.ExecAction{
+							Command: []string{
+								"/bin/bash", "-c", `kubectl label nodes ${NODENAME} gitpod.io/ws-daemon_ready_ns_${KUBE_NAMESPACE}=true`,
+							},
+						},
+					},
+					PreStop: &corev1.Handler{
+						Exec: &corev1.ExecAction{
+							Command: []string{
+								"/bin/bash", "-c", `kubectl label nodes ${NODENAME} gitpod.io/ws-daemon_ready_ns_${KUBE_NAMESPACE}-`,
+							},
+						},
+					},
+				},
 			},
 			*common.KubeRBACProxyContainer(ctx),
 		},