Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[installer] Allow configuration of resource requests and limits #9545

Merged
merged 7 commits into from
Apr 28, 2022
Merged

[installer] Allow configuration of resource requests and limits #9545

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
add0dad
Add Resources to experimental PodConfig section
Apr 25, 2022
4f26970
Add ResourceRequirements function to common
Apr 25, 2022
1981d1f
Make resources configurable for all components
Apr 25, 2022
3998444
Add unit test for rendering resources
Apr 26, 2022
da63ff8
Add `usePodAffinity` field to webapp config
Apr 28, 2022
8f34406
Use pod anti-affinity for server if flag is set
Apr 28, 2022
6141d4a
Use pod anti-affinity for proxy if flag is set
Apr 28, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions install/installer/pkg/common/common.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -351,6 +351,21 @@ func Replicas(ctx *RenderContext, component string) *int32 {
return &replicas
}

func ResourceRequirements(ctx *RenderContext, component, containerName string, defaults corev1.ResourceRequirements) corev1.ResourceRequirements {
resources := defaults

_ = ctx.WithExperimental(func(cfg *experimental.Config) error {
if cfg.Common != nil && cfg.Common.PodConfig[component] != nil {
if cfg.Common.PodConfig[component].Resources[containerName] != nil {
resources = *cfg.Common.PodConfig[component].Resources[containerName]
}
}
return nil
})

return resources
}

// ObjectHash marshals the objects to YAML and produces a sha256 hash of the output.
// This function is useful for restarting pods when the config changes.
// Takes an error as argument to make calling it more conventient. If that error is not nil,
Expand Down
104 changes: 104 additions & 0 deletions install/installer/pkg/common/render_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ import (
"github.com/gitpod-io/gitpod/installer/pkg/config/versions"
"github.com/google/go-cmp/cmp"
"github.com/stretchr/testify/require"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/utils/pointer"
)
Expand Down Expand Up @@ -79,6 +81,108 @@ func TestReplicas(t *testing.T) {
}
}

func TestResourceRequirements(t *testing.T) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

❤️

defaultResources := corev1.ResourceRequirements{
Requests: corev1.ResourceList{
"cpu": resource.MustParse("200m"),
"memory": resource.MustParse("200Mi"),
},
Limits: corev1.ResourceList{
"cpu": resource.MustParse("200m"),
"memory": resource.MustParse("200Mi"),
},
}

serverResources := corev1.ResourceRequirements{
Requests: corev1.ResourceList{
"cpu": resource.MustParse("50m"),
"memory": resource.MustParse("100Mi"),
},
Limits: corev1.ResourceList{
"cpu": resource.MustParse("500m"),
"memory": resource.MustParse("800Mi"),
},
}

dashboardResources := corev1.ResourceRequirements{
Requests: corev1.ResourceList{
"cpu": resource.MustParse("60m"),
"memory": resource.MustParse("100Mi"),
},
Limits: corev1.ResourceList{
"cpu": resource.MustParse("100m"),
"memory": resource.MustParse("500Mi"),
},
}

testCases := []struct {
Component string
ContainerName string
Name string
ExpectedResources corev1.ResourceRequirements
}{
{
Component: server.Component,
ContainerName: server.Component,
Name: "server takes resource requirements from config",
ExpectedResources: serverResources,
},
{
Component: dashboard.Component,
ContainerName: dashboard.Component,
Name: "dashboard takes resource requirements from config",
ExpectedResources: dashboardResources,
},
{
Component: content_service.Component,
Name: "content_service takes default resource requirements",
ExpectedResources: defaultResources,
},
}
ctx, err := common.NewRenderContext(config.Config{
Experimental: &experimental.Config{
Common: &experimental.CommonConfig{
PodConfig: map[string]*experimental.PodConfig{
server.Component: {
Resources: map[string]*corev1.ResourceRequirements{
server.Component: &serverResources,
},
},
dashboard.Component: {
Resources: map[string]*corev1.ResourceRequirements{
dashboard.Component: &dashboardResources,
},
},
},
},
},
}, versions.Manifest{}, "test_namespace")
require.NoError(t, err)

for _, testCase := range testCases {
t.Run(testCase.Name, func(t *testing.T) {
actualResources := common.ResourceRequirements(ctx, testCase.Component, testCase.ContainerName, defaultResources)

if actualResources.Limits["cpu"] != testCase.ExpectedResources.Limits["cpu"] {
t.Errorf("expected cpu limits for container %q in component %q to be %+v, but got %+v",
testCase.Component, testCase.ContainerName, testCase.ExpectedResources.Limits["cpu"], actualResources.Limits["cpu"])
}
if actualResources.Limits["memory"] != testCase.ExpectedResources.Limits["memory"] {
t.Errorf("expected memory limits for container %q in component %q to be %+v, but got %+v",
testCase.Component, testCase.ContainerName, testCase.ExpectedResources.Limits["memory"], actualResources.Limits["memory"])
}
if actualResources.Requests["cpu"] != testCase.ExpectedResources.Requests["cpu"] {
t.Errorf("expected cpu requests for container %q in component %q to be %+v, but got %+v",
testCase.Component, testCase.ContainerName, testCase.ExpectedResources.Requests["cpu"], actualResources.Requests["cpu"])
}
if actualResources.Requests["memory"] != testCase.ExpectedResources.Requests["memory"] {
t.Errorf("expected memory requests for container %q in component %q to be %+v, but got %+v",
testCase.Component, testCase.ContainerName, testCase.ExpectedResources.Requests["memory"], actualResources.Requests["memory"])
}
})
}
}

func TestRepoName(t *testing.T) {
type Expectation struct {
Result string
Expand Down
4 changes: 2 additions & 2 deletions install/installer/pkg/components/agent-smith/daemonset.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,12 +54,12 @@ func daemonset(ctx *common.RenderContext) ([]runtime.Object, error) {
Image: ctx.ImageName(ctx.Config.Repository, Component, ctx.VersionManifest.Components.AgentSmith.Version),
ImagePullPolicy: corev1.PullIfNotPresent,
Args: []string{"run", "--config", "/config/config.json"},
Resources: corev1.ResourceRequirements{
Resources: common.ResourceRequirements(ctx, Component, Component, corev1.ResourceRequirements{
Requests: corev1.ResourceList{
"cpu": resource.MustParse("100m"),
"memory": resource.MustParse("32Mi"),
},
},
}),
VolumeMounts: []corev1.VolumeMount{{
Name: "config",
MountPath: "/config",
Expand Down
4 changes: 2 additions & 2 deletions install/installer/pkg/components/blobserve/deployment.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,12 +100,12 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
Name: ServicePortName,
ContainerPort: ContainerPort,
}},
Resources: corev1.ResourceRequirements{
Resources: common.ResourceRequirements(ctx, Component, Component, corev1.ResourceRequirements{
Requests: corev1.ResourceList{
"cpu": resource.MustParse("100m"),
"memory": resource.MustParse("32Mi"),
},
},
}),
SecurityContext: &corev1.SecurityContext{
Privileged: pointer.Bool(false),
RunAsUser: pointer.Int64(1000),
Expand Down
4 changes: 2 additions & 2 deletions install/installer/pkg/components/content-service/deployment.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,12 +48,12 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
"--config",
"/config/config.json",
},
Resources: corev1.ResourceRequirements{
Resources: common.ResourceRequirements(ctx, Component, Component, corev1.ResourceRequirements{
Requests: corev1.ResourceList{
"cpu": resource.MustParse("100m"),
"memory": resource.MustParse("32Mi"),
},
},
}),
Ports: []corev1.ContainerPort{{
Name: RPCServiceName,
ContainerPort: RPCPort,
Expand Down
4 changes: 2 additions & 2 deletions install/installer/pkg/components/dashboard/deployment.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,12 +49,12 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
Name: Component,
Image: ctx.ImageName(ctx.Config.Repository, Component, ctx.VersionManifest.Components.Dashboard.Version),
ImagePullPolicy: corev1.PullIfNotPresent,
Resources: corev1.ResourceRequirements{
Resources: common.ResourceRequirements(ctx, Component, Component, corev1.ResourceRequirements{
Requests: corev1.ResourceList{
"cpu": resource.MustParse("100m"),
"memory": resource.MustParse("32Mi"),
},
},
}),
Ports: []corev1.ContainerPort{{
ContainerPort: ContainerPort,
Name: PortName,
Expand Down
4 changes: 2 additions & 2 deletions install/installer/pkg/components/ide-proxy/deployment.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,12 +49,12 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
Name: Component,
Image: ctx.ImageName(ctx.Config.Repository, Component, ctx.VersionManifest.Components.IDEProxy.Version),
ImagePullPolicy: corev1.PullIfNotPresent,
Resources: corev1.ResourceRequirements{
Resources: common.ResourceRequirements(ctx, Component, Component, corev1.ResourceRequirements{
Requests: corev1.ResourceList{
"cpu": resource.MustParse("100m"),
"memory": resource.MustParse("32Mi"),
},
},
}),
Ports: []corev1.ContainerPort{{
ContainerPort: ContainerPort,
Name: PortName,
Expand Down
4 changes: 2 additions & 2 deletions install/installer/pkg/components/image-builder-mk3/deployment.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -153,12 +153,12 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
common.DefaultEnv(&ctx.Config),
common.TracingEnv(ctx),
),
Resources: corev1.ResourceRequirements{
Resources: common.ResourceRequirements(ctx, Component, Component, corev1.ResourceRequirements{
Requests: corev1.ResourceList{
"cpu": resource.MustParse("100m"),
"memory": resource.MustParse("200Mi"),
},
},
}),
Ports: []corev1.ContainerPort{{
ContainerPort: RPCPort,
Name: RPCPortName,
Expand Down
11 changes: 6 additions & 5 deletions install/installer/pkg/components/openvsx-proxy/statefulset.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ func statefulset(ctx *common.RenderContext) ([]runtime.Object, error) {
return nil, err
}

const redisContainerName = "redis"
return []runtime.Object{&appsv1.StatefulSet{
TypeMeta: common.TypeMetaStatefulSet,
ObjectMeta: metav1.ObjectMeta{
Expand Down Expand Up @@ -79,12 +80,12 @@ func statefulset(ctx *common.RenderContext) ([]runtime.Object, error) {
},
},
ImagePullPolicy: v1.PullIfNotPresent,
Resources: v1.ResourceRequirements{
Resources: common.ResourceRequirements(ctx, Component, Component, v1.ResourceRequirements{
Requests: v1.ResourceList{
"cpu": resource.MustParse("1m"),
"memory": resource.MustParse("150Mi"),
},
},
}),
Ports: []v1.ContainerPort{{
Name: PortName,
ContainerPort: ContainerPort,
Expand All @@ -100,7 +101,7 @@ func statefulset(ctx *common.RenderContext) ([]runtime.Object, error) {
common.DefaultEnv(&ctx.Config),
),
}, {
Name: "redis",
Name: redisContainerName,
Image: ctx.ImageName(common.ThirdPartyContainerRepo(ctx.Config.Repository, common.DockerRegistryURL), "library/redis", "6.2"),
Command: []string{
"redis-server",
Expand All @@ -114,12 +115,12 @@ func statefulset(ctx *common.RenderContext) ([]runtime.Object, error) {
Ports: []v1.ContainerPort{{
ContainerPort: 6379,
}},
Resources: v1.ResourceRequirements{
Resources: common.ResourceRequirements(ctx, Component, redisContainerName, v1.ResourceRequirements{
Requests: v1.ResourceList{
"cpu": resource.MustParse("1m"),
"memory": resource.MustParse("150Mi"),
},
},
}),
VolumeMounts: []v1.VolumeMount{{
Name: "config",
MountPath: "/config",
Expand Down
39 changes: 33 additions & 6 deletions install/installer/pkg/components/proxy/deployment.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"fmt"

"github.com/gitpod-io/gitpod/installer/pkg/cluster"
"github.com/gitpod-io/gitpod/installer/pkg/config/v1/experimental"

"github.com/gitpod-io/gitpod/installer/pkg/common"

Expand Down Expand Up @@ -95,6 +96,29 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
return nil, err
}

var podAntiAffinity *corev1.PodAntiAffinity
_ = ctx.WithExperimental(func(cfg *experimental.Config) error {
if cfg.WebApp != nil && cfg.WebApp.UsePodAffinity {
podAntiAffinity = &corev1.PodAntiAffinity{
PreferredDuringSchedulingIgnoredDuringExecution: []corev1.WeightedPodAffinityTerm{{
Weight: 100,
PodAffinityTerm: corev1.PodAffinityTerm{
LabelSelector: &metav1.LabelSelector{
MatchExpressions: []metav1.LabelSelectorRequirement{{
Key: "component",
Operator: "In",
Values: []string{Component},
}},
},
TopologyKey: cluster.AffinityLabelMeta,
},
}},
}
}
return nil
})

const kubeRbacProxyContainerName = "kube-rbac-proxy"
return []runtime.Object{
&appsv1.Deployment{
TypeMeta: common.TypeMetaDeployment,
Expand All @@ -117,7 +141,10 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
},
},
Spec: corev1.PodSpec{
Affinity: common.NodeAffinity(cluster.AffinityLabelMeta),
Affinity: &corev1.Affinity{
NodeAffinity: common.NodeAffinity(cluster.AffinityLabelMeta).NodeAffinity,
PodAntiAffinity: podAntiAffinity,
},
PriorityClassName: common.SystemNodeCritical,
ServiceAccountName: Component,
EnableServiceLinks: pointer.Bool(false),
Expand All @@ -142,7 +169,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
},
}},
Containers: []corev1.Container{{
Name: "kube-rbac-proxy",
Name: kubeRbacProxyContainerName,
Image: ctx.ImageName(common.ThirdPartyContainerRepo(ctx.Config.Repository, KubeRBACProxyRepo), KubeRBACProxyImage, KubeRBACProxyTag),
ImagePullPolicy: corev1.PullIfNotPresent,
Args: []string{
Expand All @@ -165,12 +192,12 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
Name: MetricsContainerName,
Protocol: *common.TCPProtocol,
}},
Resources: corev1.ResourceRequirements{
Resources: common.ResourceRequirements(ctx, Component, kubeRbacProxyContainerName, corev1.ResourceRequirements{
Requests: corev1.ResourceList{
"cpu": resource.MustParse("1m"),
"memory": resource.MustParse("30Mi"),
},
},
}),
SecurityContext: &corev1.SecurityContext{
RunAsGroup: pointer.Int64(65532),
RunAsNonRoot: pointer.Bool(true),
Expand All @@ -180,12 +207,12 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
Name: Component,
Image: ctx.ImageName(ctx.Config.Repository, Component, ctx.VersionManifest.Components.Proxy.Version),
ImagePullPolicy: corev1.PullIfNotPresent,
Resources: corev1.ResourceRequirements{
Resources: common.ResourceRequirements(ctx, Component, Component, corev1.ResourceRequirements{
Requests: corev1.ResourceList{
"cpu": resource.MustParse("100m"),
"memory": resource.MustParse("200Mi"),
},
},
}),
Ports: []corev1.ContainerPort{{
ContainerPort: ContainerHTTPPort,
Name: "http",
Expand Down
4 changes: 2 additions & 2 deletions install/installer/pkg/components/public-api-server/deployment.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,12 +46,12 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
Name: Component,
Image: ctx.ImageName(ctx.Config.Repository, Component, ctx.VersionManifest.Components.PublicAPIServer.Version),
ImagePullPolicy: corev1.PullIfNotPresent,
Resources: corev1.ResourceRequirements{
Resources: common.ResourceRequirements(ctx, Component, Component, corev1.ResourceRequirements{
Requests: corev1.ResourceList{
"cpu": resource.MustParse("100m"),
"memory": resource.MustParse("32Mi"),
},
},
}),
Ports: []corev1.ContainerPort{
{
ContainerPort: HTTPContainerPort,
Expand Down
4 changes: 2 additions & 2 deletions install/installer/pkg/components/registry-facade/daemonset.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -207,12 +207,12 @@ func daemonset(ctx *common.RenderContext) ([]runtime.Object, error) {
Image: ctx.ImageName(ctx.Config.Repository, Component, ctx.VersionManifest.Components.RegistryFacade.Version),
ImagePullPolicy: corev1.PullIfNotPresent,
Args: []string{"run", "/mnt/config/config.json"},
Resources: corev1.ResourceRequirements{
Resources: common.ResourceRequirements(ctx, Component, Component, corev1.ResourceRequirements{
Requests: corev1.ResourceList{
"cpu": resource.MustParse("100m"),
"memory": resource.MustParse("32Mi"),
},
},
}),
Ports: []corev1.ContainerPort{{
Name: ContainerPortName,
ContainerPort: ContainerPort,
Expand Down
Loading