diff --git a/components/ws-daemon/pkg/cgroup/cgroup.go b/components/ws-daemon/pkg/cgroup/cgroup.go new file mode 100644 index 00000000000000..580cf8d537a7f7 --- /dev/null +++ b/components/ws-daemon/pkg/cgroup/cgroup.go @@ -0,0 +1,120 @@ +// Copyright (c) 2022 Gitpod GmbH. All rights reserved. +// Licensed under the GNU Affero General Public License (AGPL). +// See License-AGPL.txt in the project root for license information. + +package cgroup + +import ( + "context" + + "github.com/gitpod-io/gitpod/common-go/cgroups" + "github.com/gitpod-io/gitpod/common-go/log" + "github.com/gitpod-io/gitpod/ws-daemon/pkg/dispatch" + "github.com/prometheus/client_golang/prometheus" + "golang.org/x/xerrors" +) + +func NewPluginHost(cgroupBasePath string, plugins ...Plugin) (*PluginHost, error) { + var version Version + unified, err := cgroups.IsUnifiedCgroupSetup() + if err != nil { + return nil, xerrors.Errorf("could not determine cgroup setup: %w", err) + } + if unified { + version = Version2 + } else { + version = Version1 + } + + return &PluginHost{ + CGroupBasePath: cgroupBasePath, + CGroupVersion: version, + Plugins: plugins, + + pluginActivationTotalVec: prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "cgroup_plugin_activation_total", + Help: "counts the total activation of cgroup plugins", + }, []string{"plugin", "success"}), + }, nil +} + +type PluginHost struct { + CGroupBasePath string + CGroupVersion Version + Plugins []Plugin + + pluginActivationTotalVec *prometheus.CounterVec +} + +var _ dispatch.Listener = &PluginHost{} +var _ prometheus.Collector = &PluginHost{} + +func (host *PluginHost) Describe(c chan<- *prometheus.Desc) { + host.pluginActivationTotalVec.Describe(c) + for _, p := range host.Plugins { + col, ok := p.(prometheus.Collector) + if !ok { + continue + } + + col.Describe(c) + } +} + +func (host *PluginHost) Collect(c chan<- prometheus.Metric) { + host.pluginActivationTotalVec.Collect(c) + for _, p := range host.Plugins { + col, ok := p.(prometheus.Collector) + if !ok { + continue + } + + col.Collect(c) + } +} + +func (host *PluginHost) WorkspaceAdded(ctx context.Context, ws *dispatch.Workspace) (err error) { + disp := dispatch.GetFromContext(ctx) + if disp == nil { + return xerrors.Errorf("no dispatch available") + } + + cgroupPath, err := disp.Runtime.ContainerCGroupPath(context.Background(), ws.ContainerID) + if err != nil { + return xerrors.Errorf("cannot start governer: %w", err) + } + + for _, plg := range host.Plugins { + if plg.Type() != host.CGroupVersion { + continue + } + + go func(plg Plugin) { + err := plg.Apply(ctx, host.CGroupBasePath, cgroupPath) + if err == context.Canceled || err == context.DeadlineExceeded { + return + } + if err != nil { + log.WithError(err).WithFields(ws.OWI()).WithField("plugin", plg.Name()).Error("cgroup plugin failure") + host.pluginActivationTotalVec.WithLabelValues(plg.Name(), "false").Inc() + } else { + host.pluginActivationTotalVec.WithLabelValues(plg.Name(), "true").Inc() + } + }(plg) + } + + return nil +} + +type Plugin interface { + Name() string + Type() Version + Apply(ctx context.Context, basePath, cgroupPath string) error +} + +type Version int + +const ( + Version1 Version = iota + Version2 +) diff --git a/components/ws-daemon/pkg/daemon/cache_reclaim.go b/components/ws-daemon/pkg/cgroup/plugin_cachereclaim.go similarity index 61% rename from components/ws-daemon/pkg/daemon/cache_reclaim.go rename to components/ws-daemon/pkg/cgroup/plugin_cachereclaim.go index 1555c1637e85cc..c732c15c3de80b 100644 --- a/components/ws-daemon/pkg/daemon/cache_reclaim.go +++ b/components/ws-daemon/pkg/cgroup/plugin_cachereclaim.go @@ -1,8 +1,8 @@ -// Copyright (c) 2021 Gitpod GmbH. All rights reserved. +// Copyright (c) 2022 Gitpod GmbH. All rights reserved. // Licensed under the GNU Affero General Public License (AGPL). // See License-AGPL.txt in the project root for license information. -package daemon +package cgroup import ( "bufio" @@ -17,63 +17,38 @@ import ( "strings" "time" - "github.com/gitpod-io/gitpod/common-go/log" - "github.com/gitpod-io/gitpod/ws-daemon/pkg/dispatch" "golang.org/x/xerrors" ) -type CacheReclaim string +type CacheReclaim struct{} -// WorkspaceAdded will customize the cgroups for every workspace that is started -func (c CacheReclaim) WorkspaceAdded(ctx context.Context, ws *dispatch.Workspace) error { - disp := dispatch.GetFromContext(ctx) - if disp == nil { - return xerrors.Errorf("no dispatch available") - } +func (c *CacheReclaim) Name() string { return "cache-reclaim-v1" } +func (c *CacheReclaim) Type() Version { return Version1 } - cgroupPath, err := disp.Runtime.ContainerCGroupPath(context.Background(), ws.ContainerID) - if err != nil { - return xerrors.Errorf("cannot start governer: %w", err) - } +func (c *CacheReclaim) Apply(ctx context.Context, basePath, cgroupPath string) error { + memPath := filepath.Join(string(basePath), "memory", cgroupPath) + + t := time.NewTicker(10 * time.Second) + defer t.Stop() + + var lastReclaim time.Time + for { + select { + case <-ctx.Done(): + return nil + case <-t.C: + } - memPath := filepath.Join(string(c), "memory", cgroupPath) - - go func() { - owi := ws.OWI() - log.WithFields(ws.OWI()).Debug("starting page cache reclaim") - - t := time.NewTicker(10 * time.Second) - defer t.Stop() - - var lastReclaim time.Time - for { - select { - case <-ctx.Done(): - log.WithFields(owi).Debug("shutting down page cache reclaim") - return - case <-t.C: - } - - if !lastReclaim.IsZero() && time.Since(lastReclaim) < 30*time.Second { - continue - } - - stats, err := reclaimPageCache(memPath) - if err != nil { - log.WithFields(owi).WithError(err).Warn("cannot reclaim page cache") - continue - } - e := log.WithFields(owi).WithField("reclaimed_bytes", stats.Reclaimed()).WithField("stats", stats) - if stats.DidReclaim { - e.Debug("reclaimed page cache") - } else { - e.Debug("did not reclaim page cache") - } - lastReclaim = time.Now() + if !lastReclaim.IsZero() && time.Since(lastReclaim) < 30*time.Second { + continue } - }() - return nil + _, err := reclaimPageCache(memPath) + if err != nil { + continue + } + lastReclaim = time.Now() + } } type reclaimStats struct { @@ -118,7 +93,9 @@ func readLimit(memCgroupPath string) (uint64, error) { fn := filepath.Join(string(memCgroupPath), "memory.limit_in_bytes") fc, err := os.ReadFile(fn) if err != nil { - // TODO(toru): find out why the file does not exists + // We have a race between the dispatch noticing that a workspace is stopped + // and the container going away. Hence we might be running for workspace + // container which no longer exist, i.e. their cgroup files no longer exist. if errors.Is(err, fs.ErrNotExist) { return 0, nil } diff --git a/components/ws-daemon/pkg/daemon/cache_reclaim_test.go b/components/ws-daemon/pkg/cgroup/plugin_cachereclaim_test.go similarity index 99% rename from components/ws-daemon/pkg/daemon/cache_reclaim_test.go rename to components/ws-daemon/pkg/cgroup/plugin_cachereclaim_test.go index a54786b482b737..57984a402e6000 100644 --- a/components/ws-daemon/pkg/daemon/cache_reclaim_test.go +++ b/components/ws-daemon/pkg/cgroup/plugin_cachereclaim_test.go @@ -2,7 +2,7 @@ // Licensed under the GNU Affero General Public License (AGPL). // See License-AGPL.txt in the project root for license information. -package daemon +package cgroup import ( "fmt" diff --git a/components/ws-daemon/pkg/cgroup/plugin_fuse.go b/components/ws-daemon/pkg/cgroup/plugin_fuse.go new file mode 100644 index 00000000000000..f58879802c0c02 --- /dev/null +++ b/components/ws-daemon/pkg/cgroup/plugin_fuse.go @@ -0,0 +1,58 @@ +// Copyright (c) 2022 Gitpod GmbH. All rights reserved. +// Licensed under the GNU Affero General Public License (AGPL). +// See License-AGPL.txt in the project root for license information. + +package cgroup + +import ( + "context" + + "github.com/containerd/cgroups" + "github.com/opencontainers/runtime-spec/specs-go" + "golang.org/x/xerrors" +) + +var ( + fuseDeviceMajor int64 = 10 + fuseDeviceMinor int64 = 229 +) + +type FuseDeviceEnablerV1 struct{} + +func (c *FuseDeviceEnablerV1) Name() string { return "fuse-device-enabler-v1" } +func (c *FuseDeviceEnablerV1) Type() Version { return Version1 } + +func (c *FuseDeviceEnablerV1) Apply(ctx context.Context, basePath, cgroupPath string) error { + control, err := cgroups.Load(customV1(basePath), cgroups.StaticPath(cgroupPath)) + + if err != nil { + return xerrors.Errorf("error loading cgroup at path: %s %w", cgroupPath, err) + } + + res := &specs.LinuxResources{ + Devices: []specs.LinuxDeviceCgroup{ + // /dev/fuse + { + Type: "c", + Minor: &fuseDeviceMinor, + Major: &fuseDeviceMajor, + Access: "rwm", + Allow: true, + }, + }, + } + + if err := control.Update(res); err != nil { + return xerrors.Errorf("cgroup update failed: %w", err) + } + + return nil +} + +func customV1(basePath string) func() ([]cgroups.Subsystem, error) { + return func() ([]cgroups.Subsystem, error) { + return []cgroups.Subsystem{ + cgroups.NewDevices(basePath), + }, nil + } +} diff --git a/components/ws-daemon/pkg/cgroup/plugin_fuse_v2.go b/components/ws-daemon/pkg/cgroup/plugin_fuse_v2.go new file mode 100644 index 00000000000000..dc815f3829bcb1 --- /dev/null +++ b/components/ws-daemon/pkg/cgroup/plugin_fuse_v2.go @@ -0,0 +1,67 @@ +// Copyright (c) 2022 Gitpod GmbH. All rights reserved. +// Licensed under the GNU Affero General Public License (AGPL). +// See License-AGPL.txt in the project root for license information. + +package cgroup + +import ( + "context" + "path/filepath" + + "github.com/opencontainers/runc/libcontainer/cgroups/ebpf" + "github.com/opencontainers/runc/libcontainer/cgroups/ebpf/devicefilter" + "github.com/opencontainers/runc/libcontainer/devices" + "github.com/opencontainers/runc/libcontainer/specconv" + "golang.org/x/sys/unix" + "golang.org/x/xerrors" +) + +type FuseDeviceEnablerV2 struct{} + +func (c *FuseDeviceEnablerV2) Name() string { return "fuse-device-enabler-v2" } +func (c *FuseDeviceEnablerV2) Type() Version { return Version2 } + +func (c *FuseDeviceEnablerV2) Apply(ctx context.Context, basePath, cgroupPath string) error { + fullCgroupPath := filepath.Join(basePath, cgroupPath) + cgroupFD, err := unix.Open(fullCgroupPath, unix.O_DIRECTORY|unix.O_RDONLY, 0o600) + if err != nil { + return xerrors.Errorf("cannot get directory fd for %s", fullCgroupPath) + } + defer unix.Close(cgroupFD) + + insts, license, err := devicefilter.DeviceFilter(composeDeviceRules()) + if err != nil { + return xerrors.Errorf("failed to generate device filter: %w", err) + } + + _, err = ebpf.LoadAttachCgroupDeviceFilter(insts, license, cgroupFD) + if err != nil { + return xerrors.Errorf("failed to attach cgroup device filter: %w", err) + } + + return nil +} + +func composeDeviceRules() []*devices.Rule { + denyAll := devices.Rule{ + Type: 'a', + Permissions: "rwm", + Allow: false, + } + + allowFuse := devices.Rule{ + Type: 'c', + Major: fuseDeviceMajor, + Minor: fuseDeviceMinor, + Permissions: "rwm", + Allow: true, + } + + deviceRules := make([]*devices.Rule, 0) + deviceRules = append(deviceRules, &denyAll, &allowFuse) + for _, device := range specconv.AllowedDevices { + deviceRules = append(deviceRules, &device.Rule) + } + + return deviceRules +} diff --git a/components/ws-daemon/pkg/cgroup/plugin_iolimit_v2.go b/components/ws-daemon/pkg/cgroup/plugin_iolimit_v2.go new file mode 100644 index 00000000000000..76bf6a765978d9 --- /dev/null +++ b/components/ws-daemon/pkg/cgroup/plugin_iolimit_v2.go @@ -0,0 +1,67 @@ +// Copyright (c) 2022 Gitpod GmbH. All rights reserved. +// Licensed under the GNU Affero General Public License (AGPL). +// See License-AGPL.txt in the project root for license information. + +package cgroup + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/gitpod-io/gitpod/common-go/log" +) + +type IOLimiterV2 struct { + WriteBytesPerSecond int64 + ReadBytesPerSecond int64 + ReadIOPs int64 + WriteIOPs int64 +} + +func (c *IOLimiterV2) Name() string { return "iolimiter-v2" } +func (c *IOLimiterV2) Type() Version { return Version2 } + +func (c *IOLimiterV2) Apply(ctx context.Context, basePath, cgroupPath string) error { + return c.writeIOMax(filepath.Join(basePath, cgroupPath)) +} + +func (c *IOLimiterV2) writeIOMax(loc string) error { + iostat, err := os.ReadFile(filepath.Join(string(loc), "io.stat")) + if err != nil { + return err + } + var devs []string + for _, line := range strings.Split(string(iostat), "\n") { + fields := strings.Fields(line) + if len(fields) < 1 { + continue + } + devs = append(devs, fields[0]) + } + + cpuMaxPath := filepath.Join(string(loc), "io.max") + for _, dev := range devs { + err := os.WriteFile(cpuMaxPath, []byte(fmt.Sprintf( + "%s wbps=%s rbps=%s wiops=%s riops=%s", + dev, + getLimit(c.WriteBytesPerSecond), + getLimit(c.ReadBytesPerSecond), + getLimit(c.WriteIOPs), + getLimit(c.ReadIOPs), + )), 0644) + if err != nil { + log.WithField("dev", dev).WithError(err).Warn("cannot write io.max") + } + } + return nil +} + +func getLimit(v int64) string { + if v <= 0 { + return "max" + } + return fmt.Sprintf("%d", v) +} diff --git a/components/ws-daemon/pkg/daemon/cgroup_customizer.go b/components/ws-daemon/pkg/daemon/cgroup_customizer.go deleted file mode 100644 index d1f92cb465a55e..00000000000000 --- a/components/ws-daemon/pkg/daemon/cgroup_customizer.go +++ /dev/null @@ -1,149 +0,0 @@ -// Copyright (c) 2021 Gitpod GmbH. All rights reserved. -// Licensed under the GNU Affero General Public License (AGPL). -// See License-AGPL.txt in the project root for license information. - -package daemon - -import ( - "context" - "path/filepath" - - "github.com/containerd/cgroups" - "github.com/gitpod-io/gitpod/ws-daemon/pkg/container" - "github.com/gitpod-io/gitpod/ws-daemon/pkg/dispatch" - "github.com/opencontainers/runc/libcontainer/cgroups/ebpf" - "github.com/opencontainers/runc/libcontainer/cgroups/ebpf/devicefilter" - "github.com/opencontainers/runc/libcontainer/devices" - "github.com/opencontainers/runc/libcontainer/specconv" - "github.com/opencontainers/runtime-spec/specs-go" - "golang.org/x/sys/unix" - "golang.org/x/xerrors" -) - -var ( - fuseDeviceMajor int64 = 10 - fuseDeviceMinor int64 = 229 -) - -func NewCGroupCustomizer(basePath string, unified bool) dispatch.Listener { - if unified { - return &CgroupCustomizerV2{ - cgroupBasePath: basePath, - } - } else { - return &CgroupCustomizerV1{ - cgroupBasePath: basePath, - } - } -} - -type CgroupCustomizerV1 struct { - cgroupBasePath string -} - -func (c *CgroupCustomizerV1) WithCgroupBasePath(basePath string) { - c.cgroupBasePath = basePath -} - -// WorkspaceAdded will customize the cgroups for every workspace that is started -func (c *CgroupCustomizerV1) WorkspaceAdded(ctx context.Context, ws *dispatch.Workspace) error { - cgroupPath, err := retrieveCGroupPath(ctx, ws.ContainerID) - if err != nil { - return err - } - - control, err := cgroups.Load(c.customV1, cgroups.StaticPath(cgroupPath)) - - if err != nil { - return xerrors.Errorf("error loading cgroup at path: %s %w", cgroupPath, err) - } - - res := &specs.LinuxResources{ - Devices: []specs.LinuxDeviceCgroup{ - // /dev/fuse - { - Type: "c", - Minor: &fuseDeviceMinor, - Major: &fuseDeviceMajor, - Access: "rwm", - Allow: true, - }, - }, - } - - if err := control.Update(res); err != nil { - return xerrors.Errorf("cgroup update failed: %w", err) - } - - return nil -} - -func (c *CgroupCustomizerV1) customV1() ([]cgroups.Subsystem, error) { - return []cgroups.Subsystem{ - cgroups.NewDevices(c.cgroupBasePath), - }, nil -} - -type CgroupCustomizerV2 struct { - cgroupBasePath string -} - -func (c *CgroupCustomizerV2) WorkspaceAdded(ctx context.Context, ws *dispatch.Workspace) error { - cgroupPath, err := retrieveCGroupPath(ctx, ws.ContainerID) - if err != nil { - return err - } - - fullCgroupPath := filepath.Join(c.cgroupBasePath, cgroupPath) - cgroupFD, err := unix.Open(fullCgroupPath, unix.O_DIRECTORY|unix.O_RDONLY, 0o600) - if err != nil { - return xerrors.Errorf("cannot get directory fd for %s", fullCgroupPath) - } - defer unix.Close(cgroupFD) - - insts, license, err := devicefilter.DeviceFilter(composeDeviceRules()) - if err != nil { - return xerrors.Errorf("failed to generate device filter: %w", err) - } - - _, err = ebpf.LoadAttachCgroupDeviceFilter(insts, license, cgroupFD) - return err -} - -func retrieveCGroupPath(ctx context.Context, id container.ID) (string, error) { - disp := dispatch.GetFromContext(ctx) - if disp == nil { - return "", xerrors.Errorf("no dispatch available") - } - - cgroupPath, err := disp.Runtime.ContainerCGroupPath(context.Background(), id) - if err != nil { - return "", xerrors.Errorf("cannot retrieve cgroup path: %w", err) - } - - return cgroupPath, nil -} - -func composeDeviceRules() []*devices.Rule { - denyAll := devices.Rule{ - Type: 'a', - Permissions: "rwm", - Allow: false, - } - - allowFuse := devices.Rule{ - Type: 'c', - Major: fuseDeviceMajor, - Minor: fuseDeviceMinor, - Permissions: "rwm", - Allow: true, - } - - deviceRules := make([]*devices.Rule, 0) - deviceRules = append(deviceRules, &denyAll, &allowFuse) - for _, device := range specconv.AllowedDevices { - deviceRules = append(deviceRules, &device.Rule) - } - - return deviceRules -} diff --git a/components/ws-daemon/pkg/daemon/config.go b/components/ws-daemon/pkg/daemon/config.go index 2b3a57ef0a094e..d8fed4e4cfcfad 100644 --- a/components/ws-daemon/pkg/daemon/config.go +++ b/components/ws-daemon/pkg/daemon/config.go @@ -11,6 +11,7 @@ import ( "github.com/gitpod-io/gitpod/ws-daemon/pkg/diskguard" "github.com/gitpod-io/gitpod/ws-daemon/pkg/hosts" "github.com/gitpod-io/gitpod/ws-daemon/pkg/iws" + "k8s.io/apimachinery/pkg/api/resource" ) // Config configures the workspace node daemon @@ -19,7 +20,8 @@ type Config struct { Content content.Config `json:"content"` Uidmapper iws.UidmapperConfig `json:"uidmapper"` - Resources cpulimit.Config `json:"cpulimit"` + CPULimit cpulimit.Config `json:"cpulimit"` + IOLimit IOLimitConfig `json:"ioLimit"` Hosts hosts.Config `json:"hosts"` DiskSpaceGuard diskguard.Config `json:"disk"` } @@ -29,3 +31,10 @@ type RuntimeConfig struct { Kubeconfig string `json:"kubeconfig"` KubernetesNamespace string `json:"namespace"` } + +type IOLimitConfig struct { + WriteBWPerSecond resource.Quantity `json:"writeBandwidthPerSecond"` + ReadBWPerSecond resource.Quantity `json:"readBandwidthPerSecond"` + WriteIOPS int64 `json:"writeIOPS"` + ReadIOPS int64 `json:"readIOPS"` +} diff --git a/components/ws-daemon/pkg/daemon/daemon.go b/components/ws-daemon/pkg/daemon/daemon.go index 22caca9e68c701..6b0be05854ad2b 100644 --- a/components/ws-daemon/pkg/daemon/daemon.go +++ b/components/ws-daemon/pkg/daemon/daemon.go @@ -16,8 +16,8 @@ import ( "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" - "github.com/gitpod-io/gitpod/common-go/cgroups" "github.com/gitpod-io/gitpod/ws-daemon/api" + "github.com/gitpod-io/gitpod/ws-daemon/pkg/cgroup" "github.com/gitpod-io/gitpod/ws-daemon/pkg/container" "github.com/gitpod-io/gitpod/ws-daemon/pkg/content" "github.com/gitpod-io/gitpod/ws-daemon/pkg/cpulimit" @@ -51,20 +51,29 @@ func NewDaemon(config Config, reg prometheus.Registerer) (*Daemon, error) { return nil, err } - listener := []dispatch.Listener{ - cpulimit.NewDispatchListener(&config.Resources, reg), - markUnmountFallback, + cgroupPlugins, err := cgroup.NewPluginHost(config.CPULimit.CGroupBasePath, + &cgroup.CacheReclaim{}, + &cgroup.FuseDeviceEnablerV1{}, + &cgroup.FuseDeviceEnablerV2{}, + &cgroup.IOLimiterV2{ + WriteBytesPerSecond: config.IOLimit.WriteBWPerSecond.Value(), + ReadBytesPerSecond: config.IOLimit.ReadBWPerSecond.Value(), + WriteIOPs: config.IOLimit.WriteIOPS, + ReadIOPs: config.IOLimit.ReadIOPS, + }, + ) + if err != nil { + return nil, err } - - unified, err := cgroups.IsUnifiedCgroupSetup() + err = reg.Register(cgroupPlugins) if err != nil { - return nil, xerrors.Errorf("could not determine cgroup setup: %w", err) + return nil, xerrors.Errorf("cannot register cgroup plugin metrics: %w", err) } - listener = append(listener, NewCGroupCustomizer(config.Resources.CGroupBasePath, unified)) - - if !unified { - listener = append(listener, CacheReclaim(config.Resources.CGroupBasePath)) + listener := []dispatch.Listener{ + cpulimit.NewDispatchListener(&config.CPULimit, reg), + markUnmountFallback, + cgroupPlugins, } dsptch, err := dispatch.NewDispatch(containerRuntime, clientset, config.Runtime.KubernetesNamespace, nodename, listener...) @@ -79,7 +88,7 @@ func NewDaemon(config Config, reg prometheus.Registerer) (*Daemon, error) { containerRuntime, dsptch.WorkspaceExistsOnNode, &iws.Uidmapper{Config: config.Uidmapper, Runtime: containerRuntime}, - config.Resources.CGroupBasePath, + config.CPULimit.CGroupBasePath, reg, ) if err != nil { diff --git a/install/installer/pkg/components/ws-daemon/configmap.go b/install/installer/pkg/components/ws-daemon/configmap.go index 0f31217c8688b6..e45f6ddad19c7c 100644 --- a/install/installer/pkg/components/ws-daemon/configmap.go +++ b/install/installer/pkg/components/ws-daemon/configmap.go @@ -43,13 +43,22 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) { CGroupBasePath: "/mnt/node-cgroups", ControlPeriod: util.Duration(15 * time.Second), } + var ioLimitConfig daemon.IOLimitConfig ctx.WithExperimental(func(ucfg *experimental.Config) error { - if ucfg.Workspace != nil { - cpuLimitConfig.Enabled = ucfg.Workspace.CPULimits.Enabled - cpuLimitConfig.BurstLimit = ucfg.Workspace.CPULimits.BurstLimit - cpuLimitConfig.Limit = ucfg.Workspace.CPULimits.Limit - cpuLimitConfig.TotalBandwidth = ucfg.Workspace.CPULimits.NodeCPUBandwidth + if ucfg.Workspace == nil { + return nil } + + cpuLimitConfig.Enabled = ucfg.Workspace.CPULimits.Enabled + cpuLimitConfig.BurstLimit = ucfg.Workspace.CPULimits.BurstLimit + cpuLimitConfig.Limit = ucfg.Workspace.CPULimits.Limit + cpuLimitConfig.TotalBandwidth = ucfg.Workspace.CPULimits.NodeCPUBandwidth + + ioLimitConfig.WriteBWPerSecond = ucfg.Workspace.IOLimits.WriteBWPerSecond + ioLimitConfig.ReadBWPerSecond = ucfg.Workspace.IOLimits.ReadBWPerSecond + ioLimitConfig.WriteIOPS = ucfg.Workspace.IOLimits.WriteIOPS + ioLimitConfig.ReadIOPS = ucfg.Workspace.IOLimits.ReadIOPS + return nil }) @@ -97,7 +106,8 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) { Size: 70000, }}, }, - Resources: cpuLimitConfig, + CPULimit: cpuLimitConfig, + IOLimit: ioLimitConfig, Hosts: hosts.Config{ Enabled: true, NodeHostsFile: "/mnt/hosts", diff --git a/install/installer/pkg/config/v1/experimental/experimental.go b/install/installer/pkg/config/v1/experimental/experimental.go index 4f75de94b79a11..185bc64cd228bf 100644 --- a/install/installer/pkg/config/v1/experimental/experimental.go +++ b/install/installer/pkg/config/v1/experimental/experimental.go @@ -29,6 +29,12 @@ type WorkspaceConfig struct { Limit resource.Quantity `json:"limit"` BurstLimit resource.Quantity `json:"burstLimit"` } + IOLimits struct { + WriteBWPerSecond resource.Quantity `json:"writeBandwidthPerSecond"` + ReadBWPerSecond resource.Quantity `json:"readBandwidthPerSecond"` + WriteIOPS int64 `json:"writeIOPS"` + ReadIOPS int64 `json:"readIOPS"` + } `json:"ioLimits"` RegistryFacade struct { IPFSCache struct {