Skip to content

Commit

Permalink
Add ability to set CPU limits/requests on the plugin containers
Browse files Browse the repository at this point in the history
Co-authored-by: Sergii Leshchenko <sleshche@redhat.com>
  • Loading branch information
mshaposhnik and sleshchenko authored Mar 2, 2020
1 parent 2ef1d2b commit 9732139
Show file tree
Hide file tree
Showing 28 changed files with 1,263 additions and 613 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -87,22 +87,39 @@ che.workspace.maven_server_java_options=-XX:MaxRAM=128m -XX:MaxRAMFraction=1 -XX
# RAM limit default for each machine that has no RAM settings in environment.
che.workspace.default_memory_limit_mb=1024

# RAM limit default for each sidecar that has no RAM settings in Che plugin configuration.
# RAM request default for each container that has no explicit RAM settings in environment.
# this amount will be allocated on workspace container creation
# this property might not be supported by all infrastructure implementations:
# currently it is supported by k8s and openshift
# if default memory request is more than the memory limit, request will be ignored,
# and only limit will be used
che.workspace.default_memory_request_mb=200


# CPU limit default for each container that has no CPU settings in environment.
# Can be specified either in floating point cores number, e.g. 0.125 or in K8S format integer millicores e.g. 125m
che.workspace.default_cpu_limit_cores=2

# CPU request default for each container that has no CPU settings in environment.
# if default CPU request is more than the CPU limit, request will be ignored,
# and only limit will be used
che.workspace.default_cpu_request_cores=0.125

# RAM limit and request default for each sidecar that has no RAM settings in Che plugin configuration.
che.workspace.sidecar.default_memory_limit_mb=128
che.workspace.sidecar.default_memory_request_mb=64

# CPU limit and request default for each sidecar that has no CPU settings in Che plugin configuration.
# Can be specified either in floating point cores number, e.g. 0.125 or in K8S format integer millicores e.g. 125m
che.workspace.sidecar.default_cpu_limit_cores=1
che.workspace.sidecar.default_cpu_request_cores=0.115

# Define image pulling strategy for sidecars.
# Possible values are: Always, Never, IfNotPresent. Any other value
# will be interpreted as unspecified policy (Always if :latest tag is specified,
# or IfNotPresent otherwise.)
che.workspace.sidecar.image_pull_policy=Always

# RAM request default for each machine that has no explicit RAM settings in environment.
# this amount will be allocated on workspace container creation
# this property might not be supported by all infrastructure implementations:
# currently it is supported by k8s and openshift
# if default memory request is more than the memory limit request will be ignored,
# and only limit will be used
che.workspace.default_memory_request_mb=512

# Period of inactive workspaces suspend job execution.
che.workspace.activity_check_scheduler_period_s=60
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,18 @@ public interface MachineConfig {
*/
String MEMORY_REQUEST_ATTRIBUTE = "memoryRequestBytes";

/**
* Name of the attribute from {@link #getAttributes()} which if present defines CPU limit of the
* machine in cores.
*/
String CPU_LIMIT_ATTRIBUTE = "cpuLimitCores";

/**
* Name of the attribute from {@link #getAttributes()} which if present defines requested CPU
* allocation of the machine in cores.
*/
String CPU_REQUEST_ATTRIBUTE = "cpuRequestCores";

/**
* Name of the attribute from {@link #getAttributes()} which, if present, defines the entrypoint
* command to be executed in the machine/container.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.VcsSshKeysProvisioner;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.VcsSslCertificateProvisioner;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.env.EnvVarsConverter;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.limits.ram.RamLimitRequestProvisioner;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.limits.ram.ContainerResourceProvisioner;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.restartpolicy.RestartPolicyRewriter;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.server.ServersConverter;
import org.eclipse.che.workspace.infrastructure.kubernetes.server.PreviewUrlExposer;
Expand Down Expand Up @@ -64,7 +64,7 @@ class KubernetesEnvironmentProvisionerImpl
private final ServersConverter<KubernetesEnvironment> serversConverter;
private final EnvVarsConverter envVarsConverter;
private final RestartPolicyRewriter restartPolicyRewriter;
private final RamLimitRequestProvisioner ramLimitProvisioner;
private final ContainerResourceProvisioner resourceLimitRequestProvisioner;
private final LogsVolumeMachineProvisioner logsVolumeMachineProvisioner;
private final SecurityContextProvisioner securityContextProvisioner;
private final PodTerminationGracePeriodProvisioner podTerminationGracePeriodProvisioner;
Expand All @@ -86,7 +86,7 @@ public KubernetesEnvironmentProvisionerImpl(
EnvVarsConverter envVarsConverter,
RestartPolicyRewriter restartPolicyRewriter,
WorkspaceVolumesStrategy volumesStrategy,
RamLimitRequestProvisioner ramLimitProvisioner,
ContainerResourceProvisioner resourceLimitRequestProvisioner,
LogsVolumeMachineProvisioner logsVolumeMachineProvisioner,
SecurityContextProvisioner securityContextProvisioner,
PodTerminationGracePeriodProvisioner podTerminationGracePeriodProvisioner,
Expand All @@ -105,7 +105,7 @@ public KubernetesEnvironmentProvisionerImpl(
this.serversConverter = serversConverter;
this.envVarsConverter = envVarsConverter;
this.restartPolicyRewriter = restartPolicyRewriter;
this.ramLimitProvisioner = ramLimitProvisioner;
this.resourceLimitRequestProvisioner = resourceLimitRequestProvisioner;
this.logsVolumeMachineProvisioner = logsVolumeMachineProvisioner;
this.securityContextProvisioner = securityContextProvisioner;
this.podTerminationGracePeriodProvisioner = podTerminationGracePeriodProvisioner;
Expand Down Expand Up @@ -146,7 +146,7 @@ public void provision(KubernetesEnvironment k8sEnv, RuntimeIdentity identity)
LOG.debug("Provisioning environment items for workspace '{}'", workspaceId);
restartPolicyRewriter.provision(k8sEnv, identity);
uniqueNamesProvisioner.provision(k8sEnv, identity);
ramLimitProvisioner.provision(k8sEnv, identity);
resourceLimitRequestProvisioner.provision(k8sEnv, identity);
externalServerIngressTlsProvisioner.provision(k8sEnv, identity);
securityContextProvisioner.provision(k8sEnv, identity);
podTerminationGracePeriodProvisioner.provision(k8sEnv, identity);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,7 @@
import static org.eclipse.che.workspace.infrastructure.kubernetes.environment.PodMerger.DEPLOYMENT_NAME_LABEL;
import static org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesObjectUtil.setSelector;

import com.google.common.annotations.VisibleForTesting;
import io.fabric8.kubernetes.api.model.ConfigMap;
import io.fabric8.kubernetes.api.model.Container;
import io.fabric8.kubernetes.api.model.HasMetadata;
import io.fabric8.kubernetes.api.model.PersistentVolumeClaim;
import io.fabric8.kubernetes.api.model.Pod;
Expand All @@ -26,7 +24,6 @@
import io.fabric8.kubernetes.api.model.apps.Deployment;
import io.fabric8.kubernetes.api.model.extensions.Ingress;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
Expand All @@ -42,13 +39,10 @@
import org.eclipse.che.api.workspace.server.spi.environment.InternalMachineConfig;
import org.eclipse.che.api.workspace.server.spi.environment.InternalRecipe;
import org.eclipse.che.api.workspace.server.spi.environment.MachineConfigsValidator;
import org.eclipse.che.api.workspace.server.spi.environment.MemoryAttributeProvisioner;
import org.eclipse.che.api.workspace.server.spi.environment.RecipeRetriever;
import org.eclipse.che.commons.annotation.Nullable;
import org.eclipse.che.workspace.infrastructure.kubernetes.Names;
import org.eclipse.che.workspace.infrastructure.kubernetes.Warnings;
import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment.PodData;
import org.eclipse.che.workspace.infrastructure.kubernetes.util.Containers;

/**
* Parses {@link InternalEnvironment} into {@link KubernetesEnvironment}.
Expand All @@ -60,7 +54,6 @@ public class KubernetesEnvironmentFactory

private final KubernetesRecipeParser recipeParser;
private final KubernetesEnvironmentValidator envValidator;
private final MemoryAttributeProvisioner memoryProvisioner;
private final PodMerger podMerger;

@Inject
Expand All @@ -69,12 +62,10 @@ public KubernetesEnvironmentFactory(
MachineConfigsValidator machinesValidator,
KubernetesRecipeParser recipeParser,
KubernetesEnvironmentValidator envValidator,
MemoryAttributeProvisioner memoryProvisioner,
PodMerger podMerger) {
super(recipeRetriever, machinesValidator);
this.recipeParser = recipeParser;
this.envValidator = envValidator;
this.memoryProvisioner = memoryProvisioner;
this.podMerger = podMerger;
}

Expand Down Expand Up @@ -151,8 +142,6 @@ protected KubernetesEnvironment doCreate(
.setConfigMaps(configMaps)
.build();

addRamAttributes(k8sEnv.getMachines(), k8sEnv.getPodsData().values());

envValidator.validate(k8sEnv);

return k8sEnv;
Expand Down Expand Up @@ -218,22 +207,6 @@ private <T extends HasMetadata> void putInto(Map<String, T> map, String key, T v
}
}

@VisibleForTesting
void addRamAttributes(Map<String, InternalMachineConfig> machines, Collection<PodData> pods) {
for (PodData pod : pods) {
for (Container container : pod.getSpec().getContainers()) {
final String machineName = Names.machineName(pod, container);
InternalMachineConfig machineConfig;
if ((machineConfig = machines.get(machineName)) == null) {
machineConfig = new InternalMachineConfig();
machines.put(machineName, machineConfig);
}
memoryProvisioner.provision(
machineConfig, Containers.getRamLimit(container), Containers.getRamRequest(container));
}
}
}

private void checkNotNull(Object object, String errorMessage) throws ValidationException {
if (object == null) {
throw new ValidationException(errorMessage);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
/*
* Copyright (c) 2012-2018 Red Hat, Inc.
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* SPDX-License-Identifier: EPL-2.0
*
* Contributors:
* Red Hat, Inc. - initial API and implementation
*/
package org.eclipse.che.workspace.infrastructure.kubernetes.provision.limits.ram;

import static org.eclipse.che.api.core.model.workspace.config.MachineConfig.CPU_LIMIT_ATTRIBUTE;
import static org.eclipse.che.api.core.model.workspace.config.MachineConfig.CPU_REQUEST_ATTRIBUTE;
import static org.eclipse.che.api.core.model.workspace.config.MachineConfig.MEMORY_LIMIT_ATTRIBUTE;
import static org.eclipse.che.api.core.model.workspace.config.MachineConfig.MEMORY_REQUEST_ATTRIBUTE;
import static org.eclipse.che.workspace.infrastructure.kubernetes.Names.machineName;

import io.fabric8.kubernetes.api.model.Container;
import java.util.Map;
import javax.inject.Inject;
import javax.inject.Named;
import javax.inject.Singleton;
import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity;
import org.eclipse.che.api.workspace.server.spi.InfrastructureException;
import org.eclipse.che.api.workspace.server.spi.environment.InternalMachineConfig;
import org.eclipse.che.api.workspace.server.spi.environment.ResourceLimitAttributesProvisioner;
import org.eclipse.che.commons.annotation.Traced;
import org.eclipse.che.commons.tracing.TracingTags;
import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment;
import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment.PodData;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.ConfigurationProvisioner;
import org.eclipse.che.workspace.infrastructure.kubernetes.util.Containers;
import org.eclipse.che.workspace.infrastructure.kubernetes.util.KubernetesSize;

/**
* Sets or overrides Kubernetes container RAM and CPU limit and request if corresponding attributes
* are present in machine config corresponding to the container.
*
* <p>There are two memory-related properties:
*
* <ul>
* <li>che.workspace.default_memory_limit_mb - defines default machine memory limit
* <li>che.workspace.default_memory_request_mb - defines default requested machine memory
* allocation.
* </ul>
*
* <p>Similarly, there are two CPU-related properties:
*
* <ul>
* <li>che.workspace.default_cpu_limit_cores - defines default machine CPU limit
* <li>che.workspace.default_cpu_request_cores - defines default machine CPU request
* </ul>
*
* @author Anton Korneta
* @auhtor Max Shaposhnyk
*/
@Singleton
public class ContainerResourceProvisioner implements ConfigurationProvisioner {

private final ResourceLimitAttributesProvisioner resourceLimitAttributesProvisioner;

private final long defaultMachineMaxMemorySizeAttribute;
private final long defaultMachineRequestMemorySizeAttribute;
private final float defaultMachineCpuLimitAttribute;
private final float defaultMachineCpuRequestAttribute;

@Inject
public ContainerResourceProvisioner(
@Named("che.workspace.default_memory_limit_mb") long defaultMachineMaxMemorySizeAttribute,
@Named("che.workspace.default_memory_request_mb")
long defaultMachineRequestMemorySizeAttribute,
@Named("che.workspace.default_cpu_limit_cores") String defaultMachineCpuLimitAttribute,
@Named("che.workspace.default_cpu_request_cores") String defaultMachineCpuRequestAttribute,
ResourceLimitAttributesProvisioner resourceLimitAttributesProvisioner) {
this.resourceLimitAttributesProvisioner = resourceLimitAttributesProvisioner;
this.defaultMachineMaxMemorySizeAttribute = defaultMachineMaxMemorySizeAttribute * 1024 * 1024;
this.defaultMachineRequestMemorySizeAttribute =
defaultMachineRequestMemorySizeAttribute * 1024 * 1024;
this.defaultMachineCpuLimitAttribute = KubernetesSize.toCores(defaultMachineCpuLimitAttribute);
this.defaultMachineCpuRequestAttribute =
KubernetesSize.toCores(defaultMachineCpuRequestAttribute);
}

@Override
@Traced
public void provision(KubernetesEnvironment k8sEnv, RuntimeIdentity identity)
throws InfrastructureException {

TracingTags.WORKSPACE_ID.set(identity::getWorkspaceId);

final Map<String, InternalMachineConfig> machines = k8sEnv.getMachines();
for (PodData pod : k8sEnv.getPodsData().values()) {
for (Container container : pod.getSpec().getContainers()) {

// make sure that machine configs have settings for RAM limit and request
InternalMachineConfig machineConfig = machines.get(machineName(pod, container));
resourceLimitAttributesProvisioner.provisionMemory(
machineConfig,
Containers.getRamLimit(container),
Containers.getRamRequest(container),
defaultMachineMaxMemorySizeAttribute,
defaultMachineRequestMemorySizeAttribute);

// make sure that machine configs have settings for CPU limit and request
resourceLimitAttributesProvisioner.provisionCPU(
machineConfig,
Containers.getCpuLimit(container),
Containers.getCpuRequest(container),
defaultMachineCpuLimitAttribute,
defaultMachineCpuRequestAttribute);

// reapply memory and CPU settings to k8s container to make sure that provisioned
// values above are set
final Map<String, String> attributes = machineConfig.getAttributes();
Containers.addRamLimit(container, Long.parseLong(attributes.get(MEMORY_LIMIT_ATTRIBUTE)));
Containers.addRamRequest(
container, Long.parseLong(attributes.get(MEMORY_REQUEST_ATTRIBUTE)));
Containers.addCpuLimit(container, Float.parseFloat(attributes.get(CPU_LIMIT_ATTRIBUTE)));
Containers.addCpuRequest(
container, Float.parseFloat(attributes.get(CPU_REQUEST_ATTRIBUTE)));
}
}
}
}

This file was deleted.

Loading

0 comments on commit 9732139

Please sign in to comment.