Skip to content

Commit

Permalink
Plugin ephemeral volume feature for CheContainers (#14539)
Browse files Browse the repository at this point in the history
* Add plugin ephemeral volume feature

Signed-off-by: Oleksandr Andriienko <oandriie@redhat.com>

* Move plugin containers volume logic to external class.

Signed-off-by: Oleksandr Andriienko <oandriie@redhat.com>

* Fix volume solution to work with unique pvc strategy

Signed-off-by: Oleksandr Andriienko <oandriie@redhat.com>

* Clean up.

Signed-off-by: Oleksandr Andriienko <oandriie@redhat.com>

* Clean up

Signed-off-by: Oleksandr Andriienko <oandriie@redhat.com>

* Clean up.

Signed-off-by: Oleksandr Andriienko <oandriie@redhat.com>

* Rename field persistVolume to ephemeral

Signed-off-by: Oleksandr Andriienko <oandriie@redhat.com>

* Fix up.

Signed-off-by: Oleksandr Andriienko <oandriie@redhat.com>

* Apply proposed changes changes.

Signed-off-by: Oleksandr Andriienko <oandriie@redhat.com>

* Apply proposed changes + enhasments.

Signed-off-by: Oleksandr Andriienko <oandriie@redhat.com>

* Reuse pvc claim from k8sEnv

Signed-off-by: Oleksandr Andriienko <oandriie@redhat.com>

* Use released version che-plugin-broker.

Signed-off-by: Oleksandr Andriienko <oandriie@redhat.com>

* Use newer version che-init-plugin-broker

Signed-off-by: Oleksandr Andriienko <oandriie@redhat.com>
  • Loading branch information
AndrienkoAleksandr authored Sep 26, 2019
1 parent ece3257 commit dd47af6
Show file tree
Hide file tree
Showing 7 changed files with 244 additions and 94 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -607,8 +607,8 @@ che.singleport.wildcard_domain.ipless=false

# Docker image of Che plugin broker app that resolves workspace tooling configuration and copies
# plugins dependencies to a workspace
che.workspace.plugin_broker.init.image=eclipse/che-init-plugin-broker:v0.20
che.workspace.plugin_broker.unified.image=eclipse/che-unified-plugin-broker:v0.20
che.workspace.plugin_broker.init.image=eclipse/che-init-plugin-broker:v0.21
che.workspace.plugin_broker.unified.image=eclipse/che-unified-plugin-broker:v0.21

# Docker image of Che plugin broker app that resolves workspace tooling configuration and copies
# plugins dependencies to a workspace
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
/*
* Copyright (c) 2012-2018 Red Hat, Inc.
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* SPDX-License-Identifier: EPL-2.0
*
* Contributors:
* Red Hat, Inc. - initial API and implementation
*/

package org.eclipse.che.workspace.infrastructure.kubernetes.wsplugins;

import static org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesObjectUtil.newPVC;
import static org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesObjectUtil.newVolume;
import static org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesObjectUtil.newVolumeMount;

import io.fabric8.kubernetes.api.model.Container;
import io.fabric8.kubernetes.api.model.PersistentVolumeClaim;
import io.fabric8.kubernetes.api.model.PodSpec;
import io.fabric8.kubernetes.api.model.VolumeBuilder;
import java.util.Collection;
import java.util.Optional;
import javax.inject.Inject;
import javax.inject.Named;
import javax.inject.Singleton;
import org.eclipse.che.api.workspace.server.wsplugins.model.Volume;
import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment;

/**
* Components for applying workspace plugin volumes to the kubernetes {@link
* io.fabric8.kubernetes.api.model.Container}.
*/
@Singleton
public class ChePluginsVolumeApplier {

private final String pvcQuantity;
private final String pvcAccessMode;
private final String pvcStorageClassName;

@Inject
public ChePluginsVolumeApplier(
@Named("che.infra.kubernetes.pvc.quantity") String pvcQuantity,
@Named("che.infra.kubernetes.pvc.access_mode") String pvcAccessMode,
@Named("che.infra.kubernetes.pvc.storage_class_name") String pvcStorageClassName) {
this.pvcQuantity = pvcQuantity;
this.pvcAccessMode = pvcAccessMode;
this.pvcStorageClassName = pvcStorageClassName;
}

public void applyVolumes(
KubernetesEnvironment.PodData pod,
Container container,
Collection<Volume> volumes,
KubernetesEnvironment k8sEnv) {
for (Volume volume : volumes) {
String podVolumeName = provisionPodVolume(volume, pod, k8sEnv);

container.getVolumeMounts().add(newVolumeMount(podVolumeName, volume.getMountPath(), ""));
}
}

private String provisionPodVolume(
Volume volume, KubernetesEnvironment.PodData pod, KubernetesEnvironment k8sEnv) {
if (volume.isEphemeral()) {
addEmptyDirVolumeIfAbsent(pod.getSpec(), volume.getName());
return volume.getName();
} else {
return provisionPVCPodVolume(volume, pod, k8sEnv).getName();
}
}

private io.fabric8.kubernetes.api.model.Volume provisionPVCPodVolume(
Volume volume, KubernetesEnvironment.PodData pod, KubernetesEnvironment k8sEnv) {
String pvcName = volume.getName();

if (!k8sEnv.getPersistentVolumeClaims().containsKey(pvcName)) {
final PersistentVolumeClaim pvc =
newPVC(pvcName, pvcAccessMode, pvcQuantity, pvcStorageClassName);
k8sEnv.getPersistentVolumeClaims().put(pvcName, pvc);
}

PodSpec podSpec = pod.getSpec();
Optional<io.fabric8.kubernetes.api.model.Volume> volumeOpt =
podSpec
.getVolumes()
.stream()
.filter(
vm ->
vm.getPersistentVolumeClaim() != null
&& pvcName.equals(vm.getPersistentVolumeClaim().getClaimName()))
.findAny();
io.fabric8.kubernetes.api.model.Volume podVolume;
if (volumeOpt.isPresent()) {
podVolume = volumeOpt.get();
} else {
podVolume = newVolume(pvcName, pvcName);
podSpec.getVolumes().add(podVolume);
}
return podVolume;
}

private void addEmptyDirVolumeIfAbsent(PodSpec podSpec, String uniqueVolumeName) {
if (podSpec
.getVolumes()
.stream()
.noneMatch(volume -> volume.getName().equals(uniqueVolumeName))) {
podSpec
.getVolumes()
.add(
new VolumeBuilder()
.withName(uniqueVolumeName)
.withNewEmptyDir()
.endEmptyDir()
.build());
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -71,18 +71,21 @@ public class KubernetesPluginsToolingApplier implements ChePluginsApplier {
private final String sidecarImagePullPolicy;
private final boolean isAuthEnabled;
private final ProjectsRootEnvVariableProvider projectsRootEnvVariableProvider;
private final ChePluginsVolumeApplier chePluginsVolumeApplier;

@Inject
public KubernetesPluginsToolingApplier(
@Named("che.workspace.sidecar.image_pull_policy") String sidecarImagePullPolicy,
@Named("che.workspace.sidecar.default_memory_limit_mb") long defaultSidecarMemoryLimitMB,
@Named("che.agents.auth_enabled") boolean isAuthEnabled,
ProjectsRootEnvVariableProvider projectsRootEnvVariableProvider) {
ProjectsRootEnvVariableProvider projectsRootEnvVariableProvider,
ChePluginsVolumeApplier chePluginsVolumeApplier) {
this.defaultSidecarMemoryLimitBytes = String.valueOf(defaultSidecarMemoryLimitMB * 1024 * 1024);
this.isAuthEnabled = isAuthEnabled;
this.sidecarImagePullPolicy =
validImagePullPolicies.contains(sidecarImagePullPolicy) ? sidecarImagePullPolicy : null;
this.projectsRootEnvVariableProvider = projectsRootEnvVariableProvider;
this.chePluginsVolumeApplier = chePluginsVolumeApplier;
}

@Override
Expand All @@ -95,13 +98,13 @@ public void apply(
return;
}

KubernetesEnvironment kubernetesEnvironment = (KubernetesEnvironment) internalEnvironment;
KubernetesEnvironment k8sEnv = (KubernetesEnvironment) internalEnvironment;

Map<String, PodData> pods = kubernetesEnvironment.getPodsData();
Map<String, PodData> pods = k8sEnv.getPodsData();
switch (pods.size()) {
case 0:
addToolingPod(kubernetesEnvironment);
pods = kubernetesEnvironment.getPodsData();
addToolingPod(k8sEnv);
pods = k8sEnv.getPodsData();
break;
case 1:
break;
Expand All @@ -111,31 +114,27 @@ public void apply(
}
PodData pod = pods.values().iterator().next();

CommandsResolver commandsResolver = new CommandsResolver(kubernetesEnvironment);
CommandsResolver commandsResolver = new CommandsResolver(k8sEnv);
for (ChePlugin chePlugin : chePlugins) {
for (CheContainer container : chePlugin.getInitContainers()) {
pod.getSpec().getInitContainers().add(toK8sContainer(container));
Container k8sInitContainer = toK8sContainer(container);
pod.getSpec().getInitContainers().add(k8sInitContainer);
chePluginsVolumeApplier.applyVolumes(pod, k8sInitContainer, container.getVolumes(), k8sEnv);
}

Collection<CommandImpl> pluginRelatedCommands = commandsResolver.resolve(chePlugin);

for (CheContainer container : chePlugin.getContainers()) {
addSidecar(
pod,
container,
chePlugin,
kubernetesEnvironment,
pluginRelatedCommands,
runtimeIdentity);
addSidecar(pod, container, chePlugin, k8sEnv, pluginRelatedCommands, runtimeIdentity);
}
}

chePlugins.forEach(chePlugin -> populateWorkspaceEnvVars(chePlugin, kubernetesEnvironment));
chePlugins.forEach(chePlugin -> populateWorkspaceEnvVars(chePlugin, k8sEnv));

if (isAuthEnabled) {
// enable per-workspace security with JWT proxy for sidecar based workspaces
// because it is the only workspace security implementation supported for now
kubernetesEnvironment.getAttributes().putIfAbsent(SECURE_EXPOSER_IMPL_PROPERTY, "jwtproxy");
k8sEnv.getAttributes().putIfAbsent(SECURE_EXPOSER_IMPL_PROPERTY, "jwtproxy");
}
}

Expand Down Expand Up @@ -201,7 +200,7 @@ private void addSidecar(
PodData pod,
CheContainer container,
ChePlugin chePlugin,
KubernetesEnvironment kubernetesEnvironment,
KubernetesEnvironment k8sEnv,
Collection<CommandImpl> sidecarRelatedCommands,
RuntimeIdentity runtimeIdentity)
throws InfrastructureException {
Expand All @@ -211,6 +210,7 @@ private void addSidecar(
List<ChePluginEndpoint> containerEndpoints = k8sContainerResolver.getEndpoints();

Container k8sContainer = k8sContainerResolver.resolve();
chePluginsVolumeApplier.applyVolumes(pod, k8sContainer, container.getVolumes(), k8sEnv);

String machineName = k8sContainer.getName();
Names.putMachineName(pod.getMetadata(), k8sContainer.getName(), machineName);
Expand All @@ -222,7 +222,7 @@ private void addSidecar(
.setContainer(k8sContainer)
.setContainerEndpoints(containerEndpoints)
.setDefaultSidecarMemorySizeAttribute(defaultSidecarMemoryLimitBytes)
.setAttributes(kubernetesEnvironment.getAttributes())
.setAttributes(k8sEnv.getAttributes())
.setProjectsRootPathEnvVar(projectsRootEnvVariableProvider.get(runtimeIdentity))
.setPluginPublisher(chePlugin.getPublisher())
.setPluginName(chePlugin.getName())
Expand All @@ -232,7 +232,7 @@ private void addSidecar(
InternalMachineConfig machineConfig = machineResolver.resolve();
machineConfig.getAttributes().put(CONTAINER_SOURCE_ATTRIBUTE, TOOL_CONTAINER_SOURCE);
machineConfig.getAttributes().put(PLUGIN_MACHINE_ATTRIBUTE, chePlugin.getId());
kubernetesEnvironment.getMachines().put(machineName, machineConfig);
k8sEnv.getMachines().put(machineName, machineConfig);

sidecarRelatedCommands.forEach(
c ->
Expand All @@ -245,11 +245,11 @@ private void addSidecar(
.getCommands()
.stream()
.map(c -> asCommand(machineName, c))
.forEach(c -> kubernetesEnvironment.getCommands().add(c));
.forEach(c -> k8sEnv.getCommands().add(c));

SidecarServicesProvisioner sidecarServicesProvisioner =
new SidecarServicesProvisioner(containerEndpoints, pod.getMetadata().getName());
sidecarServicesProvisioner.provision(kubernetesEnvironment);
sidecarServicesProvisioner.provision(k8sEnv);
}

private CommandImpl asCommand(String machineName, Command command) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,6 @@ private void normalizeMemory(Container container, InternalMachineConfig machineC
projectsRootPathEnvVar.second,
projectsRootPathEnvVar.first));
}
result.put(volume.getName(), new VolumeImpl().withPath(volume.getMountPath()));
}
return result;
}
Expand Down
Loading

0 comments on commit dd47af6

Please sign in to comment.