Skip to content

Commit

Permalink
Make jenkins agent more flexible (#1602)
Browse files Browse the repository at this point in the history
  • Loading branch information
@Vlatombe
Vlatombe authored Sep 11, 2024
1 parent cc37496 commit 7345138
Show file tree
Hide file tree
Showing 22 changed files with 458 additions and 277 deletions.
26 changes: 16 additions & 10 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,12 @@ spec:
Multiple containers can be defined for the agent pod, with shared resources, like mounts. Ports in each container can
be accessed as in any Kubernetes pod, by using `localhost`.

The `container` step allows executing commands into each container.
One container must run the Jenkins agent. If unspecified, a container named `jnlp` will be created with the inbound-agent image.
The Jenkins agent requires a JRE to run, so you can avoid the extra container by providing a name using the `agentContainer`.
To get the Jenkins agent injected, you will also need to set `agentInjection` to `true`, and leave the command and argument fields empty for this container.
The container specified by `agentContainer` will be the one where shell steps (or any other step running remote commands on the agent) will run on.

To execute commands in another container part of the pod (different from the one running the Jenkins agent), you can use the `container` step.

**Note**
---
Expand All @@ -194,8 +199,11 @@ It is recommended to use the same uid across the different containers part of th
---

```groovy
podTemplate(containers: [
containerTemplate(name: 'maven', image: 'maven:3.8.1-jdk-8', command: 'sleep', args: '99d'),
podTemplate(
agentContainer: 'maven',
agentInjection: true,
containers: [
containerTemplate(name: 'maven', image: 'maven:3.9.9-eclipse-temurin-17'),
containerTemplate(name: 'golang', image: 'golang:1.16.5', command: 'sleep', args: '99d')
]) {
Expand Down Expand Up @@ -229,17 +237,16 @@ podTemplate(containers: [
or
```groovy
podTemplate(yaml: '''
podTemplate(
agentContainer: 'maven',
agentInjection: true,
yaml: '''
apiVersion: v1
kind: Pod
spec:
containers:
- name: maven
image: maven:3.8.1-jdk-8
command:
- sleep
args:
- 99d
image: maven:3.9.9-eclipse-temurin-17
- name: golang
image: golang:1.16.5
command:
Expand Down Expand Up @@ -269,7 +276,6 @@ podTemplate(yaml: '''
}
}
}
}
}
```
Expand Down
12 changes: 4 additions & 8 deletions examples/maven-with-cache.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,18 +7,14 @@
* two concurrent jobs with this pipeline. Or change readOnly: true after the first run
*/

podTemplate(containers: [
containerTemplate(name: 'maven', image: 'maven:3.8.1-jdk-8', command: 'sleep', args: '99d')
], volumes: [
persistentVolumeClaim(mountPath: '/root/.m2/repository', claimName: 'maven-repo', readOnly: false)
]) {
podTemplate(agentContainer: 'maven', agentInjection: true, containers: [
containerTemplate(name: 'maven', image: 'maven:3.9.9-eclipse-temurin-17')
], volumes: [genericEphemeralVolume(accessModes: 'ReadWriteOnce', mountPath: '/root/.m2/repository', requestsSize: '1Gi')]) {

node(POD_LABEL) {
stage('Build a Maven project') {
git 'https://github.com/jenkinsci/kubernetes-plugin.git'
container('maven') {
sh 'mvn -B -ntp clean package -DskipTests'
}
sh 'mvn -B -ntp clean package -DskipTests'
}
}
}
37 changes: 16 additions & 21 deletions examples/multi-container.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,32 +2,27 @@
* This pipeline describes a multi container job, running Maven and Golang builds
*/

podTemplate(yaml: '''
apiVersion: v1
kind: Pod
spec:
containers:
- name: maven
image: maven:3.8.1-jdk-8
command:
- sleep
args:
- 99d
- name: golang
image: golang:1.16.5
command:
- sleep
args:
- 99d
podTemplate(agentContainer: 'maven',
agentInjection: true,
yaml: '''
apiVersion: v1
kind: Pod
spec:
containers:
- name: maven
image: maven:3.9.9-eclipse-temurin-17
- name: golang
image: golang:1.23.1-bookworm
command:
- sleep
args:
- 99d
'''
) {

node(POD_LABEL) {
stage('Build a Maven project') {
git 'https://github.com/jenkinsci/kubernetes-plugin.git'
container('maven') {
sh 'mvn -B -ntp clean package -DskipTests'
}
sh 'mvn -B -ntp clean package -DskipTests'
}
stage('Build a Golang project') {
git url: 'https://github.com/hashicorp/terraform.git', branch: 'main'
Expand Down
25 changes: 25 additions & 0 deletions src/main/java/org/csanchez/jenkins/plugins/kubernetes/PodTemplate.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -188,6 +188,10 @@ protected static MessageDigest getLabelDigestFunction() {

private Long terminationGracePeriodSeconds;

private String agentContainer;

private boolean agentInjection;

/**
* Persisted yaml fragment
*/
Expand Down Expand Up @@ -638,6 +642,25 @@ public boolean isCapOnlyOnAlivePods() {
return capOnlyOnAlivePods;
}

@CheckForNull
public String getAgentContainer() {
return agentContainer;
}

@DataBoundSetter
public void setAgentContainer(@CheckForNull String agentContainer) {
this.agentContainer = Util.fixEmpty(agentContainer);
}

public boolean isAgentInjection() {
return agentInjection;
}

@DataBoundSetter
public void setAgentInjection(boolean agentInjection) {
this.agentInjection = agentInjection;
}

public List<TemplateEnvVar> getEnvVars() {
if (envVars == null) {
return Collections.emptyList();
Expand Down Expand Up @@ -1173,6 +1196,8 @@ public String toString() {
+ (nodeProperties == null || nodeProperties.isEmpty() ? "" : ", nodeProperties=" + nodeProperties)
+ (yamls == null || yamls.isEmpty() ? "" : ", yamls=" + yamls)
+ (!unwrapped ? "" : ", unwrapped=" + unwrapped)
+ (agentContainer == null ? "" : ", agentContainer='" + agentContainer + '\'')
+ (!agentInjection ? "" : ", agentInjection=" + agentInjection)
+ '}';
}
}
113 changes: 86 additions & 27 deletions src/main/java/org/csanchez/jenkins/plugins/kubernetes/PodTemplateBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@

package org.csanchez.jenkins.plugins.kubernetes;

import static org.csanchez.jenkins.plugins.kubernetes.ContainerTemplate.DEFAULT_WORKING_DIR;
import static org.csanchez.jenkins.plugins.kubernetes.KubernetesCloud.JNLP_NAME;
import static org.csanchez.jenkins.plugins.kubernetes.PodTemplateUtils.combine;
import static org.csanchez.jenkins.plugins.kubernetes.PodTemplateUtils.isNullOrEmpty;
Expand All @@ -41,6 +42,7 @@
import io.fabric8.kubernetes.api.model.ContainerBuilder;
import io.fabric8.kubernetes.api.model.ContainerPort;
import io.fabric8.kubernetes.api.model.EnvVar;
import io.fabric8.kubernetes.api.model.EnvVarBuilder;
import io.fabric8.kubernetes.api.model.ExecAction;
import io.fabric8.kubernetes.api.model.LocalObjectReference;
import io.fabric8.kubernetes.api.model.Pod;
Expand All @@ -51,6 +53,7 @@
import io.fabric8.kubernetes.api.model.ResourceRequirements;
import io.fabric8.kubernetes.api.model.ResourceRequirementsBuilder;
import io.fabric8.kubernetes.api.model.Volume;
import io.fabric8.kubernetes.api.model.VolumeBuilder;
import io.fabric8.kubernetes.api.model.VolumeMount;
import io.fabric8.kubernetes.api.model.VolumeMountBuilder;
import io.fabric8.kubernetes.client.utils.Serialization;
Expand Down Expand Up @@ -101,6 +104,9 @@ public class PodTemplateBuilder {
public static final String LABEL_KUBERNETES_CONTROLLER = "kubernetes.jenkins.io/controller";
static final String NO_RECONNECT_AFTER_TIMEOUT =
SystemProperties.getString(PodTemplateBuilder.class.getName() + ".noReconnectAfter", "1d");
private static final String JENKINS_AGENT_FILE_ENVVAR = "JENKINS_AGENT_FILE";
private static final String JENKINS_AGENT_AGENT_JAR = "/jenkins-agent/agent.jar";
private static final String JENKINS_AGENT_LAUNCHER_SCRIPT_LOCATION = "/jenkins-agent/jenkins-agent";

@SuppressFBWarnings(value = "MS_SHOULD_BE_FINAL", justification = "tests")
@Restricted(NoExternalUse.class)
Expand All @@ -124,7 +130,7 @@ public class PodTemplateBuilder {
}

@Restricted(NoExternalUse.class)
static final String DEFAULT_JNLP_IMAGE =
static final String DEFAULT_AGENT_IMAGE =
System.getProperty(PodTemplateStepExecution.class.getName() + ".defaultImage", defaultImageName);

static final String DEFAULT_JNLP_CONTAINER_MEMORY_REQUEST = System.getProperty(
Expand Down Expand Up @@ -309,39 +315,92 @@ public Pod build() {
}
}

// default jnlp container
Optional<Container> jnlpOpt = pod.getSpec().getContainers().stream()
.filter(c -> JNLP_NAME.equals(c.getName()))
// default agent container
String agentContainerName = StringUtils.defaultString(template.getAgentContainer(), JNLP_NAME);
Optional<Container> agentOpt = pod.getSpec().getContainers().stream()
.filter(c -> agentContainerName.equals(c.getName()))
.findFirst();
Container jnlp = jnlpOpt.orElse(new ContainerBuilder()
.withName(JNLP_NAME)
.withVolumeMounts(volumeMounts
.values()
.toArray(new VolumeMount[volumeMounts.values().size()]))
Container agentContainer = agentOpt.orElse(new ContainerBuilder()
.withName(agentContainerName)
.withVolumeMounts(volumeMounts.values().toArray(VolumeMount[]::new))
.build());
if (!jnlpOpt.isPresent()) {
pod.getSpec().getContainers().add(jnlp);
if (agentOpt.isEmpty()) {
pod.getSpec().getContainers().add(agentContainer);
}
var workingDir = agentContainer.getWorkingDir();
pod.getSpec().getContainers().stream()
.filter(c -> c.getWorkingDir() == null)
.forEach(c -> c.setWorkingDir(jnlp.getWorkingDir()));
if (StringUtils.isBlank(jnlp.getImage())) {
String jnlpImage = DEFAULT_JNLP_IMAGE;
.forEach(c -> c.setWorkingDir(workingDir));
if (StringUtils.isBlank(agentContainer.getImage())) {
String agentImage = DEFAULT_AGENT_IMAGE;
if (cloud != null && StringUtils.isNotEmpty(cloud.getJnlpregistry())) {
jnlpImage = Util.ensureEndsWith(cloud.getJnlpregistry(), "/") + jnlpImage;
agentImage = Util.ensureEndsWith(cloud.getJnlpregistry(), "/") + agentImage;
} else if (StringUtils.isNotEmpty(DEFAULT_JNLP_DOCKER_REGISTRY_PREFIX)) {
jnlpImage = Util.ensureEndsWith(DEFAULT_JNLP_DOCKER_REGISTRY_PREFIX, "/") + jnlpImage;
agentImage = Util.ensureEndsWith(DEFAULT_JNLP_DOCKER_REGISTRY_PREFIX, "/") + agentImage;
}
jnlp.setImage(jnlpImage);
agentContainer.setImage(agentImage);
}
Map<String, EnvVar> envVars = new HashMap<>();
envVars.putAll(jnlpEnvVars(jnlp.getWorkingDir()));
envVars.putAll(agentEnvVars(workingDir));
envVars.putAll(defaultEnvVars(template.getEnvVars()));
Optional.ofNullable(jnlp.getEnv()).ifPresent(jnlpEnv -> {
jnlpEnv.forEach(var -> envVars.put(var.getName(), var));
Optional.ofNullable(agentContainer.getEnv()).ifPresent(agentEnv -> {
agentEnv.forEach(var -> envVars.put(var.getName(), var));
});
jnlp.setEnv(new ArrayList<>(envVars.values()));
if (jnlp.getResources() == null) {
if (template.isAgentInjection()) {
var agentVolumeMountBuilder =
new VolumeMountBuilder().withName("jenkins-agent").withMountPath("/jenkins-agent");
var oldInitContainers = pod.getSpec().getInitContainers();
var jenkinsAgentInitContainer = new ContainerBuilder()
.withName("set-up-jenkins-agent")
.withImage(DEFAULT_AGENT_IMAGE)
.withCommand(
"/bin/sh",
"-c",
"cp $(command -v jenkins-agent) " + JENKINS_AGENT_LAUNCHER_SCRIPT_LOCATION + ";"
+ "cp /usr/share/jenkins/agent.jar " + JENKINS_AGENT_AGENT_JAR)
.withVolumeMounts(agentVolumeMountBuilder.build())
.build();
if (oldInitContainers != null) {
var newInitContainers = new ArrayList<>(oldInitContainers);
newInitContainers.add(jenkinsAgentInitContainer);
pod.getSpec().setInitContainers(newInitContainers);
} else {
pod.getSpec().setInitContainers(List.of(jenkinsAgentInitContainer));
}
var oldVolumes = pod.getSpec().getVolumes();
var jenkinsAgentSharedVolume = new VolumeBuilder()
.withName("jenkins-agent")
.withNewEmptyDir()
.and()
.build();
if (oldVolumes != null) {
var newVolumes = new ArrayList<>(oldVolumes);
newVolumes.add(jenkinsAgentSharedVolume);
pod.getSpec().setVolumes(newVolumes);
} else {
pod.getSpec().setVolumes(List.of(jenkinsAgentSharedVolume));
}
var existingVolumeMounts = agentContainer.getVolumeMounts();
if (existingVolumeMounts != null) {
var newVolumeMounts = new ArrayList<>(existingVolumeMounts);
newVolumeMounts.add(agentVolumeMountBuilder.withReadOnly().build());
agentContainer.setVolumeMounts(newVolumeMounts);
} else {
agentContainer.setVolumeMounts(
List.of(agentVolumeMountBuilder.withReadOnly().build()));
}
agentContainer.setWorkingDir(DEFAULT_WORKING_DIR);
agentContainer.setCommand(List.of(JENKINS_AGENT_LAUNCHER_SCRIPT_LOCATION));
agentContainer.setArgs(List.of());
envVars.put(
JENKINS_AGENT_FILE_ENVVAR,
new EnvVarBuilder()
.withName(JENKINS_AGENT_FILE_ENVVAR)
.withValue(JENKINS_AGENT_AGENT_JAR)
.build());
}
agentContainer.setEnv(new ArrayList<>(envVars.values()));
if (agentContainer.getResources() == null) {

Map<String, Quantity> reqMap = new HashMap<>();
Map<String, Quantity> limMap = new HashMap<>();
Expand All @@ -361,7 +420,7 @@ public Pod build() {
.withLimits(limMap)
.build();

jnlp.setResources(reqs);
agentContainer.setResources(reqs);
}
if (cloud != null) {
pod = PodDecorator.decorateAll(cloud, pod);
Expand Down Expand Up @@ -406,9 +465,9 @@ private Map<String, EnvVar> defaultEnvVars(Collection<TemplateEnvVar> globalEnvV
return envVarsMap;
}

private Map<String, EnvVar> jnlpEnvVars(String workingDir) {
private Map<String, EnvVar> agentEnvVars(String workingDir) {
if (workingDir == null) {
workingDir = ContainerTemplate.DEFAULT_WORKING_DIR;
workingDir = DEFAULT_WORKING_DIR;
}
// Last-write wins map of environment variable names to values
HashMap<String, String> env = new HashMap<>();
Expand Down Expand Up @@ -462,7 +521,7 @@ private Container createContainer(
Map<String, EnvVar> envVarsMap = new HashMap<>();
String workingDir = substituteEnv(containerTemplate.getWorkingDir());
if (JNLP_NAME.equals(containerTemplate.getName())) {
envVarsMap.putAll(jnlpEnvVars(workingDir));
envVarsMap.putAll(agentEnvVars(workingDir));
}
envVarsMap.putAll(defaultEnvVars(globalEnvVars));

Expand Down Expand Up @@ -541,7 +600,7 @@ private Container createContainer(
private VolumeMount getDefaultVolumeMount(@CheckForNull String workingDir) {
String wd = workingDir;
if (wd == null) {
wd = ContainerTemplate.DEFAULT_WORKING_DIR;
wd = DEFAULT_WORKING_DIR;
LOGGER.log(Level.FINE, "Container workingDir is null, defaulting to {0}", wd);
}
return new VolumeMountBuilder()
Expand Down
Loading

0 comments on commit 7345138

Please sign in to comment.