Merge pull request #1173 from casz/casc

Support seed job creation in Configuration as Code, updated

.gitignore

@@ -19,7 +19,7 @@ gradle-app.setting
 /local*
 build
 generated
-/out
+*/out/
 /classes
 .DS_Store
 

docs/JCasC-Job-DSL.md

@@ -0,0 +1,65 @@
+Via [configuratin-as-code-plugin](https://plugins.jenkins.io/configuration-as-code) also known as JCasC
+
+It is possible to configure initial seed jobs through a yaml config file.  
+The basics for job dsl is you have a root element called `jobs` that will be parsed to configure via job dsl
+
+Examples of config file
+
+```yml
+jobs:
+  - script: >
+      multibranchPipelineJob('configuration-as-code') {
+          branchSources {
+              git {
+                  id = 'configuration-as-code'
+                  remote('https://github.com/jenkinsci/configuration-as-code-plugin.git')
+              }
+          }
+      }
+```
+
+You can also fetch your job dsl from a file or URL
+
+```yml
+jobs:
+  - file: ./jobdsl/job.groovy
+```
+
+```yml
+jobs:
+  - url: https://raw.githubusercontent.com/jenkinsci/job-dsl-plugin/master/job-dsl-plugin/src/test/resources/javaposse/jobdsl/plugin/testjob.groovy
+```
+
+you can reference multiple scripts, files, and urls
+
+```yml
+jobs:
+  - script: >
+    job('testJob1') {
+        scm {
+            git('git://github.com/quidryan/aws-sdk-test.git')
+        }
+        triggers {
+            scm('H/15 * * * *')
+        }
+        steps {
+            maven('-e clean test')
+        }
+    }
+
+  - script: >
+    job('testJob2') {
+        scm {
+            git('git://github.com/quidryan/aws-sdk-test.git')
+        }
+        triggers {
+            scm('H/15 * * * *')
+        }
+        steps {
+            maven('-e clean test')
+        }
+    }
+
+  - file: ./jobdsl/job1.groovy
+  - file: ./jobdsl/job2.groovy
+```

job-dsl-plugin/build.gradle

@@ -95,6 +95,9 @@ dependencies {
     }
     optionalJenkinsPlugins 'org.jenkins-ci.plugins:config-file-provider:2.15.4'
     optionalJenkinsPlugins 'org.jenkinsci.plugins:managed-scripts:1.3'
+    optionalJenkinsPlugins 'io.jenkins:configuration-as-code:1.13'
+    jenkinsTest 'io.jenkins:configuration-as-code:1.13'
+    jenkinsTest 'io.jenkins:configuration-as-code:1.13:tests'
     jenkinsTest 'org.jenkins-ci.plugins:cloudbees-folder:5.14'
     jenkinsTest 'org.jenkins-ci.plugins:matrix-auth:1.3'
     jenkinsTest 'org.jenkins-ci.plugins:nested-view:1.14'

job-dsl-plugin/src/main/groovy/javaposse/jobdsl/plugin/casc/ConfigurableScriptSource.java

@@ -0,0 +1,26 @@
+package javaposse.jobdsl.plugin.casc;
+
+import io.jenkins.plugins.casc.Configurable;
+import io.jenkins.plugins.casc.ConfiguratorException;
+import io.jenkins.plugins.casc.model.CNode;
+
+public abstract class ConfigurableScriptSource extends ScriptSource implements Configurable {
+
+    @Override
+    public void configure(CNode node) throws ConfiguratorException {
+        configure(node.asScalar().getValue());
+    }
+
+    protected abstract void configure(String value);
+
+    @Override
+    public void check(CNode node) throws ConfiguratorException {
+        node.asScalar();
+    }
+
+    @Override
+    public CNode describe() throws Exception {
+        return null; // Not relevant here
+    }
+
+}

job-dsl-plugin/src/main/groovy/javaposse/jobdsl/plugin/casc/FromFileScriptSource.java

@@ -0,0 +1,31 @@
+package javaposse.jobdsl.plugin.casc;
+
+import hudson.Extension;
+import hudson.model.Descriptor;
+import org.apache.commons.io.FileUtils;
+import org.jenkinsci.Symbol;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+
+public class FromFileScriptSource extends ConfigurableScriptSource {
+
+    public String path;
+
+    @Override
+    public void configure(String path) {
+        this.path = path;
+    }
+
+    @Override
+    public String getScript() throws IOException {
+        return FileUtils.readFileToString(new File(path), StandardCharsets.UTF_8);
+    }
+
+    @Extension
+    @Symbol("file")
+    public static class DescriptorImpl extends Descriptor<ScriptSource> {
+
+    }
+}

job-dsl-plugin/src/main/groovy/javaposse/jobdsl/plugin/casc/FromUrlScriptSource.java

@@ -0,0 +1,31 @@
+package javaposse.jobdsl.plugin.casc;
+
+import hudson.Extension;
+import hudson.model.Descriptor;
+import io.jenkins.plugins.casc.Configurable;
+import org.apache.commons.io.IOUtils;
+import org.jenkinsci.Symbol;
+
+import java.io.IOException;
+import java.net.URI;
+
+public class FromUrlScriptSource extends ConfigurableScriptSource implements Configurable {
+
+    public String url;
+
+    @Override
+    public void configure(String url) {
+        this.url = url;
+    }
+
+    @Override
+    public String getScript() throws IOException {
+        return IOUtils.toString(URI.create(url));
+    }
+
+    @Extension
+    @Symbol("url")
+    public static class DescriptorImpl extends Descriptor<ScriptSource> {
+
+    }
+}

job-dsl-plugin/src/main/groovy/javaposse/jobdsl/plugin/casc/InlineGroovyScriptSource.java

@@ -0,0 +1,29 @@
+package javaposse.jobdsl.plugin.casc;
+
+import hudson.Extension;
+import hudson.model.Descriptor;
+import io.jenkins.plugins.casc.Configurable;
+import org.jenkinsci.Symbol;
+
+import java.io.IOException;
+
+public class InlineGroovyScriptSource extends ConfigurableScriptSource implements Configurable {
+
+    public String script;
+
+    @Override
+    public void configure(String script) {
+        this.script = script;
+    }
+
+    @Override
+    public String getScript() throws IOException {
+        return script;
+    }
+
+    @Extension
+    @Symbol("script")
+    public static class DescriptorImpl extends Descriptor<ScriptSource> {
+
+    }
+}

job-dsl-plugin/src/main/groovy/javaposse/jobdsl/plugin/casc/ScriptSource.java

@@ -0,0 +1,11 @@
+package javaposse.jobdsl.plugin.casc;
+
+import hudson.ExtensionPoint;
+import hudson.model.AbstractDescribableImpl;
+
+import java.io.IOException;
+
+public abstract class ScriptSource extends AbstractDescribableImpl<ScriptSource> implements ExtensionPoint {
+
+    public abstract String getScript() throws IOException;
+}

job-dsl-plugin/src/main/groovy/javaposse/jobdsl/plugin/casc/SeedJobConfigurator.java

@@ -0,0 +1,114 @@
+package javaposse.jobdsl.plugin.casc;
+
+import hudson.Extension;
+import io.jenkins.plugins.casc.Attribute;
+import io.jenkins.plugins.casc.ConfigurationContext;
+import io.jenkins.plugins.casc.ConfiguratorException;
+import io.jenkins.plugins.casc.Configurator;
+import io.jenkins.plugins.casc.RootElementConfigurator;
+import io.jenkins.plugins.casc.SecretSourceResolver;
+import io.jenkins.plugins.casc.impl.attributes.MultivaluedAttribute;
+import io.jenkins.plugins.casc.model.CNode;
+import io.jenkins.plugins.casc.model.Mapping;
+import java.util.Map;
+import javaposse.jobdsl.dsl.GeneratedItems;
+import javaposse.jobdsl.plugin.JenkinsDslScriptLoader;
+import javaposse.jobdsl.plugin.JenkinsJobManagement;
+import javaposse.jobdsl.plugin.LookupStrategy;
+import org.kohsuke.accmod.Restricted;
+import org.kohsuke.accmod.restrictions.NoExternalUse;
+
+import javax.annotation.Nonnull;
+import javax.annotation.CheckForNull;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+
+import static io.vavr.API.Try;
+import static io.vavr.API.unchecked;
+
+@Extension(optional = true, ordinal = -50) // Ordinal -50 Ensure it is loaded after GlobalJobDslSecurityConfiguration
+@Restricted(NoExternalUse.class)
+public class SeedJobConfigurator implements RootElementConfigurator<GeneratedItems[]> {
+
+    @Nonnull
+    @Override
+    public String getName() {
+        return "jobs";
+    }
+
+    @Override
+    @SuppressWarnings("unchecked")
+    public Class getTarget() {
+        return GeneratedItems[].class;
+    }
+
+    @Nonnull
+    @Override
+    @SuppressWarnings("unchecked")
+    public Set<Attribute<GeneratedItems[], ?>> describe() {
+        return Collections.singleton(new MultivaluedAttribute("", ScriptSource.class));
+    }
+
+    @Override
+    public GeneratedItems[] getTargetComponent(ConfigurationContext context) {
+        return new GeneratedItems[0]; // Doesn't really make sense
+    }
+
+    @Nonnull
+    @Override
+    @SuppressWarnings("unchecked")
+    public GeneratedItems[] configure(CNode config, ConfigurationContext context) throws ConfiguratorException {
+        JenkinsJobManagement management = new JenkinsJobManagement(System.out, System.getenv(), null, null, LookupStrategy.JENKINS_ROOT);
+        Configurator<ScriptSource> configurator = context.lookupOrFail(ScriptSource.class);
+        return config.asSequence().stream()
+            .map(source -> getActualValue(source, context))
+            .map(source -> getScriptFromSource(source, context, configurator))
+            .map(script -> generateFromScript(script, management))
+            .toArray(GeneratedItems[]::new);
+    }
+
+    @Override
+    public GeneratedItems[] check(CNode config, ConfigurationContext context) throws ConfiguratorException {
+        return new GeneratedItems[0];
+    }
+
+    @Nonnull
+    @Override
+    public List<Configurator<GeneratedItems[]>> getConfigurators(ConfigurationContext context) {
+        return Collections.singletonList(context.lookup(ScriptSource.class));
+    }
+
+    @CheckForNull
+    @Override
+    public CNode describe(GeneratedItems[] instance, ConfigurationContext context) throws Exception {
+        return null;
+    }
+
+    private CNode getActualValue(CNode config, ConfigurationContext context) {
+        return unchecked(() -> config.asMapping().entrySet().stream().findFirst()).apply()
+            .map(entry -> {
+                Mapping mapping = new Mapping();
+                mapping.put(entry.getKey(), revealSourceOrGetValue(entry, context));
+                return (CNode) mapping;
+            }).orElse(config);
+    }
+
+    private String revealSourceOrGetValue(Map.Entry<String, CNode> entry, ConfigurationContext context) {
+        String value = unchecked(() -> entry.getValue().asScalar().getValue()).apply();
+        return SecretSourceResolver.resolve(context, value);
+    }
+
+    private GeneratedItems generateFromScript(String script, JenkinsJobManagement management) {
+        return unchecked(() ->
+            Try(() -> new JenkinsDslScriptLoader(management).runScript(script))
+                .getOrElseThrow(t -> new ConfiguratorException(this, "Failed to execute script with hash " + script.hashCode(), t))).apply();
+    }
+
+    private String getScriptFromSource(CNode source, ConfigurationContext context, Configurator<ScriptSource> configurator) {
+        return unchecked(() ->
+            Try(() -> configurator.configure(source, context))
+                .getOrElseThrow(t -> new ConfiguratorException(this, "Failed to retrieve job-dsl script", t))
+                .getScript()).apply();
+    }
+}

job-dsl-plugin/src/main/resources/javaposse/jobdsl/plugin/casc/FromFileScriptSource/schema.jelly

@@ -0,0 +1,4 @@
+<?jelly escape-by-default='true'?>
+<j:jelly xmlns:j="jelly:core" xmlns:st="jelly:stapler">
+    "io.jenkins.plugins.casc.support.jobdsl.FromFileScriptSource": { "type": "string" }
+</j:jelly>

job-dsl-plugin/src/main/resources/javaposse/jobdsl/plugin/casc/FromUrlScriptSource/schema.jelly

@@ -0,0 +1,4 @@
+<?jelly escape-by-default='true'?>
+<j:jelly xmlns:j="jelly:core" xmlns:st="jelly:stapler">
+    "io.jenkins.plugins.casc.support.jobdsl.FromUrlScriptSource": { "type": "string" }
+</j:jelly>

job-dsl-plugin/src/main/resources/javaposse/jobdsl/plugin/casc/InlineGroovyScriptSource/schema.jelly

@@ -0,0 +1,4 @@
+<?jelly escape-by-default='true'?>
+<j:jelly xmlns:j="jelly:core" xmlns:st="jelly:stapler">
+    "io.jenkins.plugins.casc.support.jobdsl.InlineGroovyScriptSource": { "type": "string" }
+</j:jelly>

job-dsl-plugin/src/main/resources/javaposse/jobdsl/plugin/casc/SeedJobConfigurator/schema.jelly

@@ -0,0 +1,9 @@
+<?jelly escape-by-default='true'?>
+<j:jelly xmlns:j="jelly:core" xmlns:st="jelly:stapler">
+    "[javaposse.jobdsl.dsl.GeneratedItems;": {
+      "type": "array",
+      "items": {
+        "type": "#/definitions/io.jenkins.plugins.casc.support.jobdsl.ScriptSource"
+      }
+    }
+</j:jelly>

job-dsl-plugin/src/test/groovy/javaposse/jobdsl/plugin/ConfigurationAsCodeTest.java

@@ -0,0 +1,21 @@
+package javaposse.jobdsl.plugin;
+
+import io.jenkins.plugins.casc.misc.ConfiguredWithCode;
+import io.jenkins.plugins.casc.misc.JenkinsConfiguredWithCodeRule;
+import org.junit.ClassRule;
+import org.junit.Test;
+
+import static org.junit.Assert.assertNotNull;
+
+public class ConfigurationAsCodeTest {
+
+    @ClassRule
+    @ConfiguredWithCode("ConfigurationAsCodeTest.yaml")
+    public static JenkinsConfiguredWithCodeRule j = new JenkinsConfiguredWithCodeRule();
+
+    @Test
+    public void configure_seed_job() {
+        assertNotNull(j.jenkins.getItem("testJob1"));
+        assertNotNull(j.jenkins.getItem("testJob2"));
+    }
+}

job-dsl-plugin/src/test/resources/javaposse/jobdsl/plugin/ConfigurationAsCodeTest.yaml

@@ -0,0 +1,15 @@
+jobs:
+  - script: >
+      job('testJob1') {
+          scm {
+              git('git://github.com/quidryan/aws-sdk-test.git')
+          }
+          triggers {
+              scm('H/15 * * * *')
+          }
+          steps {
+              maven('-e clean test')
+          }
+      }
+
+  - file: ./src/test/resources/javaposse/jobdsl/plugin/testjob.groovy