Allow content variables

Dockerfile

@@ -130,11 +130,11 @@ RUN chmod +r /dist/root/ && chmod g+rw /dist/root/.config/jgit/
 # This stage builds a static binary using graal VM. For details see docs/developers.md#GraalVM
 FROM graal AS native-image
 ENV MAVEN_OPTS='-Dmaven.repo.local=/mvn'
-RUN microdnf install gnupg
 
 # Provide binaries used by apply-ng, so our runs with native-image-agent dont fail 
 # with "java.io.IOException: Cannot run program "kubectl"..." etc.
-RUN microdnf install iproute
+# jq for manually adding to  proxy-config.json
+RUN microdnf install iproute jq
 
 WORKDIR /app
 
@@ -153,14 +153,16 @@ RUN printf 'registry:\n  active: true\njenkins:\n  active: true\ncontent:\n  exa
 # Run again with different params in order to avoid NoSuchMethodException with output-config file
 RUN java -agentlib:native-image-agent=config-merge-dir=conf/ -jar gitops-playground.jar \
       --yes  --output-config-file || true
+
+# Manually add some classes that cause exceptions at runtime
+RUN jq '.[0].interfaces += ["org.apache.commons.io.filefilter.IOFileFilter"]' conf/proxy-config.json > tmp && mv tmp conf/proxy-config.json
+RUN for f in conf/*config.json; do echo "=== $f ==="; cat "$f"; echo; done
 RUN native-image -Dgroovy.grape.enable=false \
     -H:+ReportExceptionStackTraces \
     -H:ConfigurationFileDirectories=conf/ \
     -H:IncludeResourceBundles=org.eclipse.jgit.internal.JGitText \
     -H:DynamicProxyConfigurationFiles=conf/proxy-config.json \
-    -H:DynamicProxyConfigurationResources=proxy-config.json \
     -H:ReflectionConfigurationFiles=conf/reflect-config.json \
-    -H:ReflectionConfigurationResources=reflect-config.json \
     --features=com.cloudogu.gitops.graal.groovy.GroovyApplicationRegistrationFeature,com.cloudogu.gitops.graal.groovy.GroovyDgmClassesRegistrationFeature,com.cloudogu.gitops.graal.jgit.JGitReflectionFeature,com.cloudogu.gitops.graal.okhttp.OkHttpReflectionFeature \
     --static \
     --allow-incomplete-classpath \
@@ -175,7 +177,6 @@ RUN native-image -Dgroovy.grape.enable=false \
     apply-ng
 
 
-
 FROM alpine AS prod
 # copy groovy cli binary from native-image stage
 COPY --from=native-image /app/apply-ng app/apply-ng

docs/configuration.schema.json

@@ -150,6 +150,68 @@
         "examples" : {
           "type" : [ "boolean", "null" ],
           "description" : "Deploy example content: source repos, GitOps repos, Jenkins Job, Argo CD apps/project"
+        },
+        "namespaces" : {
+          "description" : "Additional kubernetes namespaces. These are authorized to Argo CD, supplied with image pull secrets, monitored by prometheus, etc. Namespaces can be templates, e.g. ${config.application.namePrefix}staging",
+          "type" : [ "array", "null" ],
+          "items" : {
+            "type" : "string"
+          }
+        },
+        "repos" : {
+          "description" : "Content repos to push into target environment",
+          "type" : [ "array", "null" ],
+          "items" : {
+            "type" : "object",
+            "properties" : {
+              "folderBased" : {
+                "type" : [ "boolean", "null" ],
+                "description" : "When true, interpret the folder structure of each repo as repos. That is, root folder becomes namespace in SCM, sub folders become repository names in SCM"
+              },
+              "overrideMode" : {
+                "anyOf" : [ {
+                  "type" : "null"
+                }, {
+                  "type" : "string",
+                  "enum" : [ "INIT", "RESET", "UPGRADE" ]
+                } ],
+                "description" : "This defines, how customer repos will be updated.\nINIT - push only if repo does not exist (SCMM Return value 409).\nRESET - delete alle files after cloning source - new files are deleted\nUPGRADE - clone and copy - existing files will be overwritten, new files are kept"
+              },
+              "password" : {
+                "type" : [ "string", "null" ],
+                "description" : "Password to authenticate against content repo"
+              },
+              "path" : {
+                "type" : [ "string", "null" ],
+                "description" : "Path within the content repo to process"
+              },
+              "ref" : {
+                "type" : [ "string", "null" ],
+                "description" : "Reference for a specific branch, tag, or commit"
+              },
+              "target" : {
+                "type" : [ "string", "null" ],
+                "description" : "Target path for the repository"
+              },
+              "templating" : {
+                "type" : [ "boolean", "null" ],
+                "description" : "When true, template all files ending in .ftl within the repo"
+              },
+              "url" : {
+                "type" : [ "string", "null" ],
+                "description" : "URL of the content repo"
+              },
+              "username" : {
+                "type" : [ "string", "null" ],
+                "description" : "Username to authenticate against content repo"
+              }
+            },
+            "additionalProperties" : false
+          }
+        },
+        "variables" : {
+          "$ref" : "#/$defs/Map(String,Object)-nullable",
+          "description" : "Additional variables to use in custom templates."
         }
       },
       "additionalProperties" : false,

docs/developers.md

@@ -13,7 +13,7 @@ The versions are also specified in the `Config.groovy` file, so it is recommende
 
 ## Table of contents
 
-<!-- Update with ` doctoc --notitle docs/developers.md --maxlevel 4      `. See https://github.com/thlorenz/doctoc -->
+<!-- Update with `doctoc --notitle docs/developers.md --maxlevel 4`. See https://github.com/thlorenz/doctoc -->
 <!-- START doctoc generated TOC please keep comment here to allow auto update -->
 <!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
 

src/main/groovy/com/cloudogu/gitops/Application.groovy

@@ -1,6 +1,9 @@
 package com.cloudogu.gitops
 
 import com.cloudogu.gitops.config.Config
+import com.cloudogu.gitops.utils.TemplatingEngine
+import freemarker.template.Configuration
+import freemarker.template.DefaultObjectWrapperBuilder
 import groovy.util.logging.Slf4j
 import jakarta.inject.Singleton
 
@@ -35,13 +38,14 @@ class Application {
 
     void setNamespaceListToConfig(Config config) {
         Set<String> namespaces = new HashSet<>()
-        String namePrefix = config.application.namePrefix
-
-        if (config.content.examples) {
-            namespaces.addAll(Arrays.asList(
-                    namePrefix + "example-apps-staging",
-                    namePrefix + "example-apps-production"
-            ))
+        def engine = new TemplatingEngine()
+
+        config.content.namespaces.each { String ns ->
+            namespaces.add(engine.template(ns, [
+                    config              : config,
+                    // Allow for using static classes inside the templates
+                    statics             : new DefaultObjectWrapperBuilder(Configuration.VERSION_2_3_32).build().getStaticModels()
+            ]))
         }
 
         //iterates over all FeatureWithImages and gets their namespaces

src/main/groovy/com/cloudogu/gitops/cli/GitopsPlaygroundCliMainScripted.groovy

@@ -92,7 +92,7 @@ class GitopsPlaygroundCliMainScripted {
                         new Jenkins(config, executor, fileSystemUtils, new GlobalPropertyManager(jenkinsApiClient),
                                 new JobManager(jenkinsApiClient), new UserManager(jenkinsApiClient),
                                 new PrometheusConfigurator(jenkinsApiClient), helmStrategy, k8sClient, networkingUtils),
-                        new Content(config, k8sClient),
+                        new Content(config, k8sClient, scmmRepoProvider, scmmApiClient),
                         new ArgoCD(config, k8sClient, helmClient, fileSystemUtils, scmmRepoProvider),
                         new IngressNginx(config, fileSystemUtils, deployer, k8sClient, airGappedUtils),
                         new CertManager(config, fileSystemUtils, deployer, k8sClient, airGappedUtils),

src/main/groovy/com/cloudogu/gitops/config/ApplicationConfigurator.groovy

@@ -52,9 +52,12 @@ class ApplicationConfigurator {
         if (newConfig.features.ingressNginx.active && !newConfig.application.baseUrl) {
             log.warn("Ingress-controller is activated without baseUrl parameter. Services will not be accessible by hostnames. To avoid this use baseUrl with ingress. ")
         }
-
-        if (newConfig.content.examples && !newConfig.registry.active) {
-            throw new RuntimeException("content.examples requires either registry.active or registry.url")
+        if (newConfig.content.examples) {
+            if  (!newConfig.registry.active) {
+                throw new RuntimeException("content.examples requires either registry.active or registry.url")
+            }
+            String prefix = newConfig.application.namePrefix
+            newConfig.content.namespaces += [ prefix+"example-apps-staging", prefix+"example-apps-production"]
         }
     }
 
@@ -251,6 +254,7 @@ class ApplicationConfigurator {
     void validateConfig(Config configToSet) {
         validateScmmAndJenkinsAreBothSet(configToSet)
         validateMirrorReposHelmChartFolderSet(configToSet)
+        validateContent(configToSet)
     }
 
     private void validateMirrorReposHelmChartFolderSet(Config configToSet) {
@@ -261,6 +265,17 @@ class ApplicationConfigurator {
                     "LOCAL_HELM_CHART_FOLDER='charts' after running 'scripts/downloadHelmCharts.sh' from the repo")
         }
     }
+
+    static void validateContent(Config config) {
+        config.content.repos.each { repo ->
+            if (!repo.url) {
+                throw new RuntimeException('content.repos requires a url parameter')
+            }
+            if (!repo.folderBased && !repo.target) {
+                throw new RuntimeException('content.repos.folderBased: false requires folder content.repos.target to be set')
+            }
+        }
+    }
 
     private void validateScmmAndJenkinsAreBothSet(Config configToSet) {
         if (configToSet.jenkins.active && 

src/main/groovy/com/cloudogu/gitops/config/Config.groovy

@@ -85,13 +85,53 @@ class Config {
     @JsonPropertyDescription(CONTENT_DESCRIPTION)
     @Mixin
     ContentSchema content = new ContentSchema()
-
+    
     class ContentSchema {
+
         @Option(names = ['--content-examples'], description = CONTENT_EXAMPLES_DESCRIPTION)
         @JsonPropertyDescription(CONTENT_EXAMPLES_DESCRIPTION)
         Boolean examples = false
+
+        @JsonPropertyDescription(CONTENT_NAMESPACES_DESCRIPTION)
+        List<String> namespaces = []
+
+        @JsonPropertyDescription(CONTENT_REPO_DESCRIPTION)
+        List<ContentRepositorySchema> repos = []
+
+        @JsonPropertyDescription(CONTENT_VARIABLES_DESCRIPTION)
+        Map<String, Object> variables = [:]
+
+        static class ContentRepositorySchema {
+            @JsonPropertyDescription(CONTENT_REPO_URL_DESCRIPTION)
+            String url = ''
+
+            @JsonPropertyDescription(CONTENT_REPO_PATH_DESCRIPTION)
+            String path = '.'
+
+            @JsonPropertyDescription(CONTENT_REPO_REF_DESCRIPTION)
+            String ref = 'main'
+
+            @JsonPropertyDescription(CONTENT_REPO_USERNAME_DESCRIPTION)
+            String username = ''
+
+            @JsonPropertyDescription(CONTENT_REPO_PASSWORD_DESCRIPTION)
+            String password = ''
+
+            @JsonPropertyDescription(CONTENT_REPO_TEMPLATING_DESCRIPTION)
+            Boolean templating = false
+
+            @JsonPropertyDescription(CONTENT_REPO_FOLDER_BASED_REPOS_DESCRIPTION)
+            Boolean folderBased = false
+
+            @JsonPropertyDescription(CONTENT_REPO_TARGET_DESCRIPTION)
+            String target = ''
+
+            @JsonPropertyDescription(CONTENT_REPO_TARGET_OVERRIDE_MODE)
+            OverrideMode overrideMode = OverrideMode.INIT // default is init a new repository
+        }
     }
 
+
     static class HelmConfig {
         @JsonPropertyDescription(HELM_CONFIG_CHART_DESCRIPTION)
         String chart = ''
@@ -733,6 +773,14 @@ class Config {
         dev, prod
     }
 
+    /**
+     * This defines, how customer repos will be updated.
+     */
+    static enum OverrideMode {
+        INIT, RESET, UPGRADE
+    }
+
+
     private static final ObjectMapper objectMapper = new ObjectMapper()
             .registerModule(new SimpleModule().addSerializer(GString, new JsonSerializer<GString>() {
                 @Override

src/main/groovy/com/cloudogu/gitops/config/ConfigConstants.groovy

@@ -29,7 +29,19 @@ interface ConfigConstants {
 
     // Content
     String CONTENT_EXAMPLES_DESCRIPTION = 'Deploy example content: source repos, GitOps repos, Jenkins Job, Argo CD apps/project'
-
+    String CONTENT_NAMESPACES_DESCRIPTION = 'Additional kubernetes namespaces. These are authorized to Argo CD, supplied with image pull secrets, monitored by prometheus, etc. Namespaces can be templates, e.g. ${config.application.namePrefix}staging'
+    String CONTENT_REPO_DESCRIPTION = "Content repos to push into target environment"
+    String CONTENT_REPO_URL_DESCRIPTION = "URL of the content repo"
+    String CONTENT_REPO_PATH_DESCRIPTION = "Path within the content repo to process"
+    String CONTENT_REPO_REF_DESCRIPTION = "Reference for a specific branch, tag, or commit"
+    String CONTENT_REPO_USERNAME_DESCRIPTION = "Username to authenticate against content repo"
+    String CONTENT_REPO_PASSWORD_DESCRIPTION = "Password to authenticate against content repo"
+    String CONTENT_REPO_TEMPLATING_DESCRIPTION = "When true, template all files ending in .ftl within the repo"
+    String CONTENT_REPO_FOLDER_BASED_REPOS_DESCRIPTION = "When true, interpret the folder structure of each repo as repos. That is, root folder becomes namespace in SCM, sub folders become repository names in SCM"
+    String CONTENT_REPO_TARGET_DESCRIPTION = "Target path for the repository"
+    String CONTENT_REPO_TARGET_OVERRIDE_MODE = "This defines, how customer repos will be updated.\nINIT - push only if repo does not exist (SCMM Return value 409).\nRESET - delete alle files after cloning source - new files are deleted\nUPGRADE - clone and copy - existing files will be overwritten, new files are kept"
+    String CONTENT_VARIABLES_DESCRIPTION = "Additional variables to use in custom templates."
+
     // group jenkins
     String JENKINS_ENABLE_DESCRIPTION = 'Installs Jenkins as CI server'
     String JENKINS_DESCRIPTION = 'Config parameters for Jenkins CI/CD Pipeline Server'