KOGITO-2374 Setup Maven repo url

Jenkinsfile.deploy

@@ -65,22 +65,26 @@ pipeline {
                 }
             }
         }
-        stage('Update Artifacts') {
+        stage('Update Maven information') {
             steps {
                 script {
-                    updateArtifactCmd = "python3 scripts/update-artifacts.py"
+                    // Update artifacts
+                    updateArtifactCmd = "python3 scripts/update-maven-information.py"
                     if(params.MAVEN_ARTIFACT_REPOSITORY != ''){
-                        updateArtifactCmd += "--repo-url ${params.MAVEN_ARTIFACT_REPOSITORY}"
+                        updateArtifactCmd += " --repo-url ${params.MAVEN_ARTIFACT_REPOSITORY}"
                     }
                     sh updateArtifactCmd
 
+                    // For debug
                     sh "cat modules/kogito-data-index/module.yaml"
                     sh "cat modules/kogito-jobs-service/module.yaml"
                     sh "cat modules/kogito-management-console/module.yaml"
+                    sh "cat tests/test-apps/clone-repo.sh"
+                    sh "cat ${HOME}/.m2/settings.xml"
                 }
             }
         }
-        stage('Validate CeKit Image and Modules descriptors'){
+        stage('Validate CeKit Image and Modules descriptors') {
             steps {
                 sh """
                     curl -Ls https://github.com/kiegroup/kie-cloud-tools/releases/download/1.0-SNAPSHOT/cekit-image-validator-runner.tgz --output cekit-image-validator-runner.tgz

modules/kogito-maven/3.6.x/added/configure-maven.sh

@@ -25,6 +25,7 @@ function configure() {
     configure_proxy
     configure_mirrors
     configure_maven_download_output
+    set_kogito_maven_repo
     add_maven_repo
 }
 
@@ -91,6 +92,13 @@ function configure_maven_download_output() {
     fi
 }
 
+function set_kogito_maven_repo() {
+    local kogito_maven_repo_url="${KOGITO_MAVEN_REPO_URL}"
+    if [ -n "${kogito_maven_repo_url}" ]; then
+        sed -i "s|https://repository.jboss.org/nexus/content/groups/public/|${kogito_maven_repo_url}|" $HOME/.m2/settings.xml
+    fi
+}
+
 function add_maven_repo() {
     # single remote repository scenario: respect fully qualified url if specified, otherwise find and use service
     local single_repo_url="${MAVEN_REPO_URL}"

modules/kogito-maven/3.6.x/module.yaml

@@ -29,8 +29,11 @@ envs:
   - name: "MAVEN_DOWNLOAD_OUTPUT"
     description: "If set to true will print the transfer logs for downloading/uploading of maven dependencies. Defaults to false"
     example: "true"
+  - name: "KOGITO_MAVEN_REPO_URL"
+    value: "https://repository.jboss.org/nexus/content/groups/public/"
+    description: "Defines the Kogito Maven repository."
   - name: "MAVEN_REPO_URL"
-    description: "Defines the new Repository address."
+    description: "Defines an extra Maven repository."
     example: "https://nexus.test.com/group/public"
   - name: "MAVEN_REPO_ID"
     description: "Defines the id of the new Repository"

scripts/common.py

@@ -7,7 +7,7 @@
 
 from ruamel.yaml import YAML
 
-# all kogito-image modules that points to the kogito version.
+# All kogito-image modules that have the kogito version.
 MODULES = {"kogito-data-index", "kogito-image-dependencies",
            "kogito-infinispan-properties", "kogito-jobs-service",
            "kogito-jq", "kogito-kubernetes-client",
@@ -17,15 +17,13 @@
            "kogito-quarkus-s2i", "kogito-s2i-core",
            "kogito-springboot", "kogito-springboot-s2i",
            "kogito-system-user"}
+MODULE_FILENAME = "module.yaml"
+MODULES_DIR = "modules"
 
 # imagestream file that contains all images, this file aldo needs to be updated.
-IMAGE_STREAM = "kogito-imagestream.yaml"
-
+IMAGE_STREAM_FILENAME = "kogito-imagestream.yaml"
 # image.yaml file definition that needs to be updated
-IMAGE = "image.yaml"
-
-# declared envs on modules.yaml that also needs to have its version updated
-ENVS = {"KOGITO_VERSION"}
+IMAGE_FILENAME = "image.yaml"
 
 def yaml_loader():
     """
@@ -43,17 +41,17 @@ def update_image_version(target_version):
     Update image.yaml version tag.
     :param target_version: version used to update the image.yaml file
     """
-    print("Updating Image main file version from file {0} to version {1}".format(IMAGE, target_version))
+    print("Updating Image main file version from file {0} to version {1}".format(IMAGE_FILENAME, target_version))
     try:
-        with open(IMAGE) as image:
+        with open(IMAGE_FILENAME) as image:
             data = yaml_loader().load(image)
             if 'version' in data:
                 data['version'] = target_version
             else:
                 print("Field version not found, returning...")
                 return
 
-        with open(IMAGE, 'w') as image:
+        with open(IMAGE_FILENAME, 'w') as image:
             yaml_loader().dump(data, image)
     except TypeError as err:
         print("Unexpected error:", err)
@@ -64,9 +62,9 @@ def update_image_stream(target_version):
     Update the imagestream file, it will update the tag name, version and image tag.
     :param target_version: version used to update the imagestream file;
     """
-    print("Updating ImageStream images version from file {0} to version {1}".format(IMAGE_STREAM, target_version))
+    print("Updating ImageStream images version from file {0} to version {1}".format(IMAGE_STREAM_FILENAME, target_version))
     try:
-        with open(IMAGE_STREAM) as imagestream:
+        with open(IMAGE_STREAM_FILENAME) as imagestream:
             data = yaml_loader().load(imagestream)
             for item_index, item in enumerate(data['items'], start=0):
                 for tag_index, tag in enumerate(item['spec']['tags'], start=0):
@@ -77,58 +75,100 @@ def update_image_stream(target_version):
                     updatedImageName = imageDict[0] + ':' + target_version
                     data['items'][item_index]['spec']['tags'][tag_index]['from']['name'] = updatedImageName
 
-        with open(IMAGE_STREAM, 'w') as imagestream:
+        with open(IMAGE_STREAM_FILENAME, 'w') as imagestream:
             yaml_loader().dump(data, imagestream)
 
     except TypeError:
         raise
 
+def get_all_module_dirs():
+    modules = []
+
+    # r=>root, d=>directories, f=>files
+    for r, d, f in os.walk(MODULES_DIR):
+        for item in f:
+            if MODULE_FILENAME == item:
+                modules.append(os.path.dirname(os.path.join(r, item)))
+
+    return modules
+
+def get_kogito_module_dirs():
+    modules = []
 
-def update_kogito_modules(target_version):
+    for moduleName in MODULES:
+        modules.append(os.path.join(MODULES_DIR, moduleName))
+
+    return modules
+
+def update_modules_version(target_version):
     """
-    Update every module.yaml file listed on MODULES as well the envs listed on ENVS.
-    :param target_version:  version used to update all needed module.yaml files
+    Update every Kogito module.yaml to the given version.
+    :param target_version: version used to update all Kogito module.yaml files
     """
-    modules_dir = "modules"
-    try:
+    for module_dir in get_kogito_module_dirs():
+        update_module_version(module_dir, target_version)
 
-        for module in MODULES:
-            module = module + "/module.yaml"
-            with open(os.path.join(modules_dir, module)) as m:
-                data = yaml_loader().load(m)
-                print(
-                    "Updating module {0} version from {1} to {2}".format(data['name'], data['version'], target_version))
-                data['version'] = target_version
+def update_module_version(moduleDir, target_version):
+    """
+    Set Kogito module.yaml to given version.
+    :param target_version: version to set into the module
+    """
+    try:
+        moduleFile = os.path.join(moduleDir, "module.yaml")
+        with open(moduleFile) as module:
+            data = yaml_loader().load(module)
+            print(
+                "Updating module {0} version from {1} to {2}".format(data['name'], data['version'], target_version))
+            data['version'] = target_version
 
-            with open(os.path.join(modules_dir, module), 'w') as m:
-                yaml_loader().dump(data, m)
+        with open(moduleFile, 'w') as module:
+            yaml_loader().dump(data, module)
 
     except TypeError:
         raise
 
-    update_kogito_version_in_modules(target_version)
 
-def update_kogito_version_in_modules(target_version):
+def update_kogito_version_env_in_modules(target_version):
     """
-    Update every module.yaml file listed on MODULES as well the envs listed on ENVS.
-    :param target_version:  version used to update all needed module.yaml files
+    Update all modules which contains the `KOGITO_VERSION` env var.
+    :param target_version: kogito version used to update all modules which contains the `KOGITO_VERSION` env var
     """
-    modules_dir = "modules"
-    try:
+    update_env_in_all_modules("KOGITO_VERSION", target_version)
 
-        for module in MODULES:
-            module = module + "/module.yaml"
-            with open(os.path.join(modules_dir, module)) as m:
-                data = yaml_loader().load(m)
-                if 'envs' in data:
-                    for index, env in enumerate(data['envs'], start=0):
-                        for target_env in ENVS:
-                            if target_env == env['name']:
-                                print("Updating module {0} env var {1} with value {2}".format(data['name'], target_env, target_version))
-                                data['envs'][index]['value'] = target_version
+def update_env_in_all_modules(envKey, envValue):
+    """
+    Update all modules which contains the given envKey to the given envValue.
+    :param envKey: Environment variable key to update
+    :param envValue: Environment variable value to set
+    """
+    for module_dir in get_all_module_dirs():
+        update_env_in_module(module_dir, envKey, envValue)
 
-            with open(os.path.join(modules_dir, module), 'w') as m:
-                yaml_loader().dump(data, m)
+def update_env_in_module(module_dir, envKey, envValue):
+    """
+    Update a module if it contains the given envKey to the given envValue.
+    :param envKey: Environment variable key to update if exists
+    :param envValue: Environment variable value to set if exists
+    """
+    try:
+        moduleFile = os.path.join(module_dir, "module.yaml")
+        changed = False
+        with open(moduleFile) as module:
+            data = yaml_loader().load(module)
+            if 'envs' in data:
+                for index, env in enumerate(data['envs'], start=0):
+                    if envKey == env['name']:
+                        print("Updating module {0} env var {1} with value {2}".format(data['name'], envKey, envValue))
+                        data['envs'][index]['value'] = envValue
+                        changed = True
+
+        if (changed):
+            with open(moduleFile, 'w') as module:
+                yaml_loader().dump(data, module)
 
     except TypeError:
         raise
+
+if __name__ == "__main__":
+    for m in get_kogito_module_dirs():
+        print("module {}".format(m))

scripts/manage-kogito-version.py

@@ -94,7 +94,8 @@ def update_test_apps_clone_repo(target_branch):
 
             common.update_image_version(args.bump_to)
             common.update_image_stream(args.bump_to)
-            common.update_kogito_modules(args.bump_to)
+            common.update_modules_version(args.bump_to)
+            common.update_kogito_version_env_in_modules(args.bump_to)
             update_behave_tests(tests_branch)
             update_test_apps_clone_repo(tests_branch)
         else:

scripts/push-staging.py

@@ -138,7 +138,8 @@ def get_next_rc_version(current_rc_version):
     version = get_next_rc_version(find_current_rc_version())
     common.update_image_version(version)
     common.update_image_stream(version)
-    common.update_kogito_modules(version)
+    common.update_modules_version(version)
+    common.update_kogito_version_env_in_modules(version)
 
     find_next_tag()
     tag_and_push_images()