config: Transformer for old version of config (#2147)

Refs: #2130
- Generic transformer for cnf-testsuite.yml configs to newer versions.
- Extendable through addition of new transformation rules.
- The current functionality transforms to configv2, the structure of which
was proposed in #2129.

Signed-off-by: svteb <slavo.valko@tietoevry.com>

CNF_TESTSUITE_YML_USAGE.md

@@ -43,6 +43,14 @@ Prereqs: You must have kubernetes cluster, curl, and helm 3.1.1 or greater on yo
 - Generate a cnf-testsuite.yml based on a directory of manifest files:  `./cnf-testsuite generate_config config-src=<your-manifest-directory> output-file=./cnf-testsuite.yml`
 - Inspect the cnf-testsuite.yml file for accuracy
 
+### Updating config from older versions
+
+New releases may change the format of cnf-testsuite.yml. To update your older configs automatically to the latest version, use the `update_config` task.
+
+```
+./cnf-testsuite update_config input_config=OLD_CONFIG_PATH output_config=NEW_CONFIG_PATH
+```
+
 ### Keys and Values
 
 #### allowlist_helm_chart_container_names

src/tasks/update_config.cr

@@ -0,0 +1,45 @@
+require "sam"
+require "totem"
+require "colorize"
+require "./utils/cnf_installation/config_updater/config_updater"
+require "./utils/cnf_installation/config"
+
+desc "Updates an old configuration file to the latest version and saves it to the specified location"
+task "update_config" do |_, args|
+  # Ensure both arguments are provided
+  if !((args.named.keys.includes? "input_config") && (args.named.keys.includes? "output_config"))
+    stdout_warning "Usage: update_config input_config=OLD_CONFIG_PATH output_config=NEW_CONFIG_PATH"
+    exit(0)
+  end
+
+  input_config = args.named["input_config"].as(String)
+  output_config = args.named["output_config"].as(String)
+
+  # Check if the input config file exists
+  unless File.exists?(input_config)
+    stdout_failure "The input config file '#{input_config}' does not exist."
+    exit(1)
+  end
+
+  begin
+    raw_input_config = File.read(input_config)
+
+    # Verify that config is not the latest version
+    if CNFInstall::Config.config_version_is_latest?(raw_input_config)
+      stdout_warning "Input config is the latest version."
+      exit(0)
+    end
+
+    # Initialize the ConfigUpdater
+    updater = CNFInstall::Config::ConfigUpdater.new(raw_input_config)
+    updater.transform
+
+    # Serialize the updated config to the new file
+    updater.serialize_to_file(output_config)
+
+    stdout_success "Configuration successfully updated and saved to '#{output_config}'."
+  rescue ex : CNFInstall::Config::UnsupportedConfigVersionError
+    stdout_failure ex.message
+    exit(1)
+  end
+end

src/tasks/utils/cnf_installation/config.cr

@@ -28,5 +28,26 @@ module CNFInstall
 
       config
     end
+
+    # Detects the config version.
+    def self.detect_version(tmp_content : String) : ConfigVersion
+      yaml_content = YAML.parse(tmp_content).as_h
+      version_value = yaml_content["config_version"]?.try(&.to_s)
+
+      if version_value
+        begin
+          ConfigVersion.parse(version_value.upcase)
+        rescue ex : ArgumentError
+          raise UnsupportedConfigVersionError.new(version_value)
+        end
+      else
+        # Default to V1 if no version is specified
+        ConfigVersion::V1
+      end
+    end
+
+    def self.config_version_is_latest?(tmp_content : String) : Bool
+      detect_version(tmp_content) == ConfigVersion::Latest
+    end
   end
 end

src/tasks/utils/cnf_installation/config_updater/config_updater.cr

@@ -0,0 +1,76 @@
+require "yaml"
+
+module CNFInstall
+  module Config
+    class ConfigUpdater
+      @output_config : YAML::Any
+      @input_config : ConfigBase
+      @version : ConfigVersion
+
+      # This approach could be extended in future by making use of abstract classes,
+      # which would remove the need for hashes. 
+      # Define transformation rules at the top of the class
+      # REQUIRES FUTURE EXTENSION in case of new config format.
+      VERSION_TRANSFORMATIONS = {
+        ConfigVersion::V1 => ->(input_config : ConfigBase) { V1ToV2Transformation.new(input_config.as(ConfigV1)).transform }
+      }
+
+      # Define parsing rules at the top of the class
+      # REQUIRES FUTURE EXTENSION in case of new config format.
+      VERSION_PARSERS = {
+        ConfigVersion::V1 => ->(raw_input_config : String) { ConfigV1.from_yaml(raw_input_config) }
+      }
+
+      def initialize(raw_input_config : String)
+        # Automatic version detection to streamline the transformation
+        @version = CNFInstall::Config.detect_version(raw_input_config)
+        @output_config = YAML::Any.new({} of YAML::Any => YAML::Any)
+        @input_config = parse_input_config(raw_input_config)
+      end
+
+      # Serialize the updated config to a string.
+      def serialize_to_string : String
+        YAML.dump(@output_config)
+      end
+
+      # Serialize the updated config to a file and return the file path.
+      def serialize_to_file(file_path : String) : String
+        File.write(file_path, serialize_to_string)
+        file_path
+      end
+
+      # Parses the config to the correct class.
+      # Uses the VERSION_PARSERS hash.
+      private def parse_input_config(raw_input_config : String) : ConfigBase
+        parser = VERSION_PARSERS[@version]
+        if parser
+          begin
+            parser.call(raw_input_config)
+          rescue ex : YAML::ParseException
+            stdout_failure "Failed to parse config: #{ex.message}."
+            exit(1)
+          end
+        else
+          raise UnsupportedConfigVersionError.new(@version)
+        end
+      end
+
+      # Performs the transformation from Vx to Vy.
+      # Uses the VERSION_TRANSFORMATIONS hash.
+      def transform
+        transformer = VERSION_TRANSFORMATIONS[@version]
+        if transformer
+          @output_config = transformer.call(@input_config)
+        else
+          raise UnsupportedConfigVersionError.new(@version)
+        end
+      end
+    end
+
+    class UnsupportedConfigVersionError < Exception
+      def initialize(version : ConfigVersion | String)
+        super "Unsupported configuration version detected: #{version.is_a?(ConfigVersion) ? version.to_s.downcase : version}"
+      end
+    end
+  end
+end    

src/tasks/utils/cnf_installation/config_updater/transformation_base.cr

@@ -0,0 +1,37 @@
+module CNFInstall
+  module Config
+    # The rules need to be somewhat explicit, different approaches have been attempted
+    # but due to crystals strict typing system they have not been viable/would be too complicated.
+    # 
+    # In case of future extension, create a new transformation rules class (VxToVyTransformation),
+    # This class should inherit the TransformationBase class and make use of process_data
+    # function at the end of its transform function.
+    class TransformationBase
+      @output_config : YAML::Any
+
+      def initialize()
+        @output_config = YAML::Any.new({} of YAML::Any => YAML::Any)
+      end
+
+      # Recursively remove any empty hashes/arrays/values and convert data to YAML::Any.
+      private def process_data(data : Hash | Array | String | Nil) : YAML::Any?
+        case data
+        when Array
+          processed_array = data.map { |item| process_data(item) }.compact
+          processed_array.empty? ? nil : YAML::Any.new(processed_array)
+        when Hash
+          processed_hash = Hash(YAML::Any, YAML::Any).new
+          data.each do |k, v|
+            processed_value = process_data(v)
+            processed_hash[YAML::Any.new(k)] = processed_value unless processed_value.nil?
+          end
+          processed_hash.empty? ? nil : YAML::Any.new(processed_hash)
+        when String
+          YAML::Any.new(data)
+        else
+          nil
+        end
+      end
+    end
+  end
+end

src/tasks/utils/cnf_installation/config_updater/v1_to_v2_transformation.cr

@@ -0,0 +1,124 @@
+module CNFInstall
+  module Config
+    # Rules for configV1 to configV2 transformation
+    class V1ToV2Transformation < TransformationBase
+      def initialize(@input_config : ConfigV1)
+        super()
+      end
+
+      def transform : YAML::Any
+        output_config_hash = {
+          "config_version" => "v2",
+          "common" => transform_common,
+          "dynamic" => transform_dynamic,
+          "deployments" => transform_deployments,
+        }
+
+        # Convert the entire native hash to stripped YAML::Any at the end.
+        @output_config = process_data(output_config_hash).not_nil!
+      end
+
+      private def transform_common : Hash(String, Array(Hash(String, String | Nil)) | Array(String) | Hash(String, String | Nil))
+        common = {} of String => Array(Hash(String, String | Nil)) | Array(String) | Hash(String, String | Nil)
+
+        common = {
+          "white_list_container_names" => @input_config.white_list_container_names,
+          "docker_insecure_registries" => @input_config.docker_insecure_registries,
+          "image_registry_fqdns" => @input_config.image_registry_fqdns,
+          "container_names" => transform_container_names,
+          "five_g_parameters" => transform_five_g_parameters
+        }.compact
+
+        common
+      end
+
+      private def transform_container_names : Array(Hash(String, String | Nil))
+        if @input_config.container_names
+          containers = @input_config.container_names.not_nil!.map do |container|
+            {
+              "name" => container.name,
+              "rollback_from_tag" => container.rollback_from_tag,
+              "rolling_update_test_tag" => container.rolling_update_test_tag,
+              "rolling_downgrade_test_tag" => container.rolling_downgrade_test_tag,
+              "rolling_version_change_test_tag" => container.rolling_version_change_test_tag
+            }
+          end
+
+          return containers
+        end
+
+        [] of Hash(String, String | Nil)
+      end
+
+      private def transform_dynamic : Hash(String, String | Nil)
+        {
+          "source_cnf_dir" => @input_config.source_cnf_dir,
+          "destination_cnf_dir" => @input_config.destination_cnf_dir
+        }
+      end
+
+      private def transform_deployments : Hash(String, Array(Hash(String, String | Nil)))
+        deployments = {} of String => Array(Hash(String, String | Nil))
+
+        if @input_config.manifest_directory
+          deployments["manifests"] = [{
+            "name" => @input_config.release_name,
+            "manifest_directory" => @input_config.manifest_directory
+          }]
+        elsif @input_config.helm_directory
+          deployments["helm_dirs"] = [{
+            "name" => @input_config.release_name,
+            "helm_directory" => @input_config.helm_directory,
+            "helm_values" => @input_config.helm_values,
+            "namespace" => @input_config.helm_install_namespace
+          }]
+        elsif @input_config.helm_chart
+          helm_chart_data = {
+            "name" => @input_config.release_name,
+            "helm_chart_name" => @input_config.helm_chart,
+            "helm_values" => @input_config.helm_values,
+            "namespace" => @input_config.helm_install_namespace
+          }
+
+          if @input_config.helm_repository
+            helm_chart_data["helm_repo_name"] = @input_config.helm_repository.not_nil!.name
+            helm_chart_data["helm_repo_url"] = @input_config.helm_repository.not_nil!.repo_url
+          end
+
+          deployments["helm_charts"] = [helm_chart_data]
+        end
+
+        deployments
+      end
+
+      private def transform_five_g_parameters : Hash(String, String | Nil)
+        {
+          "core" => @input_config.core,
+          "amf_label" => @input_config.amf_label,
+          "smf_label" => @input_config.smf_label,
+          "upf_label" => @input_config.upf_label,
+          "ric_label" => @input_config.ric_label,
+          "amf_service_name" => @input_config.amf_service_name,
+          "mmc" => @input_config.mmc,
+          "mnc" => @input_config.mnc,
+          "sst" => @input_config.sst,
+          "sd" => @input_config.sd,
+          "tac" => @input_config.tac,
+          "protectionScheme" => @input_config.protectionScheme,
+          "publicKey" => @input_config.publicKey,
+          "publicKeyId" => @input_config.publicKeyId,
+          "routingIndicator" => @input_config.routingIndicator,
+          "enabled" => @input_config.enabled,
+          "count" => @input_config.count,
+          "initialMSISDN" => @input_config.initialMSISDN,
+          "key" => @input_config.key,
+          "op" => @input_config.op,
+          "opType" => @input_config.opType,
+          "type" => @input_config.type,
+          "apn" => @input_config.apn,
+          "emergency" => @input_config.emergency
+        }
+      end
+    end
+  end
+end

src/tasks/utils/cnf_installation/config_versions/config_v1.cr

@@ -0,0 +1,68 @@
+module CNFInstall
+  module Config
+    @[YAML::Serializable::Options(emit_nulls: true)]
+    class ConfigV1 < ConfigBase
+      getter config_version : String?
+      getter destination_cnf_dir : String?
+      getter source_cnf_dir : String?
+      getter manifest_directory : String?
+      getter helm_directory : String?
+      getter release_name : String?
+      getter helm_repository : HelmRepository?
+      getter helm_chart : String?
+      getter helm_values : String?
+      getter helm_install_namespace : String?
+      getter container_names : Array(Container)?
+      getter white_list_container_names : Array(String)?
+      getter docker_insecure_registries : Array(String)?
+      getter image_registry_fqdns : Hash(String, String?)?
+
+      # Unused properties
+      getter install_script : String?
+      getter service_name : String?
+      getter git_clone_url : String?
+      getter docker_repository : String?
+
+      # 5G related properties
+      getter amf_label : String?
+      getter smf_label : String?
+      getter upf_label : String?
+      getter ric_label : String?
+      getter core : String?
+      getter amf_service_name : String?
+      getter mmc : String?
+      getter mnc : String?
+      getter sst : String?
+      getter sd : String?
+      getter tac : String?
+      getter protectionScheme : String?
+      getter publicKey : String?
+      getter publicKeyId : String?
+      getter routingIndicator : String?
+      getter enabled : String?
+      getter count : String?
+      getter initialMSISDN : String?
+      getter key : String?
+      getter op : String?
+      getter opType : String?
+      getter type : String?
+      getter apn : String?
+      getter emergency : String?
+
+      # Nested class for Helm Repository details
+      class HelmRepository < ConfigBase
+        getter name : String?
+        getter repo_url : String?
+      end
+
+      # Nested class for Container details
+      class Container < ConfigBase
+        getter name : String?
+        getter rollback_from_tag : String?
+        getter rolling_update_test_tag : String?
+        getter rolling_downgrade_test_tag : String?
+        getter rolling_version_change_test_tag : String?
+      end
+    end
+  end
+end