Support using multiple images sources in our CLI (#142)

* Initial stab at making the multiple registry support work

* Update TODO

* update TODO

* merge fixes

* add logic to find registry credentials

* Code cleanup and adding unit tests

* updates to the registry class

* In Progress work

* Working publish

* Move _push_image_from_local_registry and copy_image into the registry class

* tidy up changes

* Namespace and style fixes

* Minor input config change

* unit tests

* Do not push image namespace to target ACR

* Make sure not to break the nexus part of the code

* Update CHANGELOG

* Fix error not being caught correctly

* Minor comment change

* Markups

* Update warning

* Change how we treat namespace and find images

* markups

* modify how we create the registry list

---------

Co-authored-by: Andy Churchard <andy.churchard@metaswitch.com>

src/aosm/HISTORY.rst

@@ -5,6 +5,8 @@ Release History
 
 Unreleased
 ++++++++
+* Added: Users can specify multiple image sources from all types of registries (not just ACRs). General improvements in how CNF image sources are handled. 
+* Fixed: Namespace appeared twice in the `artifacts.json` file, leading to errors in the publish step of the CLI.
 * Changed configurationType for NF Resources from Secret to Open
 
 1.0.0b10

src/aosm/azext_aosm/build_processors/helm_chart_processor.py

@@ -17,6 +17,7 @@
 )
 from azext_aosm.definition_folder.builder.local_file_builder import LocalFileBuilder
 from azext_aosm.inputs.helm_chart_input import HelmChartInput
+from azext_aosm.common.registry import ContainerRegistryHandler
 from azext_aosm.vendored_sdks.models import (
     ApplicationEnablement,
     ArtifactType,
@@ -53,12 +54,10 @@ def __init__(
         self,
         name: str,
         input_artifact: HelmChartInput,
-        source_registry: str,
-        source_registry_namespace: str,
+        registry_handler: ContainerRegistryHandler,
     ):
         super().__init__(name, input_artifact)
-        self.source_registry = source_registry
-        self.source_registry_namespace = source_registry_namespace
+        self.registry_handler = registry_handler
         self.input_artifact: HelmChartInput = input_artifact
 
     def get_artifact_manifest_list(self) -> List[ManifestArtifactFormat]:
@@ -114,13 +113,20 @@ def get_artifact_details(
         artifact_details.append(helm_chart_details)
         for image_name, image_version in self._find_chart_images():
             # We only support remote ACR artifacts for container images
+
+            registry, namespace = self.registry_handler.find_registry_for_image(
+                image_name, image_version
+            )
+            if registry is None or namespace is None:
+                continue
+
             artifact_details.append(
                 RemoteACRArtifact(
                     artifact_name=image_name,
                     artifact_type=ArtifactType.OCI_ARTIFACT.value,
                     artifact_version=image_version,
-                    source_registry=self.source_registry,
-                    source_registry_namespace=self.source_registry_namespace,
+                    source_registry=registry,
+                    registry_namespace=namespace,
                 )
             )
 

src/aosm/azext_aosm/build_processors/nexus_image_processor.py

@@ -10,23 +10,28 @@
 from knack.log import get_logger
 
 from azext_aosm.build_processors.base_processor import BaseInputProcessor
-from azext_aosm.common.artifact import (BaseArtifact,
-                                        RemoteACRArtifact)
+from azext_aosm.common.artifact import BaseArtifact, RemoteACRArtifact
 from azext_aosm.definition_folder.builder.local_file_builder import LocalFileBuilder
 from azext_aosm.inputs.nexus_image_input import NexusImageFileInput
 from azext_aosm.vendored_sdks.models import (
     AzureOperatorNexusNetworkFunctionImageApplication,
     AzureOperatorNexusImageArtifactProfile,
     AzureOperatorNexusImageDeployMappingRuleProfile,
-    ImageArtifactProfile, ImageMappingRuleProfile,
-    ApplicationEnablement, ArtifactType,
+    ImageArtifactProfile,
+    ImageMappingRuleProfile,
+    ApplicationEnablement,
+    ArtifactType,
     DependsOnProfile,
-    ManifestArtifactFormat, ReferencedResource, ResourceElementTemplate,
+    ManifestArtifactFormat,
+    ReferencedResource,
+    ResourceElementTemplate,
 )
+from azext_aosm.common.registry import AzureContainerRegistry
 from azext_aosm.common.constants import (
     VNF_OUTPUT_FOLDER_FILENAME,
     NF_DEFINITION_FOLDER_NAME,
-    NEXUS_IMAGE_PARAMETERS_FILENAME)
+    NEXUS_IMAGE_PARAMETERS_FILENAME,
+)
 
 logger = get_logger(__name__)
 
@@ -74,19 +79,25 @@ def get_artifact_details(
         artifacts: List[BaseArtifact] = []
         file_builders: List[LocalFileBuilder] = []
 
-        # We only support remote ACR artifacts for container images
+        # We only support remote ACR artifacts for nexus container images
+        source_registry = AzureContainerRegistry(
+            self.input_artifact.source_acr_registry
+        )
+
         artifacts.append(
             RemoteACRArtifact(
                 artifact_name=self.input_artifact.artifact_name,
                 artifact_type=ArtifactType.IMAGE_FILE.value,
                 artifact_version=self.input_artifact.artifact_version,
-                source_registry=self.input_artifact.source_acr_registry,
-                source_registry_namespace="",
+                source_registry=source_registry,
+                registry_namespace="",
             )
         )
         return artifacts, file_builders
 
-    def generate_nf_application(self) -> AzureOperatorNexusNetworkFunctionImageApplication:
+    def generate_nf_application(
+        self,
+    ) -> AzureOperatorNexusNetworkFunctionImageApplication:
         """
         Generate the NF application.
 
@@ -97,8 +108,9 @@ def generate_nf_application(self) -> AzureOperatorNexusNetworkFunctionImageAppli
 
         return AzureOperatorNexusNetworkFunctionImageApplication(
             name=self.name,
-            depends_on_profile=DependsOnProfile(install_depends_on=[],
-                                                uninstall_depends_on=[], update_depends_on=[]),
+            depends_on_profile=DependsOnProfile(
+                install_depends_on=[], uninstall_depends_on=[], update_depends_on=[]
+            ),
             artifact_profile=self._generate_artifact_profile(),
             deploy_parameters_mapping_rule_profile=self._generate_mapping_rule_profile(),
         )
@@ -109,8 +121,10 @@ def generate_resource_element_template(self) -> ResourceElementTemplate:
 
         :raises NotImplementedError: NSDs do not support deployment of Nexus images.
         """
-        raise NotImplementedError("NSDs do not support deployment of Nexus images directly, "
-                                  "they must be provided in the NF.")
+        raise NotImplementedError(
+            "NSDs do not support deployment of Nexus images directly, "
+            "they must be provided in the NF."
+        )
 
     def _generate_artifact_profile(self) -> AzureOperatorNexusImageArtifactProfile:
         """
@@ -154,27 +168,25 @@ def _generate_mapping_rule_profile(
         )
 
     def generate_parameters_file(self) -> LocalFileBuilder:
-        """ Generate parameters file. """
+        """Generate parameters file."""
         mapping_rule_profile = self._generate_mapping_rule_profile()
         if (
             mapping_rule_profile.image_mapping_rule_profile
             and mapping_rule_profile.image_mapping_rule_profile.user_configuration
         ):
-            params = (
-                mapping_rule_profile.image_mapping_rule_profile.user_configuration
-            )
-            logger.info(
-                "Created parameters file for Nexus image."
-            )
+            params = mapping_rule_profile.image_mapping_rule_profile.user_configuration
+            logger.info("Created parameters file for Nexus image.")
         # We still want to create an empty params file,
         # otherwise the nf definition bicep will refer to files that don't exist
         else:
-            params = '{}'
+            params = "{}"
         return LocalFileBuilder(
             Path(
                 VNF_OUTPUT_FOLDER_FILENAME,
                 NF_DEFINITION_FOLDER_NAME,
-                self.input_artifact.artifact_name + '-' + NEXUS_IMAGE_PARAMETERS_FILENAME,
+                self.input_artifact.artifact_name
+                + "-"
+                + NEXUS_IMAGE_PARAMETERS_FILENAME,
             ),
             json.dumps(json.loads(params), indent=4),
         )

src/aosm/azext_aosm/cli_handlers/onboarding_cnf_handler.py

@@ -51,6 +51,7 @@
 from azext_aosm.inputs.helm_chart_input import HelmChartInput
 
 from .onboarding_nfd_base_handler import OnboardingNFDBaseCLIHandler
+from azext_aosm.common.registry import ContainerRegistryHandler
 from azext_aosm.common.utils import render_bicep_contents_from_j2, get_template_path
 
 logger = get_logger(__name__)
@@ -73,7 +74,9 @@ def output_folder_file_name(self) -> str:
         """Get the output folder file name."""
         return CNF_OUTPUT_FOLDER_FILENAME
 
-    def _get_input_config(self, input_config: Optional[dict] = None) -> OnboardingCNFInputConfig:
+    def _get_input_config(
+        self, input_config: Optional[dict] = None
+    ) -> OnboardingCNFInputConfig:
         """Get the configuration for the command."""
         if input_config is None:
             input_config = {}
@@ -89,8 +92,11 @@ def _get_params_config(self, config_file: Path) -> CNFCommonParametersConfig:
 
     def _get_processor_list(self) -> list[HelmChartProcessor]:
         processor_list = []
-        # for each helm package, instantiate helm processor
         assert isinstance(self.config, OnboardingCNFInputConfig)
+
+        registry_handler = ContainerRegistryHandler(self.config.image_sources)
+
+        # for each helm package, instantiate helm processor
         for helm_package in self.config.helm_packages:
             if helm_package.default_values:
                 if Path(helm_package.default_values).exists():
@@ -110,10 +116,7 @@ def _get_processor_list(self) -> list[HelmChartProcessor]:
                 default_config_path=helm_package.default_values,
             )
             helm_processor = HelmChartProcessor(
-                helm_package.name,
-                helm_input,
-                self.config.images.source_registry,
-                self.config.images.source_registry_namespace,
+                helm_package.name, helm_input, registry_handler
             )
             processor_list.append(helm_processor)
         return processor_list
@@ -126,9 +129,9 @@ def _validate_helm_template(self):
             try:
                 helm_processor.input_artifact.validate_template()
             except TemplateValidationError as error:
-                validation_errors[
-                    helm_processor.input_artifact.artifact_name
-                ] = str(error)
+                validation_errors[helm_processor.input_artifact.artifact_name] = str(
+                    error
+                )
 
         if validation_errors:
             # Create an error file using a j2 template
@@ -203,10 +206,7 @@ def build_manifest_bicep(self) -> BicepDefinitionElementBuilder:
             CNF_TEMPLATE_FOLDER_NAME, CNF_MANIFEST_TEMPLATE_FILENAME
         )
 
-        params = {
-            "acr_artifacts": artifact_list,
-            "sa_artifacts": []
-        }
+        params = {"acr_artifacts": artifact_list, "sa_artifacts": []}
         bicep_contents = render_bicep_contents_from_j2(template_path, params)
 
         # Create Bicep element with manifest contents