From 28013e03c60a5686c4c338483e144514e90f54f8 Mon Sep 17 00:00:00 2001 From: Jordan Date: Thu, 25 Jan 2024 14:45:05 +0000 Subject: [PATCH 01/37] added nexus arm processor; added nfvi type to vnfinput.json; added condition for calling each arm processor --- .../build_processors/arm_processor.py | 74 ++++++++++++++----- .../cli_handlers/onboarding_vnf_handler.py | 13 +++- .../onboarding_vnf_input_config.py | 5 +- 3 files changed, 70 insertions(+), 22 deletions(-) diff --git a/src/aosm/azext_aosm/build_processors/arm_processor.py b/src/aosm/azext_aosm/build_processors/arm_processor.py index da0aa7f4bf4..6b7e62f7775 100644 --- a/src/aosm/azext_aosm/build_processors/arm_processor.py +++ b/src/aosm/azext_aosm/build_processors/arm_processor.py @@ -14,6 +14,7 @@ from azext_aosm.common.local_file_builder import LocalFileBuilder from azext_aosm.inputs.arm_template_input import ArmTemplateInput from azext_aosm.vendored_sdks.models import ( + ArtifactType, ApplicationEnablement, ArmResourceDefinitionResourceElementTemplate, ArmResourceDefinitionResourceElementTemplateDetails, ArmTemplateArtifactProfile, ArmTemplateMappingRuleProfile, @@ -21,7 +22,9 @@ AzureCoreArmTemplateDeployMappingRuleProfile, AzureCoreArtifactType, AzureCoreNetworkFunctionArmTemplateApplication, DependsOnProfile, ManifestArtifactFormat, NetworkFunctionApplication, NSDArtifactProfile, - ReferencedResource, ResourceElementTemplate, TemplateType) + ReferencedResource, ResourceElementTemplate, TemplateType, ArtifactProfile, + AzureOperatorNexusNetworkFunctionApplication, AzureOperatorNexusArtifactType, + AzureOperatorNexusArmTemplateDeployMappingRuleProfile, AzureOperatorNexusArmTemplateArtifactProfile) logger = get_logger(__name__) @@ -60,7 +63,7 @@ def get_artifact_manifest_list(self) -> List[ManifestArtifactFormat]: return [ ManifestArtifactFormat( artifact_name=self.input_artifact.artifact_name, - artifact_type=AzureCoreArtifactType.ARM_TEMPLATE.value, + artifact_type=ArtifactType.ARM_TEMPLATE.value, artifact_version=self.input_artifact.artifact_version, ) ] @@ -82,7 +85,7 @@ def get_artifact_details( [ LocalFileACRArtifact( artifact_name=self.input_artifact.artifact_name, - artifact_type=AzureCoreArtifactType.ARM_TEMPLATE.value, + artifact_type=ArtifactType.ARM_TEMPLATE.value, artifact_version=self.input_artifact.artifact_version, file_path=self.input_artifact.template_path, ) @@ -94,22 +97,15 @@ def get_artifact_details( def generate_nf_application(self) -> NetworkFunctionApplication: return self.generate_nfvi_specific_nf_application() - def generate_artifact_profile(self) -> AzureCoreArmTemplateArtifactProfile: - artifact_profile = ArmTemplateArtifactProfile( - template_name=self.input_artifact.artifact_name, - template_version=self.input_artifact.artifact_version, - ) - return AzureCoreArmTemplateArtifactProfile( - artifact_store=ReferencedResource(id=""), - template_artifact_profile=artifact_profile, - ) - @abstractmethod def generate_nfvi_specific_nf_application(self): pass def generate_resource_element_template(self) -> ResourceElementTemplate: - """Generate the resource element template.""" + """Generate the resource element template. + + Note: There is no Nexus specific RET, arm RET can deploy anything (except NFs) + """ parameter_values = self.generate_values_mappings( self.input_artifact.get_schema(), self.input_artifact.get_defaults(), True ) @@ -165,11 +161,55 @@ def _generate_mapping_rule_profile( template_mapping_rule_profile=mapping_profile, ) + def generate_artifact_profile(self) -> AzureCoreArmTemplateArtifactProfile: + artifact_profile = ArmTemplateArtifactProfile( + template_name=self.input_artifact.artifact_name, + template_version=self.input_artifact.artifact_version, + ) + return AzureCoreArmTemplateArtifactProfile( + artifact_store=ReferencedResource(id=""), + template_artifact_profile=artifact_profile, + ) class NexusArmBuildProcessor(BaseArmBuildProcessor): """ - Not implemented yet. This class represents a processor for generating ARM templates specific to Nexus. + This class represents a processor for generating ARM templates specific to Nexus. """ + # TODO: JORDAN check if we need Arm_template.value here? + def generate_nfvi_specific_nf_application( + self, + ) -> AzureOperatorNexusNetworkFunctionApplication: + return AzureOperatorNexusNetworkFunctionApplication( + name=self.name, + depends_on_profile=DependsOnProfile(install_depends_on=[], + uninstall_depends_on=[], update_depends_on=[]), + artifact_type=AzureOperatorNexusArtifactType.ARM_TEMPLATE, + artifact_profile=self.generate_artifact_profile(), + deploy_parameters_mapping_rule_profile=self._generate_mapping_rule_profile(), + ) - def generate_nfvi_specific_nf_application(self): - return NotImplementedError + def _generate_mapping_rule_profile( + self, + ) -> AzureOperatorNexusArmTemplateDeployMappingRuleProfile: + template_parameters = self.generate_values_mappings( + self.input_artifact.get_schema(), self.input_artifact.get_defaults() + ) + + mapping_profile = ArmTemplateMappingRuleProfile( + template_parameters=json.dumps(template_parameters) + ) + + return AzureOperatorNexusArmTemplateDeployMappingRuleProfile( + application_enablement=ApplicationEnablement.ENABLED, + template_mapping_rule_profile=mapping_profile, + ) + + def generate_artifact_profile(self) -> AzureOperatorNexusArmTemplateArtifactProfile: + artifact_profile = ArmTemplateArtifactProfile( + template_name=self.input_artifact.artifact_name, + template_version=self.input_artifact.artifact_version, + ) + return AzureOperatorNexusArmTemplateArtifactProfile( + artifact_store=ReferencedResource(id=""), + template_artifact_profile=artifact_profile, + ) diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_vnf_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_vnf_handler.py index f122af0a85f..71bfc9b60f8 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_vnf_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_vnf_handler.py @@ -8,7 +8,7 @@ from knack.log import get_logger from azext_aosm.build_processors.arm_processor import ( - AzureCoreArmBuildProcessor, BaseArmBuildProcessor) + AzureCoreArmBuildProcessor, BaseArmBuildProcessor, NexusArmBuildProcessor) from azext_aosm.build_processors.vhd_processor import VHDProcessor from azext_aosm.common.constants import (ARTIFACT_LIST_FILENAME, BASE_FOLDER_NAME, @@ -87,9 +87,14 @@ def _get_processor_list(self): template_path=Path(arm_template.file_path).absolute(), ) # TODO: generalise for nexus in nexus ready stories - processor_list.append( - AzureCoreArmBuildProcessor(arm_input.artifact_name, arm_input) - ) + if self.config.nfvi_type == "AzureCore": + processor_list.append( + AzureCoreArmBuildProcessor(arm_input.artifact_name, arm_input) + ) + elif self.config.nfvi_type == "AzureOperatorNexus": + processor_list.append( + NexusArmBuildProcessor(arm_input.artifact_name, arm_input) + ) # Instantiate vhd processor if not self.config.vhd.artifact_name: diff --git a/src/aosm/azext_aosm/configuration_models/onboarding_vnf_input_config.py b/src/aosm/azext_aosm/configuration_models/onboarding_vnf_input_config.py index 561dab3fe44..1ad54226c4f 100644 --- a/src/aosm/azext_aosm/configuration_models/onboarding_vnf_input_config.py +++ b/src/aosm/azext_aosm/configuration_models/onboarding_vnf_input_config.py @@ -114,6 +114,8 @@ class OnboardingVNFInputConfig(OnboardingNFDBaseInputConfig): ) }, ) + nfvi_type: str = field(default="AzureCore", + metadata={"comment": "NFVI type. Defaults to AzureCore. Can be AzureCore or AzureOperatorNexus"}) # TODO: Add better comments arm_templates: List[ArmTemplatePropertiesConfig] = field( @@ -147,7 +149,8 @@ def sa_manifest_name(self) -> str: def validate(self): """Validate the configuration.""" super().validate() - + if not self.nfvi_type: + raise ValidationError("nfvi_type must be set") if not self.image_name_parameter: raise ValidationError("image_name_parameter must be set") if not self.arm_templates: From 7f08186c7856e0ce8b7d6f0970ab89cf2b97c268 Mon Sep 17 00:00:00 2001 From: Jordan Date: Thu, 25 Jan 2024 18:11:03 +0000 Subject: [PATCH 02/37] pipe through nfvi type from vnf input to template; added nexus input; fixed mistakes in armprocessor --- .../build_processors/arm_processor.py | 5 +- .../build_processors/nexus_image_processor.py | 170 ++++++++++++++++++ .../cli_handlers/onboarding_vnf_handler.py | 9 +- .../templates/vnf/vnfdefinition.bicep.j2 | 2 +- .../azext_aosm/inputs/nexus_image_input.py | 84 +++++++++ 5 files changed, 264 insertions(+), 6 deletions(-) create mode 100644 src/aosm/azext_aosm/build_processors/nexus_image_processor.py create mode 100644 src/aosm/azext_aosm/inputs/nexus_image_input.py diff --git a/src/aosm/azext_aosm/build_processors/arm_processor.py b/src/aosm/azext_aosm/build_processors/arm_processor.py index 6b7e62f7775..cc6cf20e676 100644 --- a/src/aosm/azext_aosm/build_processors/arm_processor.py +++ b/src/aosm/azext_aosm/build_processors/arm_processor.py @@ -15,6 +15,7 @@ from azext_aosm.inputs.arm_template_input import ArmTemplateInput from azext_aosm.vendored_sdks.models import ( ArtifactType, + AzureOperatorNexusNetworkFunctionArmTemplateApplication, ApplicationEnablement, ArmResourceDefinitionResourceElementTemplate, ArmResourceDefinitionResourceElementTemplateDetails, ArmTemplateArtifactProfile, ArmTemplateMappingRuleProfile, @@ -178,8 +179,8 @@ class NexusArmBuildProcessor(BaseArmBuildProcessor): # TODO: JORDAN check if we need Arm_template.value here? def generate_nfvi_specific_nf_application( self, - ) -> AzureOperatorNexusNetworkFunctionApplication: - return AzureOperatorNexusNetworkFunctionApplication( + ) -> AzureOperatorNexusNetworkFunctionArmTemplateApplication: + return AzureOperatorNexusNetworkFunctionArmTemplateApplication( name=self.name, depends_on_profile=DependsOnProfile(install_depends_on=[], uninstall_depends_on=[], update_depends_on=[]), diff --git a/src/aosm/azext_aosm/build_processors/nexus_image_processor.py b/src/aosm/azext_aosm/build_processors/nexus_image_processor.py new file mode 100644 index 00000000000..6e7e56b909f --- /dev/null +++ b/src/aosm/azext_aosm/build_processors/nexus_image_processor.py @@ -0,0 +1,170 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import json +from typing import List, Tuple + +from knack.log import get_logger + +from azext_aosm.build_processors.base_processor import BaseInputProcessor +from azext_aosm.common.artifact import (BaseArtifact, + BlobStorageAccountArtifact, + LocalFileStorageAccountArtifact) +from azext_aosm.common.local_file_builder import LocalFileBuilder +from azext_aosm.inputs.nexus_image_input import NexusImageFileInput +from azext_aosm.vendored_sdks.models import ( + AzureOperatorNexusNetworkFunctionImageApplication, + AzureOperatorNexusImageArtifactProfile, + AzureOperatorNexusImageDeployMappingRuleProfile, + ImageArtifactProfile, ImageMappingRuleProfile, + ApplicationEnablement, ArtifactType, + DependsOnProfile, + ManifestArtifactFormat, ReferencedResource, ResourceElementTemplate, + ) + +logger = get_logger(__name__) + + +class NexusImageProcessor(BaseInputProcessor): + """ + A class for processing VHD inputs. + + :param name: The name of the artifact. + :type name: str + :param input_artifact: The input artifact. + :type input_artifact: VHDFileInput + """ + + def __init__(self, name: str, input_artifact: NexusImageFileInput): + super().__init__(name, input_artifact) + self.input_artifact: NexusImageFileInput = input_artifact + + def get_artifact_manifest_list(self) -> List[ManifestArtifactFormat]: + """ + Get the list of artifacts for the artifact manifest. + + :return: A list of artifacts for the artifact manifest. + :rtype: List[ManifestArtifactFormat] + """ + logger.info("Getting artifact manifest list for VHD input.") + return [ + ManifestArtifactFormat( + artifact_name=self.input_artifact.artifact_name, + artifact_type=ArtifactType.IMAGE_FILE.value, + artifact_version=self.input_artifact.artifact_version, + ) + ] + + # TODO: figure out what artifact type this can be + def get_artifact_details( + self, + ) -> Tuple[List[BaseArtifact], List[LocalFileBuilder]]: + """ + Get the artifact details for publishing. + + :return: A tuple containing the list of artifacts and the list of local file builders. + :rtype: Tuple[List[BaseArtifact], List[LocalFileBuilder]] + """ + logger.info("Getting artifact details for VHD input.") + artifacts: List[BaseArtifact] = [] + file_builders: List[LocalFileBuilder] = [] + + if self.input_artifact.file_path: + logger.debug( + "VHD input has a file path. Adding LocalFileStorageAccountArtifact." + ) + artifacts.append( + LocalFileStorageAccountArtifact( + artifact_name=self.input_artifact.artifact_name, + artifact_type=ArtifactType.IMAGE_FILE.value, + artifact_version=self.input_artifact.artifact_version, + file_path=self.input_artifact.file_path, + ) + ) + # elif self.input_artifact.blob_sas_uri: + # logger.debug( + # "VHD input has a blob SAS URI. Adding BlobStorageAccountArtifact." + # ) + # artifacts.append( + # BlobStorageAccountArtifact( + # artifact_manifest=artifact_manifest, + # blob_sas_uri=self.input_artifact.blob_sas_uri, + # ) + # ) + else: + # TODO: change error once file input defined + logger.error("ImageFileInput must have either a file path or a blob SAS URI.") + raise ValueError( + "VHDFileInput must have either a file path or a blob SAS URI." + ) + + return artifacts, file_builders + + def generate_nf_application(self) -> AzureOperatorNexusNetworkFunctionImageApplication: + """ + Generate the NF application. + + :return: The NF application. + :rtype: AzureCoreNetworkFunctionVhdApplication + """ + logger.info("Generating NF application for VHD input.") + + return AzureOperatorNexusNetworkFunctionImageApplication( + name=self.name, + depends_on_profile=DependsOnProfile(install_depends_on=[], + uninstall_depends_on=[], update_depends_on=[]), + artifact_profile=self._generate_artifact_profile(), + deploy_parameters_mapping_rule_profile=self._generate_mapping_rule_profile(), + ) + + def generate_resource_element_template(self) -> ResourceElementTemplate: + """ + Generate the resource element template. + + :raises NotImplementedError: NSDs do not support deployment of VHDs. + """ + raise NotImplementedError("NSDs do not support deployment of VHDs.") + + def _generate_artifact_profile(self) -> AzureOperatorNexusImageArtifactProfile: + """ + Generate the artifact profile. + + :return: The artifact profile. + :rtype: AzureOperatorNexusImageArtifactProfile + """ + logger.debug("Generating artifact profile for VHD input.") + # TODO: JORDAN check what inputs this takes + artifact_profile = ImageArtifactProfile( + image_name=self.input_artifact.artifact_name, + image_version=self.input_artifact.artifact_version, + ) + + return AzureOperatorNexusImageArtifactProfile( + artifact_store=ReferencedResource(id=""), + image_artifact_profile=artifact_profile, + ) + + def _generate_mapping_rule_profile( + self, + ) -> AzureOperatorNexusImageDeployMappingRuleProfile: + """ + Generate the mapping rule profile. + + :return: The mapping rule profile. + :rtype: AzureOperatorNexusImageDeployMappingRuleProfile + """ + logger.debug("Generating mapping rule profile for VHD input.") + user_configuration = self.generate_values_mappings( + self.input_artifact.get_schema(), self.input_artifact.get_defaults() + ) + + mapping = ImageMappingRuleProfile( + user_configuration=json.dumps(user_configuration), + ) + + return AzureOperatorNexusImageDeployMappingRuleProfile( + application_enablement=ApplicationEnablement.ENABLED, + image_mapping_rule_profile=mapping, + ) diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_vnf_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_vnf_handler.py index 71bfc9b60f8..c547b7f996a 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_vnf_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_vnf_handler.py @@ -191,6 +191,7 @@ def build_resource_bicep(self): schema_properties = {} for processor in self.processors: + # Generate NF Application nf_application = processor.generate_nf_application() logger.debug("Created nf application %s", nf_application.name) @@ -202,7 +203,8 @@ def build_resource_bicep(self): if isinstance(processor, BaseArmBuildProcessor): acr_nf_application_list.append(nf_application) - + print(nf_application) + print(nf_application.deploy_parameters_mapping_rule_profile) # Generate local file for template_parameters + add to supporting files list params = ( nf_application.deploy_parameters_mapping_rule_profile.template_mapping_rule_profile.template_parameters @@ -212,9 +214,9 @@ def build_resource_bicep(self): "Created templatateParameters as supporting file for nfDefinition bicep" ) elif isinstance(processor, VHDProcessor): - # Generate NF Application - # nf_application = processor.generate_nf_application() sa_nf_application_list.append(nf_application) + print(nf_application) + print(nf_application.deploy_parameters_mapping_rule_profile) # Generate local file for vhd_parameters params = ( nf_application.deploy_parameters_mapping_rule_profile.vhd_image_mapping_rule_profile.user_configuration @@ -239,6 +241,7 @@ def build_resource_bicep(self): ) params = { + "nfvi_type": self.config.nfvi_type, "acr_nf_applications": acr_nf_application_list, "sa_nf_application": sa_nf_application_list[0], "deployment_parameters_file": DEPLOYMENT_PARAMETERS_FILENAME, diff --git a/src/aosm/azext_aosm/common/templates/vnf/vnfdefinition.bicep.j2 b/src/aosm/azext_aosm/common/templates/vnf/vnfdefinition.bicep.j2 index dc3f640a6dd..adc8f4896c5 100644 --- a/src/aosm/azext_aosm/common/templates/vnf/vnfdefinition.bicep.j2 +++ b/src/aosm/azext_aosm/common/templates/vnf/vnfdefinition.bicep.j2 @@ -53,7 +53,7 @@ resource nfdv 'Microsoft.Hybridnetwork/publishers/networkfunctiondefinitiongroup deployParameters: string(loadJsonContent('{{deployment_parameters_file}}')) networkFunctionType: 'VirtualNetworkFunction' networkFunctionTemplate: { - nfviType: 'AzureCore' + nfviType: '{{ nfvi_type }}' networkFunctionApplications: [ { artifactType: 'VhdImageFile' diff --git a/src/aosm/azext_aosm/inputs/nexus_image_input.py b/src/aosm/azext_aosm/inputs/nexus_image_input.py new file mode 100644 index 00000000000..70e11ea93a7 --- /dev/null +++ b/src/aosm/azext_aosm/inputs/nexus_image_input.py @@ -0,0 +1,84 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- +import copy +import json +from pathlib import Path +from typing import Any, Dict, Optional + +from knack.log import get_logger + +from azext_aosm.common.constants import BASE_SCHEMA +from azext_aosm.inputs.base_input import BaseInput + +logger = get_logger(__name__) + + +class NexusImageFileInput(BaseInput): + """ + A utility class for working with VHD file inputs. + + :param artifact_name: The name of the artifact. + :type artifact_name: str + :param artifact_version: The version of the artifact. + :type artifact_version: str + :param file_path: The path to the VHD file. + :type file_path: Path + :param default_config: The default configuration. + :type default_config: Optional[Dict[str, Any]] + :param blob_sas_uri: The blob SAS URI. + :type blob_sas_uri: Optional[str] + """ + + def __init__( + self, + artifact_name: str, + artifact_version: str, + # file_path: Optional[Path] = None, + # blob_sas_uri: Optional[str] = None, + default_config: Optional[Dict[str, Any]] = None, + ): + super().__init__(artifact_name, artifact_version, default_config) + # self.file_path = file_path + # self.blob_sas_uri = blob_sas_uri + + def get_defaults(self) -> Dict[str, Any]: + """ + Gets the default values for configuring the input. + + :return: A dictionary containing the default values. + :rtype: Dict[str, Any] + """ + # logger.info("Getting default values for VHD file input") + # default_config = self.default_config or {} + # logger.debug( + # "Default values for VHD file Input: %s", + # json.dumps(default_config, indent=4), + # ) + # return copy.deepcopy(default_config) + + def get_schema(self) -> Dict[str, Any]: + """ + Gets the schema for the VHD file input. + + :return: A dictionary containing the schema. + :rtype: Dict[str, Any] + """ + # logger.info("Getting schema for VHD file input") + # vhd_properties = { + # "imageName": {"type": "string"}, + # "azureDeployLocation": {"type": "string"}, + # "imageDiskSizeGB": {"type": "integer"}, + # "imageOsState": {"type": "string"}, + # "imageHyperVGeneration": {"type": "string"}, + # "apiVersion": {"type": "string"}, + # } + # vhd_required = ["imageName"] + + # schema = copy.deepcopy(BASE_SCHEMA) + # schema["properties"].update(vhd_properties) + # schema["required"] += vhd_required + + # logger.debug("Schema for VHD file input: %s", json.dumps(schema, indent=4)) + # return copy.deepcopy(schema) From 30979f52ae4f76e9d5ef7cc502b3cef08d77bdbc Mon Sep 17 00:00:00 2001 From: Jordan Date: Tue, 30 Jan 2024 11:56:49 +0000 Subject: [PATCH 03/37] initial commit for children vnf handlers --- src/aosm/azext_aosm/_params.py | 6 + .../onboarding_core_vnf_handler.py | 298 ++++++++++++ .../onboarding_nexus_vnf_handler.py | 291 ++++++++++++ .../cli_handlers/onboarding_vnf_handler.py | 424 +++++++++--------- src/aosm/azext_aosm/custom.py | 20 +- 5 files changed, 823 insertions(+), 216 deletions(-) create mode 100644 src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py create mode 100644 src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py diff --git a/src/aosm/azext_aosm/_params.py b/src/aosm/azext_aosm/_params.py index eb23ad7e1d4..7790a23bb2e 100644 --- a/src/aosm/azext_aosm/_params.py +++ b/src/aosm/azext_aosm/_params.py @@ -91,6 +91,12 @@ def load_arguments(self: AzCommandsLoader, _): "Requires Docker to be installed locally." ), ) + c.argument( + "nexus", + options_list=["--nexus"], + required=False, + help="Flag to specify if the VNF is a Nexus VNF.", + ) with self.argument_context("aosm nsd") as c: c.argument( diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py new file mode 100644 index 00000000000..83e316b86e5 --- /dev/null +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py @@ -0,0 +1,298 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- +import json +from pathlib import Path +from typing import Dict, Any +from knack.log import get_logger + +from azext_aosm.build_processors.arm_processor import ( + AzureCoreArmBuildProcessor, BaseArmBuildProcessor, NexusArmBuildProcessor) +from azext_aosm.build_processors.vhd_processor import VHDProcessor +from azext_aosm.common.constants import (ARTIFACT_LIST_FILENAME, + BASE_FOLDER_NAME, + MANIFEST_FOLDER_NAME, + NF_DEFINITION_FOLDER_NAME, + VNF_BASE_TEMPLATE_FILENAME, + VNF_TEMPLATE_FOLDER_NAME, + VNF_DEFINITION_TEMPLATE_FILENAME, + VNF_INPUT_FILENAME, + VNF_MANIFEST_TEMPLATE_FILENAME, + VNF_OUTPUT_FOLDER_FILENAME, + DEPLOYMENT_PARAMETERS_FILENAME, + VHD_PARAMETERS_FILENAME, + TEMPLATE_PARAMETERS_FILENAME) +from azext_aosm.common.local_file_builder import LocalFileBuilder +from azext_aosm.configuration_models.onboarding_vnf_input_config import ( + OnboardingVNFInputConfig, +) +from azext_aosm.configuration_models.common_parameters_config import ( + VNFCommonParametersConfig, +) +from azext_aosm.definition_folder.builder.artifact_builder import ( + ArtifactDefinitionElementBuilder, +) +from azext_aosm.definition_folder.builder.bicep_builder import ( + BicepDefinitionElementBuilder, +) +from azext_aosm.definition_folder.builder.json_builder import ( + JSONDefinitionElementBuilder, +) +from azext_aosm.inputs.arm_template_input import ArmTemplateInput +from azext_aosm.inputs.vhd_file_input import VHDFileInput + +# from .onboarding_nfd_base_handler import OnboardingNFDBaseCLIHandler +from .onboarding_vnf_handler import OnboardingVNFCLIHandler +logger = get_logger(__name__) + + +class OnboardingCoreVNFCLIHandler(OnboardingVNFCLIHandler): + """CLI handler for publishing NFDs.""" + + @property + def default_config_file_name(self) -> str: + """Get the default configuration file name.""" + return VNF_INPUT_FILENAME + + @property + def output_folder_file_name(self) -> str: + """Get the output folder file name.""" + return VNF_OUTPUT_FOLDER_FILENAME + + def _get_input_config(self, input_config: Dict[str, Any] = None) -> OnboardingVNFInputConfig: + """Get the configuration for the command.""" + if input_config is None: + input_config = {} + return OnboardingVNFInputConfig(**input_config) + + def _get_params_config( + self, config_file: dict = None + ) -> VNFCommonParametersConfig: + """Get the configuration for the command.""" + with open(config_file, "r", encoding="utf-8") as _file: + params_dict = json.load(_file) + if params_dict is None: + params_dict = {} + return VNFCommonParametersConfig(**params_dict) + + def _get_processor_list(self): + print("in get processor") + processor_list = [] + # for each arm template, instantiate arm processor + for arm_template in self.config.arm_templates: + arm_input = ArmTemplateInput( + artifact_name=arm_template.artifact_name, + artifact_version=arm_template.version, + default_config=None, + template_path=Path(arm_template.file_path).absolute(), + ) + # TODO: test azure core vnf still works + processor_list.append( + AzureCoreArmBuildProcessor(arm_input.artifact_name, arm_input) + ) + + # Instantiate vhd processor + if not self.config.vhd.artifact_name: + self.config.vhd.artifact_name = self.config.nf_name + "-vhd" + vhd_processor = VHDProcessor( + name=self.config.vhd.artifact_name, + input_artifact=VHDFileInput( + artifact_name=self.config.vhd.artifact_name, + artifact_version=self.config.vhd.version, + default_config=self._get_default_config(self.config.vhd), + file_path=Path(self.config.vhd.file_path).absolute(), + blob_sas_uri=self.config.vhd.blob_sas_url, + ), + ) + processor_list.append(vhd_processor) + return processor_list + + def build_base_bicep(self): + """Build the base bicep file.""" + # Build manifest bicep contents, with j2 template + template_path = self._get_template_path( + VNF_TEMPLATE_FOLDER_NAME, VNF_BASE_TEMPLATE_FILENAME + ) + bicep_contents = self._render_base_bicep_contents(template_path) + # Create Bicep element with manifest contents + bicep_file = BicepDefinitionElementBuilder( + Path(VNF_OUTPUT_FOLDER_FILENAME, BASE_FOLDER_NAME), bicep_contents + ) + return bicep_file + + def build_manifest_bicep(self): + """Build the manifest bicep file.""" + acr_artifact_list = [] + + logger.info("Creating artifact manifest bicep") + + for processor in self.processors: + if isinstance(processor, BaseArmBuildProcessor): + acr_artifact_list.extend(processor.get_artifact_manifest_list()) + logger.debug( + "Created list of artifacts from %s arm template(s) provided: %s", + len(self.config.arm_templates), + acr_artifact_list, + ) + elif isinstance(processor, VHDProcessor): + sa_artifact_list = processor.get_artifact_manifest_list() + logger.debug( + "Created list of artifacts from vhd image provided: %s", + sa_artifact_list, + ) + + # Build manifest bicep contents, with j2 template + template_path = self._get_template_path( + VNF_TEMPLATE_FOLDER_NAME, VNF_MANIFEST_TEMPLATE_FILENAME + ) + bicep_contents = self._render_manifest_bicep_contents( + template_path, acr_artifact_list, sa_artifact_list + ) + # Create Bicep element with manifest contents + bicep_file = BicepDefinitionElementBuilder( + Path(VNF_OUTPUT_FOLDER_FILENAME, MANIFEST_FOLDER_NAME), + bicep_contents, + ) + + logger.info("Created artifact manifest bicep element") + return bicep_file + + def build_artifact_list(self): + """Build the artifact list.""" + logger.info("Creating artifacts list for artifacts.json") + artifact_list = [] + # For each arm template, get list of artifacts and combine + for processor in self.processors: + (artifacts, _) = processor.get_artifact_details() + if artifacts not in artifact_list: + artifact_list.extend(artifacts) + logger.debug( + "Created list of artifact details from %s arm template(s) and the vhd image provided: %s", + len(self.config.arm_templates), + artifact_list, + ) + + # Generate Artifact Element with artifact list (of arm template and vhd images) + return ArtifactDefinitionElementBuilder( + Path(VNF_OUTPUT_FOLDER_FILENAME, ARTIFACT_LIST_FILENAME), artifact_list + ) + + def build_resource_bicep(self): + """Build the resource bicep file.""" + logger.info("Creating artifacts list for artifacts.json") + acr_nf_application_list = [] + sa_nf_application_list = [] + supporting_files = [] + schema_properties = {} + + for processor in self.processors: + # Generate NF Application + nf_application = processor.generate_nf_application() + logger.debug("Created nf application %s", nf_application.name) + + # Generate deploymentParameters schema properties + params_schema = processor.generate_params_schema() + schema_properties.update(params_schema) + + # For each arm template, generate nf application + if isinstance(processor, BaseArmBuildProcessor): + + acr_nf_application_list.append(nf_application) + print(nf_application) + print(nf_application.deploy_parameters_mapping_rule_profile) + # Generate local file for template_parameters + add to supporting files list + params = ( + nf_application.deploy_parameters_mapping_rule_profile.template_mapping_rule_profile.template_parameters + ) + template_name = TEMPLATE_PARAMETERS_FILENAME + logger.info( + "Created templatateParameters as supporting file for nfDefinition bicep" + ) + elif isinstance(processor, VHDProcessor): + sa_nf_application_list.append(nf_application) + print(nf_application) + print(nf_application.deploy_parameters_mapping_rule_profile) + # Generate local file for vhd_parameters + params = ( + nf_application.deploy_parameters_mapping_rule_profile.vhd_image_mapping_rule_profile.user_configuration + ) + template_name = VHD_PARAMETERS_FILENAME + else: + raise TypeError(f"Type: {type(processor)} is not valid") + + parameters_file = LocalFileBuilder( + Path( + VNF_OUTPUT_FOLDER_FILENAME, + NF_DEFINITION_FOLDER_NAME, + template_name, + ), + json.dumps(json.loads(params), indent=4), + ) + supporting_files.append(parameters_file) + + # Create bicep contents using vnf defintion j2 template + template_path = self._get_template_path( + VNF_TEMPLATE_FOLDER_NAME, VNF_DEFINITION_TEMPLATE_FILENAME + ) + + params = { + "nfvi_type": self.config.nfvi_type, + "acr_nf_applications": acr_nf_application_list, + "sa_nf_application": sa_nf_application_list[0], + "deployment_parameters_file": DEPLOYMENT_PARAMETERS_FILENAME, + "vhd_parameters_file": VHD_PARAMETERS_FILENAME, + "template_parameters_file": TEMPLATE_PARAMETERS_FILENAME + } + bicep_contents = self._render_definition_bicep_contents( + template_path, params + ) + + # Create a bicep element + # + add its supporting files (deploymentParameters, vhdParameters and templateParameters) + bicep_file = BicepDefinitionElementBuilder( + Path(VNF_OUTPUT_FOLDER_FILENAME, NF_DEFINITION_FOLDER_NAME), + bicep_contents, + ) + for supporting_file in supporting_files: + bicep_file.add_supporting_file(supporting_file) + + # Add the deploymentParameters schema file + bicep_file.add_supporting_file( + self._render_deployment_params_schema( + schema_properties, VNF_OUTPUT_FOLDER_FILENAME, NF_DEFINITION_FOLDER_NAME + ) + ) + return bicep_file + + def build_all_parameters_json(self): + params_content = { + "location": self.config.location, + "publisherName": self.config.publisher_name, + "publisherResourceGroupName": self.config.publisher_resource_group_name, + "acrArtifactStoreName": self.config.acr_artifact_store_name, + "saArtifactStoreName": self.config.blob_artifact_store_name, + "acrManifestName": self.config.acr_artifact_store_name + "-manifest", + "saManifestName": self.config.blob_artifact_store_name + "-manifest", + "nfDefinitionGroup": self.config.nf_name, + "nfDefinitionVersion": self.config.version + } + base_file = JSONDefinitionElementBuilder( + Path(VNF_OUTPUT_FOLDER_FILENAME), json.dumps(params_content, indent=4) + ) + return base_file + + def _get_default_config(self, vhd): + default_config = {} + if vhd.image_disk_size_GB: + default_config.update({"image_disk_size_GB": vhd.image_disk_size_GB}) + if vhd.image_hyper_v_generation: + default_config.update( + {"image_hyper_v_generation": vhd.image_hyper_v_generation} + ) + else: + # Default to V1 if not specified + default_config.update({"image_hyper_v_generation": "V1"}) + if vhd.image_api_version: + default_config.update({"image_api_version": vhd.image_api_version}) + return default_config diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py new file mode 100644 index 00000000000..99ca9a047b0 --- /dev/null +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py @@ -0,0 +1,291 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- +import json +from pathlib import Path +from typing import Dict, Any +from knack.log import get_logger + +from azext_aosm.build_processors.arm_processor import ( + AzureCoreArmBuildProcessor, BaseArmBuildProcessor, NexusArmBuildProcessor) +# from azext_aosm.build_processors.vhd_processor import VHDProcessor +from azext_aosm.build_processors.nexus_image_processor import NexusImageProcessor +from azext_aosm.common.constants import (ARTIFACT_LIST_FILENAME, + BASE_FOLDER_NAME, + MANIFEST_FOLDER_NAME, + NF_DEFINITION_FOLDER_NAME, + VNF_BASE_TEMPLATE_FILENAME, + VNF_TEMPLATE_FOLDER_NAME, + VNF_DEFINITION_TEMPLATE_FILENAME, + VNF_INPUT_FILENAME, + VNF_MANIFEST_TEMPLATE_FILENAME, + VNF_OUTPUT_FOLDER_FILENAME, + DEPLOYMENT_PARAMETERS_FILENAME, + VHD_PARAMETERS_FILENAME, + TEMPLATE_PARAMETERS_FILENAME) +from azext_aosm.common.local_file_builder import LocalFileBuilder +from azext_aosm.configuration_models.onboarding_vnf_input_config import ( + OnboardingVNFInputConfig, +) +from azext_aosm.configuration_models.common_parameters_config import ( + VNFCommonParametersConfig, +) +from azext_aosm.definition_folder.builder.artifact_builder import ( + ArtifactDefinitionElementBuilder, +) +from azext_aosm.definition_folder.builder.bicep_builder import ( + BicepDefinitionElementBuilder, +) +from azext_aosm.definition_folder.builder.json_builder import ( + JSONDefinitionElementBuilder, +) +from azext_aosm.inputs.arm_template_input import ArmTemplateInput +from azext_aosm.inputs.vhd_file_input import VHDFileInput + +from .onboarding_vnf_handler import OnboardingVNFCLIHandler +logger = get_logger(__name__) + + +class OnboardingNexusVNFCLIHandler(OnboardingVNFCLIHandler): + """CLI handler for publishing NFDs.""" + + @property + def default_config_file_name(self) -> str: + """Get the default configuration file name.""" + return VNF_INPUT_FILENAME + + @property + def output_folder_file_name(self) -> str: + """Get the output folder file name.""" + return VNF_OUTPUT_FOLDER_FILENAME + + def _get_input_config(self, input_config: Dict[str, Any] = None) -> OnboardingVNFInputConfig: + """Get the configuration for the command.""" + if input_config is None: + input_config = {} + return OnboardingVNFInputConfig(**input_config) + + def _get_params_config( + self, config_file: dict = None + ) -> VNFCommonParametersConfig: + """Get the configuration for the command.""" + with open(config_file, "r", encoding="utf-8") as _file: + params_dict = json.load(_file) + if params_dict is None: + params_dict = {} + return VNFCommonParametersConfig(**params_dict) + + def _get_processor_list(self): + processor_list = [] + # for each arm template, instantiate arm processor + for arm_template in self.config.arm_templates: + arm_input = ArmTemplateInput( + artifact_name=arm_template.artifact_name, + artifact_version=arm_template.version, + default_config=None, + template_path=Path(arm_template.file_path).absolute(), + ) + # TODO: test azure core vnf still works + processor_list.append( + NexusArmBuildProcessor(arm_input.artifact_name, arm_input) + ) + + # TODO: add artifact for image file, after making config + # # Instantiate vhd processor + # if not self.config.vhd.artifact_name: + # self.config.vhd.artifact_name = self.config.nf_name + "-vhd" + # vhd_processor = VHDProcessor( + # name=self.config.vhd.artifact_name, + # input_artifact=VHDFileInput( + # artifact_name=self.config.vhd.artifact_name, + # artifact_version=self.config.vhd.version, + # default_config=self._get_default_config(self.config.vhd), + # file_path=Path(self.config.vhd.file_path).absolute(), + # blob_sas_uri=self.config.vhd.blob_sas_url, + # ), + # ) + # processor_list.append(vhd_processor) + return processor_list + + def build_base_bicep(self): + """Build the base bicep file.""" + # Build manifest bicep contents, with j2 template + template_path = self._get_template_path( + VNF_TEMPLATE_FOLDER_NAME, VNF_BASE_TEMPLATE_FILENAME + ) + bicep_contents = self._render_base_bicep_contents(template_path) + # Create Bicep element with manifest contents + bicep_file = BicepDefinitionElementBuilder( + Path(VNF_OUTPUT_FOLDER_FILENAME, BASE_FOLDER_NAME), bicep_contents + ) + return bicep_file + + def build_manifest_bicep(self): + """Build the manifest bicep file.""" + acr_artifact_list = [] + + logger.info("Creating artifact manifest bicep") + + for processor in self.processors: + acr_artifact_list.extend(processor.get_artifact_manifest_list()) + logger.debug( + "Created list of artifacts from arm template(s) and image files(s) provided: %s", + acr_artifact_list, + ) + + # Build manifest bicep contents, with j2 template + template_path = self._get_template_path( + VNF_TEMPLATE_FOLDER_NAME, VNF_MANIFEST_TEMPLATE_FILENAME + ) + bicep_contents = self._render_manifest_bicep_contents( + template_path, acr_artifact_list + ) + # Create Bicep element with manifest contents + bicep_file = BicepDefinitionElementBuilder( + Path(VNF_OUTPUT_FOLDER_FILENAME, MANIFEST_FOLDER_NAME), + bicep_contents, + ) + + logger.info("Created artifact manifest bicep element") + return bicep_file + + def build_artifact_list(self): + """Build the artifact list.""" + logger.info("Creating artifacts list for artifacts.json") + artifact_list = [] + # For each arm template, get list of artifacts and combine + for processor in self.processors: + (artifacts, _) = processor.get_artifact_details() + if artifacts not in artifact_list: + artifact_list.extend(artifacts) + logger.debug( + "Created list of artifact details from arm template(s) and image(s) provided: %s", + artifact_list, + ) + + # Generate Artifact Element with artifact list (of arm template and vhd images) + return ArtifactDefinitionElementBuilder( + Path(VNF_OUTPUT_FOLDER_FILENAME, ARTIFACT_LIST_FILENAME), artifact_list + ) + + def build_resource_bicep(self): + """Build the resource bicep file.""" + logger.info("Creating artifacts list for artifacts.json") + acr_nf_application_list = [] + sa_nf_application_list = [] + supporting_files = [] + schema_properties = {} + + # TODO: fx this for nexus + for processor in self.processors: + # Generate NF Application + nf_application = processor.generate_nf_application() + logger.debug("Created nf application %s", nf_application.name) + + # Generate deploymentParameters schema properties + params_schema = processor.generate_params_schema() + schema_properties.update(params_schema) + + # For each arm template, generate nf application + if isinstance(processor, BaseArmBuildProcessor): + + acr_nf_application_list.append(nf_application) + print(nf_application) + print(nf_application.deploy_parameters_mapping_rule_profile) + # Generate local file for template_parameters + add to supporting files list + params = ( + nf_application.deploy_parameters_mapping_rule_profile.template_mapping_rule_profile.template_parameters + ) + template_name = TEMPLATE_PARAMETERS_FILENAME + logger.info( + "Created templatateParameters as supporting file for nfDefinition bicep" + ) + # elif isinstance(processor, NexusImageProcessor): + # TODO: fix this + # sa_nf_application_list.append(nf_application) + # print(nf_application) + # print(nf_application.deploy_parameters_mapping_rule_profile) + # # Generate local file for vhd_parameters + # params = ( + # nf_application.deploy_parameters_mapping_rule_profile.vhd_image_mapping_rule_profile.user_configuration + # ) + # template_name = VHD_PARAMETERS_FILENAME + else: + raise TypeError(f"Type: {type(processor)} is not valid") + + parameters_file = LocalFileBuilder( + Path( + VNF_OUTPUT_FOLDER_FILENAME, + NF_DEFINITION_FOLDER_NAME, + template_name, + ), + json.dumps(json.loads(params), indent=4), + ) + supporting_files.append(parameters_file) + + # Create bicep contents using vnf defintion j2 template + template_path = self._get_template_path( + VNF_TEMPLATE_FOLDER_NAME, VNF_DEFINITION_TEMPLATE_FILENAME + ) + + params = { + "nfvi_type": self.config.nfvi_type, + "acr_nf_applications": acr_nf_application_list, + "sa_nf_application": sa_nf_application_list[0], + "deployment_parameters_file": DEPLOYMENT_PARAMETERS_FILENAME, + "vhd_parameters_file": VHD_PARAMETERS_FILENAME, + "template_parameters_file": TEMPLATE_PARAMETERS_FILENAME + } + bicep_contents = self._render_definition_bicep_contents( + template_path, params + ) + + # Create a bicep element + # + add its supporting files (deploymentParameters, vhdParameters and templateParameters) + bicep_file = BicepDefinitionElementBuilder( + Path(VNF_OUTPUT_FOLDER_FILENAME, NF_DEFINITION_FOLDER_NAME), + bicep_contents, + ) + for supporting_file in supporting_files: + bicep_file.add_supporting_file(supporting_file) + + # Add the deploymentParameters schema file + bicep_file.add_supporting_file( + self._render_deployment_params_schema( + schema_properties, VNF_OUTPUT_FOLDER_FILENAME, NF_DEFINITION_FOLDER_NAME + ) + ) + return bicep_file + + def build_all_parameters_json(self): + params_content = { + "location": self.config.location, + "publisherName": self.config.publisher_name, + "publisherResourceGroupName": self.config.publisher_resource_group_name, + "acrArtifactStoreName": self.config.acr_artifact_store_name, + # "saArtifactStoreName": self.config.blob_artifact_store_name, + "acrManifestName": self.config.acr_artifact_store_name + "-manifest", + # "saManifestName": self.config.blob_artifact_store_name + "-manifest", + "nfDefinitionGroup": self.config.nf_name, + "nfDefinitionVersion": self.config.version + } + base_file = JSONDefinitionElementBuilder( + Path(VNF_OUTPUT_FOLDER_FILENAME), json.dumps(params_content, indent=4) + ) + return base_file + + # def _get_default_config(self, vhd): + # default_config = {} + # if vhd.image_disk_size_GB: + # default_config.update({"image_disk_size_GB": vhd.image_disk_size_GB}) + # if vhd.image_hyper_v_generation: + # default_config.update( + # {"image_hyper_v_generation": vhd.image_hyper_v_generation} + # ) + # else: + # # Default to V1 if not specified + # default_config.update({"image_hyper_v_generation": "V1"}) + # if vhd.image_api_version: + # default_config.update({"image_api_version": vhd.image_api_version}) + # return default_config diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_vnf_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_vnf_handler.py index c547b7f996a..9f342fb5320 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_vnf_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_vnf_handler.py @@ -66,237 +66,237 @@ def _get_input_config(self, input_config: Dict[str, Any] = None) -> OnboardingVN input_config = {} return OnboardingVNFInputConfig(**input_config) - def _get_params_config( - self, config_file: dict = None - ) -> VNFCommonParametersConfig: - """Get the configuration for the command.""" - with open(config_file, "r", encoding="utf-8") as _file: - params_dict = json.load(_file) - if params_dict is None: - params_dict = {} - return VNFCommonParametersConfig(**params_dict) + # def _get_params_config( + # self, config_file: dict = None + # ) -> VNFCommonParametersConfig: + # """Get the configuration for the command.""" + # with open(config_file, "r", encoding="utf-8") as _file: + # params_dict = json.load(_file) + # if params_dict is None: + # params_dict = {} + # return VNFCommonParametersConfig(**params_dict) - def _get_processor_list(self): - processor_list = [] - # for each arm template, instantiate arm processor - for arm_template in self.config.arm_templates: - arm_input = ArmTemplateInput( - artifact_name=arm_template.artifact_name, - artifact_version=arm_template.version, - default_config=None, - template_path=Path(arm_template.file_path).absolute(), - ) - # TODO: generalise for nexus in nexus ready stories - if self.config.nfvi_type == "AzureCore": - processor_list.append( - AzureCoreArmBuildProcessor(arm_input.artifact_name, arm_input) - ) - elif self.config.nfvi_type == "AzureOperatorNexus": - processor_list.append( - NexusArmBuildProcessor(arm_input.artifact_name, arm_input) - ) + # def _get_processor_list(self): + # processor_list = [] + # # for each arm template, instantiate arm processor + # for arm_template in self.config.arm_templates: + # arm_input = ArmTemplateInput( + # artifact_name=arm_template.artifact_name, + # artifact_version=arm_template.version, + # default_config=None, + # template_path=Path(arm_template.file_path).absolute(), + # ) + # # TODO: generalise for nexus in nexus ready stories + # if self.config.nfvi_type == "AzureCore": + # processor_list.append( + # AzureCoreArmBuildProcessor(arm_input.artifact_name, arm_input) + # ) + # elif self.config.nfvi_type == "AzureOperatorNexus": + # processor_list.append( + # NexusArmBuildProcessor(arm_input.artifact_name, arm_input) + # ) - # Instantiate vhd processor - if not self.config.vhd.artifact_name: - self.config.vhd.artifact_name = self.config.nf_name + "-vhd" - vhd_processor = VHDProcessor( - name=self.config.vhd.artifact_name, - input_artifact=VHDFileInput( - artifact_name=self.config.vhd.artifact_name, - artifact_version=self.config.vhd.version, - default_config=self._get_default_config(self.config.vhd), - file_path=Path(self.config.vhd.file_path).absolute(), - blob_sas_uri=self.config.vhd.blob_sas_url, - ), - ) - processor_list.append(vhd_processor) - return processor_list + # # Instantiate vhd processor + # if not self.config.vhd.artifact_name: + # self.config.vhd.artifact_name = self.config.nf_name + "-vhd" + # vhd_processor = VHDProcessor( + # name=self.config.vhd.artifact_name, + # input_artifact=VHDFileInput( + # artifact_name=self.config.vhd.artifact_name, + # artifact_version=self.config.vhd.version, + # default_config=self._get_default_config(self.config.vhd), + # file_path=Path(self.config.vhd.file_path).absolute(), + # blob_sas_uri=self.config.vhd.blob_sas_url, + # ), + # ) + # processor_list.append(vhd_processor) + # return processor_list - def build_base_bicep(self): - """Build the base bicep file.""" - # Build manifest bicep contents, with j2 template - template_path = self._get_template_path( - VNF_TEMPLATE_FOLDER_NAME, VNF_BASE_TEMPLATE_FILENAME - ) - bicep_contents = self._render_base_bicep_contents(template_path) - # Create Bicep element with manifest contents - bicep_file = BicepDefinitionElementBuilder( - Path(VNF_OUTPUT_FOLDER_FILENAME, BASE_FOLDER_NAME), bicep_contents - ) - return bicep_file + # def build_base_bicep(self): + # """Build the base bicep file.""" + # # Build manifest bicep contents, with j2 template + # template_path = self._get_template_path( + # VNF_TEMPLATE_FOLDER_NAME, VNF_BASE_TEMPLATE_FILENAME + # ) + # bicep_contents = self._render_base_bicep_contents(template_path) + # # Create Bicep element with manifest contents + # bicep_file = BicepDefinitionElementBuilder( + # Path(VNF_OUTPUT_FOLDER_FILENAME, BASE_FOLDER_NAME), bicep_contents + # ) + # return bicep_file - def build_manifest_bicep(self): - """Build the manifest bicep file.""" - acr_artifact_list = [] + # def build_manifest_bicep(self): + # """Build the manifest bicep file.""" + # acr_artifact_list = [] - logger.info("Creating artifact manifest bicep") + # logger.info("Creating artifact manifest bicep") - for processor in self.processors: - if isinstance(processor, BaseArmBuildProcessor): - acr_artifact_list.extend(processor.get_artifact_manifest_list()) - logger.debug( - "Created list of artifacts from %s arm template(s) provided: %s", - len(self.config.arm_templates), - acr_artifact_list, - ) - elif isinstance(processor, VHDProcessor): - sa_artifact_list = processor.get_artifact_manifest_list() - logger.debug( - "Created list of artifacts from vhd image provided: %s", - sa_artifact_list, - ) + # for processor in self.processors: + # if isinstance(processor, BaseArmBuildProcessor): + # acr_artifact_list.extend(processor.get_artifact_manifest_list()) + # logger.debug( + # "Created list of artifacts from %s arm template(s) provided: %s", + # len(self.config.arm_templates), + # acr_artifact_list, + # ) + # elif isinstance(processor, VHDProcessor): + # sa_artifact_list = processor.get_artifact_manifest_list() + # logger.debug( + # "Created list of artifacts from vhd image provided: %s", + # sa_artifact_list, + # ) - # Build manifest bicep contents, with j2 template - template_path = self._get_template_path( - VNF_TEMPLATE_FOLDER_NAME, VNF_MANIFEST_TEMPLATE_FILENAME - ) - bicep_contents = self._render_manifest_bicep_contents( - template_path, acr_artifact_list, sa_artifact_list - ) - # Create Bicep element with manifest contents - bicep_file = BicepDefinitionElementBuilder( - Path(VNF_OUTPUT_FOLDER_FILENAME, MANIFEST_FOLDER_NAME), - bicep_contents, - ) + # # Build manifest bicep contents, with j2 template + # template_path = self._get_template_path( + # VNF_TEMPLATE_FOLDER_NAME, VNF_MANIFEST_TEMPLATE_FILENAME + # ) + # bicep_contents = self._render_manifest_bicep_contents( + # template_path, acr_artifact_list, sa_artifact_list + # ) + # # Create Bicep element with manifest contents + # bicep_file = BicepDefinitionElementBuilder( + # Path(VNF_OUTPUT_FOLDER_FILENAME, MANIFEST_FOLDER_NAME), + # bicep_contents, + # ) - logger.info("Created artifact manifest bicep element") - return bicep_file + # logger.info("Created artifact manifest bicep element") + # return bicep_file - def build_artifact_list(self): - """Build the artifact list.""" - logger.info("Creating artifacts list for artifacts.json") - artifact_list = [] - # For each arm template, get list of artifacts and combine - for processor in self.processors: - (artifacts, _) = processor.get_artifact_details() - if artifacts not in artifact_list: - artifact_list.extend(artifacts) - logger.debug( - "Created list of artifact details from %s arm template(s) and the vhd image provided: %s", - len(self.config.arm_templates), - artifact_list, - ) + # def build_artifact_list(self): + # """Build the artifact list.""" + # logger.info("Creating artifacts list for artifacts.json") + # artifact_list = [] + # # For each arm template, get list of artifacts and combine + # for processor in self.processors: + # (artifacts, _) = processor.get_artifact_details() + # if artifacts not in artifact_list: + # artifact_list.extend(artifacts) + # logger.debug( + # "Created list of artifact details from %s arm template(s) and the vhd image provided: %s", + # len(self.config.arm_templates), + # artifact_list, + # ) - # Generate Artifact Element with artifact list (of arm template and vhd images) - return ArtifactDefinitionElementBuilder( - Path(VNF_OUTPUT_FOLDER_FILENAME, ARTIFACT_LIST_FILENAME), artifact_list - ) + # # Generate Artifact Element with artifact list (of arm template and vhd images) + # return ArtifactDefinitionElementBuilder( + # Path(VNF_OUTPUT_FOLDER_FILENAME, ARTIFACT_LIST_FILENAME), artifact_list + # ) - def build_resource_bicep(self): - """Build the resource bicep file.""" - logger.info("Creating artifacts list for artifacts.json") - acr_nf_application_list = [] - sa_nf_application_list = [] - supporting_files = [] - schema_properties = {} + # def build_resource_bicep(self): + # """Build the resource bicep file.""" + # logger.info("Creating artifacts list for artifacts.json") + # acr_nf_application_list = [] + # sa_nf_application_list = [] + # supporting_files = [] + # schema_properties = {} - for processor in self.processors: - # Generate NF Application - nf_application = processor.generate_nf_application() - logger.debug("Created nf application %s", nf_application.name) + # for processor in self.processors: + # # Generate NF Application + # nf_application = processor.generate_nf_application() + # logger.debug("Created nf application %s", nf_application.name) - # Generate deploymentParameters schema properties - params_schema = processor.generate_params_schema() - schema_properties.update(params_schema) + # # Generate deploymentParameters schema properties + # params_schema = processor.generate_params_schema() + # schema_properties.update(params_schema) - # For each arm template, generate nf application - if isinstance(processor, BaseArmBuildProcessor): + # # For each arm template, generate nf application + # if isinstance(processor, BaseArmBuildProcessor): - acr_nf_application_list.append(nf_application) - print(nf_application) - print(nf_application.deploy_parameters_mapping_rule_profile) - # Generate local file for template_parameters + add to supporting files list - params = ( - nf_application.deploy_parameters_mapping_rule_profile.template_mapping_rule_profile.template_parameters - ) - template_name = TEMPLATE_PARAMETERS_FILENAME - logger.info( - "Created templatateParameters as supporting file for nfDefinition bicep" - ) - elif isinstance(processor, VHDProcessor): - sa_nf_application_list.append(nf_application) - print(nf_application) - print(nf_application.deploy_parameters_mapping_rule_profile) - # Generate local file for vhd_parameters - params = ( - nf_application.deploy_parameters_mapping_rule_profile.vhd_image_mapping_rule_profile.user_configuration - ) - template_name = VHD_PARAMETERS_FILENAME - else: - raise TypeError(f"Type: {type(processor)} is not valid") + # acr_nf_application_list.append(nf_application) + # print(nf_application) + # print(nf_application.deploy_parameters_mapping_rule_profile) + # # Generate local file for template_parameters + add to supporting files list + # params = ( + # nf_application.deploy_parameters_mapping_rule_profile.template_mapping_rule_profile.template_parameters + # ) + # template_name = TEMPLATE_PARAMETERS_FILENAME + # logger.info( + # "Created templatateParameters as supporting file for nfDefinition bicep" + # ) + # elif isinstance(processor, VHDProcessor): + # sa_nf_application_list.append(nf_application) + # print(nf_application) + # print(nf_application.deploy_parameters_mapping_rule_profile) + # # Generate local file for vhd_parameters + # params = ( + # nf_application.deploy_parameters_mapping_rule_profile.vhd_image_mapping_rule_profile.user_configuration + # ) + # template_name = VHD_PARAMETERS_FILENAME + # else: + # raise TypeError(f"Type: {type(processor)} is not valid") - parameters_file = LocalFileBuilder( - Path( - VNF_OUTPUT_FOLDER_FILENAME, - NF_DEFINITION_FOLDER_NAME, - template_name, - ), - json.dumps(json.loads(params), indent=4), - ) - supporting_files.append(parameters_file) + # parameters_file = LocalFileBuilder( + # Path( + # VNF_OUTPUT_FOLDER_FILENAME, + # NF_DEFINITION_FOLDER_NAME, + # template_name, + # ), + # json.dumps(json.loads(params), indent=4), + # ) + # supporting_files.append(parameters_file) - # Create bicep contents using vnf defintion j2 template - template_path = self._get_template_path( - VNF_TEMPLATE_FOLDER_NAME, VNF_DEFINITION_TEMPLATE_FILENAME - ) + # # Create bicep contents using vnf defintion j2 template + # template_path = self._get_template_path( + # VNF_TEMPLATE_FOLDER_NAME, VNF_DEFINITION_TEMPLATE_FILENAME + # ) - params = { - "nfvi_type": self.config.nfvi_type, - "acr_nf_applications": acr_nf_application_list, - "sa_nf_application": sa_nf_application_list[0], - "deployment_parameters_file": DEPLOYMENT_PARAMETERS_FILENAME, - "vhd_parameters_file": VHD_PARAMETERS_FILENAME, - "template_parameters_file": TEMPLATE_PARAMETERS_FILENAME - } - bicep_contents = self._render_definition_bicep_contents( - template_path, params - ) + # params = { + # "nfvi_type": self.config.nfvi_type, + # "acr_nf_applications": acr_nf_application_list, + # "sa_nf_application": sa_nf_application_list[0], + # "deployment_parameters_file": DEPLOYMENT_PARAMETERS_FILENAME, + # "vhd_parameters_file": VHD_PARAMETERS_FILENAME, + # "template_parameters_file": TEMPLATE_PARAMETERS_FILENAME + # } + # bicep_contents = self._render_definition_bicep_contents( + # template_path, params + # ) - # Create a bicep element - # + add its supporting files (deploymentParameters, vhdParameters and templateParameters) - bicep_file = BicepDefinitionElementBuilder( - Path(VNF_OUTPUT_FOLDER_FILENAME, NF_DEFINITION_FOLDER_NAME), - bicep_contents, - ) - for supporting_file in supporting_files: - bicep_file.add_supporting_file(supporting_file) + # # Create a bicep element + # # + add its supporting files (deploymentParameters, vhdParameters and templateParameters) + # bicep_file = BicepDefinitionElementBuilder( + # Path(VNF_OUTPUT_FOLDER_FILENAME, NF_DEFINITION_FOLDER_NAME), + # bicep_contents, + # ) + # for supporting_file in supporting_files: + # bicep_file.add_supporting_file(supporting_file) - # Add the deploymentParameters schema file - bicep_file.add_supporting_file( - self._render_deployment_params_schema( - schema_properties, VNF_OUTPUT_FOLDER_FILENAME, NF_DEFINITION_FOLDER_NAME - ) - ) - return bicep_file + # # Add the deploymentParameters schema file + # bicep_file.add_supporting_file( + # self._render_deployment_params_schema( + # schema_properties, VNF_OUTPUT_FOLDER_FILENAME, NF_DEFINITION_FOLDER_NAME + # ) + # ) + # return bicep_file - def build_all_parameters_json(self): - params_content = { - "location": self.config.location, - "publisherName": self.config.publisher_name, - "publisherResourceGroupName": self.config.publisher_resource_group_name, - "acrArtifactStoreName": self.config.acr_artifact_store_name, - "saArtifactStoreName": self.config.blob_artifact_store_name, - "acrManifestName": self.config.acr_artifact_store_name + "-manifest", - "saManifestName": self.config.blob_artifact_store_name + "-manifest", - "nfDefinitionGroup": self.config.nf_name, - "nfDefinitionVersion": self.config.version - } - base_file = JSONDefinitionElementBuilder( - Path(VNF_OUTPUT_FOLDER_FILENAME), json.dumps(params_content, indent=4) - ) - return base_file + # def build_all_parameters_json(self): + # params_content = { + # "location": self.config.location, + # "publisherName": self.config.publisher_name, + # "publisherResourceGroupName": self.config.publisher_resource_group_name, + # "acrArtifactStoreName": self.config.acr_artifact_store_name, + # "saArtifactStoreName": self.config.blob_artifact_store_name, + # "acrManifestName": self.config.acr_artifact_store_name + "-manifest", + # "saManifestName": self.config.blob_artifact_store_name + "-manifest", + # "nfDefinitionGroup": self.config.nf_name, + # "nfDefinitionVersion": self.config.version + # } + # base_file = JSONDefinitionElementBuilder( + # Path(VNF_OUTPUT_FOLDER_FILENAME), json.dumps(params_content, indent=4) + # ) + # return base_file - def _get_default_config(self, vhd): - default_config = {} - if vhd.image_disk_size_GB: - default_config.update({"image_disk_size_GB": vhd.image_disk_size_GB}) - if vhd.image_hyper_v_generation: - default_config.update( - {"image_hyper_v_generation": vhd.image_hyper_v_generation} - ) - else: - # Default to V1 if not specified - default_config.update({"image_hyper_v_generation": "V1"}) - if vhd.image_api_version: - default_config.update({"image_api_version": vhd.image_api_version}) - return default_config + # def _get_default_config(self, vhd): + # default_config = {} + # if vhd.image_disk_size_GB: + # default_config.update({"image_disk_size_GB": vhd.image_disk_size_GB}) + # if vhd.image_hyper_v_generation: + # default_config.update( + # {"image_hyper_v_generation": vhd.image_hyper_v_generation} + # ) + # else: + # # Default to V1 if not specified + # default_config.update({"image_hyper_v_generation": "V1"}) + # if vhd.image_api_version: + # default_config.update({"image_api_version": vhd.image_api_version}) + # return default_config diff --git a/src/aosm/azext_aosm/custom.py b/src/aosm/azext_aosm/custom.py index 293c99f3615..79d623e24c2 100644 --- a/src/aosm/azext_aosm/custom.py +++ b/src/aosm/azext_aosm/custom.py @@ -7,6 +7,8 @@ from pathlib import Path from azext_aosm.cli_handlers.onboarding_cnf_handler import OnboardingCNFCLIHandler from azext_aosm.cli_handlers.onboarding_vnf_handler import OnboardingVNFCLIHandler +from azext_aosm.cli_handlers.onboarding_core_vnf_handler import OnboardingCoreVNFCLIHandler +from azext_aosm.cli_handlers.onboarding_nexus_vnf_handler import OnboardingNexusVNFCLIHandler from azext_aosm.cli_handlers.onboarding_nsd_handler import OnboardingNSDCLIHandler from azext_aosm.common.command_context import CommandContext from azext_aosm.common.constants import ALL_PARAMETERS_FILE_NAME, CNF, VNF @@ -14,25 +16,35 @@ from azure.cli.core.azclierror import UnrecognizedArgumentError -def onboard_nfd_generate_config(definition_type: str, output_file: str | None): +def onboard_nfd_generate_config(definition_type: str, output_file: str | None, nexus: bool = False): """Generate config file for onboarding NFs.""" if definition_type == CNF: handler = OnboardingCNFCLIHandler() handler.generate_config(output_file) elif definition_type == VNF: - handler = OnboardingVNFCLIHandler() + if nexus: + handler = OnboardingNexusVNFCLIHandler() + else: + print("Generating config file for Core VNF") + handler = OnboardingCoreVNFCLIHandler() + handler.generate_config(output_file) else: raise UnrecognizedArgumentError("Invalid definition type") -def onboard_nfd_build(definition_type: str, config_file: Path, skip: str = None): +def onboard_nfd_build(definition_type: str, config_file: Path, skip: str = None, nexus: bool = False): """Build the NF definition.""" if definition_type == CNF: handler = OnboardingCNFCLIHandler(Path(config_file), skip=skip) handler.build() elif definition_type == VNF: - handler = OnboardingVNFCLIHandler(Path(config_file)) + if nexus: + handler = OnboardingNexusVNFCLIHandler(Path(config_file)) + else: + print("Generating config file for Core VNF") + handler = OnboardingCoreVNFCLIHandler(Path(config_file)) + handler.build() else: raise UnrecognizedArgumentError("Invalid definition type") From 68f46f2220477a9835aab8b16f35bcc14db026c2 Mon Sep 17 00:00:00 2001 From: Jordan Date: Tue, 30 Jan 2024 15:55:14 +0000 Subject: [PATCH 04/37] added core and nexus config logic --- .../onboarding_core_vnf_handler.py | 48 ++++++++----------- .../onboarding_nexus_vnf_handler.py | 8 ++-- .../cli_handlers/onboarding_vnf_handler.py | 40 ++++++++-------- src/aosm/azext_aosm/common/artifact.py | 12 ++--- .../common_parameters_config.py | 6 ++- .../reader/bicep_definition.py | 6 +-- 6 files changed, 57 insertions(+), 63 deletions(-) diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py index 83e316b86e5..56ea5309dc5 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py @@ -8,7 +8,7 @@ from knack.log import get_logger from azext_aosm.build_processors.arm_processor import ( - AzureCoreArmBuildProcessor, BaseArmBuildProcessor, NexusArmBuildProcessor) + AzureCoreArmBuildProcessor, BaseArmBuildProcessor) from azext_aosm.build_processors.vhd_processor import VHDProcessor from azext_aosm.common.constants import (ARTIFACT_LIST_FILENAME, BASE_FOLDER_NAME, @@ -28,7 +28,7 @@ OnboardingVNFInputConfig, ) from azext_aosm.configuration_models.common_parameters_config import ( - VNFCommonParametersConfig, + CoreVNFCommonParametersConfig, ) from azext_aosm.definition_folder.builder.artifact_builder import ( ArtifactDefinitionElementBuilder, @@ -50,16 +50,6 @@ class OnboardingCoreVNFCLIHandler(OnboardingVNFCLIHandler): """CLI handler for publishing NFDs.""" - @property - def default_config_file_name(self) -> str: - """Get the default configuration file name.""" - return VNF_INPUT_FILENAME - - @property - def output_folder_file_name(self) -> str: - """Get the output folder file name.""" - return VNF_OUTPUT_FOLDER_FILENAME - def _get_input_config(self, input_config: Dict[str, Any] = None) -> OnboardingVNFInputConfig: """Get the configuration for the command.""" if input_config is None: @@ -68,13 +58,13 @@ def _get_input_config(self, input_config: Dict[str, Any] = None) -> OnboardingVN def _get_params_config( self, config_file: dict = None - ) -> VNFCommonParametersConfig: + ) -> CoreVNFCommonParametersConfig: """Get the configuration for the command.""" with open(config_file, "r", encoding="utf-8") as _file: params_dict = json.load(_file) if params_dict is None: params_dict = {} - return VNFCommonParametersConfig(**params_dict) + return CoreVNFCommonParametersConfig(**params_dict) def _get_processor_list(self): print("in get processor") @@ -108,18 +98,18 @@ def _get_processor_list(self): processor_list.append(vhd_processor) return processor_list - def build_base_bicep(self): - """Build the base bicep file.""" - # Build manifest bicep contents, with j2 template - template_path = self._get_template_path( - VNF_TEMPLATE_FOLDER_NAME, VNF_BASE_TEMPLATE_FILENAME - ) - bicep_contents = self._render_base_bicep_contents(template_path) - # Create Bicep element with manifest contents - bicep_file = BicepDefinitionElementBuilder( - Path(VNF_OUTPUT_FOLDER_FILENAME, BASE_FOLDER_NAME), bicep_contents - ) - return bicep_file + # def build_base_bicep(self): + # """Build the base bicep file.""" + # # Build manifest bicep contents, with j2 template + # template_path = self._get_template_path( + # VNF_TEMPLATE_FOLDER_NAME, VNF_BASE_TEMPLATE_FILENAME + # ) + # bicep_contents = self._render_base_bicep_contents(template_path) + # # Create Bicep element with manifest contents + # bicep_file = BicepDefinitionElementBuilder( + # Path(VNF_OUTPUT_FOLDER_FILENAME, BASE_FOLDER_NAME), bicep_contents + # ) + # return bicep_file def build_manifest_bicep(self): """Build the manifest bicep file.""" @@ -199,8 +189,6 @@ def build_resource_bicep(self): if isinstance(processor, BaseArmBuildProcessor): acr_nf_application_list.append(nf_application) - print(nf_application) - print(nf_application.deploy_parameters_mapping_rule_profile) # Generate local file for template_parameters + add to supporting files list params = ( nf_application.deploy_parameters_mapping_rule_profile.template_mapping_rule_profile.template_parameters @@ -266,13 +254,14 @@ def build_resource_bicep(self): return bicep_file def build_all_parameters_json(self): + """Create object for all_parameters.json""" params_content = { "location": self.config.location, "publisherName": self.config.publisher_name, "publisherResourceGroupName": self.config.publisher_resource_group_name, "acrArtifactStoreName": self.config.acr_artifact_store_name, "saArtifactStoreName": self.config.blob_artifact_store_name, - "acrManifestName": self.config.acr_artifact_store_name + "-manifest", + "acrManifestName": self.config.acr_artifact_store_name + "-manifest", "saManifestName": self.config.blob_artifact_store_name + "-manifest", "nfDefinitionGroup": self.config.nf_name, "nfDefinitionVersion": self.config.version @@ -283,6 +272,7 @@ def build_all_parameters_json(self): return base_file def _get_default_config(self, vhd): + """Get default VHD config for Azure Core VNF.""" default_config = {} if vhd.image_disk_size_GB: default_config.update({"image_disk_size_GB": vhd.image_disk_size_GB}) diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py index 99ca9a047b0..d13688df496 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py @@ -29,7 +29,7 @@ OnboardingVNFInputConfig, ) from azext_aosm.configuration_models.common_parameters_config import ( - VNFCommonParametersConfig, + NexusVNFCommonParametersConfig, ) from azext_aosm.definition_folder.builder.artifact_builder import ( ArtifactDefinitionElementBuilder, @@ -68,13 +68,13 @@ def _get_input_config(self, input_config: Dict[str, Any] = None) -> OnboardingVN def _get_params_config( self, config_file: dict = None - ) -> VNFCommonParametersConfig: + ) -> NexusVNFCommonParametersConfig: """Get the configuration for the command.""" with open(config_file, "r", encoding="utf-8") as _file: params_dict = json.load(_file) if params_dict is None: params_dict = {} - return VNFCommonParametersConfig(**params_dict) + return NexusVNFCommonParametersConfig(**params_dict) def _get_processor_list(self): processor_list = [] @@ -265,7 +265,7 @@ def build_all_parameters_json(self): "publisherResourceGroupName": self.config.publisher_resource_group_name, "acrArtifactStoreName": self.config.acr_artifact_store_name, # "saArtifactStoreName": self.config.blob_artifact_store_name, - "acrManifestName": self.config.acr_artifact_store_name + "-manifest", + "acrManifestName": self.config.acr_artifact_store_name + "-manifest", # "saManifestName": self.config.blob_artifact_store_name + "-manifest", "nfDefinitionGroup": self.config.nf_name, "nfDefinitionVersion": self.config.version diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_vnf_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_vnf_handler.py index 9f342fb5320..013bf46428d 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_vnf_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_vnf_handler.py @@ -27,9 +27,9 @@ from azext_aosm.configuration_models.onboarding_vnf_input_config import ( OnboardingVNFInputConfig, ) -from azext_aosm.configuration_models.common_parameters_config import ( - VNFCommonParametersConfig, -) +# from azext_aosm.configuration_models.common_parameters_config import ( +# VNFCommonParametersConfig, +# ) from azext_aosm.definition_folder.builder.artifact_builder import ( ArtifactDefinitionElementBuilder, ) @@ -60,11 +60,11 @@ def output_folder_file_name(self) -> str: """Get the output folder file name.""" return VNF_OUTPUT_FOLDER_FILENAME - def _get_input_config(self, input_config: Dict[str, Any] = None) -> OnboardingVNFInputConfig: - """Get the configuration for the command.""" - if input_config is None: - input_config = {} - return OnboardingVNFInputConfig(**input_config) + # def _get_input_config(self, input_config: Dict[str, Any] = None) -> OnboardingVNFInputConfig: + # """Get the configuration for the command.""" + # if input_config is None: + # input_config = {} + # return OnboardingVNFInputConfig(**input_config) # def _get_params_config( # self, config_file: dict = None @@ -112,18 +112,18 @@ def _get_input_config(self, input_config: Dict[str, Any] = None) -> OnboardingVN # processor_list.append(vhd_processor) # return processor_list - # def build_base_bicep(self): - # """Build the base bicep file.""" - # # Build manifest bicep contents, with j2 template - # template_path = self._get_template_path( - # VNF_TEMPLATE_FOLDER_NAME, VNF_BASE_TEMPLATE_FILENAME - # ) - # bicep_contents = self._render_base_bicep_contents(template_path) - # # Create Bicep element with manifest contents - # bicep_file = BicepDefinitionElementBuilder( - # Path(VNF_OUTPUT_FOLDER_FILENAME, BASE_FOLDER_NAME), bicep_contents - # ) - # return bicep_file + def build_base_bicep(self): + """Build the base bicep file.""" + # Build manifest bicep contents, with j2 template + template_path = self._get_template_path( + VNF_TEMPLATE_FOLDER_NAME, VNF_BASE_TEMPLATE_FILENAME + ) + bicep_contents = self._render_base_bicep_contents(template_path) + # Create Bicep element with manifest contents + bicep_file = BicepDefinitionElementBuilder( + Path(VNF_OUTPUT_FOLDER_FILENAME, BASE_FOLDER_NAME), bicep_contents + ) + return bicep_file # def build_manifest_bicep(self): # """Build the manifest bicep file.""" diff --git a/src/aosm/azext_aosm/common/artifact.py b/src/aosm/azext_aosm/common/artifact.py index c7f2d22df31..c61215d27b3 100644 --- a/src/aosm/azext_aosm/common/artifact.py +++ b/src/aosm/azext_aosm/common/artifact.py @@ -16,7 +16,7 @@ from azext_aosm.vendored_sdks.models import ManifestArtifactFormat from azext_aosm.vendored_sdks import HybridNetworkManagementClient from azext_aosm.common.command_context import CommandContext -from azext_aosm.configuration_models.common_parameters_config import BaseCommonParametersConfig, VNFCommonParametersConfig +from azext_aosm.configuration_models.common_parameters_config import BaseCommonParametersConfig, NFDCommonParametersConfig from azext_aosm.vendored_sdks import HybridNetworkManagementClient from knack.util import CLIError from knack.log import get_logger @@ -490,11 +490,11 @@ class BaseStorageAccountArtifact(BaseArtifact): """Abstract base class for storage account artifacts.""" @abstractmethod - def upload(self, config: VNFCommonParametersConfig, command_context: CommandContext): + def upload(self, config: NFDCommonParametersConfig, command_context: CommandContext): """Upload the artifact.""" pass - def _get_blob_client(self, config: VNFCommonParametersConfig, command_context: CommandContext) -> BlobClient: + def _get_blob_client(self, config: NFDCommonParametersConfig, command_context: CommandContext) -> BlobClient: container_basename = self.artifact_name.replace("-", "") container_name = f"{container_basename}-{self.artifact_version}" # For AOSM to work VHD blobs must have the suffix .vhd @@ -530,8 +530,8 @@ def __init__(self, artifact_name, artifact_type, artifact_version, file_path: Pa super().__init__(artifact_name, artifact_type, artifact_version) self.file_path = str(file_path) # TODO: Jordan cast this to str here, `str(file_path)`, check output file isn't broken, and/or is it used as a Path elsewhere? - - def upload(self, config: VNFCommonParametersConfig, command_context: CommandContext): + # TODO: test that making this nfdconfig is ok + def upload(self, config: NFDCommonParametersConfig, command_context: CommandContext): """Upload the artifact.""" logger.debug("LocalFileStorageAccountArtifact config: %s", config) blob_client = self._get_blob_client(config=config, command_context=command_context) @@ -581,7 +581,7 @@ def __init__(self, artifact_manifest: ManifestArtifactFormat, blob_sas_uri: str) super().__init__(artifact_manifest) self.blob_sas_uri = blob_sas_uri - def upload(self, config: VNFCommonParametersConfig, command_context: CommandContext): + def upload(self, config: NFDCommonParametersConfig, command_context: CommandContext): """Upload the artifact.""" logger.info("Copy from SAS URL to blob store") diff --git a/src/aosm/azext_aosm/configuration_models/common_parameters_config.py b/src/aosm/azext_aosm/configuration_models/common_parameters_config.py index 12479490171..5bf985f3266 100644 --- a/src/aosm/azext_aosm/configuration_models/common_parameters_config.py +++ b/src/aosm/azext_aosm/configuration_models/common_parameters_config.py @@ -32,12 +32,16 @@ class NFDCommonParametersConfig(BaseCommonParametersConfig): @dataclass(frozen=True) -class VNFCommonParametersConfig(NFDCommonParametersConfig): +class CoreVNFCommonParametersConfig(NFDCommonParametersConfig): """Common parameters configuration for VNFs.""" saArtifactStoreName: str saManifestName: str +@dataclass(frozen=True) +class NexusVNFCommonParametersConfig(NFDCommonParametersConfig): + """Common parameters configuration for VNFs.""" + @dataclass(frozen=True) class CNFCommonParametersConfig(NFDCommonParametersConfig): diff --git a/src/aosm/azext_aosm/definition_folder/reader/bicep_definition.py b/src/aosm/azext_aosm/definition_folder/reader/bicep_definition.py index 4f4a97ba939..78f622e4f78 100644 --- a/src/aosm/azext_aosm/definition_folder/reader/bicep_definition.py +++ b/src/aosm/azext_aosm/definition_folder/reader/bicep_definition.py @@ -13,7 +13,7 @@ from azext_aosm.common.command_context import CommandContext from azext_aosm.common.utils import convert_bicep_to_arm from azext_aosm.configuration_models.common_parameters_config import \ - BaseCommonParametersConfig, VNFCommonParametersConfig + BaseCommonParametersConfig, CoreVNFCommonParametersConfig from azext_aosm.definition_folder.reader.base_definition import \ BaseDefinitionElement from azext_aosm.common.constants import ManifestsExist @@ -129,8 +129,8 @@ def _artifact_manifests_exist( acr_manifest_exists = True except azure_exceptions.ResourceNotFoundError: acr_manifest_exists = False - - if isinstance(config, VNFCommonParametersConfig): + # TODO: test config type change works + if isinstance(config, CoreVNFCommonParametersConfig): try: command_context.aosm_client.artifact_manifests.get( resource_group_name=config.publisherResourceGroupName, From e81e482130da9a9b9ae886298921cdaad49d3916 Mon Sep 17 00:00:00 2001 From: Jordan Date: Thu, 1 Feb 2024 17:47:56 +0000 Subject: [PATCH 05/37] fixed nexus input and nexus processor; fixed templates --- .../build_processors/nexus_image_processor.py | 62 +++++-------- .../onboarding_core_vnf_handler.py | 13 +-- .../onboarding_nexus_vnf_handler.py | 91 ++++++++----------- .../cli_handlers/onboarding_vnf_handler.py | 6 +- src/aosm/azext_aosm/common/constants.py | 1 + .../vnf/vnfartifactmanifest.bicep.j2 | 9 ++ .../templates/vnf/vnfdefinition.bicep.j2 | 35 +++++-- .../onboarding_vnf_input_config.py | 69 ++++++++++++-- .../azext_aosm/inputs/nexus_image_input.py | 11 ++- 9 files changed, 178 insertions(+), 119 deletions(-) diff --git a/src/aosm/azext_aosm/build_processors/nexus_image_processor.py b/src/aosm/azext_aosm/build_processors/nexus_image_processor.py index 6e7e56b909f..f793db595a6 100644 --- a/src/aosm/azext_aosm/build_processors/nexus_image_processor.py +++ b/src/aosm/azext_aosm/build_processors/nexus_image_processor.py @@ -10,8 +10,7 @@ from azext_aosm.build_processors.base_processor import BaseInputProcessor from azext_aosm.common.artifact import (BaseArtifact, - BlobStorageAccountArtifact, - LocalFileStorageAccountArtifact) + RemoteACRArtifact) from azext_aosm.common.local_file_builder import LocalFileBuilder from azext_aosm.inputs.nexus_image_input import NexusImageFileInput from azext_aosm.vendored_sdks.models import ( @@ -22,19 +21,19 @@ ApplicationEnablement, ArtifactType, DependsOnProfile, ManifestArtifactFormat, ReferencedResource, ResourceElementTemplate, - ) +) logger = get_logger(__name__) class NexusImageProcessor(BaseInputProcessor): """ - A class for processing VHD inputs. + A class for processing Nexus image inputs. :param name: The name of the artifact. :type name: str :param input_artifact: The input artifact. - :type input_artifact: VHDFileInput + :type input_artifact: NexusImageFileInput """ def __init__(self, name: str, input_artifact: NexusImageFileInput): @@ -48,7 +47,7 @@ def get_artifact_manifest_list(self) -> List[ManifestArtifactFormat]: :return: A list of artifacts for the artifact manifest. :rtype: List[ManifestArtifactFormat] """ - logger.info("Getting artifact manifest list for VHD input.") + logger.info("Getting artifact manifest list for Nexus image input.") return [ ManifestArtifactFormat( artifact_name=self.input_artifact.artifact_name, @@ -67,39 +66,20 @@ def get_artifact_details( :return: A tuple containing the list of artifacts and the list of local file builders. :rtype: Tuple[List[BaseArtifact], List[LocalFileBuilder]] """ - logger.info("Getting artifact details for VHD input.") + logger.info("Getting artifact details for Nexus image input.") artifacts: List[BaseArtifact] = [] file_builders: List[LocalFileBuilder] = [] - if self.input_artifact.file_path: - logger.debug( - "VHD input has a file path. Adding LocalFileStorageAccountArtifact." - ) - artifacts.append( - LocalFileStorageAccountArtifact( - artifact_name=self.input_artifact.artifact_name, - artifact_type=ArtifactType.IMAGE_FILE.value, - artifact_version=self.input_artifact.artifact_version, - file_path=self.input_artifact.file_path, - ) - ) - # elif self.input_artifact.blob_sas_uri: - # logger.debug( - # "VHD input has a blob SAS URI. Adding BlobStorageAccountArtifact." - # ) - # artifacts.append( - # BlobStorageAccountArtifact( - # artifact_manifest=artifact_manifest, - # blob_sas_uri=self.input_artifact.blob_sas_uri, - # ) - # ) - else: - # TODO: change error once file input defined - logger.error("ImageFileInput must have either a file path or a blob SAS URI.") - raise ValueError( - "VHDFileInput must have either a file path or a blob SAS URI." + # We only support remote ACR artifacts for container images + artifacts.append( + RemoteACRArtifact( + artifact_name=self.input_artifact.artifact_name, + artifact_type=ArtifactType.IMAGE_FILE.value, + artifact_version=self.input_artifact.artifact_version, + source_registry=self.input_artifact.source_acr_registry, + source_registry_namespace="", ) - + ) return artifacts, file_builders def generate_nf_application(self) -> AzureOperatorNexusNetworkFunctionImageApplication: @@ -107,9 +87,9 @@ def generate_nf_application(self) -> AzureOperatorNexusNetworkFunctionImageAppli Generate the NF application. :return: The NF application. - :rtype: AzureCoreNetworkFunctionVhdApplication + :rtype: AzureOperatorNexusNetworkFunctionImageApplication """ - logger.info("Generating NF application for VHD input.") + logger.info("Generating NF application for Nexus image input.") return AzureOperatorNexusNetworkFunctionImageApplication( name=self.name, @@ -123,9 +103,9 @@ def generate_resource_element_template(self) -> ResourceElementTemplate: """ Generate the resource element template. - :raises NotImplementedError: NSDs do not support deployment of VHDs. + :raises NotImplementedError: NSDs do not support deployment of Nexus images. """ - raise NotImplementedError("NSDs do not support deployment of VHDs.") + raise NotImplementedError("NSDs do not support deployment of Nexus images.") def _generate_artifact_profile(self) -> AzureOperatorNexusImageArtifactProfile: """ @@ -134,7 +114,7 @@ def _generate_artifact_profile(self) -> AzureOperatorNexusImageArtifactProfile: :return: The artifact profile. :rtype: AzureOperatorNexusImageArtifactProfile """ - logger.debug("Generating artifact profile for VHD input.") + logger.debug("Generating artifact profile for Nexus image input.") # TODO: JORDAN check what inputs this takes artifact_profile = ImageArtifactProfile( image_name=self.input_artifact.artifact_name, @@ -155,7 +135,7 @@ def _generate_mapping_rule_profile( :return: The mapping rule profile. :rtype: AzureOperatorNexusImageDeployMappingRuleProfile """ - logger.debug("Generating mapping rule profile for VHD input.") + logger.debug("Generating mapping rule profile for Nexus image input.") user_configuration = self.generate_values_mappings( self.input_artifact.get_schema(), self.input_artifact.get_defaults() ) diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py index 56ea5309dc5..fabb91b5215 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py @@ -25,7 +25,7 @@ TEMPLATE_PARAMETERS_FILENAME) from azext_aosm.common.local_file_builder import LocalFileBuilder from azext_aosm.configuration_models.onboarding_vnf_input_config import ( - OnboardingVNFInputConfig, + OnboardingCoreVNFInputConfig, ) from azext_aosm.configuration_models.common_parameters_config import ( CoreVNFCommonParametersConfig, @@ -50,11 +50,11 @@ class OnboardingCoreVNFCLIHandler(OnboardingVNFCLIHandler): """CLI handler for publishing NFDs.""" - def _get_input_config(self, input_config: Dict[str, Any] = None) -> OnboardingVNFInputConfig: + def _get_input_config(self, input_config: Dict[str, Any] = None) -> OnboardingCoreVNFInputConfig: """Get the configuration for the command.""" if input_config is None: input_config = {} - return OnboardingVNFInputConfig(**input_config) + return OnboardingCoreVNFInputConfig(**input_config) def _get_params_config( self, config_file: dict = None @@ -200,7 +200,7 @@ def build_resource_bicep(self): elif isinstance(processor, VHDProcessor): sa_nf_application_list.append(nf_application) print(nf_application) - print(nf_application.deploy_parameters_mapping_rule_profile) + print(nf_application.deploy_parameters_mapping_rule_profile.application_enablement.value) # Generate local file for vhd_parameters params = ( nf_application.deploy_parameters_mapping_rule_profile.vhd_image_mapping_rule_profile.user_configuration @@ -225,9 +225,10 @@ def build_resource_bicep(self): ) params = { - "nfvi_type": self.config.nfvi_type, + "nfvi_type": 'AzureCore', "acr_nf_applications": acr_nf_application_list, - "sa_nf_application": sa_nf_application_list[0], + "sa_nf_applications": sa_nf_application_list, + "nexus_image_nf_applications": [], "deployment_parameters_file": DEPLOYMENT_PARAMETERS_FILENAME, "vhd_parameters_file": VHD_PARAMETERS_FILENAME, "template_parameters_file": TEMPLATE_PARAMETERS_FILENAME diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py index d13688df496..3d489fe375c 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py @@ -22,11 +22,11 @@ VNF_MANIFEST_TEMPLATE_FILENAME, VNF_OUTPUT_FOLDER_FILENAME, DEPLOYMENT_PARAMETERS_FILENAME, - VHD_PARAMETERS_FILENAME, + NEXUS_IMAGE_PARAMETERS_FILENAME, TEMPLATE_PARAMETERS_FILENAME) from azext_aosm.common.local_file_builder import LocalFileBuilder from azext_aosm.configuration_models.onboarding_vnf_input_config import ( - OnboardingVNFInputConfig, + OnboardingNexusVNFInputConfig, ) from azext_aosm.configuration_models.common_parameters_config import ( NexusVNFCommonParametersConfig, @@ -41,7 +41,7 @@ JSONDefinitionElementBuilder, ) from azext_aosm.inputs.arm_template_input import ArmTemplateInput -from azext_aosm.inputs.vhd_file_input import VHDFileInput +from azext_aosm.inputs.nexus_image_input import NexusImageFileInput from .onboarding_vnf_handler import OnboardingVNFCLIHandler logger = get_logger(__name__) @@ -60,11 +60,11 @@ def output_folder_file_name(self) -> str: """Get the output folder file name.""" return VNF_OUTPUT_FOLDER_FILENAME - def _get_input_config(self, input_config: Dict[str, Any] = None) -> OnboardingVNFInputConfig: + def _get_input_config(self, input_config: Dict[str, Any] = None) -> OnboardingNexusVNFInputConfig: """Get the configuration for the command.""" if input_config is None: input_config = {} - return OnboardingVNFInputConfig(**input_config) + return OnboardingNexusVNFInputConfig(**input_config) def _get_params_config( self, config_file: dict = None @@ -90,22 +90,19 @@ def _get_processor_list(self): processor_list.append( NexusArmBuildProcessor(arm_input.artifact_name, arm_input) ) - # TODO: add artifact for image file, after making config - # # Instantiate vhd processor - # if not self.config.vhd.artifact_name: - # self.config.vhd.artifact_name = self.config.nf_name + "-vhd" - # vhd_processor = VHDProcessor( - # name=self.config.vhd.artifact_name, - # input_artifact=VHDFileInput( - # artifact_name=self.config.vhd.artifact_name, - # artifact_version=self.config.vhd.version, - # default_config=self._get_default_config(self.config.vhd), - # file_path=Path(self.config.vhd.file_path).absolute(), - # blob_sas_uri=self.config.vhd.blob_sas_url, - # ), - # ) - # processor_list.append(vhd_processor) + for image in self.config.images: + (source_acr_registry, name, version) = self._split_image_path(image) + image_input = NexusImageFileInput( + artifact_name=name, + artifact_version=version, + default_config=None, + source_acr_registry=source_acr_registry, + ) + processor_list.append( + NexusImageProcessor(image_input.artifact_name, image_input) + ) + return processor_list def build_base_bicep(self): @@ -129,6 +126,7 @@ def build_manifest_bicep(self): for processor in self.processors: acr_artifact_list.extend(processor.get_artifact_manifest_list()) + print("ACRLSIT", acr_artifact_list) logger.debug( "Created list of artifacts from arm template(s) and image files(s) provided: %s", acr_artifact_list, @@ -172,8 +170,8 @@ def build_artifact_list(self): def build_resource_bicep(self): """Build the resource bicep file.""" logger.info("Creating artifacts list for artifacts.json") - acr_nf_application_list = [] - sa_nf_application_list = [] + arm_nf_application_list = [] + image_nf_application_list = [] supporting_files = [] schema_properties = {} @@ -190,7 +188,7 @@ def build_resource_bicep(self): # For each arm template, generate nf application if isinstance(processor, BaseArmBuildProcessor): - acr_nf_application_list.append(nf_application) + arm_nf_application_list.append(nf_application) print(nf_application) print(nf_application.deploy_parameters_mapping_rule_profile) # Generate local file for template_parameters + add to supporting files list @@ -201,16 +199,16 @@ def build_resource_bicep(self): logger.info( "Created templatateParameters as supporting file for nfDefinition bicep" ) - # elif isinstance(processor, NexusImageProcessor): + elif isinstance(processor, NexusImageProcessor): # TODO: fix this - # sa_nf_application_list.append(nf_application) - # print(nf_application) - # print(nf_application.deploy_parameters_mapping_rule_profile) - # # Generate local file for vhd_parameters - # params = ( - # nf_application.deploy_parameters_mapping_rule_profile.vhd_image_mapping_rule_profile.user_configuration - # ) - # template_name = VHD_PARAMETERS_FILENAME + image_nf_application_list.append(nf_application) + print(nf_application) + print(nf_application.deploy_parameters_mapping_rule_profile.image_mapping_rule_profile) + # Generate local file for vhd_parameters + params = ( + nf_application.deploy_parameters_mapping_rule_profile.image_mapping_rule_profile.user_configuration + ) + template_name = NEXUS_IMAGE_PARAMETERS_FILENAME else: raise TypeError(f"Type: {type(processor)} is not valid") @@ -230,12 +228,13 @@ def build_resource_bicep(self): ) params = { - "nfvi_type": self.config.nfvi_type, - "acr_nf_applications": acr_nf_application_list, - "sa_nf_application": sa_nf_application_list[0], + "nfvi_type": 'AzureOperatorNexus', + "acr_nf_applications": arm_nf_application_list, + "sa_nf_applications": [], + "nexus_image_nf_applications": image_nf_application_list, "deployment_parameters_file": DEPLOYMENT_PARAMETERS_FILENAME, - "vhd_parameters_file": VHD_PARAMETERS_FILENAME, - "template_parameters_file": TEMPLATE_PARAMETERS_FILENAME + "template_parameters_file": TEMPLATE_PARAMETERS_FILENAME, + "image_parameters_file": NEXUS_IMAGE_PARAMETERS_FILENAME } bicep_contents = self._render_definition_bicep_contents( template_path, params @@ -275,17 +274,7 @@ def build_all_parameters_json(self): ) return base_file - # def _get_default_config(self, vhd): - # default_config = {} - # if vhd.image_disk_size_GB: - # default_config.update({"image_disk_size_GB": vhd.image_disk_size_GB}) - # if vhd.image_hyper_v_generation: - # default_config.update( - # {"image_hyper_v_generation": vhd.image_hyper_v_generation} - # ) - # else: - # # Default to V1 if not specified - # default_config.update({"image_hyper_v_generation": "V1"}) - # if vhd.image_api_version: - # default_config.update({"image_api_version": vhd.image_api_version}) - # return default_config + def _split_image_path(self, image): + (source_acr_registry, name_and_version) = image.split("/", 2) + (name, version) = name_and_version.split(":", 2) + return (source_acr_registry, name, version) diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_vnf_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_vnf_handler.py index 013bf46428d..28e3be83ebd 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_vnf_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_vnf_handler.py @@ -24,9 +24,9 @@ VHD_PARAMETERS_FILENAME, TEMPLATE_PARAMETERS_FILENAME) from azext_aosm.common.local_file_builder import LocalFileBuilder -from azext_aosm.configuration_models.onboarding_vnf_input_config import ( - OnboardingVNFInputConfig, -) +# from azext_aosm.configuration_models.onboarding_vnf_input_config import ( +# OnboardingVNFInputConfig, +# ) # from azext_aosm.configuration_models.common_parameters_config import ( # VNFCommonParametersConfig, # ) diff --git a/src/aosm/azext_aosm/common/constants.py b/src/aosm/azext_aosm/common/constants.py index f2ec18e1b5a..3a125fe5b63 100644 --- a/src/aosm/azext_aosm/common/constants.py +++ b/src/aosm/azext_aosm/common/constants.py @@ -51,6 +51,7 @@ class ManifestsExist(str, Enum): DEPLOYMENT_PARAMETERS_FILENAME = "deploymentParameters.json" TEMPLATE_PARAMETERS_FILENAME = "templateParameters.json" VHD_PARAMETERS_FILENAME = "vhdParameters.json" +NEXUS_IMAGE_PARAMETERS_FILENAME = "imageParameters.json" NSD_OUTPUT_FOLDER_FILENAME = "nsd-cli-output" NSD_INPUT_FILENAME = "nsd-input.jsonc" diff --git a/src/aosm/azext_aosm/common/templates/vnf/vnfartifactmanifest.bicep.j2 b/src/aosm/azext_aosm/common/templates/vnf/vnfartifactmanifest.bicep.j2 index 2a1d3de8b86..daf9072180e 100644 --- a/src/aosm/azext_aosm/common/templates/vnf/vnfartifactmanifest.bicep.j2 +++ b/src/aosm/azext_aosm/common/templates/vnf/vnfartifactmanifest.bicep.j2 @@ -6,12 +6,16 @@ param location string param publisherName string @description('Name of an existing ACR-backed Artifact Store, deployed under the publisher.') param acrArtifactStoreName string +{%- if sa_artifacts %} @description('Name of an existing Storage Account-backed Artifact Store, deployed under the publisher.') param saArtifactStoreName string +{%- endif %} @description('Name of the manifest to deploy for the ACR-backed Artifact Store') param acrManifestName string +{%- if sa_artifacts %} @description('Name of the manifest to deploy for the Storage Account-backed Artifact Store') param saManifestName string +{%- endif %} // The publisher resource is the top level AOSM resource under which all other designer resources // are created. @@ -27,12 +31,14 @@ resource acrArtifactStore 'Microsoft.HybridNetwork/publishers/artifactStores@202 name: acrArtifactStoreName } +{%- if sa_artifacts %} // The storage account is the resource in which the VHD images are stored. // If using publish command, this is created from deploying the vnfbase.bicep resource saArtifactStore 'Microsoft.HybridNetwork/publishers/artifactStores@2023-09-01' existing = { parent: publisher name: saArtifactStoreName } +{%- endif %} resource acrArtifactManifest 'Microsoft.Hybridnetwork/publishers/artifactStores/artifactManifests@2023-09-01' = { parent: acrArtifactStore @@ -51,6 +57,7 @@ resource acrArtifactManifest 'Microsoft.Hybridnetwork/publishers/artifactStores/ } } +{%- if sa_artifacts %} resource saArtifactManifest 'Microsoft.Hybridnetwork/publishers/artifactStores/artifactManifests@2023-09-01' = { parent: saArtifactStore name: saManifestName @@ -67,3 +74,5 @@ resource saArtifactManifest 'Microsoft.Hybridnetwork/publishers/artifactStores/a ] } } +{%- endif %} + diff --git a/src/aosm/azext_aosm/common/templates/vnf/vnfdefinition.bicep.j2 b/src/aosm/azext_aosm/common/templates/vnf/vnfdefinition.bicep.j2 index adc8f4896c5..bef03f26fa1 100644 --- a/src/aosm/azext_aosm/common/templates/vnf/vnfdefinition.bicep.j2 +++ b/src/aosm/azext_aosm/common/templates/vnf/vnfdefinition.bicep.j2 @@ -55,14 +55,15 @@ resource nfdv 'Microsoft.Hybridnetwork/publishers/networkfunctiondefinitiongroup networkFunctionTemplate: { nfviType: '{{ nfvi_type }}' networkFunctionApplications: [ + {%- for configuration in sa_nf_applications %} { artifactType: 'VhdImageFile' - name: '{{ sa_nf_application.artifact_profile.vhd_artifact_profile.vhd_name }}' + name: '{{ configuration.artifact_profile.vhd_artifact_profile.vhd_name }}' dependsOnProfile: null artifactProfile: { vhdArtifactProfile: { - vhdName: '{{ sa_nf_application.artifact_profile.vhd_artifact_profile.vhd_name }}' - vhdVersion: '{{ sa_nf_application.artifact_profile.vhd_artifact_profile.vhd_version }}' + vhdName: '{{ configuration.artifact_profile.vhd_artifact_profile.vhd_name }}' + vhdVersion: '{{ configuration.artifact_profile.vhd_artifact_profile.vhd_version }}' } artifactStore: { id: saArtifactStore.id @@ -73,10 +74,10 @@ resource nfdv 'Microsoft.Hybridnetwork/publishers/networkfunctiondefinitiongroup vhdImageMappingRuleProfile: { userConfiguration: string(loadJsonContent('{{vhd_parameters_file}}')) } - // ?? - applicationEnablement: 'Unknown' + applicationEnablement: '{{ configuration.deploy_parameters_mapping_rule_profile.application_enablement.value }}' } } + {%- endfor %} {%- for configuration in acr_nf_applications %} { artifactType: 'ArmTemplate' @@ -95,7 +96,29 @@ resource nfdv 'Microsoft.Hybridnetwork/publishers/networkfunctiondefinitiongroup templateMappingRuleProfile: { templateParameters: string(loadJsonContent('{{template_parameters_file}}')) } - applicationEnablement: 'Unknown' + applicationEnablement: '{{ configuration.deploy_parameters_mapping_rule_profile.application_enablement.value }}' + } + } + {%- endfor %} + {%- for configuration in nexus_image_nf_applications %} + { + artifactType: 'ImageFile' + name: '{{ configuration.name }}' + dependsOnProfile: null + artifactProfile: { + imageArtifactProfile: { + imageName: '{{ configuration.artifact_profile.image_artifact_profile.image_name }}' + imageVersion: '{{ configuration.artifact_profile.image_artifact_profile.image_version }}' + } + artifactStore: { + id: acrArtifactStore.id + } + } + deployParametersMappingRuleProfile: { + imageMappingRuleProfile: { + userConfiguration: string(loadJsonContent('{{image_parameters_file}}')) + } + applicationEnablement: '{{ configuration.deploy_parameters_mapping_rule_profile.application_enablement.value }}' } } {%- endfor %} diff --git a/src/aosm/azext_aosm/configuration_models/onboarding_vnf_input_config.py b/src/aosm/azext_aosm/configuration_models/onboarding_vnf_input_config.py index a58e18a8598..7a81dcb5b56 100644 --- a/src/aosm/azext_aosm/configuration_models/onboarding_vnf_input_config.py +++ b/src/aosm/azext_aosm/configuration_models/onboarding_vnf_input_config.py @@ -91,9 +91,8 @@ def validate(self): "One of file_path or sas_blob_url must be set for vhd." ) - @dataclass -class OnboardingVNFInputConfig(OnboardingNFDBaseInputConfig): +class OnboardingCoreVNFInputConfig(OnboardingNFDBaseInputConfig): """Input configuration for onboarding VNFs.""" blob_artifact_store_name: str | None = field( @@ -114,8 +113,6 @@ class OnboardingVNFInputConfig(OnboardingNFDBaseInputConfig): ) }, ) - nfvi_type: str = field(default="AzureCore", - metadata={"comment": "NFVI type. Defaults to AzureCore. Can be AzureCore or AzureOperatorNexus"}) # TODO: Add better comments arm_templates: List[ArmTemplatePropertiesConfig] = field( @@ -149,8 +146,6 @@ def sa_manifest_name(self) -> str: def validate(self): """Validate the configuration.""" super().validate() - if not self.nfvi_type: - raise ValidationError("nfvi_type must be set") if not self.image_name_parameter: raise ValidationError("image_name_parameter must be set") if not self.arm_templates: @@ -161,4 +156,64 @@ def validate(self): raise ValidationError("You must include at least one arm template") for arm_template in self.arm_templates: arm_template.validate() - self.vhd.validate() \ No newline at end of file + self.vhd.validate() + + +@dataclass +class OnboardingNexusVNFInputConfig(OnboardingNFDBaseInputConfig): + """Input configuration for onboarding VNFs.""" + + image_name_parameter: str = field( + default="", + metadata={ + "comment": ( + "The parameter name in the VM ARM template which " + "specifies the name of the image to use for the VM." + ) + }, + ) + + arm_templates: List[ArmTemplatePropertiesConfig] = field( + default_factory=lambda: [ArmTemplatePropertiesConfig()], + metadata={"comment": "ARM template configuration. The ARM templates given here would deploy a VM if run. They will be used to generate the VNF."}, + ) + + images: List[str] = field( + default_factory=lambda: [], + metadata={ + "comment": ( + "List of images to be pulled from the acr registry.\n" + "You must provide the source acr registry, the image name and the version.\n" + "For example: 'sourceacr.azurecr.io/imagename:imageversion'." + ) + }, + ) + + def __post_init__(self): + arm_list = [] + for arm_template in self.arm_templates: + if arm_template and isinstance(arm_template, dict): + arm_list.append(ArmTemplatePropertiesConfig(**arm_template)) + else: + arm_list.append(arm_template) + self.arm_templates = arm_list + + @property + def sa_manifest_name(self) -> str: + """Return the Storage account manifest name from the NFD name.""" + sanitized_nf_name = self.nf_name.lower().replace("_", "-") + return f"{sanitized_nf_name}-sa-manifest-{self.version.replace('.', '-')}" + + def validate(self): + """Validate the configuration.""" + super().validate() + if not self.image_name_parameter: + raise ValidationError("image_name_parameter must be set") + if not self.arm_templates: + raise ValidationError("arm_template must be set") + if not self.images: + raise ValidationError("You must include at least one image") + if not self.arm_templates: + raise ValidationError("You must include at least one arm template") + for arm_template in self.arm_templates: + arm_template.validate() diff --git a/src/aosm/azext_aosm/inputs/nexus_image_input.py b/src/aosm/azext_aosm/inputs/nexus_image_input.py index 70e11ea93a7..c4a3830d7ee 100644 --- a/src/aosm/azext_aosm/inputs/nexus_image_input.py +++ b/src/aosm/azext_aosm/inputs/nexus_image_input.py @@ -36,12 +36,13 @@ def __init__( artifact_name: str, artifact_version: str, # file_path: Optional[Path] = None, - # blob_sas_uri: Optional[str] = None, + source_acr_registry: str, default_config: Optional[Dict[str, Any]] = None, ): super().__init__(artifact_name, artifact_version, default_config) # self.file_path = file_path # self.blob_sas_uri = blob_sas_uri + self.source_acr_registry = source_acr_registry def get_defaults(self) -> Dict[str, Any]: """ @@ -51,12 +52,12 @@ def get_defaults(self) -> Dict[str, Any]: :rtype: Dict[str, Any] """ # logger.info("Getting default values for VHD file input") - # default_config = self.default_config or {} + default_config = self.default_config or {} # logger.debug( # "Default values for VHD file Input: %s", # json.dumps(default_config, indent=4), # ) - # return copy.deepcopy(default_config) + return copy.deepcopy(default_config) def get_schema(self) -> Dict[str, Any]: """ @@ -76,9 +77,9 @@ def get_schema(self) -> Dict[str, Any]: # } # vhd_required = ["imageName"] - # schema = copy.deepcopy(BASE_SCHEMA) + schema = copy.deepcopy(BASE_SCHEMA) # schema["properties"].update(vhd_properties) # schema["required"] += vhd_required # logger.debug("Schema for VHD file input: %s", json.dumps(schema, indent=4)) - # return copy.deepcopy(schema) + return copy.deepcopy(schema) From 99a713b4d1f1549c587d26c6b0f6b6408e8d010a Mon Sep 17 00:00:00 2001 From: Jordan Date: Thu, 1 Feb 2024 18:02:49 +0000 Subject: [PATCH 06/37] generalised render manifest and render definition bicep functions to use one --- .../cli_handlers/onboarding_base_handler.py | 23 +------------------ .../cli_handlers/onboarding_cnf_handler.py | 12 ++++++---- .../onboarding_core_vnf_handler.py | 11 +++++---- .../onboarding_nexus_vnf_handler.py | 11 ++++++--- .../cli_handlers/onboarding_nsd_handler.py | 10 ++++---- 5 files changed, 30 insertions(+), 37 deletions(-) diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_base_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_base_handler.py index 330c2df8c6c..ba435183b7c 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_base_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_base_handler.py @@ -183,7 +183,7 @@ def _render_base_bicep_contents(self, template_path): bicep_contents: str = template.render() return bicep_contents - def _render_definition_bicep_contents(self, template_path: Path, params): + def _render_bicep_contents_from_j2(self, template_path: Path, params): """Write the definition bicep file from given template.""" with open(template_path, "r", encoding="UTF-8") as f: template: Template = Template( @@ -194,27 +194,6 @@ def _render_definition_bicep_contents(self, template_path: Path, params): bicep_contents: str = template.render(params) return bicep_contents - def _render_manifest_bicep_contents( - self, - template_path: Path, - acr_artifact_list: list, - sa_artifact_list: list = None, - ): - """Write the manifest bicep file from given template. - - Returns bicep content as string - """ - with open(template_path, "r", encoding="UTF-8") as f: - template: Template = Template( - f.read(), - undefined=StrictUndefined, - ) - - bicep_contents: str = template.render( - acr_artifacts=acr_artifact_list, sa_artifacts=sa_artifact_list - ) - return bicep_contents - def _get_template_path(self, definition_type: str, template_name: str) -> Path: """Get the path to a template.""" return ( diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_cnf_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_cnf_handler.py index de6c46bebc6..8c4332de470 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_cnf_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_cnf_handler.py @@ -197,9 +197,13 @@ def build_manifest_bicep(self): template_path = self._get_template_path( CNF_TEMPLATE_FOLDER_NAME, CNF_MANIFEST_TEMPLATE_FILENAME ) - bicep_contents = self._render_manifest_bicep_contents( - template_path, artifact_list - ) + + params = { + "acr_artifacts": artifact_list, + "sa_artifacts": [] + } + bicep_contents = self._render_bicep_contents_from_j2(template_path, params) + # Create Bicep element with manifest contents bicep_file = BicepDefinitionElementBuilder( Path(CNF_OUTPUT_FOLDER_FILENAME, MANIFEST_FOLDER_NAME), bicep_contents @@ -265,7 +269,7 @@ def build_resource_bicep(self): "acr_nf_applications": nf_application_list, "deployment_parameters_file": DEPLOYMENT_PARAMETERS_FILENAME, } - bicep_contents = self._render_definition_bicep_contents(template_path, params) + bicep_contents = self._render_bicep_contents_from_j2(template_path, params) # Create a bicep element + add its supporting mapping files bicep_file = BicepDefinitionElementBuilder( diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py index fabb91b5215..1eebaccf419 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py @@ -136,9 +136,12 @@ def build_manifest_bicep(self): template_path = self._get_template_path( VNF_TEMPLATE_FOLDER_NAME, VNF_MANIFEST_TEMPLATE_FILENAME ) - bicep_contents = self._render_manifest_bicep_contents( - template_path, acr_artifact_list, sa_artifact_list - ) + params = { + "acr_artifacts": acr_artifact_list, + "sa_artifacts": sa_artifact_list, + } + bicep_contents = self._render_bicep_contents_from_j2(template_path, params) + # Create Bicep element with manifest contents bicep_file = BicepDefinitionElementBuilder( Path(VNF_OUTPUT_FOLDER_FILENAME, MANIFEST_FOLDER_NAME), @@ -233,7 +236,7 @@ def build_resource_bicep(self): "vhd_parameters_file": VHD_PARAMETERS_FILENAME, "template_parameters_file": TEMPLATE_PARAMETERS_FILENAME } - bicep_contents = self._render_definition_bicep_contents( + bicep_contents = self._render_bicep_contents_from_j2( template_path, params ) diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py index 3d489fe375c..28b0b698bca 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py @@ -136,9 +136,14 @@ def build_manifest_bicep(self): template_path = self._get_template_path( VNF_TEMPLATE_FOLDER_NAME, VNF_MANIFEST_TEMPLATE_FILENAME ) - bicep_contents = self._render_manifest_bicep_contents( - template_path, acr_artifact_list + params = { + "acr_artifacts": acr_artifact_list, + "sa_artifacts": [] + } + bicep_contents = self._render_bicep_contents_from_j2( + template_path, params ) + # Create Bicep element with manifest contents bicep_file = BicepDefinitionElementBuilder( Path(VNF_OUTPUT_FOLDER_FILENAME, MANIFEST_FOLDER_NAME), @@ -236,7 +241,7 @@ def build_resource_bicep(self): "template_parameters_file": TEMPLATE_PARAMETERS_FILENAME, "image_parameters_file": NEXUS_IMAGE_PARAMETERS_FILENAME } - bicep_contents = self._render_definition_bicep_contents( + bicep_contents = self._render_bicep_contents_from_j2( template_path, params ) diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_nsd_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_nsd_handler.py index 9866d9de315..2612e68d6b7 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_nsd_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_nsd_handler.py @@ -144,9 +144,11 @@ def build_manifest_bicep(self): template_path = self._get_template_path( NSD_TEMPLATE_FOLDER_NAME, NSD_MANIFEST_TEMPLATE_FILENAME ) - bicep_contents = self._render_manifest_bicep_contents( - template_path, artifact_list - ) + params = { + "acr_artifacts": artifact_list, + "sa_artifacts": [] + } + bicep_contents = self._render_bicep_contents_from_j2(template_path, params) bicep_file = BicepDefinitionElementBuilder( Path(NSD_OUTPUT_FOLDER_FILENAME, MANIFEST_FOLDER_NAME), bicep_contents @@ -222,7 +224,7 @@ def build_resource_bicep(self): "template_parameters_file": TEMPLATE_PARAMETERS_FILENAME, } - bicep_contents = self._render_definition_bicep_contents( + bicep_contents = self._render_bicep_contents_from_j2( template_path, params ) # Generate the nsd bicep file From 01100dc0fcaabcf6f2d121af9afd9107a90e9b07 Mon Sep 17 00:00:00 2001 From: Jordan Date: Thu, 1 Feb 2024 18:12:56 +0000 Subject: [PATCH 07/37] added incorrect config error handling --- src/aosm/azext_aosm/cli_handlers/onboarding_base_handler.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_base_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_base_handler.py index ba435183b7c..4392641a66a 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_base_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_base_handler.py @@ -49,7 +49,10 @@ def __init__( # If config file is the input.jsonc for build command if provided_input_path.suffix == ".jsonc": config_dict = self._read_input_config_from_file(provided_input_path) - self.config = self._get_input_config(config_dict) + try: + self.config = self._get_input_config(config_dict) + except Exception as e: + raise UnclassifiedUserFault(f"The input file provided contains an incorrect input.\nPlease fix the problem parameter: - {e}") from e # Validate config before getting processor list, # in case error with input artifacts i.e helm package self.config.validate() From 7196d3466ab20f31c6eaf3fbcfd85e84cf8bde2e Mon Sep 17 00:00:00 2001 From: Jordan Date: Fri, 2 Feb 2024 11:13:12 +0000 Subject: [PATCH 08/37] added nexus flag to publish; added error handling for incorrect vnf type (if forgot to put nexus flag case); tidied nexus inputs --- .../cli_handlers/onboarding_base_handler.py | 7 +++- src/aosm/azext_aosm/custom.py | 10 ++--- .../azext_aosm/inputs/nexus_image_input.py | 37 ++++--------------- 3 files changed, 17 insertions(+), 37 deletions(-) diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_base_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_base_handler.py index 4392641a66a..427b3f4824c 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_base_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_base_handler.py @@ -52,14 +52,17 @@ def __init__( try: self.config = self._get_input_config(config_dict) except Exception as e: - raise UnclassifiedUserFault(f"The input file provided contains an incorrect input.\nPlease fix the problem parameter: - {e}") from e + raise UnclassifiedUserFault(f"The input file provided contains an incorrect input.\nPlease fix the problem parameter:\n{e}") from e # Validate config before getting processor list, # in case error with input artifacts i.e helm package self.config.validate() self.processors = self._get_processor_list() # If config file is the all parameters json file for publish/delete elif provided_input_path.suffix == ".json": - self.config = self._get_params_config(provided_input_path) + try: + self.config = self._get_params_config(provided_input_path) + except Exception as e: + raise UnclassifiedUserFault(f"The all_deploy.parameters.json in the folder provided contains an incorrect input.\nPlease check if you have provided the correct folder for the definition/design type:\n{e}") from e else: raise UnclassifiedUserFault("Invalid input") # TODO: Change this to work with publish? diff --git a/src/aosm/azext_aosm/custom.py b/src/aosm/azext_aosm/custom.py index 578ee0ba45b..95b1182ed9b 100644 --- a/src/aosm/azext_aosm/custom.py +++ b/src/aosm/azext_aosm/custom.py @@ -25,7 +25,6 @@ def onboard_nfd_generate_config(definition_type: str, output_file: str | None, n if nexus: handler = OnboardingNexusVNFCLIHandler() else: - print("Generating config file for Core VNF") handler = OnboardingCoreVNFCLIHandler() handler.generate_config(output_file) @@ -42,7 +41,6 @@ def onboard_nfd_build(definition_type: str, config_file: Path, skip: str = None, if nexus: handler = OnboardingNexusVNFCLIHandler(Path(config_file)) else: - print("Generating config file for Core VNF") handler = OnboardingCoreVNFCLIHandler(Path(config_file)) handler.build() @@ -55,6 +53,7 @@ def onboard_nfd_publish( definition_type: str, build_output_folder: Path, no_subscription_permissions: bool = False, + nexus: bool = False ): """Publish the NF definition.""" command_context = CommandContext( @@ -70,9 +69,10 @@ def onboard_nfd_publish( ) handler.publish(command_context=command_context) elif definition_type == VNF: - handler = OnboardingVNFCLIHandler( - Path(build_output_folder, ALL_PARAMETERS_FILE_NAME) - ) + if nexus: + handler = OnboardingNexusVNFCLIHandler(Path(build_output_folder, ALL_PARAMETERS_FILE_NAME)) + else: + handler = OnboardingCoreVNFCLIHandler(Path(build_output_folder, ALL_PARAMETERS_FILE_NAME)) handler.publish(command_context=command_context) else: raise UnrecognizedArgumentError("Invalid definition type") diff --git a/src/aosm/azext_aosm/inputs/nexus_image_input.py b/src/aosm/azext_aosm/inputs/nexus_image_input.py index c4a3830d7ee..bc7880f47a6 100644 --- a/src/aosm/azext_aosm/inputs/nexus_image_input.py +++ b/src/aosm/azext_aosm/inputs/nexus_image_input.py @@ -35,51 +35,28 @@ def __init__( self, artifact_name: str, artifact_version: str, - # file_path: Optional[Path] = None, source_acr_registry: str, default_config: Optional[Dict[str, Any]] = None, ): super().__init__(artifact_name, artifact_version, default_config) - # self.file_path = file_path - # self.blob_sas_uri = blob_sas_uri self.source_acr_registry = source_acr_registry def get_defaults(self) -> Dict[str, Any]: """ Gets the default values for configuring the input. - :return: A dictionary containing the default values. + For Nexus images, there are no defaults. + :return: An empty dictionary. :rtype: Dict[str, Any] """ - # logger.info("Getting default values for VHD file input") - default_config = self.default_config or {} - # logger.debug( - # "Default values for VHD file Input: %s", - # json.dumps(default_config, indent=4), - # ) - return copy.deepcopy(default_config) + return {} def get_schema(self) -> Dict[str, Any]: """ - Gets the schema for the VHD file input. + Gets the schema for the file input. - :return: A dictionary containing the schema. + For Nexus images, there is no schema. + :return: An empty dictionary. :rtype: Dict[str, Any] """ - # logger.info("Getting schema for VHD file input") - # vhd_properties = { - # "imageName": {"type": "string"}, - # "azureDeployLocation": {"type": "string"}, - # "imageDiskSizeGB": {"type": "integer"}, - # "imageOsState": {"type": "string"}, - # "imageHyperVGeneration": {"type": "string"}, - # "apiVersion": {"type": "string"}, - # } - # vhd_required = ["imageName"] - - schema = copy.deepcopy(BASE_SCHEMA) - # schema["properties"].update(vhd_properties) - # schema["required"] += vhd_required - - # logger.debug("Schema for VHD file input: %s", json.dumps(schema, indent=4)) - return copy.deepcopy(schema) + return {} From 53b1ad0b5a1518713cfcc9dd8c64aa6865dc0200 Mon Sep 17 00:00:00 2001 From: Jordan Date: Fri, 2 Feb 2024 14:52:14 +0000 Subject: [PATCH 09/37] added vnf nexus base bicep; fixed vnf definition template; temp fix for image versions --- .../onboarding_core_vnf_handler.py | 26 +- .../onboarding_nexus_vnf_handler.py | 39 ++- .../cli_handlers/onboarding_vnf_handler.py | 283 +----------------- src/aosm/azext_aosm/common/constants.py | 6 +- .../templates/vnf/vnfdefinition.bicep.j2 | 4 + .../common/templates/vnf/vnfnexusbase.bicep | 35 +++ 6 files changed, 84 insertions(+), 309 deletions(-) create mode 100644 src/aosm/azext_aosm/common/templates/vnf/vnfnexusbase.bicep diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py index 1eebaccf419..f1fdba56451 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py @@ -14,7 +14,7 @@ BASE_FOLDER_NAME, MANIFEST_FOLDER_NAME, NF_DEFINITION_FOLDER_NAME, - VNF_BASE_TEMPLATE_FILENAME, + VNF_CORE_BASE_TEMPLATE_FILENAME, VNF_TEMPLATE_FOLDER_NAME, VNF_DEFINITION_TEMPLATE_FILENAME, VNF_INPUT_FILENAME, @@ -98,18 +98,18 @@ def _get_processor_list(self): processor_list.append(vhd_processor) return processor_list - # def build_base_bicep(self): - # """Build the base bicep file.""" - # # Build manifest bicep contents, with j2 template - # template_path = self._get_template_path( - # VNF_TEMPLATE_FOLDER_NAME, VNF_BASE_TEMPLATE_FILENAME - # ) - # bicep_contents = self._render_base_bicep_contents(template_path) - # # Create Bicep element with manifest contents - # bicep_file = BicepDefinitionElementBuilder( - # Path(VNF_OUTPUT_FOLDER_FILENAME, BASE_FOLDER_NAME), bicep_contents - # ) - # return bicep_file + def build_base_bicep(self): + """Build the base bicep file.""" + # Build manifest bicep contents, with j2 template + template_path = self._get_template_path( + VNF_TEMPLATE_FOLDER_NAME, VNF_CORE_BASE_TEMPLATE_FILENAME + ) + bicep_contents = self._render_base_bicep_contents(template_path) + # Create Bicep element with manifest contents + bicep_file = BicepDefinitionElementBuilder( + Path(VNF_OUTPUT_FOLDER_FILENAME, BASE_FOLDER_NAME), bicep_contents + ) + return bicep_file def build_manifest_bicep(self): """Build the manifest bicep file.""" diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py index 28b0b698bca..b3aa954e962 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py @@ -3,19 +3,17 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- import json +import re from pathlib import Path from typing import Dict, Any from knack.log import get_logger -from azext_aosm.build_processors.arm_processor import ( - AzureCoreArmBuildProcessor, BaseArmBuildProcessor, NexusArmBuildProcessor) -# from azext_aosm.build_processors.vhd_processor import VHDProcessor +from azext_aosm.build_processors.arm_processor import NexusArmBuildProcessor from azext_aosm.build_processors.nexus_image_processor import NexusImageProcessor from azext_aosm.common.constants import (ARTIFACT_LIST_FILENAME, BASE_FOLDER_NAME, MANIFEST_FOLDER_NAME, NF_DEFINITION_FOLDER_NAME, - VNF_BASE_TEMPLATE_FILENAME, VNF_TEMPLATE_FOLDER_NAME, VNF_DEFINITION_TEMPLATE_FILENAME, VNF_INPUT_FILENAME, @@ -23,7 +21,9 @@ VNF_OUTPUT_FOLDER_FILENAME, DEPLOYMENT_PARAMETERS_FILENAME, NEXUS_IMAGE_PARAMETERS_FILENAME, - TEMPLATE_PARAMETERS_FILENAME) + TEMPLATE_PARAMETERS_FILENAME, + VNF_NEXUS_BASE_TEMPLATE_FILENAME, + SEMVER_REGEX) from azext_aosm.common.local_file_builder import LocalFileBuilder from azext_aosm.configuration_models.onboarding_vnf_input_config import ( OnboardingNexusVNFInputConfig, @@ -42,8 +42,8 @@ ) from azext_aosm.inputs.arm_template_input import ArmTemplateInput from azext_aosm.inputs.nexus_image_input import NexusImageFileInput - from .onboarding_vnf_handler import OnboardingVNFCLIHandler + logger = get_logger(__name__) @@ -60,7 +60,9 @@ def output_folder_file_name(self) -> str: """Get the output folder file name.""" return VNF_OUTPUT_FOLDER_FILENAME - def _get_input_config(self, input_config: Dict[str, Any] = None) -> OnboardingNexusVNFInputConfig: + def _get_input_config( + self, input_config: Dict[str, Any] = None + ) -> OnboardingNexusVNFInputConfig: """Get the configuration for the command.""" if input_config is None: input_config = {} @@ -86,13 +88,16 @@ def _get_processor_list(self): default_config=None, template_path=Path(arm_template.file_path).absolute(), ) - # TODO: test azure core vnf still works processor_list.append( NexusArmBuildProcessor(arm_input.artifact_name, arm_input) ) - # TODO: add artifact for image file, after making config + # For each image, instantiate image processor for image in self.config.images: (source_acr_registry, name, version) = self._split_image_path(image) + + # TEMP FIX FOR INVALID VERSIONS + version = self._create_semver_compatible_version(version) + image_input = NexusImageFileInput( artifact_name=name, artifact_version=version, @@ -109,7 +114,7 @@ def build_base_bicep(self): """Build the base bicep file.""" # Build manifest bicep contents, with j2 template template_path = self._get_template_path( - VNF_TEMPLATE_FOLDER_NAME, VNF_BASE_TEMPLATE_FILENAME + VNF_TEMPLATE_FOLDER_NAME, VNF_NEXUS_BASE_TEMPLATE_FILENAME ) bicep_contents = self._render_base_bicep_contents(template_path) # Create Bicep element with manifest contents @@ -191,7 +196,7 @@ def build_resource_bicep(self): schema_properties.update(params_schema) # For each arm template, generate nf application - if isinstance(processor, BaseArmBuildProcessor): + if isinstance(processor, NexusArmBuildProcessor): arm_nf_application_list.append(nf_application) print(nf_application) @@ -263,14 +268,13 @@ def build_resource_bicep(self): return bicep_file def build_all_parameters_json(self): + """Build the all parameters json file.""" params_content = { "location": self.config.location, "publisherName": self.config.publisher_name, "publisherResourceGroupName": self.config.publisher_resource_group_name, "acrArtifactStoreName": self.config.acr_artifact_store_name, - # "saArtifactStoreName": self.config.blob_artifact_store_name, "acrManifestName": self.config.acr_artifact_store_name + "-manifest", - # "saManifestName": self.config.blob_artifact_store_name + "-manifest", "nfDefinitionGroup": self.config.nf_name, "nfDefinitionVersion": self.config.version } @@ -280,6 +284,15 @@ def build_all_parameters_json(self): return base_file def _split_image_path(self, image): + """Split the image path into source acr registry, name and version.""" (source_acr_registry, name_and_version) = image.split("/", 2) (name, version) = name_and_version.split(":", 2) return (source_acr_registry, name, version) + + def _create_semver_compatible_version(self, version): + """Create a semver compatible version.""" + if re.search(SEMVER_REGEX, version): + return version + else: + print(f"Invalid version {version}, using 1.0.0 as default") + return '1.0.0' diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_vnf_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_vnf_handler.py index 28e3be83ebd..3fcb68d3406 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_vnf_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_vnf_handler.py @@ -2,50 +2,10 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- -import json -from pathlib import Path -from typing import Dict, Any -from knack.log import get_logger - -from azext_aosm.build_processors.arm_processor import ( - AzureCoreArmBuildProcessor, BaseArmBuildProcessor, NexusArmBuildProcessor) -from azext_aosm.build_processors.vhd_processor import VHDProcessor -from azext_aosm.common.constants import (ARTIFACT_LIST_FILENAME, - BASE_FOLDER_NAME, - MANIFEST_FOLDER_NAME, - NF_DEFINITION_FOLDER_NAME, - VNF_BASE_TEMPLATE_FILENAME, - VNF_TEMPLATE_FOLDER_NAME, - VNF_DEFINITION_TEMPLATE_FILENAME, - VNF_INPUT_FILENAME, - VNF_MANIFEST_TEMPLATE_FILENAME, - VNF_OUTPUT_FOLDER_FILENAME, - DEPLOYMENT_PARAMETERS_FILENAME, - VHD_PARAMETERS_FILENAME, - TEMPLATE_PARAMETERS_FILENAME) -from azext_aosm.common.local_file_builder import LocalFileBuilder -# from azext_aosm.configuration_models.onboarding_vnf_input_config import ( -# OnboardingVNFInputConfig, -# ) -# from azext_aosm.configuration_models.common_parameters_config import ( -# VNFCommonParametersConfig, -# ) -from azext_aosm.definition_folder.builder.artifact_builder import ( - ArtifactDefinitionElementBuilder, -) -from azext_aosm.definition_folder.builder.bicep_builder import ( - BicepDefinitionElementBuilder, -) -from azext_aosm.definition_folder.builder.json_builder import ( - JSONDefinitionElementBuilder, -) -from azext_aosm.inputs.arm_template_input import ArmTemplateInput -from azext_aosm.inputs.vhd_file_input import VHDFileInput +from azext_aosm.common.constants import VNF_INPUT_FILENAME, VNF_OUTPUT_FOLDER_FILENAME from .onboarding_nfd_base_handler import OnboardingNFDBaseCLIHandler -logger = get_logger(__name__) - class OnboardingVNFCLIHandler(OnboardingNFDBaseCLIHandler): """CLI handler for publishing NFDs.""" @@ -59,244 +19,3 @@ def default_config_file_name(self) -> str: def output_folder_file_name(self) -> str: """Get the output folder file name.""" return VNF_OUTPUT_FOLDER_FILENAME - - # def _get_input_config(self, input_config: Dict[str, Any] = None) -> OnboardingVNFInputConfig: - # """Get the configuration for the command.""" - # if input_config is None: - # input_config = {} - # return OnboardingVNFInputConfig(**input_config) - - # def _get_params_config( - # self, config_file: dict = None - # ) -> VNFCommonParametersConfig: - # """Get the configuration for the command.""" - # with open(config_file, "r", encoding="utf-8") as _file: - # params_dict = json.load(_file) - # if params_dict is None: - # params_dict = {} - # return VNFCommonParametersConfig(**params_dict) - - # def _get_processor_list(self): - # processor_list = [] - # # for each arm template, instantiate arm processor - # for arm_template in self.config.arm_templates: - # arm_input = ArmTemplateInput( - # artifact_name=arm_template.artifact_name, - # artifact_version=arm_template.version, - # default_config=None, - # template_path=Path(arm_template.file_path).absolute(), - # ) - # # TODO: generalise for nexus in nexus ready stories - # if self.config.nfvi_type == "AzureCore": - # processor_list.append( - # AzureCoreArmBuildProcessor(arm_input.artifact_name, arm_input) - # ) - # elif self.config.nfvi_type == "AzureOperatorNexus": - # processor_list.append( - # NexusArmBuildProcessor(arm_input.artifact_name, arm_input) - # ) - - # # Instantiate vhd processor - # if not self.config.vhd.artifact_name: - # self.config.vhd.artifact_name = self.config.nf_name + "-vhd" - # vhd_processor = VHDProcessor( - # name=self.config.vhd.artifact_name, - # input_artifact=VHDFileInput( - # artifact_name=self.config.vhd.artifact_name, - # artifact_version=self.config.vhd.version, - # default_config=self._get_default_config(self.config.vhd), - # file_path=Path(self.config.vhd.file_path).absolute(), - # blob_sas_uri=self.config.vhd.blob_sas_url, - # ), - # ) - # processor_list.append(vhd_processor) - # return processor_list - - def build_base_bicep(self): - """Build the base bicep file.""" - # Build manifest bicep contents, with j2 template - template_path = self._get_template_path( - VNF_TEMPLATE_FOLDER_NAME, VNF_BASE_TEMPLATE_FILENAME - ) - bicep_contents = self._render_base_bicep_contents(template_path) - # Create Bicep element with manifest contents - bicep_file = BicepDefinitionElementBuilder( - Path(VNF_OUTPUT_FOLDER_FILENAME, BASE_FOLDER_NAME), bicep_contents - ) - return bicep_file - - # def build_manifest_bicep(self): - # """Build the manifest bicep file.""" - # acr_artifact_list = [] - - # logger.info("Creating artifact manifest bicep") - - # for processor in self.processors: - # if isinstance(processor, BaseArmBuildProcessor): - # acr_artifact_list.extend(processor.get_artifact_manifest_list()) - # logger.debug( - # "Created list of artifacts from %s arm template(s) provided: %s", - # len(self.config.arm_templates), - # acr_artifact_list, - # ) - # elif isinstance(processor, VHDProcessor): - # sa_artifact_list = processor.get_artifact_manifest_list() - # logger.debug( - # "Created list of artifacts from vhd image provided: %s", - # sa_artifact_list, - # ) - - # # Build manifest bicep contents, with j2 template - # template_path = self._get_template_path( - # VNF_TEMPLATE_FOLDER_NAME, VNF_MANIFEST_TEMPLATE_FILENAME - # ) - # bicep_contents = self._render_manifest_bicep_contents( - # template_path, acr_artifact_list, sa_artifact_list - # ) - # # Create Bicep element with manifest contents - # bicep_file = BicepDefinitionElementBuilder( - # Path(VNF_OUTPUT_FOLDER_FILENAME, MANIFEST_FOLDER_NAME), - # bicep_contents, - # ) - - # logger.info("Created artifact manifest bicep element") - # return bicep_file - - # def build_artifact_list(self): - # """Build the artifact list.""" - # logger.info("Creating artifacts list for artifacts.json") - # artifact_list = [] - # # For each arm template, get list of artifacts and combine - # for processor in self.processors: - # (artifacts, _) = processor.get_artifact_details() - # if artifacts not in artifact_list: - # artifact_list.extend(artifacts) - # logger.debug( - # "Created list of artifact details from %s arm template(s) and the vhd image provided: %s", - # len(self.config.arm_templates), - # artifact_list, - # ) - - # # Generate Artifact Element with artifact list (of arm template and vhd images) - # return ArtifactDefinitionElementBuilder( - # Path(VNF_OUTPUT_FOLDER_FILENAME, ARTIFACT_LIST_FILENAME), artifact_list - # ) - - # def build_resource_bicep(self): - # """Build the resource bicep file.""" - # logger.info("Creating artifacts list for artifacts.json") - # acr_nf_application_list = [] - # sa_nf_application_list = [] - # supporting_files = [] - # schema_properties = {} - - # for processor in self.processors: - # # Generate NF Application - # nf_application = processor.generate_nf_application() - # logger.debug("Created nf application %s", nf_application.name) - - # # Generate deploymentParameters schema properties - # params_schema = processor.generate_params_schema() - # schema_properties.update(params_schema) - - # # For each arm template, generate nf application - # if isinstance(processor, BaseArmBuildProcessor): - - # acr_nf_application_list.append(nf_application) - # print(nf_application) - # print(nf_application.deploy_parameters_mapping_rule_profile) - # # Generate local file for template_parameters + add to supporting files list - # params = ( - # nf_application.deploy_parameters_mapping_rule_profile.template_mapping_rule_profile.template_parameters - # ) - # template_name = TEMPLATE_PARAMETERS_FILENAME - # logger.info( - # "Created templatateParameters as supporting file for nfDefinition bicep" - # ) - # elif isinstance(processor, VHDProcessor): - # sa_nf_application_list.append(nf_application) - # print(nf_application) - # print(nf_application.deploy_parameters_mapping_rule_profile) - # # Generate local file for vhd_parameters - # params = ( - # nf_application.deploy_parameters_mapping_rule_profile.vhd_image_mapping_rule_profile.user_configuration - # ) - # template_name = VHD_PARAMETERS_FILENAME - # else: - # raise TypeError(f"Type: {type(processor)} is not valid") - - # parameters_file = LocalFileBuilder( - # Path( - # VNF_OUTPUT_FOLDER_FILENAME, - # NF_DEFINITION_FOLDER_NAME, - # template_name, - # ), - # json.dumps(json.loads(params), indent=4), - # ) - # supporting_files.append(parameters_file) - - # # Create bicep contents using vnf defintion j2 template - # template_path = self._get_template_path( - # VNF_TEMPLATE_FOLDER_NAME, VNF_DEFINITION_TEMPLATE_FILENAME - # ) - - # params = { - # "nfvi_type": self.config.nfvi_type, - # "acr_nf_applications": acr_nf_application_list, - # "sa_nf_application": sa_nf_application_list[0], - # "deployment_parameters_file": DEPLOYMENT_PARAMETERS_FILENAME, - # "vhd_parameters_file": VHD_PARAMETERS_FILENAME, - # "template_parameters_file": TEMPLATE_PARAMETERS_FILENAME - # } - # bicep_contents = self._render_definition_bicep_contents( - # template_path, params - # ) - - # # Create a bicep element - # # + add its supporting files (deploymentParameters, vhdParameters and templateParameters) - # bicep_file = BicepDefinitionElementBuilder( - # Path(VNF_OUTPUT_FOLDER_FILENAME, NF_DEFINITION_FOLDER_NAME), - # bicep_contents, - # ) - # for supporting_file in supporting_files: - # bicep_file.add_supporting_file(supporting_file) - - # # Add the deploymentParameters schema file - # bicep_file.add_supporting_file( - # self._render_deployment_params_schema( - # schema_properties, VNF_OUTPUT_FOLDER_FILENAME, NF_DEFINITION_FOLDER_NAME - # ) - # ) - # return bicep_file - - # def build_all_parameters_json(self): - # params_content = { - # "location": self.config.location, - # "publisherName": self.config.publisher_name, - # "publisherResourceGroupName": self.config.publisher_resource_group_name, - # "acrArtifactStoreName": self.config.acr_artifact_store_name, - # "saArtifactStoreName": self.config.blob_artifact_store_name, - # "acrManifestName": self.config.acr_artifact_store_name + "-manifest", - # "saManifestName": self.config.blob_artifact_store_name + "-manifest", - # "nfDefinitionGroup": self.config.nf_name, - # "nfDefinitionVersion": self.config.version - # } - # base_file = JSONDefinitionElementBuilder( - # Path(VNF_OUTPUT_FOLDER_FILENAME), json.dumps(params_content, indent=4) - # ) - # return base_file - - # def _get_default_config(self, vhd): - # default_config = {} - # if vhd.image_disk_size_GB: - # default_config.update({"image_disk_size_GB": vhd.image_disk_size_GB}) - # if vhd.image_hyper_v_generation: - # default_config.update( - # {"image_hyper_v_generation": vhd.image_hyper_v_generation} - # ) - # else: - # # Default to V1 if not specified - # default_config.update({"image_hyper_v_generation": "V1"}) - # if vhd.image_api_version: - # default_config.update({"image_api_version": vhd.image_api_version}) - # return default_config diff --git a/src/aosm/azext_aosm/common/constants.py b/src/aosm/azext_aosm/common/constants.py index 3a125fe5b63..ebe9f0baa2d 100644 --- a/src/aosm/azext_aosm/common/constants.py +++ b/src/aosm/azext_aosm/common/constants.py @@ -65,7 +65,8 @@ class ManifestsExist(str, Enum): VNF_INPUT_FILENAME = "vnf-input.jsonc" VNF_DEFINITION_TEMPLATE_FILENAME = "vnfdefinition.bicep.j2" VNF_MANIFEST_TEMPLATE_FILENAME = "vnfartifactmanifest.bicep.j2" -VNF_BASE_TEMPLATE_FILENAME = "vnfbase.bicep" +VNF_CORE_BASE_TEMPLATE_FILENAME = "vnfbase.bicep" +VNF_NEXUS_BASE_TEMPLATE_FILENAME = "vnfnexusbase.bicep" VNF_TEMPLATE_FOLDER_NAME = "vnf" CNF_OUTPUT_FOLDER_FILENAME = "cnf-cli-output" @@ -77,6 +78,9 @@ class ManifestsExist(str, Enum): CNF_VALUES_SCHEMA_FILENAME = "values.schema.json" CNF_TEMPLATE_FOLDER_NAME = "cnf" + +SEMVER_REGEX = r"^(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)(?:-(?P(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$" + ################# # OLD CONSTANTS # ################# diff --git a/src/aosm/azext_aosm/common/templates/vnf/vnfdefinition.bicep.j2 b/src/aosm/azext_aosm/common/templates/vnf/vnfdefinition.bicep.j2 index bef03f26fa1..74b5bdce2d0 100644 --- a/src/aosm/azext_aosm/common/templates/vnf/vnfdefinition.bicep.j2 +++ b/src/aosm/azext_aosm/common/templates/vnf/vnfdefinition.bicep.j2 @@ -6,8 +6,10 @@ param location string param publisherName string @description('Name of an existing ACR-backed Artifact Store, deployed under the publisher.') param acrArtifactStoreName string +{%- if sa_nf_applications %} @description('Name of an existing Storage Account-backed Artifact Store, deployed under the publisher.') param saArtifactStoreName string +{%- endif %} @description('Name of an existing Network Function Definition Group') param nfDefinitionGroup string @description('The version of the NFDV you want to deploy, in format A.B.C') @@ -27,12 +29,14 @@ resource acrArtifactStore 'Microsoft.HybridNetwork/publishers/artifactStores@202 name: acrArtifactStoreName } +{%- if sa_nf_applications %} // The storage account is the resource in which the VHD images are stored. // If using publish command, this is created from deploying the vnfbase.bicep resource saArtifactStore 'Microsoft.HybridNetwork/publishers/artifactStores@2023-09-01' existing = { parent: publisher name: saArtifactStoreName } +{%- endif %} // The NFD Group is the parent resource under which all NFD versions will be created. // If using publish command, this is created from deploying the vnfbase.bicep diff --git a/src/aosm/azext_aosm/common/templates/vnf/vnfnexusbase.bicep b/src/aosm/azext_aosm/common/templates/vnf/vnfnexusbase.bicep new file mode 100644 index 00000000000..be16ebaf457 --- /dev/null +++ b/src/aosm/azext_aosm/common/templates/vnf/vnfnexusbase.bicep @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft Corporation. + +// This file creates the base AOSM resources for a Nexus VNF +param location string +@description('Name of a publisher, expected to be in the resource group where you deploy the template') +param publisherName string +@description('Name of an ACR-backed Artifact Store, deployed under the publisher.') +param acrArtifactStoreName string +@description('Name of a Network Function Definition Group') +param nfDefinitionGroup string + +// The publisher resource is the top level AOSM resource under which all other designer resources +// are created. +resource publisher 'Microsoft.HybridNetwork/publishers@2023-09-01' = { + name: publisherName + location: location + properties: { scope: 'Private'} +} + +// The artifact store is the resource in which all the artifacts required to deploy the NF are stored. +resource acrArtifactStore 'Microsoft.HybridNetwork/publishers/artifactStores@2023-09-01' = { + parent: publisher + name: acrArtifactStoreName + location: location + properties: { + storeType: 'AzureContainerRegistry' + } +} + +// The NFD Group is the parent resource under which all NFD versions will be created. +resource nfdg 'Microsoft.Hybridnetwork/publishers/networkfunctiondefinitiongroups@2023-09-01' = { + parent: publisher + name: nfDefinitionGroup + location: location +} From a723cdb0aa422a2217500926dbd2735375476bc6 Mon Sep 17 00:00:00 2001 From: Jordan Date: Fri, 2 Feb 2024 15:03:04 +0000 Subject: [PATCH 10/37] moved render bicep to common/utils; replaced build base bicep with render bicep --- .../cli_handlers/onboarding_base_handler.py | 20 ---------------- .../cli_handlers/onboarding_cnf_handler.py | 17 ++++++------- .../onboarding_core_vnf_handler.py | 22 ++++++++--------- .../onboarding_nexus_vnf_handler.py | 13 +++++----- .../cli_handlers/onboarding_nsd_handler.py | 19 +++++++-------- src/aosm/azext_aosm/common/utils.py | 24 ++++++++++++++++--- 6 files changed, 57 insertions(+), 58 deletions(-) diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_base_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_base_handler.py index 427b3f4824c..cfb84ec852b 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_base_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_base_handler.py @@ -189,26 +189,6 @@ def _render_base_bicep_contents(self, template_path): bicep_contents: str = template.render() return bicep_contents - def _render_bicep_contents_from_j2(self, template_path: Path, params): - """Write the definition bicep file from given template.""" - with open(template_path, "r", encoding="UTF-8") as f: - template: Template = Template( - f.read(), - undefined=StrictUndefined, - ) - - bicep_contents: str = template.render(params) - return bicep_contents - - def _get_template_path(self, definition_type: str, template_name: str) -> Path: - """Get the path to a template.""" - return ( - Path(__file__).parent.parent - / "common" - / "templates" - / definition_type - / template_name - ) def _serialize(self, dataclass, indent_count=1): """ diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_cnf_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_cnf_handler.py index 8c4332de470..fe70e00c758 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_cnf_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_cnf_handler.py @@ -50,6 +50,7 @@ ) from .onboarding_nfd_base_handler import OnboardingNFDBaseCLIHandler +from azext_aosm.common.utils import render_bicep_contents_from_j2, get_template_path logger = get_logger(__name__) yaml_processor = ruamel.yaml.YAML(typ="safe", pure=True) @@ -127,7 +128,7 @@ def _validate_helm_template(self): if validation_errors: # Create an error file using a j2 template - error_output_template_path = self._get_template_path( + error_output_template_path = get_template_path( CNF_TEMPLATE_FOLDER_NAME, CNF_HELM_VALIDATION_ERRORS_TEMPLATE_FILENAME ) @@ -168,10 +169,10 @@ def pre_validate_build(self): def build_base_bicep(self): """Build the base bicep file.""" # Build manifest bicep contents, with j2 template - template_path = self._get_template_path( + template_path = get_template_path( CNF_TEMPLATE_FOLDER_NAME, CNF_BASE_TEMPLATE_FILENAME ) - bicep_contents = self._render_base_bicep_contents(template_path) + bicep_contents = render_bicep_contents_from_j2(template_path, {}) # Create Bicep element with base contents bicep_file = BicepDefinitionElementBuilder( Path(CNF_OUTPUT_FOLDER_FILENAME, BASE_FOLDER_NAME), bicep_contents @@ -194,15 +195,15 @@ def build_manifest_bicep(self): artifact_list, ) # Build manifest bicep contents, with j2 template - template_path = self._get_template_path( + template_path = get_template_path( CNF_TEMPLATE_FOLDER_NAME, CNF_MANIFEST_TEMPLATE_FILENAME ) - + params = { "acr_artifacts": artifact_list, "sa_artifacts": [] } - bicep_contents = self._render_bicep_contents_from_j2(template_path, params) + bicep_contents = render_bicep_contents_from_j2(template_path, params) # Create Bicep element with manifest contents bicep_file = BicepDefinitionElementBuilder( @@ -261,7 +262,7 @@ def build_resource_bicep(self): mappings_files.append(mapping_file) # Create bicep contents using cnf defintion j2 template - template_path = self._get_template_path( + template_path = get_template_path( CNF_TEMPLATE_FOLDER_NAME, CNF_DEFINITION_TEMPLATE_FILENAME ) @@ -269,7 +270,7 @@ def build_resource_bicep(self): "acr_nf_applications": nf_application_list, "deployment_parameters_file": DEPLOYMENT_PARAMETERS_FILENAME, } - bicep_contents = self._render_bicep_contents_from_j2(template_path, params) + bicep_contents = render_bicep_contents_from_j2(template_path, params) # Create a bicep element + add its supporting mapping files bicep_file = BicepDefinitionElementBuilder( diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py index f1fdba56451..8669db2ad0a 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py @@ -8,7 +8,7 @@ from knack.log import get_logger from azext_aosm.build_processors.arm_processor import ( - AzureCoreArmBuildProcessor, BaseArmBuildProcessor) + AzureCoreArmBuildProcessor) from azext_aosm.build_processors.vhd_processor import VHDProcessor from azext_aosm.common.constants import (ARTIFACT_LIST_FILENAME, BASE_FOLDER_NAME, @@ -41,7 +41,7 @@ ) from azext_aosm.inputs.arm_template_input import ArmTemplateInput from azext_aosm.inputs.vhd_file_input import VHDFileInput - +from azext_aosm.common.utils import render_bicep_contents_from_j2, get_template_path # from .onboarding_nfd_base_handler import OnboardingNFDBaseCLIHandler from .onboarding_vnf_handler import OnboardingVNFCLIHandler logger = get_logger(__name__) @@ -101,10 +101,10 @@ def _get_processor_list(self): def build_base_bicep(self): """Build the base bicep file.""" # Build manifest bicep contents, with j2 template - template_path = self._get_template_path( + template_path = get_template_path( VNF_TEMPLATE_FOLDER_NAME, VNF_CORE_BASE_TEMPLATE_FILENAME ) - bicep_contents = self._render_base_bicep_contents(template_path) + bicep_contents = render_bicep_contents_from_j2(template_path, {}) # Create Bicep element with manifest contents bicep_file = BicepDefinitionElementBuilder( Path(VNF_OUTPUT_FOLDER_FILENAME, BASE_FOLDER_NAME), bicep_contents @@ -118,7 +118,7 @@ def build_manifest_bicep(self): logger.info("Creating artifact manifest bicep") for processor in self.processors: - if isinstance(processor, BaseArmBuildProcessor): + if isinstance(processor, AzureCoreArmBuildProcessor): acr_artifact_list.extend(processor.get_artifact_manifest_list()) logger.debug( "Created list of artifacts from %s arm template(s) provided: %s", @@ -133,14 +133,14 @@ def build_manifest_bicep(self): ) # Build manifest bicep contents, with j2 template - template_path = self._get_template_path( + template_path = get_template_path( VNF_TEMPLATE_FOLDER_NAME, VNF_MANIFEST_TEMPLATE_FILENAME ) params = { "acr_artifacts": acr_artifact_list, "sa_artifacts": sa_artifact_list, } - bicep_contents = self._render_bicep_contents_from_j2(template_path, params) + bicep_contents = render_bicep_contents_from_j2(template_path, params) # Create Bicep element with manifest contents bicep_file = BicepDefinitionElementBuilder( @@ -189,7 +189,7 @@ def build_resource_bicep(self): schema_properties.update(params_schema) # For each arm template, generate nf application - if isinstance(processor, BaseArmBuildProcessor): + if isinstance(processor, AzureCoreArmBuildProcessor): acr_nf_application_list.append(nf_application) # Generate local file for template_parameters + add to supporting files list @@ -223,7 +223,7 @@ def build_resource_bicep(self): supporting_files.append(parameters_file) # Create bicep contents using vnf defintion j2 template - template_path = self._get_template_path( + template_path = get_template_path( VNF_TEMPLATE_FOLDER_NAME, VNF_DEFINITION_TEMPLATE_FILENAME ) @@ -236,7 +236,7 @@ def build_resource_bicep(self): "vhd_parameters_file": VHD_PARAMETERS_FILENAME, "template_parameters_file": TEMPLATE_PARAMETERS_FILENAME } - bicep_contents = self._render_bicep_contents_from_j2( + bicep_contents = render_bicep_contents_from_j2( template_path, params ) @@ -258,7 +258,7 @@ def build_resource_bicep(self): return bicep_file def build_all_parameters_json(self): - """Create object for all_parameters.json""" + """Create object for all_parameters.json.""" params_content = { "location": self.config.location, "publisherName": self.config.publisher_name, diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py index b3aa954e962..fa8dc36fd7d 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py @@ -43,6 +43,7 @@ from azext_aosm.inputs.arm_template_input import ArmTemplateInput from azext_aosm.inputs.nexus_image_input import NexusImageFileInput from .onboarding_vnf_handler import OnboardingVNFCLIHandler +from azext_aosm.common.utils import render_bicep_contents_from_j2, get_template_path logger = get_logger(__name__) @@ -113,10 +114,10 @@ def _get_processor_list(self): def build_base_bicep(self): """Build the base bicep file.""" # Build manifest bicep contents, with j2 template - template_path = self._get_template_path( + template_path = get_template_path( VNF_TEMPLATE_FOLDER_NAME, VNF_NEXUS_BASE_TEMPLATE_FILENAME ) - bicep_contents = self._render_base_bicep_contents(template_path) + bicep_contents = render_bicep_contents_from_j2(template_path, {}) # Create Bicep element with manifest contents bicep_file = BicepDefinitionElementBuilder( Path(VNF_OUTPUT_FOLDER_FILENAME, BASE_FOLDER_NAME), bicep_contents @@ -138,14 +139,14 @@ def build_manifest_bicep(self): ) # Build manifest bicep contents, with j2 template - template_path = self._get_template_path( + template_path = get_template_path( VNF_TEMPLATE_FOLDER_NAME, VNF_MANIFEST_TEMPLATE_FILENAME ) params = { "acr_artifacts": acr_artifact_list, "sa_artifacts": [] } - bicep_contents = self._render_bicep_contents_from_j2( + bicep_contents = render_bicep_contents_from_j2( template_path, params ) @@ -233,7 +234,7 @@ def build_resource_bicep(self): supporting_files.append(parameters_file) # Create bicep contents using vnf defintion j2 template - template_path = self._get_template_path( + template_path = get_template_path( VNF_TEMPLATE_FOLDER_NAME, VNF_DEFINITION_TEMPLATE_FILENAME ) @@ -246,7 +247,7 @@ def build_resource_bicep(self): "template_parameters_file": TEMPLATE_PARAMETERS_FILENAME, "image_parameters_file": NEXUS_IMAGE_PARAMETERS_FILENAME } - bicep_contents = self._render_bicep_contents_from_j2( + bicep_contents = render_bicep_contents_from_j2( template_path, params ) diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_nsd_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_nsd_handler.py index 2612e68d6b7..288a8ee9c38 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_nsd_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_nsd_handler.py @@ -33,9 +33,8 @@ JSONDefinitionElementBuilder from azext_aosm.inputs.arm_template_input import ArmTemplateInput from azext_aosm.inputs.nfd_input import NFDInput -from azext_aosm.vendored_sdks.models import ( - ManifestArtifactFormat, NetworkFunctionDefinitionVersion, - NetworkFunctionDefinitionVersionPropertiesFormat) +from azext_aosm.vendored_sdks.models import NetworkFunctionDefinitionVersion +from azext_aosm.common.utils import render_bicep_contents_from_j2, get_template_path logger = get_logger(__name__) @@ -122,10 +121,10 @@ def _get_processor_list(self): def build_base_bicep(self): """Build the base bicep file.""" # Build base bicep contents, with bicep template - template_path = self._get_template_path( + template_path = get_template_path( NSD_TEMPLATE_FOLDER_NAME, NSD_BASE_TEMPLATE_FILENAME ) - bicep_contents = self._render_base_bicep_contents(template_path) + bicep_contents = render_bicep_contents_from_j2(template_path, {}) # Create Bicep element with manifest contents bicep_file = BicepDefinitionElementBuilder( Path(NSD_OUTPUT_FOLDER_FILENAME, BASE_FOLDER_NAME), bicep_contents @@ -141,14 +140,14 @@ def build_manifest_bicep(self): "Created list of artifacts from resource element(s) provided: %s", artifact_list, ) - template_path = self._get_template_path( + template_path = get_template_path( NSD_TEMPLATE_FOLDER_NAME, NSD_MANIFEST_TEMPLATE_FILENAME ) params = { "acr_artifacts": artifact_list, "sa_artifacts": [] } - bicep_contents = self._render_bicep_contents_from_j2(template_path, params) + bicep_contents = render_bicep_contents_from_j2(template_path, params) bicep_file = BicepDefinitionElementBuilder( Path(NSD_OUTPUT_FOLDER_FILENAME, MANIFEST_FOLDER_NAME), bicep_contents @@ -210,7 +209,7 @@ def build_resource_bicep(self): # List of NF RET names, for adding to required part of CGS nf_names.append(processor.name) - template_path = self._get_template_path( + template_path = get_template_path( NSD_TEMPLATE_FOLDER_NAME, NSD_DEFINITION_TEMPLATE_FILENAME ) @@ -224,7 +223,7 @@ def build_resource_bicep(self): "template_parameters_file": TEMPLATE_PARAMETERS_FILENAME, } - bicep_contents = self._render_bicep_contents_from_j2( + bicep_contents = render_bicep_contents_from_j2( template_path, params ) # Generate the nsd bicep file @@ -244,7 +243,7 @@ def build_resource_bicep(self): return bicep_file def build_all_parameters_json(self): - # TODO: add common params for build resource bicep + """Build all parameters json.""" params_content = { "location": self.config.location, "publisherName": self.config.publisher_name, diff --git a/src/aosm/azext_aosm/common/utils.py b/src/aosm/azext_aosm/common/utils.py index 6e6fc7fe15c..39f39683b43 100644 --- a/src/aosm/azext_aosm/common/utils.py +++ b/src/aosm/azext_aosm/common/utils.py @@ -6,6 +6,7 @@ import os import tarfile from pathlib import Path +from jinja2 import StrictUndefined, Template from typing import Any, Dict, Iterable import json import shutil @@ -59,10 +60,27 @@ def convert_bicep_to_arm(bicep_template_path: Path) -> dict: return arm_json +def render_bicep_contents_from_j2(template_path: Path, params): + """Write the definition bicep file from given template.""" + with open(template_path, "r", encoding="UTF-8") as f: + template: Template = Template( + f.read(), + undefined=StrictUndefined, + ) + + bicep_contents: str = template.render(params) + return bicep_contents + -def create_bicep_from_template(): - # Take j2 template, take params, return bicep file - return NotImplementedError +def get_template_path(definition_type: str, template_name: str) -> Path: + """Get the path to a template.""" + return ( + Path(__file__).parent.parent + / "common" + / "templates" + / definition_type + / template_name + ) def extract_tarfile(file_path: Path, target_dir: Path) -> Path: From 5e229d5da5dec6409666512a7a9dcc4e1ece2023 Mon Sep 17 00:00:00 2001 From: Jordan Date: Fri, 2 Feb 2024 15:20:01 +0000 Subject: [PATCH 11/37] general tidyup: removed prints, added return types --- .../cli_handlers/onboarding_cnf_handler.py | 10 +++---- .../onboarding_core_vnf_handler.py | 30 +++++++++---------- .../onboarding_nexus_vnf_handler.py | 23 ++++++-------- .../cli_handlers/onboarding_nsd_handler.py | 12 ++++---- src/aosm/azext_aosm/common/utils.py | 1 + 5 files changed, 35 insertions(+), 41 deletions(-) diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_cnf_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_cnf_handler.py index fe70e00c758..ec3f9fcbad1 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_cnf_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_cnf_handler.py @@ -166,7 +166,7 @@ def pre_validate_build(self): if self.skip != HELM_TEMPLATE: self._validate_helm_template() - def build_base_bicep(self): + def build_base_bicep(self) -> BicepDefinitionElementBuilder: """Build the base bicep file.""" # Build manifest bicep contents, with j2 template template_path = get_template_path( @@ -179,7 +179,7 @@ def build_base_bicep(self): ) return bicep_file - def build_manifest_bicep(self): + def build_manifest_bicep(self) -> BicepDefinitionElementBuilder: """Build the manifest bicep file.""" artifact_list = [] logger.info("Creating artifact manifest bicep") @@ -212,7 +212,7 @@ def build_manifest_bicep(self): return bicep_file - def build_artifact_list(self): + def build_artifact_list(self) -> ArtifactDefinitionElementBuilder: """Build the artifact list.""" artifact_list = [] # For each helm package, get list of artifacts and combine @@ -231,7 +231,7 @@ def build_artifact_list(self): Path(CNF_OUTPUT_FOLDER_FILENAME, ARTIFACT_LIST_FILENAME), artifact_list ) - def build_resource_bicep(self): + def build_resource_bicep(self) -> BicepDefinitionElementBuilder: """Build the resource bicep file.""" logger.info("Creating artifacts list for artifacts.json") nf_application_list = [] @@ -287,7 +287,7 @@ def build_resource_bicep(self): ) return bicep_file - def build_all_parameters_json(self): + def build_all_parameters_json(self) -> JSONDefinitionElementBuilder: """Build the all parameters json file.""" params_content = { "location": self.config.location, diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py index 8669db2ad0a..dc70f1c330a 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py @@ -7,9 +7,9 @@ from typing import Dict, Any from knack.log import get_logger -from azext_aosm.build_processors.arm_processor import ( - AzureCoreArmBuildProcessor) +from azext_aosm.build_processors.arm_processor import AzureCoreArmBuildProcessor from azext_aosm.build_processors.vhd_processor import VHDProcessor +from azext_aosm.build_processors.base_processor import BaseInputProcessor from azext_aosm.common.constants import (ARTIFACT_LIST_FILENAME, BASE_FOLDER_NAME, MANIFEST_FOLDER_NAME, @@ -17,7 +17,6 @@ VNF_CORE_BASE_TEMPLATE_FILENAME, VNF_TEMPLATE_FOLDER_NAME, VNF_DEFINITION_TEMPLATE_FILENAME, - VNF_INPUT_FILENAME, VNF_MANIFEST_TEMPLATE_FILENAME, VNF_OUTPUT_FOLDER_FILENAME, DEPLOYMENT_PARAMETERS_FILENAME, @@ -42,7 +41,7 @@ from azext_aosm.inputs.arm_template_input import ArmTemplateInput from azext_aosm.inputs.vhd_file_input import VHDFileInput from azext_aosm.common.utils import render_bicep_contents_from_j2, get_template_path -# from .onboarding_nfd_base_handler import OnboardingNFDBaseCLIHandler + from .onboarding_vnf_handler import OnboardingVNFCLIHandler logger = get_logger(__name__) @@ -50,7 +49,9 @@ class OnboardingCoreVNFCLIHandler(OnboardingVNFCLIHandler): """CLI handler for publishing NFDs.""" - def _get_input_config(self, input_config: Dict[str, Any] = None) -> OnboardingCoreVNFInputConfig: + def _get_input_config( + self, input_config: Dict[str, Any] = None + ) -> OnboardingCoreVNFInputConfig: """Get the configuration for the command.""" if input_config is None: input_config = {} @@ -66,8 +67,8 @@ def _get_params_config( params_dict = {} return CoreVNFCommonParametersConfig(**params_dict) - def _get_processor_list(self): - print("in get processor") + def _get_processor_list(self) -> [BaseInputProcessor]: + """Get the list of processors.""" processor_list = [] # for each arm template, instantiate arm processor for arm_template in self.config.arm_templates: @@ -77,7 +78,6 @@ def _get_processor_list(self): default_config=None, template_path=Path(arm_template.file_path).absolute(), ) - # TODO: test azure core vnf still works processor_list.append( AzureCoreArmBuildProcessor(arm_input.artifact_name, arm_input) ) @@ -98,7 +98,7 @@ def _get_processor_list(self): processor_list.append(vhd_processor) return processor_list - def build_base_bicep(self): + def build_base_bicep(self) -> BicepDefinitionElementBuilder: """Build the base bicep file.""" # Build manifest bicep contents, with j2 template template_path = get_template_path( @@ -111,7 +111,7 @@ def build_base_bicep(self): ) return bicep_file - def build_manifest_bicep(self): + def build_manifest_bicep(self) -> BicepDefinitionElementBuilder: """Build the manifest bicep file.""" acr_artifact_list = [] @@ -151,7 +151,7 @@ def build_manifest_bicep(self): logger.info("Created artifact manifest bicep element") return bicep_file - def build_artifact_list(self): + def build_artifact_list(self) -> ArtifactDefinitionElementBuilder: """Build the artifact list.""" logger.info("Creating artifacts list for artifacts.json") artifact_list = [] @@ -171,7 +171,7 @@ def build_artifact_list(self): Path(VNF_OUTPUT_FOLDER_FILENAME, ARTIFACT_LIST_FILENAME), artifact_list ) - def build_resource_bicep(self): + def build_resource_bicep(self) -> BicepDefinitionElementBuilder: """Build the resource bicep file.""" logger.info("Creating artifacts list for artifacts.json") acr_nf_application_list = [] @@ -202,8 +202,6 @@ def build_resource_bicep(self): ) elif isinstance(processor, VHDProcessor): sa_nf_application_list.append(nf_application) - print(nf_application) - print(nf_application.deploy_parameters_mapping_rule_profile.application_enablement.value) # Generate local file for vhd_parameters params = ( nf_application.deploy_parameters_mapping_rule_profile.vhd_image_mapping_rule_profile.user_configuration @@ -257,7 +255,7 @@ def build_resource_bicep(self): ) return bicep_file - def build_all_parameters_json(self): + def build_all_parameters_json(self) -> JSONDefinitionElementBuilder: """Create object for all_parameters.json.""" params_content = { "location": self.config.location, @@ -275,7 +273,7 @@ def build_all_parameters_json(self): ) return base_file - def _get_default_config(self, vhd): + def _get_default_config(self, vhd) -> Dict[str, Any]: """Get default VHD config for Azure Core VNF.""" default_config = {} if vhd.image_disk_size_GB: diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py index fa8dc36fd7d..40097fea2cb 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py @@ -10,6 +10,7 @@ from azext_aosm.build_processors.arm_processor import NexusArmBuildProcessor from azext_aosm.build_processors.nexus_image_processor import NexusImageProcessor +from azext_aosm.build_processors.base_processor import BaseInputProcessor from azext_aosm.common.constants import (ARTIFACT_LIST_FILENAME, BASE_FOLDER_NAME, MANIFEST_FOLDER_NAME, @@ -79,7 +80,7 @@ def _get_params_config( params_dict = {} return NexusVNFCommonParametersConfig(**params_dict) - def _get_processor_list(self): + def _get_processor_list(self) -> [BaseInputProcessor]: processor_list = [] # for each arm template, instantiate arm processor for arm_template in self.config.arm_templates: @@ -111,7 +112,7 @@ def _get_processor_list(self): return processor_list - def build_base_bicep(self): + def build_base_bicep(self) -> BicepDefinitionElementBuilder: """Build the base bicep file.""" # Build manifest bicep contents, with j2 template template_path = get_template_path( @@ -124,7 +125,7 @@ def build_base_bicep(self): ) return bicep_file - def build_manifest_bicep(self): + def build_manifest_bicep(self) -> BicepDefinitionElementBuilder: """Build the manifest bicep file.""" acr_artifact_list = [] @@ -132,7 +133,6 @@ def build_manifest_bicep(self): for processor in self.processors: acr_artifact_list.extend(processor.get_artifact_manifest_list()) - print("ACRLSIT", acr_artifact_list) logger.debug( "Created list of artifacts from arm template(s) and image files(s) provided: %s", acr_artifact_list, @@ -159,7 +159,7 @@ def build_manifest_bicep(self): logger.info("Created artifact manifest bicep element") return bicep_file - def build_artifact_list(self): + def build_artifact_list(self) -> ArtifactDefinitionElementBuilder: """Build the artifact list.""" logger.info("Creating artifacts list for artifacts.json") artifact_list = [] @@ -178,7 +178,7 @@ def build_artifact_list(self): Path(VNF_OUTPUT_FOLDER_FILENAME, ARTIFACT_LIST_FILENAME), artifact_list ) - def build_resource_bicep(self): + def build_resource_bicep(self) -> BicepDefinitionElementBuilder: """Build the resource bicep file.""" logger.info("Creating artifacts list for artifacts.json") arm_nf_application_list = [] @@ -186,7 +186,6 @@ def build_resource_bicep(self): supporting_files = [] schema_properties = {} - # TODO: fx this for nexus for processor in self.processors: # Generate NF Application nf_application = processor.generate_nf_application() @@ -200,8 +199,6 @@ def build_resource_bicep(self): if isinstance(processor, NexusArmBuildProcessor): arm_nf_application_list.append(nf_application) - print(nf_application) - print(nf_application.deploy_parameters_mapping_rule_profile) # Generate local file for template_parameters + add to supporting files list params = ( nf_application.deploy_parameters_mapping_rule_profile.template_mapping_rule_profile.template_parameters @@ -213,8 +210,6 @@ def build_resource_bicep(self): elif isinstance(processor, NexusImageProcessor): # TODO: fix this image_nf_application_list.append(nf_application) - print(nf_application) - print(nf_application.deploy_parameters_mapping_rule_profile.image_mapping_rule_profile) # Generate local file for vhd_parameters params = ( nf_application.deploy_parameters_mapping_rule_profile.image_mapping_rule_profile.user_configuration @@ -268,7 +263,7 @@ def build_resource_bicep(self): ) return bicep_file - def build_all_parameters_json(self): + def build_all_parameters_json(self) -> JSONDefinitionElementBuilder: """Build the all parameters json file.""" params_content = { "location": self.config.location, @@ -284,13 +279,13 @@ def build_all_parameters_json(self): ) return base_file - def _split_image_path(self, image): + def _split_image_path(self, image) -> (str, str, str): """Split the image path into source acr registry, name and version.""" (source_acr_registry, name_and_version) = image.split("/", 2) (name, version) = name_and_version.split(":", 2) return (source_acr_registry, name, version) - def _create_semver_compatible_version(self, version): + def _create_semver_compatible_version(self, version) -> str: """Create a semver compatible version.""" if re.search(SEMVER_REGEX, version): return version diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_nsd_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_nsd_handler.py index 288a8ee9c38..702193c4fde 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_nsd_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_nsd_handler.py @@ -73,7 +73,7 @@ def _get_params_config( params_dict = {} return NSDCommonParametersConfig(**params_dict) - def _get_processor_list(self): + def _get_processor_list(self) -> list: processor_list = [] # for each resource element template, instantiate processor for resource_element in self.config.resource_element_templates: @@ -118,7 +118,7 @@ def _get_processor_list(self): raise ValueError(f"Invalid resource element type: {resource_element.resource_element_type}") return processor_list - def build_base_bicep(self): + def build_base_bicep(self) -> BicepDefinitionElementBuilder: """Build the base bicep file.""" # Build base bicep contents, with bicep template template_path = get_template_path( @@ -131,7 +131,7 @@ def build_base_bicep(self): ) return bicep_file - def build_manifest_bicep(self): + def build_manifest_bicep(self) -> BicepDefinitionElementBuilder: """Build the manifest bicep file.""" artifact_list = [] for processor in self.processors: @@ -154,7 +154,7 @@ def build_manifest_bicep(self): ) return bicep_file - def build_artifact_list(self): + def build_artifact_list(self) -> ArtifactDefinitionElementBuilder: """Build the artifact list.""" # Build artifact list for ArmTemplates artifact_list = [] @@ -174,7 +174,7 @@ def build_artifact_list(self): return artifact_file - def build_resource_bicep(self): + def build_resource_bicep(self) -> BicepDefinitionElementBuilder: """Build the resource bicep file.""" bicep_contents = {} schema_properties = {} @@ -242,7 +242,7 @@ def build_resource_bicep(self): return bicep_file - def build_all_parameters_json(self): + def build_all_parameters_json(self) -> JSONDefinitionElementBuilder: """Build all parameters json.""" params_content = { "location": self.config.location, diff --git a/src/aosm/azext_aosm/common/utils.py b/src/aosm/azext_aosm/common/utils.py index 39f39683b43..0dbad63b5e4 100644 --- a/src/aosm/azext_aosm/common/utils.py +++ b/src/aosm/azext_aosm/common/utils.py @@ -60,6 +60,7 @@ def convert_bicep_to_arm(bicep_template_path: Path) -> dict: return arm_json + def render_bicep_contents_from_j2(template_path: Path, params): """Write the definition bicep file from given template.""" with open(template_path, "r", encoding="UTF-8") as f: From 7628817e7b342353c9a63cc15a9e076f9df03d6d Mon Sep 17 00:00:00 2001 From: Jordan Date: Fri, 2 Feb 2024 16:13:20 +0000 Subject: [PATCH 12/37] added nfvi type to nsds --- .../cli_handlers/onboarding_nsd_handler.py | 1 + .../common/templates/nsd/nsddefinition.bicep.j2 | 2 +- .../onboarding_nsd_input_config.py | 15 ++++++++++++++- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_nsd_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_nsd_handler.py index 702193c4fde..b02c02341e9 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_nsd_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_nsd_handler.py @@ -215,6 +215,7 @@ def build_resource_bicep(self) -> BicepDefinitionElementBuilder: params = { "nsdv_description": self.config.nsdv_description, + "nfvi_type": self.config.nfvi_type, "cgs_name": CGS_NAME, "nfvi_site_name": self.nfvi_site_name, "nf_rets": ret_list, diff --git a/src/aosm/azext_aosm/common/templates/nsd/nsddefinition.bicep.j2 b/src/aosm/azext_aosm/common/templates/nsd/nsddefinition.bicep.j2 index 8460f4c3b55..96d944a5a1e 100644 --- a/src/aosm/azext_aosm/common/templates/nsd/nsddefinition.bicep.j2 +++ b/src/aosm/azext_aosm/common/templates/nsd/nsddefinition.bicep.j2 @@ -71,7 +71,7 @@ resource nsdVersion 'Microsoft.Hybridnetwork/publishers/networkservicedesigngrou nfvisFromSite: { nfvi1: { name: nfviSiteName - type: 'AzureCore' + type: '{{ nfvi_type}}' } } // This field lists the templates that will be deployed by AOSM and the config mappings diff --git a/src/aosm/azext_aosm/configuration_models/onboarding_nsd_input_config.py b/src/aosm/azext_aosm/configuration_models/onboarding_nsd_input_config.py index 6c3151f42a4..4b878fa4b90 100644 --- a/src/aosm/azext_aosm/configuration_models/onboarding_nsd_input_config.py +++ b/src/aosm/azext_aosm/configuration_models/onboarding_nsd_input_config.py @@ -175,6 +175,15 @@ class OnboardingNSDInputConfig(OnboardingBaseInputConfig): "comment": "Optional. Description of the Network Service Design Version (NSDV)." }, ) + nfvi_type: str = field( + default="AzureCore", + metadata={ + "comment": ( + "Type of NFVI (for nfvisFromSite). Defaults to 'AzureCore'.\n" + "Valid values are 'AzureCore', 'AzureOperatorNexus' or 'AzureArcKubernetes." + ) + }, + ) # # TODO: Add detailed comment for this resource_element_templates: "list[NetworkFunctionConfig | ArmTemplateConfig]" = ( @@ -196,7 +205,11 @@ def validate(self): raise ValidationError("nsd_name must be set") if not self.nsd_version: raise ValidationError("nsd_version must be set") - + print("here", self.nfvi_type) + if self.nfvi_type not in ["AzureCore", "AzureOperatorNexus", "AzureArcKubernetes"]: + raise ValidationError( + "nfvi_type must be either 'AzureCore', 'AzureOperatorNexus' or 'AzureArcKubernetes'" + ) # Validate each RET for configuration in self.resource_element_templates: configuration.validate() From 0b20c58fddec0509c09d9932a4d622e4bc2ab570 Mon Sep 17 00:00:00 2001 From: Jordan Date: Fri, 2 Feb 2024 16:15:46 +0000 Subject: [PATCH 13/37] removed old todos --- src/aosm/azext_aosm/build_processors/arm_processor.py | 1 - src/aosm/azext_aosm/build_processors/nexus_image_processor.py | 2 -- src/aosm/azext_aosm/cli_handlers/onboarding_base_handler.py | 1 - .../azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py | 1 - 4 files changed, 5 deletions(-) diff --git a/src/aosm/azext_aosm/build_processors/arm_processor.py b/src/aosm/azext_aosm/build_processors/arm_processor.py index fa01e443dfe..3062a226de5 100644 --- a/src/aosm/azext_aosm/build_processors/arm_processor.py +++ b/src/aosm/azext_aosm/build_processors/arm_processor.py @@ -176,7 +176,6 @@ class NexusArmBuildProcessor(BaseArmBuildProcessor): """ This class represents a processor for generating ARM templates specific to Nexus. """ - # TODO: JORDAN check if we need Arm_template.value here? def generate_nfvi_specific_nf_application( self, ) -> AzureOperatorNexusNetworkFunctionArmTemplateApplication: diff --git a/src/aosm/azext_aosm/build_processors/nexus_image_processor.py b/src/aosm/azext_aosm/build_processors/nexus_image_processor.py index f793db595a6..9e836189982 100644 --- a/src/aosm/azext_aosm/build_processors/nexus_image_processor.py +++ b/src/aosm/azext_aosm/build_processors/nexus_image_processor.py @@ -56,7 +56,6 @@ def get_artifact_manifest_list(self) -> List[ManifestArtifactFormat]: ) ] - # TODO: figure out what artifact type this can be def get_artifact_details( self, ) -> Tuple[List[BaseArtifact], List[LocalFileBuilder]]: @@ -115,7 +114,6 @@ def _generate_artifact_profile(self) -> AzureOperatorNexusImageArtifactProfile: :rtype: AzureOperatorNexusImageArtifactProfile """ logger.debug("Generating artifact profile for Nexus image input.") - # TODO: JORDAN check what inputs this takes artifact_profile = ImageArtifactProfile( image_name=self.input_artifact.artifact_name, image_version=self.input_artifact.artifact_version, diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_base_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_base_handler.py index cfb84ec852b..332578700f9 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_base_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_base_handler.py @@ -65,7 +65,6 @@ def __init__( raise UnclassifiedUserFault(f"The all_deploy.parameters.json in the folder provided contains an incorrect input.\nPlease check if you have provided the correct folder for the definition/design type:\n{e}") from e else: raise UnclassifiedUserFault("Invalid input") - # TODO: Change this to work with publish? # If no config file provided (for generate-config) else: self.config = self._get_input_config() diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py index 40097fea2cb..69fe1fe6dd4 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py @@ -208,7 +208,6 @@ def build_resource_bicep(self) -> BicepDefinitionElementBuilder: "Created templatateParameters as supporting file for nfDefinition bicep" ) elif isinstance(processor, NexusImageProcessor): - # TODO: fix this image_nf_application_list.append(nf_application) # Generate local file for vhd_parameters params = ( From 1c20ecb8ac9f6ea44cb885a7617c3fec6957c992 Mon Sep 17 00:00:00 2001 From: Jordan Date: Tue, 6 Feb 2024 19:27:30 +0000 Subject: [PATCH 14/37] refactored nexus handler, moved generate params code into processor --- .../build_processors/arm_processor.py | 24 ++++++++++ .../build_processors/nexus_image_processor.py | 20 +++++++++ .../onboarding_nexus_vnf_handler.py | 44 ++++--------------- 3 files changed, 52 insertions(+), 36 deletions(-) diff --git a/src/aosm/azext_aosm/build_processors/arm_processor.py b/src/aosm/azext_aosm/build_processors/arm_processor.py index 3062a226de5..3e07941b02c 100644 --- a/src/aosm/azext_aosm/build_processors/arm_processor.py +++ b/src/aosm/azext_aosm/build_processors/arm_processor.py @@ -4,6 +4,7 @@ # -------------------------------------------------------------------------------------------- import json +from pathlib import Path from abc import abstractmethod from typing import List, Tuple, final @@ -27,6 +28,11 @@ AzureOperatorNexusNetworkFunctionApplication, AzureOperatorNexusArtifactType, AzureOperatorNexusArmTemplateDeployMappingRuleProfile, AzureOperatorNexusArmTemplateArtifactProfile) +from azext_aosm.common.constants import ( + VNF_OUTPUT_FOLDER_FILENAME, + NF_DEFINITION_FOLDER_NAME, + TEMPLATE_PARAMETERS_FILENAME) + logger = get_logger(__name__) @@ -213,3 +219,21 @@ def generate_artifact_profile(self) -> AzureOperatorNexusArmTemplateArtifactProf artifact_store=ReferencedResource(id=""), template_artifact_profile=artifact_profile, ) + + def generate_parameters_file(self) -> LocalFileBuilder: + """ Generate parameters file. """ + mapping_rule_profile = self._generate_mapping_rule_profile() + params = ( + mapping_rule_profile.template_mapping_rule_profile.template_parameters + ) + return LocalFileBuilder( + Path( + VNF_OUTPUT_FOLDER_FILENAME, + NF_DEFINITION_FOLDER_NAME, + TEMPLATE_PARAMETERS_FILENAME, + ), + json.dumps(json.loads(params), indent=4), + ) + + def generate_template_parameters(self): + pass \ No newline at end of file diff --git a/src/aosm/azext_aosm/build_processors/nexus_image_processor.py b/src/aosm/azext_aosm/build_processors/nexus_image_processor.py index 9e836189982..b3f438639b8 100644 --- a/src/aosm/azext_aosm/build_processors/nexus_image_processor.py +++ b/src/aosm/azext_aosm/build_processors/nexus_image_processor.py @@ -4,6 +4,7 @@ # -------------------------------------------------------------------------------------------- import json +from pathlib import Path from typing import List, Tuple from knack.log import get_logger @@ -22,6 +23,10 @@ DependsOnProfile, ManifestArtifactFormat, ReferencedResource, ResourceElementTemplate, ) +from azext_aosm.common.constants import ( + VNF_OUTPUT_FOLDER_FILENAME, + NF_DEFINITION_FOLDER_NAME, + NEXUS_IMAGE_PARAMETERS_FILENAME) logger = get_logger(__name__) @@ -146,3 +151,18 @@ def _generate_mapping_rule_profile( application_enablement=ApplicationEnablement.ENABLED, image_mapping_rule_profile=mapping, ) + + def generate_parameters_file(self) -> LocalFileBuilder: + """ Generate parameters file. """ + mapping_rule_profile = self._generate_mapping_rule_profile() + params = ( + mapping_rule_profile.image_mapping_rule_profile.user_configuration + ) + return LocalFileBuilder( + Path( + VNF_OUTPUT_FOLDER_FILENAME, + NF_DEFINITION_FOLDER_NAME, + NEXUS_IMAGE_PARAMETERS_FILENAME, + ), + json.dumps(json.loads(params), indent=4), + ) diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py index 69fe1fe6dd4..e5664b1b085 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py @@ -52,15 +52,6 @@ class OnboardingNexusVNFCLIHandler(OnboardingVNFCLIHandler): """CLI handler for publishing NFDs.""" - @property - def default_config_file_name(self) -> str: - """Get the default configuration file name.""" - return VNF_INPUT_FILENAME - - @property - def output_folder_file_name(self) -> str: - """Get the output folder file name.""" - return VNF_OUTPUT_FOLDER_FILENAME def _get_input_config( self, input_config: Dict[str, Any] = None @@ -97,9 +88,6 @@ def _get_processor_list(self) -> [BaseInputProcessor]: for image in self.config.images: (source_acr_registry, name, version) = self._split_image_path(image) - # TEMP FIX FOR INVALID VERSIONS - version = self._create_semver_compatible_version(version) - image_input = NexusImageFileInput( artifact_name=name, artifact_version=version, @@ -159,6 +147,7 @@ def build_manifest_bicep(self) -> BicepDefinitionElementBuilder: logger.info("Created artifact manifest bicep element") return bicep_file + # JORDAN: can pull out def build_artifact_list(self) -> ArtifactDefinitionElementBuilder: """Build the artifact list.""" logger.info("Creating artifacts list for artifacts.json") @@ -195,43 +184,26 @@ def build_resource_bicep(self) -> BicepDefinitionElementBuilder: params_schema = processor.generate_params_schema() schema_properties.update(params_schema) - # For each arm template, generate nf application - if isinstance(processor, NexusArmBuildProcessor): + # Generate local file for parameters, i.e vhdParameters + parameters_file = processor.generate_parameters_file() + logger.info( + "Created parameters file as supporting file for nfDefinition bicep" + ) + if isinstance(processor, NexusArmBuildProcessor): arm_nf_application_list.append(nf_application) - # Generate local file for template_parameters + add to supporting files list - params = ( - nf_application.deploy_parameters_mapping_rule_profile.template_mapping_rule_profile.template_parameters - ) - template_name = TEMPLATE_PARAMETERS_FILENAME - logger.info( - "Created templatateParameters as supporting file for nfDefinition bicep" - ) elif isinstance(processor, NexusImageProcessor): image_nf_application_list.append(nf_application) - # Generate local file for vhd_parameters - params = ( - nf_application.deploy_parameters_mapping_rule_profile.image_mapping_rule_profile.user_configuration - ) - template_name = NEXUS_IMAGE_PARAMETERS_FILENAME else: raise TypeError(f"Type: {type(processor)} is not valid") - parameters_file = LocalFileBuilder( - Path( - VNF_OUTPUT_FOLDER_FILENAME, - NF_DEFINITION_FOLDER_NAME, - template_name, - ), - json.dumps(json.loads(params), indent=4), - ) supporting_files.append(parameters_file) # Create bicep contents using vnf defintion j2 template template_path = get_template_path( VNF_TEMPLATE_FOLDER_NAME, VNF_DEFINITION_TEMPLATE_FILENAME ) - + # JORDAN: another private method? params = { "nfvi_type": 'AzureOperatorNexus', "acr_nf_applications": arm_nf_application_list, From e2c6d68ac23a646fd21a7bd722341f347988d53e Mon Sep 17 00:00:00 2001 From: Jordan Date: Fri, 9 Feb 2024 11:03:07 +0000 Subject: [PATCH 15/37] moved logic to base vnf handler; moved more logic to processor --- .../build_processors/arm_processor.py | 47 ++++---- .../build_processors/nexus_image_processor.py | 3 + .../build_processors/vhd_processor.py | 24 +++- .../onboarding_core_vnf_handler.py | 110 +++++------------- .../onboarding_nexus_vnf_handler.py | 96 +++++---------- .../cli_handlers/onboarding_vnf_handler.py | 72 +++++++++++- 6 files changed, 175 insertions(+), 177 deletions(-) diff --git a/src/aosm/azext_aosm/build_processors/arm_processor.py b/src/aosm/azext_aosm/build_processors/arm_processor.py index 3e07941b02c..12510a97290 100644 --- a/src/aosm/azext_aosm/build_processors/arm_processor.py +++ b/src/aosm/azext_aosm/build_processors/arm_processor.py @@ -134,11 +134,33 @@ def generate_resource_element_template(self) -> ResourceElementTemplate: ), ) + def generate_parameters_file(self) -> LocalFileBuilder: + """ Generate parameters file. """ + mapping_rule_profile = self._generate_mapping_rule_profile() + params = ( + mapping_rule_profile.template_mapping_rule_profile.template_parameters + ) + print(self.input_artifact.artifact_name + '-' + TEMPLATE_PARAMETERS_FILENAME) + logger.info( + "Created parameters file for arm template." + ) + return LocalFileBuilder( + Path( + VNF_OUTPUT_FOLDER_FILENAME, + NF_DEFINITION_FOLDER_NAME, + self.input_artifact.artifact_name + '-' + TEMPLATE_PARAMETERS_FILENAME, + ), + json.dumps(json.loads(params), indent=4), + ) + + def _generate_mapping_rule_profile( + self, + ): + raise NotImplementedError("This method must be implemented in a subclass.") + class AzureCoreArmBuildProcessor(BaseArmBuildProcessor): - """ - This class represents an ARM template processor for Azure Core. - """ + """This class represents an ARM template processor for Azure Core.""" def generate_nfvi_specific_nf_application( self, @@ -178,6 +200,7 @@ def generate_artifact_profile(self) -> AzureCoreArmTemplateArtifactProfile: template_artifact_profile=artifact_profile, ) + class NexusArmBuildProcessor(BaseArmBuildProcessor): """ This class represents a processor for generating ARM templates specific to Nexus. @@ -219,21 +242,3 @@ def generate_artifact_profile(self) -> AzureOperatorNexusArmTemplateArtifactProf artifact_store=ReferencedResource(id=""), template_artifact_profile=artifact_profile, ) - - def generate_parameters_file(self) -> LocalFileBuilder: - """ Generate parameters file. """ - mapping_rule_profile = self._generate_mapping_rule_profile() - params = ( - mapping_rule_profile.template_mapping_rule_profile.template_parameters - ) - return LocalFileBuilder( - Path( - VNF_OUTPUT_FOLDER_FILENAME, - NF_DEFINITION_FOLDER_NAME, - TEMPLATE_PARAMETERS_FILENAME, - ), - json.dumps(json.loads(params), indent=4), - ) - - def generate_template_parameters(self): - pass \ No newline at end of file diff --git a/src/aosm/azext_aosm/build_processors/nexus_image_processor.py b/src/aosm/azext_aosm/build_processors/nexus_image_processor.py index b3f438639b8..13800e2328e 100644 --- a/src/aosm/azext_aosm/build_processors/nexus_image_processor.py +++ b/src/aosm/azext_aosm/build_processors/nexus_image_processor.py @@ -158,6 +158,9 @@ def generate_parameters_file(self) -> LocalFileBuilder: params = ( mapping_rule_profile.image_mapping_rule_profile.user_configuration ) + logger.info( + "Created parameters file for Nexus image." + ) return LocalFileBuilder( Path( VNF_OUTPUT_FOLDER_FILENAME, diff --git a/src/aosm/azext_aosm/build_processors/vhd_processor.py b/src/aosm/azext_aosm/build_processors/vhd_processor.py index dde6212df8a..5afd66011e7 100644 --- a/src/aosm/azext_aosm/build_processors/vhd_processor.py +++ b/src/aosm/azext_aosm/build_processors/vhd_processor.py @@ -2,7 +2,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- - +from pathlib import Path import json from typing import List, Tuple @@ -20,6 +20,10 @@ AzureCoreVhdImageDeployMappingRuleProfile, DependsOnProfile, ManifestArtifactFormat, ReferencedResource, ResourceElementTemplate, VhdImageArtifactProfile, VhdImageMappingRuleProfile) +from azext_aosm.common.constants import ( + VNF_OUTPUT_FOLDER_FILENAME, + NF_DEFINITION_FOLDER_NAME, + VHD_PARAMETERS_FILENAME) logger = get_logger(__name__) @@ -162,3 +166,21 @@ def _generate_mapping_rule_profile( application_enablement=ApplicationEnablement.ENABLED, vhd_image_mapping_rule_profile=mapping, ) + + def generate_parameters_file(self) -> LocalFileBuilder: + """ Generate parameters file. """ + mapping_rule_profile = self._generate_mapping_rule_profile() + params = ( + mapping_rule_profile.vhd_image_mapping_rule_profile.user_configuration + ) + logger.info( + "Created parameters file for Nexus image." + ) + return LocalFileBuilder( + Path( + VNF_OUTPUT_FOLDER_FILENAME, + NF_DEFINITION_FOLDER_NAME, + VHD_PARAMETERS_FILENAME, + ), + json.dumps(json.loads(params), indent=4), + ) diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py index dc70f1c330a..212d972e244 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py @@ -171,90 +171,6 @@ def build_artifact_list(self) -> ArtifactDefinitionElementBuilder: Path(VNF_OUTPUT_FOLDER_FILENAME, ARTIFACT_LIST_FILENAME), artifact_list ) - def build_resource_bicep(self) -> BicepDefinitionElementBuilder: - """Build the resource bicep file.""" - logger.info("Creating artifacts list for artifacts.json") - acr_nf_application_list = [] - sa_nf_application_list = [] - supporting_files = [] - schema_properties = {} - - for processor in self.processors: - # Generate NF Application - nf_application = processor.generate_nf_application() - logger.debug("Created nf application %s", nf_application.name) - - # Generate deploymentParameters schema properties - params_schema = processor.generate_params_schema() - schema_properties.update(params_schema) - - # For each arm template, generate nf application - if isinstance(processor, AzureCoreArmBuildProcessor): - - acr_nf_application_list.append(nf_application) - # Generate local file for template_parameters + add to supporting files list - params = ( - nf_application.deploy_parameters_mapping_rule_profile.template_mapping_rule_profile.template_parameters - ) - template_name = TEMPLATE_PARAMETERS_FILENAME - logger.info( - "Created templatateParameters as supporting file for nfDefinition bicep" - ) - elif isinstance(processor, VHDProcessor): - sa_nf_application_list.append(nf_application) - # Generate local file for vhd_parameters - params = ( - nf_application.deploy_parameters_mapping_rule_profile.vhd_image_mapping_rule_profile.user_configuration - ) - template_name = VHD_PARAMETERS_FILENAME - else: - raise TypeError(f"Type: {type(processor)} is not valid") - - parameters_file = LocalFileBuilder( - Path( - VNF_OUTPUT_FOLDER_FILENAME, - NF_DEFINITION_FOLDER_NAME, - template_name, - ), - json.dumps(json.loads(params), indent=4), - ) - supporting_files.append(parameters_file) - - # Create bicep contents using vnf defintion j2 template - template_path = get_template_path( - VNF_TEMPLATE_FOLDER_NAME, VNF_DEFINITION_TEMPLATE_FILENAME - ) - - params = { - "nfvi_type": 'AzureCore', - "acr_nf_applications": acr_nf_application_list, - "sa_nf_applications": sa_nf_application_list, - "nexus_image_nf_applications": [], - "deployment_parameters_file": DEPLOYMENT_PARAMETERS_FILENAME, - "vhd_parameters_file": VHD_PARAMETERS_FILENAME, - "template_parameters_file": TEMPLATE_PARAMETERS_FILENAME - } - bicep_contents = render_bicep_contents_from_j2( - template_path, params - ) - - # Create a bicep element - # + add its supporting files (deploymentParameters, vhdParameters and templateParameters) - bicep_file = BicepDefinitionElementBuilder( - Path(VNF_OUTPUT_FOLDER_FILENAME, NF_DEFINITION_FOLDER_NAME), - bicep_contents, - ) - for supporting_file in supporting_files: - bicep_file.add_supporting_file(supporting_file) - - # Add the deploymentParameters schema file - bicep_file.add_supporting_file( - self._render_deployment_params_schema( - schema_properties, VNF_OUTPUT_FOLDER_FILENAME, NF_DEFINITION_FOLDER_NAME - ) - ) - return bicep_file - def build_all_parameters_json(self) -> JSONDefinitionElementBuilder: """Create object for all_parameters.json.""" params_content = { @@ -288,3 +204,29 @@ def _get_default_config(self, vhd) -> Dict[str, Any]: if vhd.image_api_version: default_config.update({"image_api_version": vhd.image_api_version}) return default_config + + def _generate_type_specific_nf_application(self, processor) -> (Any, Any): + """Generate the type specific nf application.""" + arm_nf = [] + image_nf = [] + nf_application = processor.generate_nf_application() + print(nf_application) + if isinstance(processor, AzureCoreArmBuildProcessor): + arm_nf.append(nf_application) + elif isinstance(processor, VHDProcessor): + image_nf.append(nf_application) + else: + raise TypeError(f"Type: {type(processor)} is not valid") + logger.debug("Created nf application %s", nf_application.name) + return (arm_nf, image_nf) + + def _get_nfd_template_params(self, arm_nf_application_list, image_nf_application_list) -> Dict[str, Any]: + """Get the nfd template params.""" + return {"nfvi_type": 'AzureCore', + "acr_nf_applications": arm_nf_application_list, + "sa_nf_applications": image_nf_application_list, + "nexus_image_nf_applications": [], + "deployment_parameters_file": DEPLOYMENT_PARAMETERS_FILENAME, + "vhd_parameters_file": VHD_PARAMETERS_FILENAME, + "template_parameters_file": TEMPLATE_PARAMETERS_FILENAME + } diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py index e5664b1b085..552351eba6c 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py @@ -14,10 +14,7 @@ from azext_aosm.common.constants import (ARTIFACT_LIST_FILENAME, BASE_FOLDER_NAME, MANIFEST_FOLDER_NAME, - NF_DEFINITION_FOLDER_NAME, VNF_TEMPLATE_FOLDER_NAME, - VNF_DEFINITION_TEMPLATE_FILENAME, - VNF_INPUT_FILENAME, VNF_MANIFEST_TEMPLATE_FILENAME, VNF_OUTPUT_FOLDER_FILENAME, DEPLOYMENT_PARAMETERS_FILENAME, @@ -167,72 +164,6 @@ def build_artifact_list(self) -> ArtifactDefinitionElementBuilder: Path(VNF_OUTPUT_FOLDER_FILENAME, ARTIFACT_LIST_FILENAME), artifact_list ) - def build_resource_bicep(self) -> BicepDefinitionElementBuilder: - """Build the resource bicep file.""" - logger.info("Creating artifacts list for artifacts.json") - arm_nf_application_list = [] - image_nf_application_list = [] - supporting_files = [] - schema_properties = {} - - for processor in self.processors: - # Generate NF Application - nf_application = processor.generate_nf_application() - logger.debug("Created nf application %s", nf_application.name) - - # Generate deploymentParameters schema properties - params_schema = processor.generate_params_schema() - schema_properties.update(params_schema) - - # Generate local file for parameters, i.e vhdParameters - parameters_file = processor.generate_parameters_file() - logger.info( - "Created parameters file as supporting file for nfDefinition bicep" - ) - - if isinstance(processor, NexusArmBuildProcessor): - arm_nf_application_list.append(nf_application) - elif isinstance(processor, NexusImageProcessor): - image_nf_application_list.append(nf_application) - else: - raise TypeError(f"Type: {type(processor)} is not valid") - - supporting_files.append(parameters_file) - - # Create bicep contents using vnf defintion j2 template - template_path = get_template_path( - VNF_TEMPLATE_FOLDER_NAME, VNF_DEFINITION_TEMPLATE_FILENAME - ) - # JORDAN: another private method? - params = { - "nfvi_type": 'AzureOperatorNexus', - "acr_nf_applications": arm_nf_application_list, - "sa_nf_applications": [], - "nexus_image_nf_applications": image_nf_application_list, - "deployment_parameters_file": DEPLOYMENT_PARAMETERS_FILENAME, - "template_parameters_file": TEMPLATE_PARAMETERS_FILENAME, - "image_parameters_file": NEXUS_IMAGE_PARAMETERS_FILENAME - } - bicep_contents = render_bicep_contents_from_j2( - template_path, params - ) - - # Create a bicep element - # + add its supporting files (deploymentParameters, vhdParameters and templateParameters) - bicep_file = BicepDefinitionElementBuilder( - Path(VNF_OUTPUT_FOLDER_FILENAME, NF_DEFINITION_FOLDER_NAME), - bicep_contents, - ) - for supporting_file in supporting_files: - bicep_file.add_supporting_file(supporting_file) - - # Add the deploymentParameters schema file - bicep_file.add_supporting_file( - self._render_deployment_params_schema( - schema_properties, VNF_OUTPUT_FOLDER_FILENAME, NF_DEFINITION_FOLDER_NAME - ) - ) - return bicep_file def build_all_parameters_json(self) -> JSONDefinitionElementBuilder: """Build the all parameters json file.""" @@ -263,3 +194,30 @@ def _create_semver_compatible_version(self, version) -> str: else: print(f"Invalid version {version}, using 1.0.0 as default") return '1.0.0' + + def _generate_type_specific_nf_application(self, processor) -> (Any, Any): + """Generate the type specific nf application.""" + arm_nf = [] + image_nf = [] + nf_application = processor.generate_nf_application() + print(nf_application) + if isinstance(processor, NexusArmBuildProcessor): + arm_nf.append(nf_application) + elif isinstance(processor, NexusImageProcessor): + image_nf.append(nf_application) + else: + raise TypeError(f"Type: {type(processor)} is not valid") + logger.debug("Created nf application %s", nf_application.name) + return (arm_nf, image_nf) + + def _get_nfd_template_params(self, arm_nf_application_list, image_nf_application_list) -> Dict[str, Any]: + """Get the nfd template params.""" + return { + "nfvi_type": 'AzureOperatorNexus', + "acr_nf_applications": arm_nf_application_list, + "sa_nf_applications": [], + "nexus_image_nf_applications": image_nf_application_list, + "deployment_parameters_file": DEPLOYMENT_PARAMETERS_FILENAME, + "template_parameters_file": TEMPLATE_PARAMETERS_FILENAME, + "image_parameters_file": NEXUS_IMAGE_PARAMETERS_FILENAME + } diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_vnf_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_vnf_handler.py index 3fcb68d3406..7dd4431d606 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_vnf_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_vnf_handler.py @@ -2,10 +2,21 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- - +from pathlib import Path +from abc import ABC, abstractmethod from azext_aosm.common.constants import VNF_INPUT_FILENAME, VNF_OUTPUT_FOLDER_FILENAME from .onboarding_nfd_base_handler import OnboardingNFDBaseCLIHandler - +from knack.log import get_logger +from azext_aosm.common.constants import ( + NF_DEFINITION_FOLDER_NAME, + VNF_TEMPLATE_FOLDER_NAME, + VNF_DEFINITION_TEMPLATE_FILENAME, + VNF_INPUT_FILENAME) +from azext_aosm.common.utils import render_bicep_contents_from_j2, get_template_path +from azext_aosm.definition_folder.builder.bicep_builder import ( + BicepDefinitionElementBuilder, +) +logger = get_logger(__name__) class OnboardingVNFCLIHandler(OnboardingNFDBaseCLIHandler): """CLI handler for publishing NFDs.""" @@ -19,3 +30,60 @@ def default_config_file_name(self) -> str: def output_folder_file_name(self) -> str: """Get the output folder file name.""" return VNF_OUTPUT_FOLDER_FILENAME + + def build_resource_bicep(self) -> BicepDefinitionElementBuilder: + """Build the resource bicep file.""" + logger.info("Creating artifacts list for artifacts.json") + arm_nf_application_list = [] + image_nf_application_list = [] + supporting_files = [] + schema_properties = {} + + for processor in self.processors: + # Generate NF Application + add to correct list for nfd template + (arm_nf_application, image_nf_application) = self._generate_type_specific_nf_application(processor) + arm_nf_application_list.extend(arm_nf_application) + image_nf_application_list.extend(image_nf_application) + + # Generate deploymentParameters schema properties + params_schema = processor.generate_params_schema() + schema_properties.update(params_schema) + + # Generate local file for parameters, i.e imageParameters + parameters_file = processor.generate_parameters_file() + supporting_files.append(parameters_file) + + # Create bicep contents using vnf defintion j2 template + template_path = get_template_path( + VNF_TEMPLATE_FOLDER_NAME, VNF_DEFINITION_TEMPLATE_FILENAME + ) + + params = self._get_nfd_template_params(arm_nf_application_list, image_nf_application_list) + bicep_contents = render_bicep_contents_from_j2( + template_path, params + ) + + # Create a bicep element + # + add its supporting files (deploymentParameters, vhdParameters and templateParameters) + bicep_file = BicepDefinitionElementBuilder( + Path(VNF_OUTPUT_FOLDER_FILENAME, NF_DEFINITION_FOLDER_NAME), + bicep_contents, + ) + for supporting_file in supporting_files: + bicep_file.add_supporting_file(supporting_file) + + # Add the deploymentParameters schema file + bicep_file.add_supporting_file( + self._render_deployment_params_schema( + schema_properties, VNF_OUTPUT_FOLDER_FILENAME, NF_DEFINITION_FOLDER_NAME + ) + ) + return bicep_file + + @abstractmethod + def _get_nfd_template_params(self, arm_nf_application_list, image_nf_application_list): + return NotImplementedError + + @abstractmethod + def _generate_type_specific_nf_application(self, processor): + return NotImplementedError From b1d0575ce95b58a86a588d493c0a0b33a7b5a4a5 Mon Sep 17 00:00:00 2001 From: Jordan Date: Fri, 9 Feb 2024 11:06:59 +0000 Subject: [PATCH 16/37] fix template name in vnf j2 --- src/aosm/azext_aosm/common/templates/vnf/vnfdefinition.bicep.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aosm/azext_aosm/common/templates/vnf/vnfdefinition.bicep.j2 b/src/aosm/azext_aosm/common/templates/vnf/vnfdefinition.bicep.j2 index 74b5bdce2d0..c935d10136c 100644 --- a/src/aosm/azext_aosm/common/templates/vnf/vnfdefinition.bicep.j2 +++ b/src/aosm/azext_aosm/common/templates/vnf/vnfdefinition.bicep.j2 @@ -98,7 +98,7 @@ resource nfdv 'Microsoft.Hybridnetwork/publishers/networkfunctiondefinitiongroup } deployParametersMappingRuleProfile: { templateMappingRuleProfile: { - templateParameters: string(loadJsonContent('{{template_parameters_file}}')) + templateParameters: string(loadJsonContent('{{ configuration.name }}-{{template_parameters_file}}')) } applicationEnablement: '{{ configuration.deploy_parameters_mapping_rule_profile.application_enablement.value }}' } From e3d7b621b88b2023b8a065fb593e4bca0191d7e9 Mon Sep 17 00:00:00 2001 From: Jordan Date: Fri, 9 Feb 2024 11:12:05 +0000 Subject: [PATCH 17/37] fixed nexus image file --- src/aosm/azext_aosm/build_processors/nexus_image_processor.py | 2 +- src/aosm/azext_aosm/common/templates/vnf/vnfdefinition.bicep.j2 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/aosm/azext_aosm/build_processors/nexus_image_processor.py b/src/aosm/azext_aosm/build_processors/nexus_image_processor.py index 13800e2328e..df1e1e64df2 100644 --- a/src/aosm/azext_aosm/build_processors/nexus_image_processor.py +++ b/src/aosm/azext_aosm/build_processors/nexus_image_processor.py @@ -165,7 +165,7 @@ def generate_parameters_file(self) -> LocalFileBuilder: Path( VNF_OUTPUT_FOLDER_FILENAME, NF_DEFINITION_FOLDER_NAME, - NEXUS_IMAGE_PARAMETERS_FILENAME, + self.input_artifact.artifact_name + '-' + NEXUS_IMAGE_PARAMETERS_FILENAME, ), json.dumps(json.loads(params), indent=4), ) diff --git a/src/aosm/azext_aosm/common/templates/vnf/vnfdefinition.bicep.j2 b/src/aosm/azext_aosm/common/templates/vnf/vnfdefinition.bicep.j2 index c935d10136c..ab7f10875f1 100644 --- a/src/aosm/azext_aosm/common/templates/vnf/vnfdefinition.bicep.j2 +++ b/src/aosm/azext_aosm/common/templates/vnf/vnfdefinition.bicep.j2 @@ -120,7 +120,7 @@ resource nfdv 'Microsoft.Hybridnetwork/publishers/networkfunctiondefinitiongroup } deployParametersMappingRuleProfile: { imageMappingRuleProfile: { - userConfiguration: string(loadJsonContent('{{image_parameters_file}}')) + userConfiguration: string(loadJsonContent('{{ configuration.name }}-{{image_parameters_file}}')) } applicationEnablement: '{{ configuration.deploy_parameters_mapping_rule_profile.application_enablement.value }}' } From 0fe396b826b0f462217cea7bbb31dc31d140b9bb Mon Sep 17 00:00:00 2001 From: Jordan Date: Fri, 9 Feb 2024 11:33:16 +0000 Subject: [PATCH 18/37] added nfvitype to nsd nf template by making new j2 + changing how nfd processor works --- .../build_processors/nfd_processor.py | 18 ++++++++---------- src/aosm/azext_aosm/common/constants.py | 1 + .../nf_template.bicep.j2} | 2 +- 3 files changed, 10 insertions(+), 11 deletions(-) rename src/aosm/azext_aosm/common/templates/{nf_template.bicep => nsd/nf_template.bicep.j2} (93%) diff --git a/src/aosm/azext_aosm/build_processors/nfd_processor.py b/src/aosm/azext_aosm/build_processors/nfd_processor.py index 96205407c6c..9f2864922d3 100644 --- a/src/aosm/azext_aosm/build_processors/nfd_processor.py +++ b/src/aosm/azext_aosm/build_processors/nfd_processor.py @@ -20,18 +20,11 @@ NFDResourceElementTemplate from azext_aosm.vendored_sdks.models import (NSDArtifactProfile, ReferencedResource, TemplateType) -from azext_aosm.common.constants import NSD_OUTPUT_FOLDER_FILENAME +from azext_aosm.common.constants import NSD_OUTPUT_FOLDER_FILENAME, NSD_NF_TEMPLATE_FILENAME, NSD_TEMPLATE_FOLDER_NAME +from azext_aosm.common.utils import render_bicep_contents_from_j2, get_template_path logger = get_logger(__name__) -NF_BICEP_TEMPLATE_PATH = ( - Path(__file__).parent.parent / "common" / "templates" / "nf_template.bicep" -) - -NF_BICEP_TEMPLATE_PATH = ( - Path(__file__).parent.parent / "common" / "templates" / "nf_template.bicep" -) - class NFDProcessor(BaseInputProcessor): """ @@ -82,10 +75,15 @@ def get_artifact_details( file_path=self.input_artifact.arm_template_output_path.relative_to(Path(NSD_OUTPUT_FOLDER_FILENAME)), ) + template_path = get_template_path(NSD_TEMPLATE_FOLDER_NAME, NSD_NF_TEMPLATE_FILENAME) + params = { + "nfvi_type": self.input_artifact.network_function_definition.properties.network_function_template.nfvi_type + } + bicep_contents = render_bicep_contents_from_j2(template_path, params) # Create a local file builder for the ARM template file_builder = LocalFileBuilder( self.input_artifact.arm_template_output_path, - NF_BICEP_TEMPLATE_PATH.read_text(), + bicep_contents, ) return [artifact_details], [file_builder] diff --git a/src/aosm/azext_aosm/common/constants.py b/src/aosm/azext_aosm/common/constants.py index ebe9f0baa2d..25112d6dc64 100644 --- a/src/aosm/azext_aosm/common/constants.py +++ b/src/aosm/azext_aosm/common/constants.py @@ -60,6 +60,7 @@ class ManifestsExist(str, Enum): NSD_BASE_TEMPLATE_FILENAME = "nsdbase.bicep" NSD_TEMPLATE_FOLDER_NAME = "nsd" NSD_DEFINITION_FOLDER_NAME = "nsdDefinition" +NSD_NF_TEMPLATE_FILENAME = "nf_template.bicep.j2" VNF_OUTPUT_FOLDER_FILENAME = "vnf-cli-output" VNF_INPUT_FILENAME = "vnf-input.jsonc" diff --git a/src/aosm/azext_aosm/common/templates/nf_template.bicep b/src/aosm/azext_aosm/common/templates/nsd/nf_template.bicep.j2 similarity index 93% rename from src/aosm/azext_aosm/common/templates/nf_template.bicep rename to src/aosm/azext_aosm/common/templates/nsd/nf_template.bicep.j2 index 781aa874103..b6bbfbcfe27 100644 --- a/src/aosm/azext_aosm/common/templates/nf_template.bicep +++ b/src/aosm/azext_aosm/common/templates/nsd/nf_template.bicep.j2 @@ -33,7 +33,7 @@ resource nfResource 'Microsoft.HybridNetwork/networkFunctions@2023-09-01' = [for id: nfdv.id idType: 'Open' } - nfviType: (configObject.customLocationId == '') ? 'AzureCore' : 'AzureArcKubernetes' + nfviType: '{{nfvi_type}}' nfviId: (configObject.customLocationId == '') ? resourceGroupId : configObject.customLocationId allowSoftwareUpdate: true configurationType: 'Secret' From 9ffb64022321a442175e48a0645129cf3d525fd8 Mon Sep 17 00:00:00 2001 From: Jordan Date: Fri, 9 Feb 2024 11:49:19 +0000 Subject: [PATCH 19/37] added nfvitype to nsd nf template by making new j2 + changing how nfd processor works --- .../onboarding_core_vnf_handler.py | 32 +++----------- .../onboarding_nexus_vnf_handler.py | 43 +++---------------- .../cli_handlers/onboarding_vnf_handler.py | 31 +++++++++++-- 3 files changed, 37 insertions(+), 69 deletions(-) diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py index 212d972e244..01fa6bc511e 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py @@ -22,16 +22,12 @@ DEPLOYMENT_PARAMETERS_FILENAME, VHD_PARAMETERS_FILENAME, TEMPLATE_PARAMETERS_FILENAME) -from azext_aosm.common.local_file_builder import LocalFileBuilder from azext_aosm.configuration_models.onboarding_vnf_input_config import ( OnboardingCoreVNFInputConfig, ) from azext_aosm.configuration_models.common_parameters_config import ( CoreVNFCommonParametersConfig, ) -from azext_aosm.definition_folder.builder.artifact_builder import ( - ArtifactDefinitionElementBuilder, -) from azext_aosm.definition_folder.builder.bicep_builder import ( BicepDefinitionElementBuilder, ) @@ -151,26 +147,6 @@ def build_manifest_bicep(self) -> BicepDefinitionElementBuilder: logger.info("Created artifact manifest bicep element") return bicep_file - def build_artifact_list(self) -> ArtifactDefinitionElementBuilder: - """Build the artifact list.""" - logger.info("Creating artifacts list for artifacts.json") - artifact_list = [] - # For each arm template, get list of artifacts and combine - for processor in self.processors: - (artifacts, _) = processor.get_artifact_details() - if artifacts not in artifact_list: - artifact_list.extend(artifacts) - logger.debug( - "Created list of artifact details from %s arm template(s) and the vhd image provided: %s", - len(self.config.arm_templates), - artifact_list, - ) - - # Generate Artifact Element with artifact list (of arm template and vhd images) - return ArtifactDefinitionElementBuilder( - Path(VNF_OUTPUT_FOLDER_FILENAME, ARTIFACT_LIST_FILENAME), artifact_list - ) - def build_all_parameters_json(self) -> JSONDefinitionElementBuilder: """Create object for all_parameters.json.""" params_content = { @@ -205,7 +181,7 @@ def _get_default_config(self, vhd) -> Dict[str, Any]: default_config.update({"image_api_version": vhd.image_api_version}) return default_config - def _generate_type_specific_nf_application(self, processor) -> (Any, Any): + def _generate_type_specific_nf_application(self, processor) -> "tuple[list, list]": """Generate the type specific nf application.""" arm_nf = [] image_nf = [] @@ -220,9 +196,11 @@ def _generate_type_specific_nf_application(self, processor) -> (Any, Any): logger.debug("Created nf application %s", nf_application.name) return (arm_nf, image_nf) - def _get_nfd_template_params(self, arm_nf_application_list, image_nf_application_list) -> Dict[str, Any]: + def _get_nfd_template_params( + self, arm_nf_application_list, image_nf_application_list) -> Dict[str, Any]: """Get the nfd template params.""" - return {"nfvi_type": 'AzureCore', + return { + "nfvi_type": 'AzureCore', "acr_nf_applications": arm_nf_application_list, "sa_nf_applications": image_nf_application_list, "nexus_image_nf_applications": [], diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py index 552351eba6c..ee4fd84d036 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py @@ -21,17 +21,13 @@ NEXUS_IMAGE_PARAMETERS_FILENAME, TEMPLATE_PARAMETERS_FILENAME, VNF_NEXUS_BASE_TEMPLATE_FILENAME, - SEMVER_REGEX) -from azext_aosm.common.local_file_builder import LocalFileBuilder + ) from azext_aosm.configuration_models.onboarding_vnf_input_config import ( OnboardingNexusVNFInputConfig, ) from azext_aosm.configuration_models.common_parameters_config import ( NexusVNFCommonParametersConfig, ) -from azext_aosm.definition_folder.builder.artifact_builder import ( - ArtifactDefinitionElementBuilder, -) from azext_aosm.definition_folder.builder.bicep_builder import ( BicepDefinitionElementBuilder, ) @@ -49,7 +45,6 @@ class OnboardingNexusVNFCLIHandler(OnboardingVNFCLIHandler): """CLI handler for publishing NFDs.""" - def _get_input_config( self, input_config: Dict[str, Any] = None ) -> OnboardingNexusVNFInputConfig: @@ -144,27 +139,6 @@ def build_manifest_bicep(self) -> BicepDefinitionElementBuilder: logger.info("Created artifact manifest bicep element") return bicep_file - # JORDAN: can pull out - def build_artifact_list(self) -> ArtifactDefinitionElementBuilder: - """Build the artifact list.""" - logger.info("Creating artifacts list for artifacts.json") - artifact_list = [] - # For each arm template, get list of artifacts and combine - for processor in self.processors: - (artifacts, _) = processor.get_artifact_details() - if artifacts not in artifact_list: - artifact_list.extend(artifacts) - logger.debug( - "Created list of artifact details from arm template(s) and image(s) provided: %s", - artifact_list, - ) - - # Generate Artifact Element with artifact list (of arm template and vhd images) - return ArtifactDefinitionElementBuilder( - Path(VNF_OUTPUT_FOLDER_FILENAME, ARTIFACT_LIST_FILENAME), artifact_list - ) - - def build_all_parameters_json(self) -> JSONDefinitionElementBuilder: """Build the all parameters json file.""" params_content = { @@ -181,21 +155,13 @@ def build_all_parameters_json(self) -> JSONDefinitionElementBuilder: ) return base_file - def _split_image_path(self, image) -> (str, str, str): + def _split_image_path(self, image) -> "tuple[str, str, str]": """Split the image path into source acr registry, name and version.""" (source_acr_registry, name_and_version) = image.split("/", 2) (name, version) = name_and_version.split(":", 2) return (source_acr_registry, name, version) - def _create_semver_compatible_version(self, version) -> str: - """Create a semver compatible version.""" - if re.search(SEMVER_REGEX, version): - return version - else: - print(f"Invalid version {version}, using 1.0.0 as default") - return '1.0.0' - - def _generate_type_specific_nf_application(self, processor) -> (Any, Any): + def _generate_type_specific_nf_application(self, processor) -> "tuple[list, list]": """Generate the type specific nf application.""" arm_nf = [] image_nf = [] @@ -210,7 +176,8 @@ def _generate_type_specific_nf_application(self, processor) -> (Any, Any): logger.debug("Created nf application %s", nf_application.name) return (arm_nf, image_nf) - def _get_nfd_template_params(self, arm_nf_application_list, image_nf_application_list) -> Dict[str, Any]: + def _get_nfd_template_params( + self, arm_nf_application_list, image_nf_application_list) -> Dict[str, Any]: """Get the nfd template params.""" return { "nfvi_type": 'AzureOperatorNexus', diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_vnf_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_vnf_handler.py index 7dd4431d606..f104e988096 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_vnf_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_vnf_handler.py @@ -4,20 +4,23 @@ # -------------------------------------------------------------------------------------------- from pathlib import Path from abc import ABC, abstractmethod -from azext_aosm.common.constants import VNF_INPUT_FILENAME, VNF_OUTPUT_FOLDER_FILENAME + from .onboarding_nfd_base_handler import OnboardingNFDBaseCLIHandler from knack.log import get_logger -from azext_aosm.common.constants import ( - NF_DEFINITION_FOLDER_NAME, +from azext_aosm.common.constants import (NF_DEFINITION_FOLDER_NAME, VNF_TEMPLATE_FOLDER_NAME, VNF_DEFINITION_TEMPLATE_FILENAME, - VNF_INPUT_FILENAME) + VNF_INPUT_FILENAME, + VNF_OUTPUT_FOLDER_FILENAME, + ARTIFACT_LIST_FILENAME) from azext_aosm.common.utils import render_bicep_contents_from_j2, get_template_path from azext_aosm.definition_folder.builder.bicep_builder import ( BicepDefinitionElementBuilder, ) +from azext_aosm.definition_folder.builder.artifact_builder import ArtifactDefinitionElementBuilder logger = get_logger(__name__) + class OnboardingVNFCLIHandler(OnboardingNFDBaseCLIHandler): """CLI handler for publishing NFDs.""" @@ -31,6 +34,26 @@ def output_folder_file_name(self) -> str: """Get the output folder file name.""" return VNF_OUTPUT_FOLDER_FILENAME + def build_artifact_list(self) -> ArtifactDefinitionElementBuilder: + """Build the artifact list.""" + logger.info("Creating artifacts list for artifacts.json") + artifact_list = [] + # For each arm template, get list of artifacts and combine + for processor in self.processors: + (artifacts, _) = processor.get_artifact_details() + if artifacts not in artifact_list: + artifact_list.extend(artifacts) + logger.debug( + "Created list of artifact details from %s arm template(s) and the image provided: %s", + len(self.config.arm_templates), + artifact_list, + ) + + # Generate Artifact Element with artifact list (of arm template and vhd images) + return ArtifactDefinitionElementBuilder( + Path(VNF_OUTPUT_FOLDER_FILENAME, ARTIFACT_LIST_FILENAME), artifact_list + ) + def build_resource_bicep(self) -> BicepDefinitionElementBuilder: """Build the resource bicep file.""" logger.info("Creating artifacts list for artifacts.json") From 3e773476030bd2b4152f0ee5504d22bd3a95e157 Mon Sep 17 00:00:00 2001 From: Jordan Date: Fri, 9 Feb 2024 11:52:01 +0000 Subject: [PATCH 20/37] minor formatting --- .../azext_aosm/cli_handlers/onboarding_core_vnf_handler.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py index 01fa6bc511e..4bf0f7c80e3 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py @@ -136,7 +136,9 @@ def build_manifest_bicep(self) -> BicepDefinitionElementBuilder: "acr_artifacts": acr_artifact_list, "sa_artifacts": sa_artifact_list, } - bicep_contents = render_bicep_contents_from_j2(template_path, params) + bicep_contents = render_bicep_contents_from_j2( + template_path, params + ) # Create Bicep element with manifest contents bicep_file = BicepDefinitionElementBuilder( From e63ab2ff6daad48b3294c724c6c2644e6c3da04b Mon Sep 17 00:00:00 2001 From: Jordan Date: Fri, 9 Feb 2024 13:05:46 +0000 Subject: [PATCH 21/37] made vnfnexus a definition type; slight refactor of custom.py --- src/aosm/azext_aosm/_params.py | 3 +- src/aosm/azext_aosm/common/constants.py | 2 +- src/aosm/azext_aosm/custom.py | 52 +++++++++++-------------- 3 files changed, 25 insertions(+), 32 deletions(-) diff --git a/src/aosm/azext_aosm/_params.py b/src/aosm/azext_aosm/_params.py index 7790a23bb2e..20610edbc36 100644 --- a/src/aosm/azext_aosm/_params.py +++ b/src/aosm/azext_aosm/_params.py @@ -9,6 +9,7 @@ from .common.constants import ( CNF, VNF, + VNF_NEXUS, BICEP_PUBLISH, ARTIFACT_UPLOAD, IMAGE_UPLOAD, @@ -23,7 +24,7 @@ def load_arguments(self: AzCommandsLoader, _): get_three_state_flag, ) - definition_type = get_enum_type([VNF, CNF]) + definition_type = get_enum_type([VNF, CNF, VNF_NEXUS]) nf_skip_steps = get_enum_type( [BICEP_PUBLISH, ARTIFACT_UPLOAD, IMAGE_UPLOAD, HELM_TEMPLATE] ) diff --git a/src/aosm/azext_aosm/common/constants.py b/src/aosm/azext_aosm/common/constants.py index 25112d6dc64..9ed7efdfaf6 100644 --- a/src/aosm/azext_aosm/common/constants.py +++ b/src/aosm/azext_aosm/common/constants.py @@ -11,7 +11,7 @@ VNF = "vnf" CNF = "cnf" NSD = "nsd" - +VNF_NEXUS = "vnfnexus" class DeployableResourceTypes(str, Enum): VNF = VNF diff --git a/src/aosm/azext_aosm/custom.py b/src/aosm/azext_aosm/custom.py index 95b1182ed9b..695d4c8aabc 100644 --- a/src/aosm/azext_aosm/custom.py +++ b/src/aosm/azext_aosm/custom.py @@ -11,41 +11,37 @@ from azext_aosm.cli_handlers.onboarding_nexus_vnf_handler import OnboardingNexusVNFCLIHandler from azext_aosm.cli_handlers.onboarding_nsd_handler import OnboardingNSDCLIHandler from azext_aosm.common.command_context import CommandContext -from azext_aosm.common.constants import ALL_PARAMETERS_FILE_NAME, CNF, VNF +from azext_aosm.common.constants import ALL_PARAMETERS_FILE_NAME, CNF, VNF, VNF_NEXUS from azure.cli.core.commands import AzCliCommand from azure.cli.core.azclierror import UnrecognizedArgumentError -def onboard_nfd_generate_config(definition_type: str, output_file: str | None, nexus: bool = False): +def onboard_nfd_generate_config(definition_type: str, output_file: str | None): """Generate config file for onboarding NFs.""" if definition_type == CNF: handler = OnboardingCNFCLIHandler() - handler.generate_config(output_file) elif definition_type == VNF: - if nexus: - handler = OnboardingNexusVNFCLIHandler() - else: - handler = OnboardingCoreVNFCLIHandler() - - handler.generate_config(output_file) + handler = OnboardingCoreVNFCLIHandler() + elif definition_type == VNF_NEXUS: + handler = OnboardingNexusVNFCLIHandler() else: - raise UnrecognizedArgumentError("Invalid definition type") + raise UnrecognizedArgumentError( + "Invalid definition type, valid values are 'cnf', 'vnf' or 'vnfnexus'") + handler.generate_config(output_file) -def onboard_nfd_build(definition_type: str, config_file: Path, skip: str = None, nexus: bool = False): +def onboard_nfd_build(definition_type: str, config_file: Path, skip: str = None): """Build the NF definition.""" if definition_type == CNF: handler = OnboardingCNFCLIHandler(Path(config_file), skip=skip) - handler.build() elif definition_type == VNF: - if nexus: - handler = OnboardingNexusVNFCLIHandler(Path(config_file)) - else: - handler = OnboardingCoreVNFCLIHandler(Path(config_file)) - - handler.build() + handler = OnboardingCoreVNFCLIHandler(Path(config_file)) + elif definition_type == VNF_NEXUS: + handler = OnboardingNexusVNFCLIHandler(Path(config_file)) else: - raise UnrecognizedArgumentError("Invalid definition type") + raise UnrecognizedArgumentError( + "Invalid definition type, valid values are 'cnf', 'vnf' or 'vnfnexus'") + handler.build() def onboard_nfd_publish( @@ -53,7 +49,6 @@ def onboard_nfd_publish( definition_type: str, build_output_folder: Path, no_subscription_permissions: bool = False, - nexus: bool = False ): """Publish the NF definition.""" command_context = CommandContext( @@ -64,18 +59,15 @@ def onboard_nfd_publish( }, ) if definition_type == CNF: - handler = OnboardingCNFCLIHandler( - Path(build_output_folder, ALL_PARAMETERS_FILE_NAME) - ) - handler.publish(command_context=command_context) + handler = OnboardingCNFCLIHandler(Path(build_output_folder, ALL_PARAMETERS_FILE_NAME)) elif definition_type == VNF: - if nexus: - handler = OnboardingNexusVNFCLIHandler(Path(build_output_folder, ALL_PARAMETERS_FILE_NAME)) - else: - handler = OnboardingCoreVNFCLIHandler(Path(build_output_folder, ALL_PARAMETERS_FILE_NAME)) - handler.publish(command_context=command_context) + handler = OnboardingCoreVNFCLIHandler(Path(build_output_folder, ALL_PARAMETERS_FILE_NAME)) + elif definition_type == VNF_NEXUS: + handler = OnboardingNexusVNFCLIHandler(Path(build_output_folder, ALL_PARAMETERS_FILE_NAME)) else: - raise UnrecognizedArgumentError("Invalid definition type") + raise UnrecognizedArgumentError( + "Invalid definition type, valid values are 'cnf', 'vnf' or 'vnfnexus'") + handler.publish(command_context=command_context) # def onboard_nfd_delete(cmd: AzCliCommand, definition_type: str, config_file: str): From 9240657ba79964356afebd10d3b6db4207a0df30 Mon Sep 17 00:00:00 2001 From: Jordan Date: Fri, 9 Feb 2024 13:14:02 +0000 Subject: [PATCH 22/37] removed nexus param, removed prints, added commented out test file --- src/aosm/azext_aosm/_params.py | 6 ----- .../build_processors/arm_processor.py | 9 +++---- .../build_processors/base_processor.py | 3 +-- .../onboarding_core_vnf_handler.py | 2 +- .../onboarding_nexus_vnf_handler.py | 2 +- src/aosm/azext_aosm/common/artifact.py | 6 ----- .../onboarding_nsd_input_config.py | 1 - .../unit_test/test_nexus_vnf_handler.py | 27 +++++++++++++++++++ 8 files changed, 34 insertions(+), 22 deletions(-) create mode 100644 src/aosm/azext_aosm/tests/latest/unit_test/test_nexus_vnf_handler.py diff --git a/src/aosm/azext_aosm/_params.py b/src/aosm/azext_aosm/_params.py index 20610edbc36..5a9ba279370 100644 --- a/src/aosm/azext_aosm/_params.py +++ b/src/aosm/azext_aosm/_params.py @@ -92,12 +92,6 @@ def load_arguments(self: AzCommandsLoader, _): "Requires Docker to be installed locally." ), ) - c.argument( - "nexus", - options_list=["--nexus"], - required=False, - help="Flag to specify if the VNF is a Nexus VNF.", - ) with self.argument_context("aosm nsd") as c: c.argument( diff --git a/src/aosm/azext_aosm/build_processors/arm_processor.py b/src/aosm/azext_aosm/build_processors/arm_processor.py index 12510a97290..6386b44a847 100644 --- a/src/aosm/azext_aosm/build_processors/arm_processor.py +++ b/src/aosm/azext_aosm/build_processors/arm_processor.py @@ -140,7 +140,6 @@ def generate_parameters_file(self) -> LocalFileBuilder: params = ( mapping_rule_profile.template_mapping_rule_profile.template_parameters ) - print(self.input_artifact.artifact_name + '-' + TEMPLATE_PARAMETERS_FILENAME) logger.info( "Created parameters file for arm template." ) @@ -170,7 +169,7 @@ def generate_nfvi_specific_nf_application( depends_on_profile=DependsOnProfile(install_depends_on=[], uninstall_depends_on=[], update_depends_on=[]), artifact_type=AzureCoreArtifactType.ARM_TEMPLATE, - artifact_profile=self.generate_artifact_profile(), + artifact_profile=self._generate_artifact_profile(), deploy_parameters_mapping_rule_profile=self._generate_mapping_rule_profile(), ) @@ -190,7 +189,7 @@ def _generate_mapping_rule_profile( template_mapping_rule_profile=mapping_profile, ) - def generate_artifact_profile(self) -> AzureCoreArmTemplateArtifactProfile: + def _generate_artifact_profile(self) -> AzureCoreArmTemplateArtifactProfile: artifact_profile = ArmTemplateArtifactProfile( template_name=self.input_artifact.artifact_name, template_version=self.input_artifact.artifact_version, @@ -213,7 +212,7 @@ def generate_nfvi_specific_nf_application( depends_on_profile=DependsOnProfile(install_depends_on=[], uninstall_depends_on=[], update_depends_on=[]), artifact_type=AzureOperatorNexusArtifactType.ARM_TEMPLATE, - artifact_profile=self.generate_artifact_profile(), + artifact_profile=self._generate_artifact_profile(), deploy_parameters_mapping_rule_profile=self._generate_mapping_rule_profile(), ) @@ -233,7 +232,7 @@ def _generate_mapping_rule_profile( template_mapping_rule_profile=mapping_profile, ) - def generate_artifact_profile(self) -> AzureOperatorNexusArmTemplateArtifactProfile: + def _generate_artifact_profile(self) -> AzureOperatorNexusArmTemplateArtifactProfile: artifact_profile = ArmTemplateArtifactProfile( template_name=self.input_artifact.artifact_name, template_version=self.input_artifact.artifact_version, diff --git a/src/aosm/azext_aosm/build_processors/base_processor.py b/src/aosm/azext_aosm/build_processors/base_processor.py index 4e48c69ca7b..f4beaa25937 100644 --- a/src/aosm/azext_aosm/build_processors/base_processor.py +++ b/src/aosm/azext_aosm/build_processors/base_processor.py @@ -95,8 +95,7 @@ def generate_params_schema(self) -> Dict[str, Any]: ) params_schema = json.loads(base_params_schema) - # print(json.dumps(self.input_artifact.get_schema(), indent=4)) - # print(json.dumps(self.input_artifact.get_defaults(), indent=4)) + self._generate_schema( params_schema[self.name], self.input_artifact.get_schema(), diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py index 4bf0f7c80e3..d681cbefee7 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py @@ -188,7 +188,7 @@ def _generate_type_specific_nf_application(self, processor) -> "tuple[list, list arm_nf = [] image_nf = [] nf_application = processor.generate_nf_application() - print(nf_application) + if isinstance(processor, AzureCoreArmBuildProcessor): arm_nf.append(nf_application) elif isinstance(processor, VHDProcessor): diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py index ee4fd84d036..0999f986ef8 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py @@ -166,7 +166,7 @@ def _generate_type_specific_nf_application(self, processor) -> "tuple[list, list arm_nf = [] image_nf = [] nf_application = processor.generate_nf_application() - print(nf_application) + if isinstance(processor, NexusArmBuildProcessor): arm_nf.append(nf_application) elif isinstance(processor, NexusImageProcessor): diff --git a/src/aosm/azext_aosm/common/artifact.py b/src/aosm/azext_aosm/common/artifact.py index a12ecdb47e5..51ed533ea34 100644 --- a/src/aosm/azext_aosm/common/artifact.py +++ b/src/aosm/azext_aosm/common/artifact.py @@ -159,14 +159,8 @@ def upload(self, config: BaseCommonParametersConfig, command_context: CommandCon output_folder_path = command_context.cli_options["definition_folder"] resolved_file_path = output_folder_path.resolve() upload_file_path = resolved_file_path / self.file_path - print("nfp", output_folder_path) - print("rfp", resolved_file_path) - print("ufp", upload_file_path) self.file_path = upload_file_path - # self.file_path = Path(self.file_path).resolve() - print("fp", self.file_path) - if self.file_path.suffix == ".bicep": # Uploading the nf_template as part of the NSD will use this code path # This does mean we can never have a bicep file as an artifact, but that should be OK diff --git a/src/aosm/azext_aosm/configuration_models/onboarding_nsd_input_config.py b/src/aosm/azext_aosm/configuration_models/onboarding_nsd_input_config.py index 4b878fa4b90..266a386ea4e 100644 --- a/src/aosm/azext_aosm/configuration_models/onboarding_nsd_input_config.py +++ b/src/aosm/azext_aosm/configuration_models/onboarding_nsd_input_config.py @@ -205,7 +205,6 @@ def validate(self): raise ValidationError("nsd_name must be set") if not self.nsd_version: raise ValidationError("nsd_version must be set") - print("here", self.nfvi_type) if self.nfvi_type not in ["AzureCore", "AzureOperatorNexus", "AzureArcKubernetes"]: raise ValidationError( "nfvi_type must be either 'AzureCore', 'AzureOperatorNexus' or 'AzureArcKubernetes'" diff --git a/src/aosm/azext_aosm/tests/latest/unit_test/test_nexus_vnf_handler.py b/src/aosm/azext_aosm/tests/latest/unit_test/test_nexus_vnf_handler.py new file mode 100644 index 00000000000..6ef55bad11b --- /dev/null +++ b/src/aosm/azext_aosm/tests/latest/unit_test/test_nexus_vnf_handler.py @@ -0,0 +1,27 @@ +# from unittest import TestCase +# from unittest.mock import patch +# from azext_aosm.cli_handlers.onboarding_nexus_vnf_handler import OnboardingNexusVNFCLIHandler + +# class VNFNexusBuildTest(TestCase): + +# def setUp(self): +# self.nexus_handler = OnboardingNexusVNFCLIHandler() + +# def test_valid_nexus_config_provided(): +# # give it nexus specific config +# pass + +# def test_invalid_nexus_config_provided(): +# # give it nexus specific config with an error +# pass + +# def test_core_config_provided(): +# # give it core specific config +# pass + +# # def test_build_base_bicep(self): +# # with patch("pathlib.Path.write_text") as mock_write_text: +# # self.nexus_vnf_cli_handler.build_base_bicep() +# # mock_write_text.assert_called() + + \ No newline at end of file From 4fd640473bdbaebec08007bc253242ca477f1ace Mon Sep 17 00:00:00 2001 From: Jordan Date: Fri, 9 Feb 2024 16:35:40 +0000 Subject: [PATCH 23/37] fixed flake8 + pylint issues --- .../build_processors/arm_processor.py | 4 ++-- .../build_processors/nfd_processor.py | 2 +- .../cli_handlers/onboarding_base_handler.py | 1 - .../onboarding_core_vnf_handler.py | 23 +++++++++---------- .../onboarding_nexus_vnf_handler.py | 23 +++++++++---------- src/aosm/azext_aosm/common/artifact.py | 2 -- src/aosm/azext_aosm/common/constants.py | 3 --- .../common_parameters_config.py | 1 + .../onboarding_vnf_input_config.py | 8 ++++++- .../azext_aosm/inputs/nexus_image_input.py | 7 +----- 10 files changed, 34 insertions(+), 40 deletions(-) diff --git a/src/aosm/azext_aosm/build_processors/arm_processor.py b/src/aosm/azext_aosm/build_processors/arm_processor.py index 3b0133c323b..d85044aec46 100644 --- a/src/aosm/azext_aosm/build_processors/arm_processor.py +++ b/src/aosm/azext_aosm/build_processors/arm_processor.py @@ -118,7 +118,7 @@ def generate_nfvi_specific_nf_application(self): def generate_resource_element_template(self) -> ResourceElementTemplate: """Generate the resource element template. - + Note: There is no Nexus specific RET, arm RET can deploy anything (except NFs) """ parameter_values = self.generate_values_mappings( @@ -144,7 +144,7 @@ def generate_resource_element_template(self) -> ResourceElementTemplate: ) def generate_parameters_file(self) -> LocalFileBuilder: - """ Generate parameters file. """ + """Generate parameters file.""" mapping_rule_profile = self._generate_mapping_rule_profile() params = ( mapping_rule_profile.template_mapping_rule_profile.template_parameters diff --git a/src/aosm/azext_aosm/build_processors/nfd_processor.py b/src/aosm/azext_aosm/build_processors/nfd_processor.py index d6e36cea4f5..6c8d7d97743 100644 --- a/src/aosm/azext_aosm/build_processors/nfd_processor.py +++ b/src/aosm/azext_aosm/build_processors/nfd_processor.py @@ -16,7 +16,7 @@ from azext_aosm.vendored_sdks.models import ( ArmResourceDefinitionResourceElementTemplate, ArtifactType, DependsOnProfile, ManifestArtifactFormat, NetworkFunctionApplication, - NetworkFunctionDefinitionResourceElementTemplateDetails as \ + NetworkFunctionDefinitionResourceElementTemplateDetails as NFDResourceElementTemplate, NSDArtifactProfile, ReferencedResource, TemplateType) from azext_aosm.common.constants import NSD_OUTPUT_FOLDER_FILENAME, NSD_NF_TEMPLATE_FILENAME, NSD_TEMPLATE_FOLDER_NAME diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_base_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_base_handler.py index f7d4c59328a..5d04082e6b8 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_base_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_base_handler.py @@ -198,7 +198,6 @@ def _render_base_bicep_contents(template_path): bicep_contents: str = template.render() return bicep_contents - def _serialize(self, dataclass, indent_count=1): """ Convert a dataclass instance to a JSONC string. diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py index d681cbefee7..9473dc699f1 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py @@ -10,18 +10,17 @@ from azext_aosm.build_processors.arm_processor import AzureCoreArmBuildProcessor from azext_aosm.build_processors.vhd_processor import VHDProcessor from azext_aosm.build_processors.base_processor import BaseInputProcessor -from azext_aosm.common.constants import (ARTIFACT_LIST_FILENAME, - BASE_FOLDER_NAME, - MANIFEST_FOLDER_NAME, - NF_DEFINITION_FOLDER_NAME, - VNF_CORE_BASE_TEMPLATE_FILENAME, - VNF_TEMPLATE_FOLDER_NAME, - VNF_DEFINITION_TEMPLATE_FILENAME, - VNF_MANIFEST_TEMPLATE_FILENAME, - VNF_OUTPUT_FOLDER_FILENAME, - DEPLOYMENT_PARAMETERS_FILENAME, - VHD_PARAMETERS_FILENAME, - TEMPLATE_PARAMETERS_FILENAME) +from azext_aosm.common.constants import ( + BASE_FOLDER_NAME, + MANIFEST_FOLDER_NAME, + VNF_CORE_BASE_TEMPLATE_FILENAME, + VNF_TEMPLATE_FOLDER_NAME, + VNF_MANIFEST_TEMPLATE_FILENAME, + VNF_OUTPUT_FOLDER_FILENAME, + DEPLOYMENT_PARAMETERS_FILENAME, + VHD_PARAMETERS_FILENAME, + TEMPLATE_PARAMETERS_FILENAME +) from azext_aosm.configuration_models.onboarding_vnf_input_config import ( OnboardingCoreVNFInputConfig, ) diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py index 0999f986ef8..534372b2891 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py @@ -3,7 +3,6 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- import json -import re from pathlib import Path from typing import Dict, Any from knack.log import get_logger @@ -11,17 +10,17 @@ from azext_aosm.build_processors.arm_processor import NexusArmBuildProcessor from azext_aosm.build_processors.nexus_image_processor import NexusImageProcessor from azext_aosm.build_processors.base_processor import BaseInputProcessor -from azext_aosm.common.constants import (ARTIFACT_LIST_FILENAME, - BASE_FOLDER_NAME, - MANIFEST_FOLDER_NAME, - VNF_TEMPLATE_FOLDER_NAME, - VNF_MANIFEST_TEMPLATE_FILENAME, - VNF_OUTPUT_FOLDER_FILENAME, - DEPLOYMENT_PARAMETERS_FILENAME, - NEXUS_IMAGE_PARAMETERS_FILENAME, - TEMPLATE_PARAMETERS_FILENAME, - VNF_NEXUS_BASE_TEMPLATE_FILENAME, - ) +from azext_aosm.common.constants import ( + BASE_FOLDER_NAME, + MANIFEST_FOLDER_NAME, + VNF_TEMPLATE_FOLDER_NAME, + VNF_MANIFEST_TEMPLATE_FILENAME, + VNF_OUTPUT_FOLDER_FILENAME, + DEPLOYMENT_PARAMETERS_FILENAME, + NEXUS_IMAGE_PARAMETERS_FILENAME, + TEMPLATE_PARAMETERS_FILENAME, + VNF_NEXUS_BASE_TEMPLATE_FILENAME, +) from azext_aosm.configuration_models.onboarding_vnf_input_config import ( OnboardingNexusVNFInputConfig, ) diff --git a/src/aosm/azext_aosm/common/artifact.py b/src/aosm/azext_aosm/common/artifact.py index 9e13c048f59..184eaa15a50 100644 --- a/src/aosm/azext_aosm/common/artifact.py +++ b/src/aosm/azext_aosm/common/artifact.py @@ -27,8 +27,6 @@ from knack.log import get_logger from oras.client import OrasClient - - logger = get_logger(__name__) diff --git a/src/aosm/azext_aosm/common/constants.py b/src/aosm/azext_aosm/common/constants.py index d3d177f678c..3bf29f16164 100644 --- a/src/aosm/azext_aosm/common/constants.py +++ b/src/aosm/azext_aosm/common/constants.py @@ -79,9 +79,6 @@ class ManifestsExist(str, Enum): CNF_VALUES_SCHEMA_FILENAME = "values.schema.json" CNF_TEMPLATE_FOLDER_NAME = "cnf" - -SEMVER_REGEX = r"^(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)(?:-(?P(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$" - ################# # OLD CONSTANTS # ################# diff --git a/src/aosm/azext_aosm/configuration_models/common_parameters_config.py b/src/aosm/azext_aosm/configuration_models/common_parameters_config.py index d0456db9584..5c65e291345 100644 --- a/src/aosm/azext_aosm/configuration_models/common_parameters_config.py +++ b/src/aosm/azext_aosm/configuration_models/common_parameters_config.py @@ -38,6 +38,7 @@ class CoreVNFCommonParametersConfig(NFDCommonParametersConfig): saArtifactStoreName: str saManifestName: str + @dataclass(frozen=True) class NexusVNFCommonParametersConfig(NFDCommonParametersConfig): """Common parameters configuration for VNFs.""" diff --git a/src/aosm/azext_aosm/configuration_models/onboarding_vnf_input_config.py b/src/aosm/azext_aosm/configuration_models/onboarding_vnf_input_config.py index fb687aff2fc..ee7309190ed 100644 --- a/src/aosm/azext_aosm/configuration_models/onboarding_vnf_input_config.py +++ b/src/aosm/azext_aosm/configuration_models/onboarding_vnf_input_config.py @@ -98,6 +98,7 @@ def validate(self): "One of file_path or sas_blob_url must be set for vhd." ) + @dataclass class OnboardingCoreVNFInputConfig(OnboardingNFDBaseInputConfig): """Input configuration for onboarding VNFs.""" @@ -185,7 +186,12 @@ class OnboardingNexusVNFInputConfig(OnboardingNFDBaseInputConfig): arm_templates: List[ArmTemplatePropertiesConfig] = field( default_factory=lambda: [ArmTemplatePropertiesConfig()], - metadata={"comment": "ARM template configuration. The ARM templates given here would deploy a VM if run. They will be used to generate the VNF."}, + metadata={ + "comment": ( + "ARM template configuration. The ARM templates given here would deploy a VM if run." + "They will be used to generate the VNF." + ) + }, ) images: List[str] = field( diff --git a/src/aosm/azext_aosm/inputs/nexus_image_input.py b/src/aosm/azext_aosm/inputs/nexus_image_input.py index bc7880f47a6..58dfb77e052 100644 --- a/src/aosm/azext_aosm/inputs/nexus_image_input.py +++ b/src/aosm/azext_aosm/inputs/nexus_image_input.py @@ -2,14 +2,9 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- -import copy -import json -from pathlib import Path -from typing import Any, Dict, Optional +from typing import Any, Dict, Optional from knack.log import get_logger - -from azext_aosm.common.constants import BASE_SCHEMA from azext_aosm.inputs.base_input import BaseInput logger = get_logger(__name__) From b075d3be03c6b766c1b1da5b01e6ef788eba9622 Mon Sep 17 00:00:00 2001 From: jordlay <72226943+jordlay@users.noreply.github.com> Date: Mon, 12 Feb 2024 12:47:50 +0000 Subject: [PATCH 24/37] Base handler treats command inputs separately (#143) * refactored base handler init to treat inputs from different commands separately; added better validation * fixed error handling in base handler * fixed error typing --------- Co-authored-by: Jordan --- .../cli_handlers/onboarding_base_handler.py | 70 ++++++++++--------- src/aosm/azext_aosm/custom.py | 24 ++++--- 2 files changed, 51 insertions(+), 43 deletions(-) diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_base_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_base_handler.py index 5d04082e6b8..89d8631a92b 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_base_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_base_handler.py @@ -9,8 +9,8 @@ from dataclasses import fields, is_dataclass from pathlib import Path from typing import Optional, Union - -from azure.cli.core.azclierror import UnclassifiedUserFault +from json.decoder import JSONDecodeError +from azure.cli.core.azclierror import InvalidArgumentValueError, UnclassifiedUserFault, UserFault from jinja2 import StrictUndefined, Template from knack.log import get_logger @@ -37,37 +37,36 @@ class OnboardingBaseCLIHandler(ABC): def __init__( self, - provided_input_path: Optional[Path] = None, + config_file_path: Optional[Path] = None, + all_deploy_params_file_path: Optional[Path] = None, aosm_client: Optional[HybridNetworkManagementClient] = None, skip: Optional[str] = None, ): """Initialize the CLI handler.""" self.aosm_client = aosm_client self.skip = skip - # If config file provided (for build, publish and delete) - if provided_input_path: - # Explicitly define types - self.config: Union[OnboardingBaseInputConfig, BaseCommonParametersConfig] - provided_input_path = Path(provided_input_path) - # If config file is the input.jsonc for build command - if provided_input_path.suffix == ".jsonc": - config_dict = self._read_input_config_from_file(provided_input_path) - try: - self.config = self._get_input_config(config_dict) - except Exception as e: - raise UnclassifiedUserFault(f"The input file provided contains an incorrect input.\nPlease fix the problem parameter:\n{e}") from e - # Validate config before getting processor list, - # in case error with input artifacts i.e helm package - self.config.validate() - self.processors = self._get_processor_list() - # If config file is the all parameters json file for publish/delete - elif provided_input_path.suffix == ".json": - try: - self.config = self._get_params_config(provided_input_path) - except Exception as e: - raise UnclassifiedUserFault(f"The all_deploy.parameters.json in the folder provided contains an incorrect input.\nPlease check if you have provided the correct folder for the definition/design type:\n{e}") from e - else: - raise UnclassifiedUserFault("Invalid input") + # If input.jsonc file provided (therefore if build command run) + if config_file_path: + config_dict = self._read_input_config_from_file(config_file_path) + try: + self.config = self._get_input_config(config_dict) + except TypeError as e: + raise InvalidArgumentValueError( + "The input file provided contains an incorrect input.\n" + f"Please fix the problem parameter:\n{e}") from e + # Validate config before getting processor list, + # in case error with input artifacts i.e helm package + self.config.validate() + self.processors = self._get_processor_list() + # If all_deploy.parameters.json file provided (therefore if publish/delete command run) + elif all_deploy_params_file_path: + try: + self.config = self._get_params_config(all_deploy_params_file_path) + except TypeError as e: + raise InvalidArgumentValueError( + "The all_deploy.parameters.json in the folder " + "provided contains an incorrect input.\nPlease check if you have provided " + f"the correct folder for the definition/design type:\n{e}") from e # If no config file provided (for generate-config) else: self.config = self._get_input_config() @@ -180,11 +179,16 @@ def _read_input_config_from_file(input_json_path: Path) -> dict: Returns config as dictionary. """ - lines = input_json_path.read_text().splitlines() - lines = [line for line in lines if not line.strip().startswith("//")] - config_dict = json.loads("".join(lines)) - - return config_dict + try: + lines = input_json_path.read_text().splitlines() + lines = [line for line in lines if not line.strip().startswith("//")] + config_dict = json.loads("".join(lines)) + return config_dict + except FileNotFoundError as e: + raise UnclassifiedUserFault(f"Invalid config file provided.\nError: {e} ") from e + except JSONDecodeError as e: + raise UnclassifiedUserFault("Invalid JSON found in the config file provided.\n" + f"Error: {e} ") from e @staticmethod def _render_base_bicep_contents(template_path): @@ -338,4 +342,4 @@ def _build_deploy_params_schema(schema_properties): "properties": {}, } schema_contents["properties"] = schema_properties - return schema_contents + return schema_contents \ No newline at end of file diff --git a/src/aosm/azext_aosm/custom.py b/src/aosm/azext_aosm/custom.py index e7da513a9c2..17d89aef878 100644 --- a/src/aosm/azext_aosm/custom.py +++ b/src/aosm/azext_aosm/custom.py @@ -22,7 +22,7 @@ def onboard_nfd_generate_config(definition_type: str, output_file: str | None): """Generate config file for onboarding NFs.""" # Declare types explicitly - handler: OnboardingCNFCLIHandler | OnboardingVNFCLIHandler + handler: OnboardingCNFCLIHandler | OnboardingVNFCLIHandler | OnboardingNexusVNFCLIHandler if definition_type == CNF: handler = OnboardingCNFCLIHandler() elif definition_type == VNF: @@ -40,13 +40,13 @@ def onboard_nfd_build( ): """Build the NF definition.""" # Declare types explicitly - handler: OnboardingCNFCLIHandler | OnboardingVNFCLIHandler + handler: OnboardingCNFCLIHandler | OnboardingVNFCLIHandler | OnboardingNexusVNFCLIHandler if definition_type == CNF: - handler = OnboardingCNFCLIHandler(Path(config_file), skip=skip) + handler = OnboardingCNFCLIHandler(config_file_path=Path(config_file), skip=skip) elif definition_type == VNF: - handler = OnboardingCoreVNFCLIHandler(Path(config_file)) + handler = OnboardingCoreVNFCLIHandler(config_file_path=Path(config_file)) elif definition_type == VNF_NEXUS: - handler = OnboardingNexusVNFCLIHandler(Path(config_file)) + handler = OnboardingNexusVNFCLIHandler(config_file_path=Path(config_file)) else: raise UnrecognizedArgumentError( "Invalid definition type, valid values are 'cnf', 'vnf' or 'vnfnexus'") @@ -70,11 +70,14 @@ def onboard_nfd_publish( # Declare types explicitly handler: OnboardingCNFCLIHandler | OnboardingVNFCLIHandler if definition_type == CNF: - handler = OnboardingCNFCLIHandler(Path(build_output_folder, ALL_PARAMETERS_FILE_NAME)) + handler = OnboardingCNFCLIHandler( + all_deploy_params_file_path=Path(build_output_folder, ALL_PARAMETERS_FILE_NAME)) elif definition_type == VNF: - handler = OnboardingCoreVNFCLIHandler(Path(build_output_folder, ALL_PARAMETERS_FILE_NAME)) + handler = OnboardingCoreVNFCLIHandler( + all_deploy_params_file_path=Path(build_output_folder, ALL_PARAMETERS_FILE_NAME)) elif definition_type == VNF_NEXUS: - handler = OnboardingNexusVNFCLIHandler(Path(build_output_folder, ALL_PARAMETERS_FILE_NAME)) + handler = OnboardingNexusVNFCLIHandler( + all_deploy_params_file_path=Path(build_output_folder, ALL_PARAMETERS_FILE_NAME)) else: raise UnrecognizedArgumentError( "Invalid definition type, valid values are 'cnf', 'vnf' or 'vnfnexus'") @@ -103,7 +106,8 @@ def onboard_nsd_generate_config(output_file: str | None): def onboard_nsd_build(config_file: Path, cmd: AzCliCommand): """Build the NSD definition.""" command_context = CommandContext(cli_ctx=cmd.cli_ctx) - handler = OnboardingNSDCLIHandler(Path(config_file), command_context.aosm_client) + handler = OnboardingNSDCLIHandler(config_file_path=Path(config_file), + aosm_client=command_context.aosm_client) handler.build() @@ -121,7 +125,7 @@ def onboard_nsd_publish( }, ) handler = OnboardingNSDCLIHandler( - Path(build_output_folder, ALL_PARAMETERS_FILE_NAME) + all_deploy_params_file_path=Path(build_output_folder, ALL_PARAMETERS_FILE_NAME) ) handler.publish(command_context=command_context) From 17bd37bdebf0fa6cdb73c75a8fa22890a4c4047d Mon Sep 17 00:00:00 2001 From: Jordan Date: Mon, 12 Feb 2024 17:53:05 +0000 Subject: [PATCH 25/37] mypy fixes --- .../cli_handlers/onboarding_base_handler.py | 6 +++--- .../cli_handlers/onboarding_cnf_handler.py | 2 ++ .../cli_handlers/onboarding_core_vnf_handler.py | 12 +++++++----- .../cli_handlers/onboarding_nexus_vnf_handler.py | 2 ++ .../cli_handlers/onboarding_nsd_handler.py | 2 ++ src/aosm/azext_aosm/common/artifact.py | 4 ++-- src/aosm/azext_aosm/common/constants.py | 2 +- .../onboarding_vnf_input_config.py | 10 ++++++---- 8 files changed, 25 insertions(+), 15 deletions(-) diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_base_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_base_handler.py index 89d8631a92b..2beb0df8cef 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_base_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_base_handler.py @@ -8,9 +8,9 @@ from abc import ABC, abstractmethod from dataclasses import fields, is_dataclass from pathlib import Path -from typing import Optional, Union +from typing import Optional from json.decoder import JSONDecodeError -from azure.cli.core.azclierror import InvalidArgumentValueError, UnclassifiedUserFault, UserFault +from azure.cli.core.azclierror import InvalidArgumentValueError, UnclassifiedUserFault from jinja2 import StrictUndefined, Template from knack.log import get_logger @@ -342,4 +342,4 @@ def _build_deploy_params_schema(schema_properties): "properties": {}, } schema_contents["properties"] = schema_properties - return schema_contents \ No newline at end of file + return schema_contents diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_cnf_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_cnf_handler.py index 5bd144b0c7e..3cd1707770e 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_cnf_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_cnf_handler.py @@ -61,6 +61,8 @@ class OnboardingCNFCLIHandler(OnboardingNFDBaseCLIHandler): """CLI handler for publishing NFDs.""" + config: OnboardingCNFInputConfig + @property def default_config_file_name(self) -> str: """Get the default configuration file name.""" diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py index 9473dc699f1..a0f124faee0 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py @@ -4,7 +4,7 @@ # -------------------------------------------------------------------------------------------- import json from pathlib import Path -from typing import Dict, Any +from typing import Dict, Any, List, Tuple, Optional from knack.log import get_logger from azext_aosm.build_processors.arm_processor import AzureCoreArmBuildProcessor @@ -44,8 +44,10 @@ class OnboardingCoreVNFCLIHandler(OnboardingVNFCLIHandler): """CLI handler for publishing NFDs.""" + config: OnboardingCoreVNFInputConfig + def _get_input_config( - self, input_config: Dict[str, Any] = None + self, input_config: Optional[dict] = None ) -> OnboardingCoreVNFInputConfig: """Get the configuration for the command.""" if input_config is None: @@ -62,7 +64,7 @@ def _get_params_config( params_dict = {} return CoreVNFCommonParametersConfig(**params_dict) - def _get_processor_list(self) -> [BaseInputProcessor]: + def _get_processor_list(self) -> List[BaseInputProcessor]: """Get the list of processors.""" processor_list = [] # for each arm template, instantiate arm processor @@ -157,7 +159,7 @@ def build_all_parameters_json(self) -> JSONDefinitionElementBuilder: "acrArtifactStoreName": self.config.acr_artifact_store_name, "saArtifactStoreName": self.config.blob_artifact_store_name, "acrManifestName": self.config.acr_artifact_store_name + "-manifest", - "saManifestName": self.config.blob_artifact_store_name + "-manifest", + "saManifestName": self.config.sa_manifest_name, "nfDefinitionGroup": self.config.nf_name, "nfDefinitionVersion": self.config.version } @@ -182,7 +184,7 @@ def _get_default_config(self, vhd) -> Dict[str, Any]: default_config.update({"image_api_version": vhd.image_api_version}) return default_config - def _generate_type_specific_nf_application(self, processor) -> "tuple[list, list]": + def _generate_type_specific_nf_application(self, processor) -> Tuple[List, List]: """Generate the type specific nf application.""" arm_nf = [] image_nf = [] diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py index 534372b2891..2645a828274 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py @@ -44,6 +44,8 @@ class OnboardingNexusVNFCLIHandler(OnboardingVNFCLIHandler): """CLI handler for publishing NFDs.""" + config: OnboardingNexusVNFInputConfig + def _get_input_config( self, input_config: Dict[str, Any] = None ) -> OnboardingNexusVNFInputConfig: diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_nsd_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_nsd_handler.py index 26444bfb181..148e9a0a4a1 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_nsd_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_nsd_handler.py @@ -56,6 +56,8 @@ class OnboardingNSDCLIHandler(OnboardingBaseCLIHandler): """CLI handler for publishing NFDs.""" + config: OnboardingNSDInputConfig + @property def default_config_file_name(self) -> str: """Get the default configuration file name.""" diff --git a/src/aosm/azext_aosm/common/artifact.py b/src/aosm/azext_aosm/common/artifact.py index 184eaa15a50..8487b1fce2a 100644 --- a/src/aosm/azext_aosm/common/artifact.py +++ b/src/aosm/azext_aosm/common/artifact.py @@ -547,12 +547,12 @@ class BaseStorageAccountArtifact(BaseArtifact): @abstractmethod def upload( - self, config: NFDCommonParametersConfig, command_context: CommandContext + self, config: BaseCommonParametersConfig, command_context: CommandContext ): """Upload the artifact.""" def _get_blob_client( - self, config: NFDCommonParametersConfig, command_context: CommandContext + self, config: BaseCommonParametersConfig, command_context: CommandContext ) -> BlobClient: container_basename = self.artifact_name.replace("-", "") container_name = f"{container_basename}-{self.artifact_version}" diff --git a/src/aosm/azext_aosm/common/constants.py b/src/aosm/azext_aosm/common/constants.py index 3bf29f16164..7a9f9641815 100644 --- a/src/aosm/azext_aosm/common/constants.py +++ b/src/aosm/azext_aosm/common/constants.py @@ -11,7 +11,7 @@ VNF = "vnf" CNF = "cnf" NSD = "nsd" -VNF_NEXUS = "vnfnexus" +VNF_NEXUS = "vnf-nexus" class DeployableResourceTypes(str, Enum): VNF = VNF diff --git a/src/aosm/azext_aosm/configuration_models/onboarding_vnf_input_config.py b/src/aosm/azext_aosm/configuration_models/onboarding_vnf_input_config.py index ee7309190ed..13984a07d92 100644 --- a/src/aosm/azext_aosm/configuration_models/onboarding_vnf_input_config.py +++ b/src/aosm/azext_aosm/configuration_models/onboarding_vnf_input_config.py @@ -103,7 +103,7 @@ def validate(self): class OnboardingCoreVNFInputConfig(OnboardingNFDBaseInputConfig): """Input configuration for onboarding VNFs.""" - blob_artifact_store_name: str | None = field( + blob_artifact_store_name: str = field( default="", metadata={ "comment": ( @@ -122,7 +122,6 @@ class OnboardingCoreVNFInputConfig(OnboardingNFDBaseInputConfig): }, ) - # TODO: Add better comments arm_templates: List[ArmTemplatePropertiesConfig] = field( default_factory=lambda: [ArmTemplatePropertiesConfig()], metadata={ @@ -148,11 +147,14 @@ def __post_init__(self): if self.vhd and isinstance(self.vhd, dict): self.vhd = VhdImageConfig(**self.vhd) + sanitized_nf_name = self.nf_name.lower().replace("_", "-") + if not self.blob_artifact_store_name: + self.blob_artifact_store_name = sanitized_nf_name + "-sa" + @property def sa_manifest_name(self) -> str: """Return the Storage account manifest name from the NFD name.""" - sanitized_nf_name = self.nf_name.lower().replace("_", "-") - return f"{sanitized_nf_name}-sa-manifest-{self.version.replace('.', '-')}" + return f"{self.blob_artifact_store_name}-manifest-{self.version.replace('.', '-')}" def validate(self): """Validate the configuration.""" From 143f3d25b1ffa9782b78b8697c636a8bca79769c Mon Sep 17 00:00:00 2001 From: Jordan Date: Fri, 16 Feb 2024 10:55:56 +0000 Subject: [PATCH 26/37] fixed customLocation id for vnf nexus --- src/aosm/HISTORY.rst | 8 ++++++++ src/aosm/azext_aosm/inputs/nfd_input.py | 4 ++-- src/aosm/setup.py | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/aosm/HISTORY.rst b/src/aosm/HISTORY.rst index c87db19e277..fc399f6b606 100644 --- a/src/aosm/HISTORY.rst +++ b/src/aosm/HISTORY.rst @@ -4,7 +4,15 @@ Release History =============== Unreleased + +1.0.0b7 +++++++++ +* Fixed customLocation missing from Nexus + +++++++++ +1.0.0b6 ++++++++ +* Added Nexus support 1.0.0b5 ++++++++ diff --git a/src/aosm/azext_aosm/inputs/nfd_input.py b/src/aosm/azext_aosm/inputs/nfd_input.py index 1d9fbd0ddeb..f39d5fe1a2f 100644 --- a/src/aosm/azext_aosm/inputs/nfd_input.py +++ b/src/aosm/azext_aosm/inputs/nfd_input.py @@ -73,8 +73,8 @@ def get_defaults(self) -> Dict[str, Any]: } if self.network_function_definition.properties and ( - self.network_function_definition.properties.network_function_type - == "VirtualNetworkFunction" + self.network_function_definition.properties.network_function_template.nfvi_type + not in ("AzureArcKubernetes","AzureOperatorNexus") ): base_defaults["configObject"]["customLocationId"] = "" diff --git a/src/aosm/setup.py b/src/aosm/setup.py index 36a210d0207..3b3ed0c6668 100644 --- a/src/aosm/setup.py +++ b/src/aosm/setup.py @@ -17,7 +17,7 @@ # Confirm this is the right version number you want and it matches your # HISTORY.rst entry. -VERSION = "1.0.0b5" +VERSION = "1.0.0b7" # The full list of classifiers is available at From 0ff7b3d3559f647d73a49ff9fc3300270b10da6e Mon Sep 17 00:00:00 2001 From: Jordan Date: Fri, 16 Feb 2024 15:55:34 +0000 Subject: [PATCH 27/37] markups from review inc moving build manifest to parent vnf handler --- src/aosm/azext_aosm/_params.py | 2 +- .../build_processors/arm_processor.py | 9 ++- .../build_processors/nexus_image_processor.py | 3 +- .../build_processors/vhd_processor.py | 3 +- .../onboarding_core_vnf_handler.py | 65 ++++++------------- .../onboarding_nexus_vnf_handler.py | 41 ++---------- .../cli_handlers/onboarding_vnf_handler.py | 39 +++++++++++ 7 files changed, 76 insertions(+), 86 deletions(-) diff --git a/src/aosm/azext_aosm/_params.py b/src/aosm/azext_aosm/_params.py index 1b4662fa845..b36b014ee02 100644 --- a/src/aosm/azext_aosm/_params.py +++ b/src/aosm/azext_aosm/_params.py @@ -37,7 +37,7 @@ def load_arguments(self: AzCommandsLoader, _): c.argument( "definition_type", arg_type=definition_type, - help="Type of AOSM definition.", + help="Type of AOSM definition to be published.", required=True, ) c.argument( diff --git a/src/aosm/azext_aosm/build_processors/arm_processor.py b/src/aosm/azext_aosm/build_processors/arm_processor.py index d85044aec46..9d5fa569a1c 100644 --- a/src/aosm/azext_aosm/build_processors/arm_processor.py +++ b/src/aosm/azext_aosm/build_processors/arm_processor.py @@ -119,7 +119,7 @@ def generate_nfvi_specific_nf_application(self): def generate_resource_element_template(self) -> ResourceElementTemplate: """Generate the resource element template. - Note: There is no Nexus specific RET, arm RET can deploy anything (except NFs) + Note: There is no Nexus specific RET """ parameter_values = self.generate_values_mappings( self.input_artifact.get_schema(), self.input_artifact.get_defaults(), True @@ -161,10 +161,9 @@ def generate_parameters_file(self) -> LocalFileBuilder: json.dumps(json.loads(params), indent=4), ) - def _generate_mapping_rule_profile( - self, - ): - raise NotImplementedError("This method must be implemented in a subclass.") + @abstractmethod + def _generate_mapping_rule_profile(self): + pass class AzureCoreArmBuildProcessor(BaseArmBuildProcessor): diff --git a/src/aosm/azext_aosm/build_processors/nexus_image_processor.py b/src/aosm/azext_aosm/build_processors/nexus_image_processor.py index 8c34bb63c1e..22d908af529 100644 --- a/src/aosm/azext_aosm/build_processors/nexus_image_processor.py +++ b/src/aosm/azext_aosm/build_processors/nexus_image_processor.py @@ -109,7 +109,8 @@ def generate_resource_element_template(self) -> ResourceElementTemplate: :raises NotImplementedError: NSDs do not support deployment of Nexus images. """ - raise NotImplementedError("NSDs do not support deployment of Nexus images.") + raise NotImplementedError("NSDs do not support deployment of Nexus images directly, " + "they must be provided in the NF.") def _generate_artifact_profile(self) -> AzureOperatorNexusImageArtifactProfile: """ diff --git a/src/aosm/azext_aosm/build_processors/vhd_processor.py b/src/aosm/azext_aosm/build_processors/vhd_processor.py index a9167626e46..f9ddb78aac3 100644 --- a/src/aosm/azext_aosm/build_processors/vhd_processor.py +++ b/src/aosm/azext_aosm/build_processors/vhd_processor.py @@ -136,7 +136,8 @@ def generate_resource_element_template(self) -> ResourceElementTemplate: :raises NotImplementedError: NSDs do not support deployment of VHDs. """ - raise NotImplementedError("NSDs do not support deployment of VHDs.") + raise NotImplementedError("NSDs do not support deployment of VHDs directly, " + "they must be provided in the NF.") def _generate_artifact_profile(self) -> AzureCoreVhdImageArtifactProfile: """ diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py index a0f124faee0..65af6a13890 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py @@ -12,10 +12,8 @@ from azext_aosm.build_processors.base_processor import BaseInputProcessor from azext_aosm.common.constants import ( BASE_FOLDER_NAME, - MANIFEST_FOLDER_NAME, VNF_CORE_BASE_TEMPLATE_FILENAME, VNF_TEMPLATE_FOLDER_NAME, - VNF_MANIFEST_TEMPLATE_FILENAME, VNF_OUTPUT_FOLDER_FILENAME, DEPLOYMENT_PARAMETERS_FILENAME, VHD_PARAMETERS_FILENAME, @@ -108,48 +106,6 @@ def build_base_bicep(self) -> BicepDefinitionElementBuilder: ) return bicep_file - def build_manifest_bicep(self) -> BicepDefinitionElementBuilder: - """Build the manifest bicep file.""" - acr_artifact_list = [] - - logger.info("Creating artifact manifest bicep") - - for processor in self.processors: - if isinstance(processor, AzureCoreArmBuildProcessor): - acr_artifact_list.extend(processor.get_artifact_manifest_list()) - logger.debug( - "Created list of artifacts from %s arm template(s) provided: %s", - len(self.config.arm_templates), - acr_artifact_list, - ) - elif isinstance(processor, VHDProcessor): - sa_artifact_list = processor.get_artifact_manifest_list() - logger.debug( - "Created list of artifacts from vhd image provided: %s", - sa_artifact_list, - ) - - # Build manifest bicep contents, with j2 template - template_path = get_template_path( - VNF_TEMPLATE_FOLDER_NAME, VNF_MANIFEST_TEMPLATE_FILENAME - ) - params = { - "acr_artifacts": acr_artifact_list, - "sa_artifacts": sa_artifact_list, - } - bicep_contents = render_bicep_contents_from_j2( - template_path, params - ) - - # Create Bicep element with manifest contents - bicep_file = BicepDefinitionElementBuilder( - Path(VNF_OUTPUT_FOLDER_FILENAME, MANIFEST_FOLDER_NAME), - bicep_contents, - ) - - logger.info("Created artifact manifest bicep element") - return bicep_file - def build_all_parameters_json(self) -> JSONDefinitionElementBuilder: """Create object for all_parameters.json.""" params_content = { @@ -199,6 +155,27 @@ def _generate_type_specific_nf_application(self, processor) -> Tuple[List, List] logger.debug("Created nf application %s", nf_application.name) return (arm_nf, image_nf) + def _generate_type_specific_artifact_manifest(self, processor): + """Generate the type specific artifact manifest list.""" + arm_artifacts = [] + sa_artifacts = [] + + if isinstance(processor, AzureCoreArmBuildProcessor): + arm_artifacts = processor.get_artifact_manifest_list() + logger.debug( + "Created list of artifacts from %s arm template(s) provided: %s", + len(self.config.arm_templates), + arm_artifacts, + ) + elif isinstance(processor, VHDProcessor): + sa_artifacts = processor.get_artifact_manifest_list() + logger.debug( + "Created list of artifacts from vhd image provided: %s", + sa_artifacts, + ) + + return (arm_artifacts, sa_artifacts) + def _get_nfd_template_params( self, arm_nf_application_list, image_nf_application_list) -> Dict[str, Any]: """Get the nfd template params.""" diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py index 2645a828274..32a2b78df68 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py @@ -106,40 +106,6 @@ def build_base_bicep(self) -> BicepDefinitionElementBuilder: ) return bicep_file - def build_manifest_bicep(self) -> BicepDefinitionElementBuilder: - """Build the manifest bicep file.""" - acr_artifact_list = [] - - logger.info("Creating artifact manifest bicep") - - for processor in self.processors: - acr_artifact_list.extend(processor.get_artifact_manifest_list()) - logger.debug( - "Created list of artifacts from arm template(s) and image files(s) provided: %s", - acr_artifact_list, - ) - - # Build manifest bicep contents, with j2 template - template_path = get_template_path( - VNF_TEMPLATE_FOLDER_NAME, VNF_MANIFEST_TEMPLATE_FILENAME - ) - params = { - "acr_artifacts": acr_artifact_list, - "sa_artifacts": [] - } - bicep_contents = render_bicep_contents_from_j2( - template_path, params - ) - - # Create Bicep element with manifest contents - bicep_file = BicepDefinitionElementBuilder( - Path(VNF_OUTPUT_FOLDER_FILENAME, MANIFEST_FOLDER_NAME), - bicep_contents, - ) - - logger.info("Created artifact manifest bicep element") - return bicep_file - def build_all_parameters_json(self) -> JSONDefinitionElementBuilder: """Build the all parameters json file.""" params_content = { @@ -177,6 +143,13 @@ def _generate_type_specific_nf_application(self, processor) -> "tuple[list, list logger.debug("Created nf application %s", nf_application.name) return (arm_nf, image_nf) + def _generate_type_specific_artifact_manifest(self, processor): + """Generate the type specific artifact manifest list""" + arm_manifest = processor.get_artifact_manifest_list() + sa_manifest = [] + + return (arm_manifest, sa_manifest) + def _get_nfd_template_params( self, arm_nf_application_list, image_nf_application_list) -> Dict[str, Any]: """Get the nfd template params.""" diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_vnf_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_vnf_handler.py index e7f6c0c2460..af5565d4aaa 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_vnf_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_vnf_handler.py @@ -21,6 +21,8 @@ VNF_INPUT_FILENAME, VNF_OUTPUT_FOLDER_FILENAME, VNF_TEMPLATE_FOLDER_NAME, + MANIFEST_FOLDER_NAME, + VNF_MANIFEST_TEMPLATE_FILENAME ) logger = get_logger(__name__) @@ -108,6 +110,39 @@ def build_resource_bicep(self) -> BicepDefinitionElementBuilder: ) return bicep_file + def build_manifest_bicep(self) -> BicepDefinitionElementBuilder: + """Build the manifest bicep file.""" + acr_artifact_list = [] + sa_artifact_list = [] + + logger.info("Creating artifact manifest bicep") + + for processor in self.processors: + (arm_artifact, sa_artifact) = self._generate_type_specific_artifact_manifest(processor) + acr_artifact_list.extend(arm_artifact) + sa_artifact_list.extend(sa_artifact) + + # Build manifest bicep contents, with j2 template + template_path = get_template_path( + VNF_TEMPLATE_FOLDER_NAME, VNF_MANIFEST_TEMPLATE_FILENAME + ) + params = { + "acr_artifacts": acr_artifact_list, + "sa_artifacts": sa_artifact_list + } + bicep_contents = render_bicep_contents_from_j2( + template_path, params + ) + + # Create Bicep element with manifest contents + bicep_file = BicepDefinitionElementBuilder( + Path(VNF_OUTPUT_FOLDER_FILENAME, MANIFEST_FOLDER_NAME), + bicep_contents, + ) + + logger.info("Created artifact manifest bicep element") + return bicep_file + @abstractmethod def _get_nfd_template_params(self, arm_nf_application_list, image_nf_application_list): return NotImplementedError @@ -115,3 +150,7 @@ def _get_nfd_template_params(self, arm_nf_application_list, image_nf_application @abstractmethod def _generate_type_specific_nf_application(self, processor): return NotImplementedError + + @abstractmethod + def _generate_type_specific_artifact_manifest(self, processor): + return NotImplementedError From 9d3eb7bdcfc2801f963ce3f6d55fc3687af06af3 Mon Sep 17 00:00:00 2001 From: Jordan Date: Fri, 16 Feb 2024 16:02:12 +0000 Subject: [PATCH 28/37] added better docstrings --- .../cli_handlers/onboarding_vnf_handler.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_vnf_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_vnf_handler.py index af5565d4aaa..333b5d7fb95 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_vnf_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_vnf_handler.py @@ -42,7 +42,12 @@ def output_folder_file_name(self) -> str: return VNF_OUTPUT_FOLDER_FILENAME def build_artifact_list(self) -> ArtifactDefinitionElementBuilder: - """Build the artifact list.""" + """Build the artifact list. + + Gets list of artifacts to be including in the artifacts.json. + This is used during the publish command, to upload the artifacts correctly. + + """ logger.info("Creating artifacts list for artifacts.json") artifact_list = [] # For each arm template, get list of artifacts and combine @@ -62,7 +67,16 @@ def build_artifact_list(self) -> ArtifactDefinitionElementBuilder: ) def build_resource_bicep(self) -> BicepDefinitionElementBuilder: - """Build the resource bicep file.""" + """Build the resource bicep file. + + Creates nfDefinition.bicep and its supporting files. + + For each processor: + - Generates NF application for each processor + - Generates deploymentParameters (flattened to be one schema overall) + - Generates supporting parameters files (to avoid stringified JSON in template) + + """ logger.info("Creating artifacts list for artifacts.json") arm_nf_application_list = [] image_nf_application_list = [] From f5cd3206e62fd5132a7e0245133674b72fc7f347 Mon Sep 17 00:00:00 2001 From: Jordan Date: Mon, 19 Feb 2024 16:06:58 +0000 Subject: [PATCH 29/37] updated history.rst --- src/aosm/HISTORY.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/aosm/HISTORY.rst b/src/aosm/HISTORY.rst index fc399f6b606..58938f46e65 100644 --- a/src/aosm/HISTORY.rst +++ b/src/aosm/HISTORY.rst @@ -7,7 +7,8 @@ Unreleased 1.0.0b7 ++++++++ -* Fixed customLocation missing from Nexus +* Fixed: customLocation missing from Nexus +* Fixed: helm charts not uploading correctly ++++++++ 1.0.0b6 From 03c1680b6bd861deba3e94c354945aa482707729 Mon Sep 17 00:00:00 2001 From: jordlay <72226943+jordlay@users.noreply.github.com> Date: Mon, 19 Feb 2024 17:02:35 +0000 Subject: [PATCH 30/37] Achurchard/fix helm chart upload (#144) * Use helm push for Helm charts (not oras push) * Logging --------- Co-authored-by: Andy Churchard Co-authored-by: Jordan --- .../build_processors/nfd_processor.py | 2 +- src/aosm/azext_aosm/common/artifact.py | 119 ++++++++++++++---- 2 files changed, 99 insertions(+), 22 deletions(-) diff --git a/src/aosm/azext_aosm/build_processors/nfd_processor.py b/src/aosm/azext_aosm/build_processors/nfd_processor.py index 6c8d7d97743..7c0e52986ca 100644 --- a/src/aosm/azext_aosm/build_processors/nfd_processor.py +++ b/src/aosm/azext_aosm/build_processors/nfd_processor.py @@ -69,7 +69,7 @@ def get_artifact_details( # Path is relative to NSD_OUTPUT_FOLDER_FILENAME as this artifact is stored in the NSD output folder artifact_details = LocalFileACRArtifact( artifact_name=self.input_artifact.artifact_name, - artifact_type=ArtifactType.OCI_ARTIFACT.value, + artifact_type=ArtifactType.ARM_TEMPLATE.value, artifact_version=self.input_artifact.artifact_version, file_path=self.input_artifact.arm_template_output_path.relative_to( Path(NSD_OUTPUT_FOLDER_FILENAME) diff --git a/src/aosm/azext_aosm/common/artifact.py b/src/aosm/azext_aosm/common/artifact.py index 8487b1fce2a..bad5dc49daf 100644 --- a/src/aosm/azext_aosm/common/artifact.py +++ b/src/aosm/azext_aosm/common/artifact.py @@ -20,6 +20,7 @@ BlobClient, BlobType, ) +from azext_aosm.vendored_sdks.models import ArtifactType from azext_aosm.vendored_sdks import HybridNetworkManagementClient from azext_aosm.common.command_context import CommandContext from azext_aosm.common.utils import convert_bicep_to_arm @@ -186,32 +187,108 @@ def upload( target_acr = self._get_acr(oras_client) target = f"{target_acr}/{self.artifact_name}:{self.artifact_version}" logger.debug("Uploading %s to %s", self.file_path, target) - retries = 0 - while True: - try: - oras_client.push(files=[self.file_path], target=target) - break - except ValueError as error: - if retries < 20: - logger.info( - "Retrying pushing local artifact to ACR. Retries so far: %s", - retries, + + if self.artifact_type == ArtifactType.ARM_TEMPLATE.value: + retries = 0 + while True: + try: + oras_client.push(files=[self.file_path], target=target) + break + except ValueError as error: + if retries < 20: + logger.info( + "Retrying pushing local artifact to ACR. Retries so far: %s", + retries, + ) + retries += 1 + sleep(3) + continue + + logger.error( + "Failed to upload %s to %s. Check if this image exists in the" + " source registry %s.", + self.file_path, + target, + target_acr, ) - retries += 1 - sleep(3) - continue + logger.debug(error, exc_info=True) + raise error - logger.error( - "Failed to upload %s to %s. Check if this image exists in the" - " source registry %s.", + logger.info("LocalFileACRArtifact uploaded %s to %s using oras push", self.file_path, target) + + elif self.artifact_type == ArtifactType.OCI_ARTIFACT.value: + + target_acr_name = target_acr.replace(".azurecr.io", "") + target_acr_with_protocol = f"oci://{target_acr}" + username = manifest_credentials["username"] + password = manifest_credentials["acr_token"] + + # TODO: Maybe port over the "if not use_manifest_permissions" feature which means you don't need to + # install docker. Although not having to install docker feels like a marginal enhancement, as most + # people playing with containers will have docker, or won't mind installing it. Note that + # use_manifest_permissions is now in command_context.cli_options['no_subscription_permissions'] + + self._check_tool_installed("docker") + self._check_tool_installed("helm") + + # TODO: don't just dump this in /tmp + if self.file_path.is_dir(): + helm_package_cmd = [ + str(shutil.which("helm")), + "package", self.file_path, - target, + "--destination", + "/tmp", + ] + self._call_subprocess_raise_output(helm_package_cmd) + + # This seems to prevent occasional helm login failures + acr_login_cmd = [ + str(shutil.which("az")), + "acr", + "login", + "--name", + target_acr_name, + "--username", + username, + "--password", + password, + ] + self._call_subprocess_raise_output(acr_login_cmd) + + try: + helm_login_cmd = [ + str(shutil.which("helm")), + "registry", + "login", target_acr, - ) - logger.debug(error, exc_info=True) - raise error + "--username", + username, + "--password", + password, + ] + self._call_subprocess_raise_output(helm_login_cmd) + + push_command = [ + str(shutil.which("helm")), + "push", + f"/tmp/{self.artifact_name}-{self.artifact_version}.tgz", # TODO: fix up helm package to use non-tmp path + target_acr_with_protocol, + ] + self._call_subprocess_raise_output(push_command) + finally: + helm_logout_cmd = [ + str(shutil.which("helm")), + "registry", + "logout", + target_acr, + ] + self._call_subprocess_raise_output(helm_logout_cmd) + + logger.info("LocalFileACRArtifact uploaded %s to %s using helm push", self.file_path, target) - logger.info("LocalFileACRArtifact uploaded %s to %s", self.file_path, target) + else: # TODO: Make this one of the allowed Azure CLI exceptions + raise ValueError(f"Unexpected artifact type. Got {self.artifact_type}. Expected {ArtifactType.ARM_TEMPLATE.value} or {ArtifactType.OCI_ARTIFACT.value}") class RemoteACRArtifact(BaseACRArtifact): From b3d9f544a04119f462e22d1708aeeaef081272ac Mon Sep 17 00:00:00 2001 From: Jordan Date: Mon, 19 Feb 2024 17:07:14 +0000 Subject: [PATCH 31/37] bumped version --- src/aosm/HISTORY.rst | 3 +++ src/aosm/setup.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/aosm/HISTORY.rst b/src/aosm/HISTORY.rst index 58938f46e65..903bb1b9a37 100644 --- a/src/aosm/HISTORY.rst +++ b/src/aosm/HISTORY.rst @@ -5,6 +5,9 @@ Release History Unreleased +1.0.0b8 +++++++++ + 1.0.0b7 ++++++++ * Fixed: customLocation missing from Nexus diff --git a/src/aosm/setup.py b/src/aosm/setup.py index 3b3ed0c6668..09285dd7913 100644 --- a/src/aosm/setup.py +++ b/src/aosm/setup.py @@ -17,7 +17,7 @@ # Confirm this is the right version number you want and it matches your # HISTORY.rst entry. -VERSION = "1.0.0b7" +VERSION = "1.0.0b8" # The full list of classifiers is available at From b6c781be2f253047f3ef76ede06bfe5d48f415f6 Mon Sep 17 00:00:00 2001 From: jordlay <72226943+jordlay@users.noreply.github.com> Date: Mon, 26 Feb 2024 15:12:49 +0000 Subject: [PATCH 32/37] Bug: No type in schema (#148) * first fix for anyOf logic error msg * Update src/aosm/azext_aosm/build_processors/base_processor.py Co-authored-by: Cyclam <95434717+Cyclam@users.noreply.github.com> * Update src/aosm/azext_aosm/build_processors/base_processor.py Co-authored-by: Cyclam <95434717+Cyclam@users.noreply.github.com> --------- Co-authored-by: Jordan Co-authored-by: Cyclam <95434717+Cyclam@users.noreply.github.com> --- .../build_processors/base_processor.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/aosm/azext_aosm/build_processors/base_processor.py b/src/aosm/azext_aosm/build_processors/base_processor.py index 13e0031b33e..34dd9fc42ea 100644 --- a/src/aosm/azext_aosm/build_processors/base_processor.py +++ b/src/aosm/azext_aosm/build_processors/base_processor.py @@ -8,7 +8,7 @@ from typing import Any, Dict, List, Tuple from knack.log import get_logger - +from azure.cli.core.azclierror import InvalidArgumentValueError from azext_aosm.common.artifact import BaseArtifact from azext_aosm.common.constants import CGS_NAME from azext_aosm.definition_folder.builder.local_file_builder import LocalFileBuilder @@ -184,6 +184,21 @@ def generate_values_mappings( # Loop through each property in the schema. for subschema_name, subschema in schema["properties"].items(): + + if "type" not in subschema: + if "oneOf" or "anyOf" in subschema: + raise InvalidArgumentValueError( + f"The subschema '{subschema_name}' does not contain a type.\n" + "It contains 'anyOf' or 'oneOf' logic, which is not valid for AOSM.\n" + "Please remove this from your values.schema.json and provide a concrete type " + "or remove the schema and the CLI will generate a generic schema." + ) + else: + raise InvalidArgumentValueError( + f"The subschema {subschema_name} does not contain a type. This is a required field.\n" + "Please fix your values.schema.json or remove the schema and the CLI will generate a " + "generic schema." + ) # If the property is not in the values, and is required, add it to the values. if ( "required" in schema From 84e5a3b7ff4b0c4ade55c156a6ed77f879770830 Mon Sep 17 00:00:00 2001 From: jordlay <72226943+jordlay@users.noreply.github.com> Date: Mon, 26 Feb 2024 15:15:44 +0000 Subject: [PATCH 33/37] Bug: Nexus Image Version Must be Semver (#149) * added semver checking to input config validation; moved split image path to utils * changed semver regex; renamed function * fixed typo * markups * fix typo --------- Co-authored-by: Jordan --- .../onboarding_nexus_vnf_handler.py | 11 ++--------- src/aosm/azext_aosm/common/constants.py | 2 ++ src/aosm/azext_aosm/common/utils.py | 19 +++++++++++++++++-- .../onboarding_vnf_input_config.py | 7 ++++++- 4 files changed, 27 insertions(+), 12 deletions(-) diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py index 32a2b78df68..600465c30b2 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py @@ -36,7 +36,7 @@ from azext_aosm.inputs.arm_template_input import ArmTemplateInput from azext_aosm.inputs.nexus_image_input import NexusImageFileInput from .onboarding_vnf_handler import OnboardingVNFCLIHandler -from azext_aosm.common.utils import render_bicep_contents_from_j2, get_template_path +from azext_aosm.common.utils import render_bicep_contents_from_j2, get_template_path, split_image_path logger = get_logger(__name__) @@ -79,8 +79,7 @@ def _get_processor_list(self) -> [BaseInputProcessor]: ) # For each image, instantiate image processor for image in self.config.images: - (source_acr_registry, name, version) = self._split_image_path(image) - + (source_acr_registry, name, version) = split_image_path(image) image_input = NexusImageFileInput( artifact_name=name, artifact_version=version, @@ -122,12 +121,6 @@ def build_all_parameters_json(self) -> JSONDefinitionElementBuilder: ) return base_file - def _split_image_path(self, image) -> "tuple[str, str, str]": - """Split the image path into source acr registry, name and version.""" - (source_acr_registry, name_and_version) = image.split("/", 2) - (name, version) = name_and_version.split(":", 2) - return (source_acr_registry, name, version) - def _generate_type_specific_nf_application(self, processor) -> "tuple[list, list]": """Generate the type specific nf application.""" arm_nf = [] diff --git a/src/aosm/azext_aosm/common/constants.py b/src/aosm/azext_aosm/common/constants.py index 7a9f9641815..fd04aafaa2d 100644 --- a/src/aosm/azext_aosm/common/constants.py +++ b/src/aosm/azext_aosm/common/constants.py @@ -79,6 +79,8 @@ class ManifestsExist(str, Enum): CNF_VALUES_SCHEMA_FILENAME = "values.schema.json" CNF_TEMPLATE_FOLDER_NAME = "cnf" +NEXUS_IMAGE_REGEX = r"^[\~]?(\d+)\.(\d+)\.(\d+)$" +# SEMVER_REGEX = r"^(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)(?:-(?P(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$" ################# # OLD CONSTANTS # ################# diff --git a/src/aosm/azext_aosm/common/utils.py b/src/aosm/azext_aosm/common/utils.py index ec1dd8c5663..493e2d59b6e 100644 --- a/src/aosm/azext_aosm/common/utils.py +++ b/src/aosm/azext_aosm/common/utils.py @@ -2,7 +2,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- - +import re import os import tarfile from pathlib import Path @@ -13,7 +13,7 @@ import tempfile from knack.log import get_logger - +from azext_aosm.common.constants import NEXUS_IMAGE_REGEX from azext_aosm.common.exceptions import InvalidFileTypeError, MissingDependency logger = get_logger(__name__) @@ -123,3 +123,18 @@ def check_tool_installed(tool_name: str) -> None: """ if shutil.which(tool_name) is None: raise MissingDependency(f"You must install {tool_name} to use this command.") + +def split_image_path(image) -> "tuple[str, str, str]": + """Split the image path into source acr registry, name and version.""" + (source_acr_registry, name_and_version) = image.split("/", 2) + (name, version) = name_and_version.split(":", 2) + return (source_acr_registry, name, version) + +def is_valid_nexus_image_version(string): + """Check if image version is valid. + + This is based on validation in pez repo. + It requires the image version to be major.minor.patch, + but does not enforce full semver validation. + """ + return re.match(NEXUS_IMAGE_REGEX, string) is not None diff --git a/src/aosm/azext_aosm/configuration_models/onboarding_vnf_input_config.py b/src/aosm/azext_aosm/configuration_models/onboarding_vnf_input_config.py index 13984a07d92..83246efd358 100644 --- a/src/aosm/azext_aosm/configuration_models/onboarding_vnf_input_config.py +++ b/src/aosm/azext_aosm/configuration_models/onboarding_vnf_input_config.py @@ -3,7 +3,6 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- from __future__ import annotations - from dataclasses import dataclass, field from typing import List @@ -13,6 +12,7 @@ from azext_aosm.configuration_models.onboarding_nfd_base_input_config import ( OnboardingNFDBaseInputConfig, ) +from azext_aosm.common.utils import split_image_path, is_valid_nexus_image_version @dataclass @@ -231,6 +231,11 @@ def validate(self): raise ValidationError("arm_template must be set") if not self.images: raise ValidationError("You must include at least one image") + for image in self.images: + (_, _, version) = split_image_path(image) + if not is_valid_nexus_image_version(version): + raise ValidationError(f"{image} has invalid version '{version}'.\n" + "Allowed format is major.minor.patch") if not self.arm_templates: raise ValidationError("You must include at least one arm template") for arm_template in self.arm_templates: From 4d9024509a955a42989ba8678e1240bf5bd73181 Mon Sep 17 00:00:00 2001 From: jordlay <72226943+jordlay@users.noreply.github.com> Date: Mon, 26 Feb 2024 15:16:47 +0000 Subject: [PATCH 34/37] Create RG if it doesn't exist (#150) * add validation resource group exists function * tidied up code * Update src/aosm/azext_aosm/definition_folder/reader/definition_folder.py Co-authored-by: Cyclam <95434717+Cyclam@users.noreply.github.com> * renaming from markups --------- Co-authored-by: Jordan Co-authored-by: Cyclam <95434717+Cyclam@users.noreply.github.com> --- .../reader/definition_folder.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/aosm/azext_aosm/definition_folder/reader/definition_folder.py b/src/aosm/azext_aosm/definition_folder/reader/definition_folder.py index 7dfc61c70dc..95cf98813e7 100644 --- a/src/aosm/azext_aosm/definition_folder/reader/definition_folder.py +++ b/src/aosm/azext_aosm/definition_folder/reader/definition_folder.py @@ -7,7 +7,7 @@ from typing import Any, Dict, List from knack.log import get_logger - +from azure.mgmt.resource.resources.models import ResourceGroup from azext_aosm.common.command_context import CommandContext from azext_aosm.configuration_models.common_parameters_config import ( BaseCommonParametersConfig, @@ -71,10 +71,26 @@ def _parse_index_file(self, file_content: str) -> List[Dict[str, Any]]: ) return parsed_elements + def _create_or_confirm_existence_of_resource_group(self, config, command_context): + """Ensure resource group exists before deploying of elements begins. + + Using ResourceManagementClient: + - Check for existence of resource group specified in allDeployParameters.json. + - Create resource group if doesn't exist. + """ + resources_client = command_context.resources_client + if not resources_client.resource_groups.check_existence(config.publisherResourceGroupName): + rg_params = ResourceGroup(location=config.location) + resources_client.resource_groups.create_or_update( + resource_group_name=config.publisherResourceGroupName, + parameters=rg_params + ) + def deploy( self, config: BaseCommonParametersConfig, command_context: CommandContext ): """Deploy the resources defined in the folder.""" + self._create_or_confirm_existence_of_resource_group(config, command_context) for element in self.elements: logger.debug( "Deploying definition element %s of type %s", From 8cfdd0c9c937e3b3101542cddb6d7c4f439b5525 Mon Sep 17 00:00:00 2001 From: jordlay <72226943+jordlay@users.noreply.github.com> Date: Mon, 26 Feb 2024 15:19:50 +0000 Subject: [PATCH 35/37] Bug Fix: NFD and NSD Manifest Names Clash (#147) * added sa_manifest and acr_manifest property, added sa and acr to manifest names; added nsd to nsd manifest name * create manifest name from nf/nsd name instead of acr/sa; fixed docs strings --------- Co-authored-by: Jordan --- src/aosm/azext_aosm/cli_handlers/onboarding_cnf_handler.py | 2 +- .../azext_aosm/cli_handlers/onboarding_core_vnf_handler.py | 2 +- .../cli_handlers/onboarding_nexus_vnf_handler.py | 2 +- src/aosm/azext_aosm/cli_handlers/onboarding_nsd_handler.py | 2 +- .../onboarding_nfd_base_input_config.py | 6 ++++++ .../configuration_models/onboarding_nsd_input_config.py | 6 ++++++ .../configuration_models/onboarding_vnf_input_config.py | 7 ++++--- 7 files changed, 20 insertions(+), 7 deletions(-) diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_cnf_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_cnf_handler.py index 3cd1707770e..1b0ea8a487c 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_cnf_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_cnf_handler.py @@ -298,7 +298,7 @@ def build_all_parameters_json(self) -> JSONDefinitionElementBuilder: "publisherName": self.config.publisher_name, "publisherResourceGroupName": self.config.publisher_resource_group_name, "acrArtifactStoreName": self.config.acr_artifact_store_name, - "acrManifestName": self.config.acr_artifact_store_name + "-manifest", + "acrManifestName": self.config.acr_manifest_name, "nfDefinitionGroup": self.config.nf_name, "nfDefinitionVersion": self.config.version, } diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py index 65af6a13890..f35d7df5ad2 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py @@ -114,7 +114,7 @@ def build_all_parameters_json(self) -> JSONDefinitionElementBuilder: "publisherResourceGroupName": self.config.publisher_resource_group_name, "acrArtifactStoreName": self.config.acr_artifact_store_name, "saArtifactStoreName": self.config.blob_artifact_store_name, - "acrManifestName": self.config.acr_artifact_store_name + "-manifest", + "acrManifestName": self.config.acr_manifest_name, "saManifestName": self.config.sa_manifest_name, "nfDefinitionGroup": self.config.nf_name, "nfDefinitionVersion": self.config.version diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py index 600465c30b2..d475f126c51 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py @@ -112,7 +112,7 @@ def build_all_parameters_json(self) -> JSONDefinitionElementBuilder: "publisherName": self.config.publisher_name, "publisherResourceGroupName": self.config.publisher_resource_group_name, "acrArtifactStoreName": self.config.acr_artifact_store_name, - "acrManifestName": self.config.acr_artifact_store_name + "-manifest", + "acrManifestName": self.config.acr_manifest_name, "nfDefinitionGroup": self.config.nf_name, "nfDefinitionVersion": self.config.version } diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_nsd_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_nsd_handler.py index 148e9a0a4a1..a8d8cb626be 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_nsd_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_nsd_handler.py @@ -270,7 +270,7 @@ def build_all_parameters_json(self) -> JSONDefinitionElementBuilder: "publisherName": self.config.publisher_name, "publisherResourceGroupName": self.config.publisher_resource_group_name, "acrArtifactStoreName": self.config.acr_artifact_store_name, - "acrManifestName": self.config.acr_artifact_store_name + "-manifest", + "acrManifestName": self.config.acr_manifest_name, "nsDesignGroup": self.config.nsd_name, "nsDesignVersion": self.config.nsd_version, "nfviSiteName": self.nfvi_site_name, diff --git a/src/aosm/azext_aosm/configuration_models/onboarding_nfd_base_input_config.py b/src/aosm/azext_aosm/configuration_models/onboarding_nfd_base_input_config.py index 9129946aa0c..ce8a2c7a16a 100644 --- a/src/aosm/azext_aosm/configuration_models/onboarding_nfd_base_input_config.py +++ b/src/aosm/azext_aosm/configuration_models/onboarding_nfd_base_input_config.py @@ -26,6 +26,12 @@ class OnboardingNFDBaseInputConfig(OnboardingBaseInputConfig): }, ) + @property + def acr_manifest_name(self) -> str: + """Return the ACR manifest name from the NFD name and version.""" + sanitized_nf_name = self.nf_name.lower().replace("_", "-") + return f"{sanitized_nf_name}-acr-manifest-{self.version.replace('.', '-')}" + def validate(self): """Validate the configuration.""" super().validate() diff --git a/src/aosm/azext_aosm/configuration_models/onboarding_nsd_input_config.py b/src/aosm/azext_aosm/configuration_models/onboarding_nsd_input_config.py index 7744db69ebf..9ad8c209083 100644 --- a/src/aosm/azext_aosm/configuration_models/onboarding_nsd_input_config.py +++ b/src/aosm/azext_aosm/configuration_models/onboarding_nsd_input_config.py @@ -193,6 +193,12 @@ class OnboardingNSDInputConfig(OnboardingBaseInputConfig): ) ) + @property + def acr_manifest_name(self) -> str: + """Return the ACR manifest name from the NSD name and version.""" + sanitized_nsd_name = self.nsd_name.lower().replace("_", "-") + return f"{sanitized_nsd_name}-nsd-manifest-{self.nsd_version.replace('.', '-')}" + def validate(self): """Validate the configuration.""" super().validate() diff --git a/src/aosm/azext_aosm/configuration_models/onboarding_vnf_input_config.py b/src/aosm/azext_aosm/configuration_models/onboarding_vnf_input_config.py index 83246efd358..97f52dea77f 100644 --- a/src/aosm/azext_aosm/configuration_models/onboarding_vnf_input_config.py +++ b/src/aosm/azext_aosm/configuration_models/onboarding_vnf_input_config.py @@ -153,8 +153,9 @@ def __post_init__(self): @property def sa_manifest_name(self) -> str: - """Return the Storage account manifest name from the NFD name.""" - return f"{self.blob_artifact_store_name}-manifest-{self.version.replace('.', '-')}" + """Return the Storage account manifest name from the NFD name and version.""" + sanitized_nf_name = self.nf_name.lower().replace("_", "-") + return f"{sanitized_nf_name}-sa-manifest-{self.version.replace('.', '-')}" def validate(self): """Validate the configuration.""" @@ -218,7 +219,7 @@ def __post_init__(self): @property def sa_manifest_name(self) -> str: - """Return the Storage account manifest name from the NFD name.""" + """Return the Storage account manifest name from the NFD name and version.""" sanitized_nf_name = self.nf_name.lower().replace("_", "-") return f"{sanitized_nf_name}-sa-manifest-{self.version.replace('.', '-')}" From 81d31a33ac6773c6114c4fee1a529c2221ccb848 Mon Sep 17 00:00:00 2001 From: jordlay <72226943+jordlay@users.noreply.github.com> Date: Tue, 27 Feb 2024 17:11:39 +0000 Subject: [PATCH 36/37] Fix Nexus Linting + Add Unit Tests (#146) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fixed pylint + flake8 errors * fix mypy errors * fixed artifact builder tests * temp commit for unit testing * temp push of broken tests * added new mocks (not perfect) for vnfs£ * fixed artifact write failure * More mypy fixes * fixed style issues --------- Co-authored-by: Jordan Co-authored-by: Andy Churchard --- .../build_processors/base_processor.py | 13 +- .../build_processors/nexus_image_processor.py | 20 ++- .../build_processors/nfd_processor.py | 34 ++++- .../build_processors/vhd_processor.py | 12 +- .../cli_handlers/onboarding_base_handler.py | 2 + .../onboarding_core_vnf_handler.py | 8 +- .../onboarding_nexus_vnf_handler.py | 16 +-- .../cli_handlers/onboarding_nsd_handler.py | 8 +- .../cli_handlers/onboarding_vnf_handler.py | 12 +- src/aosm/azext_aosm/common/artifact.py | 12 +- src/aosm/azext_aosm/common/constants.py | 3 +- src/aosm/azext_aosm/common/utils.py | 4 +- .../builder/artifact_builder.py | 9 +- src/aosm/azext_aosm/inputs/nfd_input.py | 25 +++- .../input_with_filepath copy.json} | 0 .../mock_core_vnf/input_with_filepath.jsonc | 56 +++++++++ .../input_with_fp.json | 0 .../input_with_sas.json | 0 .../input_with_sas_token.json | 0 .../ubuntu-template.json | 0 .../mock_nexus_vnf/input_with_filepath.json | 18 +++ .../latest/mock_nexus_vnf/input_with_fp.json | 18 +++ .../latest/mock_nexus_vnf/input_with_sas.json | 21 ++++ .../mock_nexus_vnf/input_with_sas_token.json | 21 ++++ .../mock_nexus_vnf/ubuntu-template.json | 118 ++++++++++++++++++ .../input_with_filepath.json | 18 +++ .../mock_vnf_OUTDATED/input_with_fp.json | 18 +++ .../mock_vnf_OUTDATED/input_with_sas.json | 21 ++++ .../input_with_sas_token.json | 21 ++++ .../mock_vnf_OUTDATED/ubuntu-template.json | 118 ++++++++++++++++++ .../latest/unit_test/test_artifact_builder.py | 81 ++++++------ .../latest/unit_test/test_core_vnf_handler.py | 105 ++++++++++++++++ .../unit_test/test_nexus_vnf_handler.py | 76 +++++++---- .../latest/unit_test/test_nsd_cli_handler.py | 2 +- .../test_nexus_arm_processor.py | 24 ++++ 35 files changed, 800 insertions(+), 114 deletions(-) rename src/aosm/azext_aosm/tests/latest/{mock_vnf/input_with_filepath.json => mock_core_vnf/input_with_filepath copy.json} (100%) create mode 100644 src/aosm/azext_aosm/tests/latest/mock_core_vnf/input_with_filepath.jsonc rename src/aosm/azext_aosm/tests/latest/{mock_vnf => mock_core_vnf}/input_with_fp.json (100%) rename src/aosm/azext_aosm/tests/latest/{mock_vnf => mock_core_vnf}/input_with_sas.json (100%) rename src/aosm/azext_aosm/tests/latest/{mock_vnf => mock_core_vnf}/input_with_sas_token.json (100%) rename src/aosm/azext_aosm/tests/latest/{mock_vnf => mock_core_vnf}/ubuntu-template.json (100%) create mode 100644 src/aosm/azext_aosm/tests/latest/mock_nexus_vnf/input_with_filepath.json create mode 100644 src/aosm/azext_aosm/tests/latest/mock_nexus_vnf/input_with_fp.json create mode 100644 src/aosm/azext_aosm/tests/latest/mock_nexus_vnf/input_with_sas.json create mode 100644 src/aosm/azext_aosm/tests/latest/mock_nexus_vnf/input_with_sas_token.json create mode 100644 src/aosm/azext_aosm/tests/latest/mock_nexus_vnf/ubuntu-template.json create mode 100644 src/aosm/azext_aosm/tests/latest/mock_vnf_OUTDATED/input_with_filepath.json create mode 100644 src/aosm/azext_aosm/tests/latest/mock_vnf_OUTDATED/input_with_fp.json create mode 100644 src/aosm/azext_aosm/tests/latest/mock_vnf_OUTDATED/input_with_sas.json create mode 100644 src/aosm/azext_aosm/tests/latest/mock_vnf_OUTDATED/input_with_sas_token.json create mode 100644 src/aosm/azext_aosm/tests/latest/mock_vnf_OUTDATED/ubuntu-template.json create mode 100644 src/aosm/azext_aosm/tests/latest/unit_test/test_core_vnf_handler.py create mode 100644 src/aosm/azext_aosm/tests/latest/unit_test/test_processors/test_nexus_arm_processor.py diff --git a/src/aosm/azext_aosm/build_processors/base_processor.py b/src/aosm/azext_aosm/build_processors/base_processor.py index 34dd9fc42ea..82de0a04a8e 100644 --- a/src/aosm/azext_aosm/build_processors/base_processor.py +++ b/src/aosm/azext_aosm/build_processors/base_processor.py @@ -186,19 +186,18 @@ def generate_values_mappings( for subschema_name, subschema in schema["properties"].items(): if "type" not in subschema: - if "oneOf" or "anyOf" in subschema: + if ["oneOf", "anyOf"] in subschema: raise InvalidArgumentValueError( f"The subschema '{subschema_name}' does not contain a type.\n" "It contains 'anyOf' or 'oneOf' logic, which is not valid for AOSM.\n" "Please remove this from your values.schema.json and provide a concrete type " "or remove the schema and the CLI will generate a generic schema." ) - else: - raise InvalidArgumentValueError( - f"The subschema {subschema_name} does not contain a type. This is a required field.\n" - "Please fix your values.schema.json or remove the schema and the CLI will generate a " - "generic schema." - ) + raise InvalidArgumentValueError( + f"The subschema {subschema_name} does not contain a type. This is a required field.\n" + "Please fix your values.schema.json or remove the schema and the CLI will generate a " + "generic schema." + ) # If the property is not in the values, and is required, add it to the values. if ( "required" in schema diff --git a/src/aosm/azext_aosm/build_processors/nexus_image_processor.py b/src/aosm/azext_aosm/build_processors/nexus_image_processor.py index 22d908af529..83406289188 100644 --- a/src/aosm/azext_aosm/build_processors/nexus_image_processor.py +++ b/src/aosm/azext_aosm/build_processors/nexus_image_processor.py @@ -156,12 +156,20 @@ def _generate_mapping_rule_profile( def generate_parameters_file(self) -> LocalFileBuilder: """ Generate parameters file. """ mapping_rule_profile = self._generate_mapping_rule_profile() - params = ( - mapping_rule_profile.image_mapping_rule_profile.user_configuration - ) - logger.info( - "Created parameters file for Nexus image." - ) + if ( + mapping_rule_profile.image_mapping_rule_profile + and mapping_rule_profile.image_mapping_rule_profile.user_configuration + ): + params = ( + mapping_rule_profile.image_mapping_rule_profile.user_configuration + ) + logger.info( + "Created parameters file for Nexus image." + ) + # We still want to create an empty params file, + # otherwise the nf definition bicep will refer to files that don't exist + else: + params = '{}' return LocalFileBuilder( Path( VNF_OUTPUT_FOLDER_FILENAME, diff --git a/src/aosm/azext_aosm/build_processors/nfd_processor.py b/src/aosm/azext_aosm/build_processors/nfd_processor.py index 7c0e52986ca..37bb56f5493 100644 --- a/src/aosm/azext_aosm/build_processors/nfd_processor.py +++ b/src/aosm/azext_aosm/build_processors/nfd_processor.py @@ -8,7 +8,7 @@ from typing import Any, Dict, List, Tuple from knack.log import get_logger - +from azure.cli.core.azclierror import ResourceNotFoundError from azext_aosm.build_processors.base_processor import BaseInputProcessor from azext_aosm.common.artifact import (BaseArtifact, LocalFileACRArtifact) from azext_aosm.definition_folder.builder.local_file_builder import LocalFileBuilder @@ -18,10 +18,10 @@ DependsOnProfile, ManifestArtifactFormat, NetworkFunctionApplication, NetworkFunctionDefinitionResourceElementTemplateDetails as NFDResourceElementTemplate, NSDArtifactProfile, - ReferencedResource, TemplateType) + ReferencedResource, TemplateType, ContainerizedNetworkFunctionDefinitionVersion, + VirtualNetworkFunctionDefinitionVersion) from azext_aosm.common.constants import NSD_OUTPUT_FOLDER_FILENAME, NSD_NF_TEMPLATE_FILENAME, NSD_TEMPLATE_FOLDER_NAME from azext_aosm.common.utils import render_bicep_contents_from_j2, get_template_path - logger = get_logger(__name__) @@ -77,9 +77,31 @@ def get_artifact_details( ) template_path = get_template_path(NSD_TEMPLATE_FOLDER_NAME, NSD_NF_TEMPLATE_FILENAME) - params = { - "nfvi_type": self.input_artifact.network_function_definition.properties.network_function_template.nfvi_type - } + + # This horrendous if statement is required because: + # - the 'properties' and 'network_function_template' attributes are optional + # - the isinstance check is because the base NetworkFunctionDefinitionVersionPropertiesFormat class + # doesn't define the network_function_template attribute, even though both subclasses do. + # Not switching to EAFP style because mypy doesn't account for `except AttributeError` (for good reason). + # Similar test required in the NFD input, but we can't deduplicate the code because mypy doesn't + # propagate type narrowing from isinstance(). + if ( + self.input_artifact.network_function_definition.properties + and isinstance( + self.input_artifact.network_function_definition.properties, + ( + ContainerizedNetworkFunctionDefinitionVersion, + VirtualNetworkFunctionDefinitionVersion, + ), + ) + and self.input_artifact.network_function_definition.properties.network_function_template + ): + params = { + "nfvi_type": + self.input_artifact.network_function_definition.properties.network_function_template.nfvi_type + } + else: + raise ResourceNotFoundError("The NFDV provided has no nfvi type.") bicep_contents = render_bicep_contents_from_j2(template_path, params) # Create a local file builder for the ARM template file_builder = LocalFileBuilder( diff --git a/src/aosm/azext_aosm/build_processors/vhd_processor.py b/src/aosm/azext_aosm/build_processors/vhd_processor.py index f9ddb78aac3..9458b585a67 100644 --- a/src/aosm/azext_aosm/build_processors/vhd_processor.py +++ b/src/aosm/azext_aosm/build_processors/vhd_processor.py @@ -183,9 +183,15 @@ def _generate_mapping_rule_profile( def generate_parameters_file(self) -> LocalFileBuilder: """ Generate parameters file. """ mapping_rule_profile = self._generate_mapping_rule_profile() - params = ( - mapping_rule_profile.vhd_image_mapping_rule_profile.user_configuration - ) + if (mapping_rule_profile.vhd_image_mapping_rule_profile + and mapping_rule_profile.vhd_image_mapping_rule_profile.user_configuration): + params = ( + mapping_rule_profile.vhd_image_mapping_rule_profile.user_configuration + ) + # We still want to create an empty params file, + # otherwise the nf definition bicep will refer to files that don't exist + else: + params = '{}' logger.info( "Created parameters file for Nexus image." ) diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_base_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_base_handler.py index 2beb0df8cef..0ebb3db8f3d 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_base_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_base_handler.py @@ -35,6 +35,8 @@ class OnboardingBaseCLIHandler(ABC): """Abstract base class for CLI handlers.""" + config: OnboardingBaseInputConfig | BaseCommonParametersConfig + def __init__( self, config_file_path: Optional[Path] = None, diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py index f35d7df5ad2..89ab6d9ff2b 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_core_vnf_handler.py @@ -2,6 +2,8 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- +from __future__ import annotations + import json from pathlib import Path from typing import Dict, Any, List, Tuple, Optional @@ -53,7 +55,7 @@ def _get_input_config( return OnboardingCoreVNFInputConfig(**input_config) def _get_params_config( - self, config_file: dict = None + self, config_file: Path ) -> CoreVNFCommonParametersConfig: """Get the configuration for the command.""" with open(config_file, "r", encoding="utf-8") as _file: @@ -62,9 +64,9 @@ def _get_params_config( params_dict = {} return CoreVNFCommonParametersConfig(**params_dict) - def _get_processor_list(self) -> List[BaseInputProcessor]: + def _get_processor_list(self) -> List[AzureCoreArmBuildProcessor | VHDProcessor]: """Get the list of processors.""" - processor_list = [] + processor_list: List[AzureCoreArmBuildProcessor | VHDProcessor] = [] # for each arm template, instantiate arm processor for arm_template in self.config.arm_templates: arm_input = ArmTemplateInput( diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py index d475f126c51..9be8928cb87 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_nexus_vnf_handler.py @@ -2,9 +2,11 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- +from __future__ import annotations + import json from pathlib import Path -from typing import Dict, Any +from typing import Dict, Any, List, Optional from knack.log import get_logger from azext_aosm.build_processors.arm_processor import NexusArmBuildProcessor @@ -12,9 +14,7 @@ from azext_aosm.build_processors.base_processor import BaseInputProcessor from azext_aosm.common.constants import ( BASE_FOLDER_NAME, - MANIFEST_FOLDER_NAME, VNF_TEMPLATE_FOLDER_NAME, - VNF_MANIFEST_TEMPLATE_FILENAME, VNF_OUTPUT_FOLDER_FILENAME, DEPLOYMENT_PARAMETERS_FILENAME, NEXUS_IMAGE_PARAMETERS_FILENAME, @@ -45,9 +45,9 @@ class OnboardingNexusVNFCLIHandler(OnboardingVNFCLIHandler): """CLI handler for publishing NFDs.""" config: OnboardingNexusVNFInputConfig - + def _get_input_config( - self, input_config: Dict[str, Any] = None + self, input_config: Optional[dict] = None ) -> OnboardingNexusVNFInputConfig: """Get the configuration for the command.""" if input_config is None: @@ -55,7 +55,7 @@ def _get_input_config( return OnboardingNexusVNFInputConfig(**input_config) def _get_params_config( - self, config_file: dict = None + self, config_file: Path ) -> NexusVNFCommonParametersConfig: """Get the configuration for the command.""" with open(config_file, "r", encoding="utf-8") as _file: @@ -64,8 +64,8 @@ def _get_params_config( params_dict = {} return NexusVNFCommonParametersConfig(**params_dict) - def _get_processor_list(self) -> [BaseInputProcessor]: - processor_list = [] + def _get_processor_list(self) -> List[NexusArmBuildProcessor | NexusImageProcessor]: + processor_list: List[NexusArmBuildProcessor | NexusImageProcessor] = [] # for each arm template, instantiate arm processor for arm_template in self.config.arm_templates: arm_input = ArmTemplateInput( diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_nsd_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_nsd_handler.py index a8d8cb626be..f0b30f9fb7b 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_nsd_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_nsd_handler.py @@ -49,7 +49,8 @@ from azext_aosm.vendored_sdks.models import NetworkFunctionDefinitionVersion from azext_aosm.common.utils import render_bicep_contents_from_j2, get_template_path from azext_aosm.vendored_sdks import HybridNetworkManagementClient - +from azext_aosm.configuration_models.common_input import ArmTemplatePropertiesConfig +from azext_aosm.configuration_models.onboarding_nsd_input_config import NetworkFunctionPropertiesConfig logger = get_logger(__name__) @@ -89,10 +90,11 @@ def _get_params_config(self, config_file: Path) -> NSDCommonParametersConfig: return NSDCommonParametersConfig(**params_dict) def _get_processor_list(self) -> list: - processor_list = [] + processor_list: list[AzureCoreArmBuildProcessor | NFDProcessor] = [] # for each resource element template, instantiate processor for resource_element in self.config.resource_element_templates: if resource_element.resource_element_type == "ArmTemplate": + assert isinstance(resource_element.properties, ArmTemplatePropertiesConfig) arm_input = ArmTemplateInput( artifact_name=resource_element.properties.artifact_name, artifact_version=resource_element.properties.version, @@ -106,6 +108,7 @@ def _get_processor_list(self) -> list: AzureCoreArmBuildProcessor(arm_input.artifact_name, arm_input) ) elif resource_element.resource_element_type == "NF": + assert isinstance(resource_element.properties, NetworkFunctionPropertiesConfig) # TODO: change artifact name and version to the nfd name and version or justify why it was this # in the first place # AC4 note: I couldn't find a reference in the old code, but this @@ -198,7 +201,6 @@ def build_artifact_list(self) -> ArtifactDefinitionElementBuilder: def build_resource_bicep(self) -> BicepDefinitionElementBuilder: """Build the resource bicep file.""" - bicep_contents = {} schema_properties = {} nf_names = [] ret_list = [] diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_vnf_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_vnf_handler.py index 333b5d7fb95..5ed33c14336 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_vnf_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_vnf_handler.py @@ -2,13 +2,18 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- +from __future__ import annotations + from pathlib import Path from abc import abstractmethod from .onboarding_nfd_base_handler import OnboardingNFDBaseCLIHandler from knack.log import get_logger - +# from azext_aosm.configuration_models.onboarding_vnf_input_config import ( +# OnboardingBaseVNFInputConfig, +# ) from azext_aosm.common.utils import render_bicep_contents_from_j2, get_template_path +from azext_aosm.configuration_models.onboarding_vnf_input_config import (OnboardingCoreVNFInputConfig, OnboardingNexusVNFInputConfig) from azext_aosm.definition_folder.builder.bicep_builder import ( BicepDefinitionElementBuilder, ) @@ -31,6 +36,8 @@ class OnboardingVNFCLIHandler(OnboardingNFDBaseCLIHandler): """CLI handler for publishing NFDs.""" + config: OnboardingCoreVNFInputConfig | OnboardingNexusVNFInputConfig + @property def default_config_file_name(self) -> str: """Get the default configuration file name.""" @@ -49,6 +56,7 @@ def build_artifact_list(self) -> ArtifactDefinitionElementBuilder: """ logger.info("Creating artifacts list for artifacts.json") + # assert isinstance(self.config, OnboardingBaseVNFInputConfig) artifact_list = [] # For each arm template, get list of artifacts and combine for processor in self.processors: @@ -75,7 +83,7 @@ def build_resource_bicep(self) -> BicepDefinitionElementBuilder: - Generates NF application for each processor - Generates deploymentParameters (flattened to be one schema overall) - Generates supporting parameters files (to avoid stringified JSON in template) - + """ logger.info("Creating artifacts list for artifacts.json") arm_nf_application_list = [] diff --git a/src/aosm/azext_aosm/common/artifact.py b/src/aosm/azext_aosm/common/artifact.py index bad5dc49daf..c74056f696a 100644 --- a/src/aosm/azext_aosm/common/artifact.py +++ b/src/aosm/azext_aosm/common/artifact.py @@ -15,6 +15,7 @@ from azext_aosm.configuration_models.common_parameters_config import ( BaseCommonParametersConfig, NFDCommonParametersConfig, + CoreVNFCommonParametersConfig ) from azext_aosm.vendored_sdks.azure_storagev2.blob.v2022_11_02 import ( BlobClient, @@ -268,11 +269,11 @@ def upload( password, ] self._call_subprocess_raise_output(helm_login_cmd) - + # TODO: fix up helm package to use non-tmp path push_command = [ str(shutil.which("helm")), "push", - f"/tmp/{self.artifact_name}-{self.artifact_version}.tgz", # TODO: fix up helm package to use non-tmp path + f"/tmp/{self.artifact_name}-{self.artifact_version}.tgz", target_acr_with_protocol, ] self._call_subprocess_raise_output(push_command) @@ -288,7 +289,8 @@ def upload( logger.info("LocalFileACRArtifact uploaded %s to %s using helm push", self.file_path, target) else: # TODO: Make this one of the allowed Azure CLI exceptions - raise ValueError(f"Unexpected artifact type. Got {self.artifact_type}. Expected {ArtifactType.ARM_TEMPLATE.value} or {ArtifactType.OCI_ARTIFACT.value}") + raise ValueError(f"Unexpected artifact type. Got {self.artifact_type}. " + "Expected {ArtifactType.ARM_TEMPLATE.value} or {ArtifactType.OCI_ARTIFACT.value}") class RemoteACRArtifact(BaseACRArtifact): @@ -640,7 +642,9 @@ def _get_blob_client( blob_name = container_name logger.debug("container name: %s, blob name: %s", container_name, blob_name) - + # Liskov substitution dictates we must accept BaseCommonParametersConfig, but we should + # never be calling upload on this class unless we've got CoreVNFCommonParametersConfig + assert isinstance(config, CoreVNFCommonParametersConfig) manifest_credentials = ( command_context.aosm_client.artifact_manifests.list_credential( resource_group_name=config.publisherResourceGroupName, diff --git a/src/aosm/azext_aosm/common/constants.py b/src/aosm/azext_aosm/common/constants.py index fd04aafaa2d..570c4ef0d28 100644 --- a/src/aosm/azext_aosm/common/constants.py +++ b/src/aosm/azext_aosm/common/constants.py @@ -13,6 +13,7 @@ NSD = "nsd" VNF_NEXUS = "vnf-nexus" + class DeployableResourceTypes(str, Enum): VNF = VNF CNF = CNF @@ -80,7 +81,7 @@ class ManifestsExist(str, Enum): CNF_TEMPLATE_FOLDER_NAME = "cnf" NEXUS_IMAGE_REGEX = r"^[\~]?(\d+)\.(\d+)\.(\d+)$" -# SEMVER_REGEX = r"^(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)(?:-(?P(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$" + ################# # OLD CONSTANTS # ################# diff --git a/src/aosm/azext_aosm/common/utils.py b/src/aosm/azext_aosm/common/utils.py index 493e2d59b6e..025fa8c8030 100644 --- a/src/aosm/azext_aosm/common/utils.py +++ b/src/aosm/azext_aosm/common/utils.py @@ -124,15 +124,17 @@ def check_tool_installed(tool_name: str) -> None: if shutil.which(tool_name) is None: raise MissingDependency(f"You must install {tool_name} to use this command.") + def split_image_path(image) -> "tuple[str, str, str]": """Split the image path into source acr registry, name and version.""" (source_acr_registry, name_and_version) = image.split("/", 2) (name, version) = name_and_version.split(":", 2) return (source_acr_registry, name, version) + def is_valid_nexus_image_version(string): """Check if image version is valid. - + This is based on validation in pez repo. It requires the image version to be major.minor.patch, but does not enforce full semver validation. diff --git a/src/aosm/azext_aosm/definition_folder/builder/artifact_builder.py b/src/aosm/azext_aosm/definition_folder/builder/artifact_builder.py index 0495e2a49ec..575d71cc982 100644 --- a/src/aosm/azext_aosm/definition_folder/builder/artifact_builder.py +++ b/src/aosm/azext_aosm/definition_folder/builder/artifact_builder.py @@ -37,11 +37,12 @@ def write(self): # TODO: Handle converting path to string that doesn't couple this code to the artifact. # Probably should be in to_dict method. for artifact in self.artifacts: - logger.debug( - "Writing artifact %s as: %s", artifact.artifact_name, artifact.to_dict() - ) if hasattr(artifact, "file_path") and artifact.file_path is not None: artifact.file_path = str(artifact.file_path) - artifacts_list.append(artifact.to_dict()) + artifact_dict = artifact.to_dict() + artifacts_list.append(artifact_dict) + logger.debug( + "Writing artifact %s as: %s", artifact.artifact_name, artifact_dict + ) (self.path / "artifacts.json").write_text(json.dumps(artifacts_list, indent=4)) self._write_supporting_files() diff --git a/src/aosm/azext_aosm/inputs/nfd_input.py b/src/aosm/azext_aosm/inputs/nfd_input.py index f39d5fe1a2f..de8504c3a6f 100644 --- a/src/aosm/azext_aosm/inputs/nfd_input.py +++ b/src/aosm/azext_aosm/inputs/nfd_input.py @@ -11,7 +11,11 @@ from azext_aosm.common.constants import BASE_SCHEMA from azext_aosm.inputs.base_input import BaseInput -from azext_aosm.vendored_sdks.models import NetworkFunctionDefinitionVersion +from azext_aosm.vendored_sdks.models import ( + NetworkFunctionDefinitionVersion, + ContainerizedNetworkFunctionDefinitionVersion, + VirtualNetworkFunctionDefinitionVersion, +) logger = get_logger(__name__) @@ -72,9 +76,22 @@ def get_defaults(self) -> Dict[str, Any]: } } - if self.network_function_definition.properties and ( - self.network_function_definition.properties.network_function_template.nfvi_type - not in ("AzureArcKubernetes","AzureOperatorNexus") + # This horrendous if statement is required because: + # - the 'properties' and 'network_function_template' attributes are optional + # - the isinstance check is because the base NetworkFunctionDefinitionVersionPropertiesFormat class + # doesn't define the network_function_template attribute, even though both subclasses do. + # Not switching to EAFP style because mypy doesn't account for `except AttributeError` (for good reason). + # Similar test required in the NFD processor, but we can't deduplicate the code because mypy doesn't + # propagate type narrowing from isinstance(). + if ( + self.network_function_definition.properties + and isinstance( + self.network_function_definition.properties, + (ContainerizedNetworkFunctionDefinitionVersion, VirtualNetworkFunctionDefinitionVersion), + ) + and self.network_function_definition.properties.network_function_template + and self.network_function_definition.properties.network_function_template.nfvi_type + not in ("AzureArcKubernetes", "AzureOperatorNexus") ): base_defaults["configObject"]["customLocationId"] = "" diff --git a/src/aosm/azext_aosm/tests/latest/mock_vnf/input_with_filepath.json b/src/aosm/azext_aosm/tests/latest/mock_core_vnf/input_with_filepath copy.json similarity index 100% rename from src/aosm/azext_aosm/tests/latest/mock_vnf/input_with_filepath.json rename to src/aosm/azext_aosm/tests/latest/mock_core_vnf/input_with_filepath copy.json diff --git a/src/aosm/azext_aosm/tests/latest/mock_core_vnf/input_with_filepath.jsonc b/src/aosm/azext_aosm/tests/latest/mock_core_vnf/input_with_filepath.jsonc new file mode 100644 index 00000000000..d9eb7e128e4 --- /dev/null +++ b/src/aosm/azext_aosm/tests/latest/mock_core_vnf/input_with_filepath.jsonc @@ -0,0 +1,56 @@ +{ + // Azure location to use when creating resources. + "location": "eastus", + // Name of the Publisher resource you want your definition published to. + // Will be created if it does not exist. + "publisher_name": "jamie-mobile-publisher", + // Optional. Resource group for the Publisher resource. + // Will be created if it does not exist (with a default name if none is supplied). + "publisher_resource_group_name": "Jamie-publisher", + // Optional. Name of the ACR Artifact Store resource. + // Will be created if it does not exist (with a default name if none is supplied). + "acr_artifact_store_name": "ubuntu-acr", + // Name of NF definition. + "nf_name": "ubuntu-vm", + // Version of the NF definition in A.B.C format. + "version": "1.0.0", + // Optional. Name of the storage account Artifact Store resource. + // Will be created if it does not exist (with a default name if none is supplied). + "blob_artifact_store_name": "ubuntu-blob-store", + // The parameter name in the VM ARM template which specifies the name of the image to use for the VM. + "image_name_parameter": "imageName", + // ARM template configuration. + "arm_templates": [ + { + // Name of the artifact. + "artifact_name": "test-art", + // Version of the artifact in A.B.C format. + "version": "1.0.0", + // File path of the artifact you wish to upload from your local disk. + // Relative paths are relative to the configuration file. On Windows escape any backslash with another backslash. + "file_path": "ubuntu-template.json" + } + ], + // VHD image configuration. + "vhd": { + // Optional. Name of the artifact. + "artifact_name": "", + // Version of the artifact in A-B-C format. + "version": "1-0-0", + // Optional. File path of the artifact you wish to upload from your local disk. Delete if not required. + // Relative paths are relative to the configuration file. On Windows escape any backslash with another backslash. + "file_path": "livecd.ubuntu-cpc.azure.vhd", + // Optional. SAS URL of the blob artifact you wish to copy to your Artifact Store. + // Delete if not required. On Windows escape any backslash with another backslash. + "blob_sas_url": "", + // Optional. Specifies the size of empty data disks in gigabytes. + // This value cannot be larger than 1023 GB. Delete if not required. + "image_disk_size_GB": "", + // Optional. Specifies the HyperVGenerationType of the VirtualMachine created from the image. + // Valid values are V1 and V2. V1 is the default if not specified. Delete if not required. + "image_hyper_v_generation": "", + // Optional. The ARM API version used to create the Microsoft.Compute/images resource. + // Delete if not required. + "image_api_version": "" + } +} \ No newline at end of file diff --git a/src/aosm/azext_aosm/tests/latest/mock_vnf/input_with_fp.json b/src/aosm/azext_aosm/tests/latest/mock_core_vnf/input_with_fp.json similarity index 100% rename from src/aosm/azext_aosm/tests/latest/mock_vnf/input_with_fp.json rename to src/aosm/azext_aosm/tests/latest/mock_core_vnf/input_with_fp.json diff --git a/src/aosm/azext_aosm/tests/latest/mock_vnf/input_with_sas.json b/src/aosm/azext_aosm/tests/latest/mock_core_vnf/input_with_sas.json similarity index 100% rename from src/aosm/azext_aosm/tests/latest/mock_vnf/input_with_sas.json rename to src/aosm/azext_aosm/tests/latest/mock_core_vnf/input_with_sas.json diff --git a/src/aosm/azext_aosm/tests/latest/mock_vnf/input_with_sas_token.json b/src/aosm/azext_aosm/tests/latest/mock_core_vnf/input_with_sas_token.json similarity index 100% rename from src/aosm/azext_aosm/tests/latest/mock_vnf/input_with_sas_token.json rename to src/aosm/azext_aosm/tests/latest/mock_core_vnf/input_with_sas_token.json diff --git a/src/aosm/azext_aosm/tests/latest/mock_vnf/ubuntu-template.json b/src/aosm/azext_aosm/tests/latest/mock_core_vnf/ubuntu-template.json similarity index 100% rename from src/aosm/azext_aosm/tests/latest/mock_vnf/ubuntu-template.json rename to src/aosm/azext_aosm/tests/latest/mock_core_vnf/ubuntu-template.json diff --git a/src/aosm/azext_aosm/tests/latest/mock_nexus_vnf/input_with_filepath.json b/src/aosm/azext_aosm/tests/latest/mock_nexus_vnf/input_with_filepath.json new file mode 100644 index 00000000000..b3a3b991a1d --- /dev/null +++ b/src/aosm/azext_aosm/tests/latest/mock_nexus_vnf/input_with_filepath.json @@ -0,0 +1,18 @@ +{ + "publisher_name": "jamie-mobile-publisher", + "publisher_resource_group_name": "Jamie-publisher", + "nf_name": "ubuntu-vm", + "version": "1.0.0", + "acr_artifact_store_name": "ubuntu-acr", + "location": "eastus", + "blob_artifact_store_name": "ubuntu-blob-store", + "image_name_parameter": "imageName", + "arm_template": { + "file_path": "ubuntu-template.json", + "version": "1.0.0" + }, + "vhd": { + "file_path": "livecd.ubuntu-cpc.azure.vhd", + "version": "1-0-0" + } +} diff --git a/src/aosm/azext_aosm/tests/latest/mock_nexus_vnf/input_with_fp.json b/src/aosm/azext_aosm/tests/latest/mock_nexus_vnf/input_with_fp.json new file mode 100644 index 00000000000..b3a3b991a1d --- /dev/null +++ b/src/aosm/azext_aosm/tests/latest/mock_nexus_vnf/input_with_fp.json @@ -0,0 +1,18 @@ +{ + "publisher_name": "jamie-mobile-publisher", + "publisher_resource_group_name": "Jamie-publisher", + "nf_name": "ubuntu-vm", + "version": "1.0.0", + "acr_artifact_store_name": "ubuntu-acr", + "location": "eastus", + "blob_artifact_store_name": "ubuntu-blob-store", + "image_name_parameter": "imageName", + "arm_template": { + "file_path": "ubuntu-template.json", + "version": "1.0.0" + }, + "vhd": { + "file_path": "livecd.ubuntu-cpc.azure.vhd", + "version": "1-0-0" + } +} diff --git a/src/aosm/azext_aosm/tests/latest/mock_nexus_vnf/input_with_sas.json b/src/aosm/azext_aosm/tests/latest/mock_nexus_vnf/input_with_sas.json new file mode 100644 index 00000000000..5222d940186 --- /dev/null +++ b/src/aosm/azext_aosm/tests/latest/mock_nexus_vnf/input_with_sas.json @@ -0,0 +1,21 @@ +{ + "publisher_name": "jamie-mobile-publisher", + "publisher_resource_group_name": "Jamie-publisher", + "nf_name": "ubuntu-vm", + "version": "1.0.0", + "acr_artifact_store_name": "ubuntu-acr", + "location": "eastus", + "blob_artifact_store_name": "ubuntu-blob-store", + "image_name_parameter": "imageName", + "arm_template": { + "file_path": "ubuntu-template.json", + "version": "1.0.0" + }, + "vhd": { + "blob_sas_url": "https://a/dummy/sas-url", + "version": "1-0-0", + "image_disk_size_GB": 30, + "image_hyper_v_generation": "V1", + "image_api_version": "2023-03-01" + } +} \ No newline at end of file diff --git a/src/aosm/azext_aosm/tests/latest/mock_nexus_vnf/input_with_sas_token.json b/src/aosm/azext_aosm/tests/latest/mock_nexus_vnf/input_with_sas_token.json new file mode 100644 index 00000000000..5222d940186 --- /dev/null +++ b/src/aosm/azext_aosm/tests/latest/mock_nexus_vnf/input_with_sas_token.json @@ -0,0 +1,21 @@ +{ + "publisher_name": "jamie-mobile-publisher", + "publisher_resource_group_name": "Jamie-publisher", + "nf_name": "ubuntu-vm", + "version": "1.0.0", + "acr_artifact_store_name": "ubuntu-acr", + "location": "eastus", + "blob_artifact_store_name": "ubuntu-blob-store", + "image_name_parameter": "imageName", + "arm_template": { + "file_path": "ubuntu-template.json", + "version": "1.0.0" + }, + "vhd": { + "blob_sas_url": "https://a/dummy/sas-url", + "version": "1-0-0", + "image_disk_size_GB": 30, + "image_hyper_v_generation": "V1", + "image_api_version": "2023-03-01" + } +} \ No newline at end of file diff --git a/src/aosm/azext_aosm/tests/latest/mock_nexus_vnf/ubuntu-template.json b/src/aosm/azext_aosm/tests/latest/mock_nexus_vnf/ubuntu-template.json new file mode 100644 index 00000000000..378927a3fe5 --- /dev/null +++ b/src/aosm/azext_aosm/tests/latest/mock_nexus_vnf/ubuntu-template.json @@ -0,0 +1,118 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "bicep", + "version": "0.15.31.15270", + "templateHash": "1656082395923655778" + } + }, + "parameters": { + "location": { + "type": "string", + "defaultValue": "[resourceGroup().location]" + }, + "subnetName": { + "type": "string" + }, + "ubuntuVmName": { + "type": "string", + "defaultValue": "ubuntu-vm" + }, + "virtualNetworkId": { + "type": "string" + }, + "sshPublicKeyAdmin": { + "type": "string" + }, + "imageName": { + "type": "string" + } + }, + "variables": { + "imageResourceGroup": "[resourceGroup().name]", + "subscriptionId": "[subscription().subscriptionId]", + "vmSizeSku": "Standard_D2s_v3" + }, + "resources": [ + { + "type": "Microsoft.Network/networkInterfaces", + "apiVersion": "2021-05-01", + "name": "[format('{0}_nic', parameters('ubuntuVmName'))]", + "location": "[parameters('location')]", + "properties": { + "ipConfigurations": [ + { + "name": "ipconfig1", + "properties": { + "subnet": { + "id": "[format('{0}/subnets/{1}', parameters('virtualNetworkId'), parameters('subnetName'))]" + }, + "primary": true, + "privateIPAddressVersion": "IPv4" + } + } + ] + } + }, + { + "type": "Microsoft.Compute/virtualMachines", + "apiVersion": "2021-07-01", + "name": "[parameters('ubuntuVmName')]", + "location": "[parameters('location')]", + "properties": { + "hardwareProfile": { + "vmSize": "[variables('vmSizeSku')]" + }, + "storageProfile": { + "imageReference": { + "id": "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', variables('subscriptionId'), variables('imageResourceGroup')), 'Microsoft.Compute/images', parameters('imageName'))]" + }, + "osDisk": { + "osType": "Linux", + "name": "[format('{0}_disk', parameters('ubuntuVmName'))]", + "createOption": "FromImage", + "caching": "ReadWrite", + "writeAcceleratorEnabled": false, + "managedDisk": "[json('{\"storageAccountType\": \"Premium_LRS\"}')]", + "deleteOption": "Delete", + "diskSizeGB": 30 + } + }, + "osProfile": { + "computerName": "[parameters('ubuntuVmName')]", + "adminUsername": "azureuser", + "linuxConfiguration": { + "disablePasswordAuthentication": true, + "ssh": { + "publicKeys": [ + { + "path": "/home/azureuser/.ssh/authorized_keys", + "keyData": "[parameters('sshPublicKeyAdmin')]" + } + ] + }, + "provisionVMAgent": true, + "patchSettings": { + "patchMode": "ImageDefault", + "assessmentMode": "ImageDefault" + } + }, + "secrets": [], + "allowExtensionOperations": true + }, + "networkProfile": { + "networkInterfaces": [ + { + "id": "[resourceId('Microsoft.Network/networkInterfaces', format('{0}_nic', parameters('ubuntuVmName')))]" + } + ] + } + }, + "dependsOn": [ + "[resourceId('Microsoft.Network/networkInterfaces', format('{0}_nic', parameters('ubuntuVmName')))]" + ] + } + ] +} \ No newline at end of file diff --git a/src/aosm/azext_aosm/tests/latest/mock_vnf_OUTDATED/input_with_filepath.json b/src/aosm/azext_aosm/tests/latest/mock_vnf_OUTDATED/input_with_filepath.json new file mode 100644 index 00000000000..b3a3b991a1d --- /dev/null +++ b/src/aosm/azext_aosm/tests/latest/mock_vnf_OUTDATED/input_with_filepath.json @@ -0,0 +1,18 @@ +{ + "publisher_name": "jamie-mobile-publisher", + "publisher_resource_group_name": "Jamie-publisher", + "nf_name": "ubuntu-vm", + "version": "1.0.0", + "acr_artifact_store_name": "ubuntu-acr", + "location": "eastus", + "blob_artifact_store_name": "ubuntu-blob-store", + "image_name_parameter": "imageName", + "arm_template": { + "file_path": "ubuntu-template.json", + "version": "1.0.0" + }, + "vhd": { + "file_path": "livecd.ubuntu-cpc.azure.vhd", + "version": "1-0-0" + } +} diff --git a/src/aosm/azext_aosm/tests/latest/mock_vnf_OUTDATED/input_with_fp.json b/src/aosm/azext_aosm/tests/latest/mock_vnf_OUTDATED/input_with_fp.json new file mode 100644 index 00000000000..b3a3b991a1d --- /dev/null +++ b/src/aosm/azext_aosm/tests/latest/mock_vnf_OUTDATED/input_with_fp.json @@ -0,0 +1,18 @@ +{ + "publisher_name": "jamie-mobile-publisher", + "publisher_resource_group_name": "Jamie-publisher", + "nf_name": "ubuntu-vm", + "version": "1.0.0", + "acr_artifact_store_name": "ubuntu-acr", + "location": "eastus", + "blob_artifact_store_name": "ubuntu-blob-store", + "image_name_parameter": "imageName", + "arm_template": { + "file_path": "ubuntu-template.json", + "version": "1.0.0" + }, + "vhd": { + "file_path": "livecd.ubuntu-cpc.azure.vhd", + "version": "1-0-0" + } +} diff --git a/src/aosm/azext_aosm/tests/latest/mock_vnf_OUTDATED/input_with_sas.json b/src/aosm/azext_aosm/tests/latest/mock_vnf_OUTDATED/input_with_sas.json new file mode 100644 index 00000000000..5222d940186 --- /dev/null +++ b/src/aosm/azext_aosm/tests/latest/mock_vnf_OUTDATED/input_with_sas.json @@ -0,0 +1,21 @@ +{ + "publisher_name": "jamie-mobile-publisher", + "publisher_resource_group_name": "Jamie-publisher", + "nf_name": "ubuntu-vm", + "version": "1.0.0", + "acr_artifact_store_name": "ubuntu-acr", + "location": "eastus", + "blob_artifact_store_name": "ubuntu-blob-store", + "image_name_parameter": "imageName", + "arm_template": { + "file_path": "ubuntu-template.json", + "version": "1.0.0" + }, + "vhd": { + "blob_sas_url": "https://a/dummy/sas-url", + "version": "1-0-0", + "image_disk_size_GB": 30, + "image_hyper_v_generation": "V1", + "image_api_version": "2023-03-01" + } +} \ No newline at end of file diff --git a/src/aosm/azext_aosm/tests/latest/mock_vnf_OUTDATED/input_with_sas_token.json b/src/aosm/azext_aosm/tests/latest/mock_vnf_OUTDATED/input_with_sas_token.json new file mode 100644 index 00000000000..5222d940186 --- /dev/null +++ b/src/aosm/azext_aosm/tests/latest/mock_vnf_OUTDATED/input_with_sas_token.json @@ -0,0 +1,21 @@ +{ + "publisher_name": "jamie-mobile-publisher", + "publisher_resource_group_name": "Jamie-publisher", + "nf_name": "ubuntu-vm", + "version": "1.0.0", + "acr_artifact_store_name": "ubuntu-acr", + "location": "eastus", + "blob_artifact_store_name": "ubuntu-blob-store", + "image_name_parameter": "imageName", + "arm_template": { + "file_path": "ubuntu-template.json", + "version": "1.0.0" + }, + "vhd": { + "blob_sas_url": "https://a/dummy/sas-url", + "version": "1-0-0", + "image_disk_size_GB": 30, + "image_hyper_v_generation": "V1", + "image_api_version": "2023-03-01" + } +} \ No newline at end of file diff --git a/src/aosm/azext_aosm/tests/latest/mock_vnf_OUTDATED/ubuntu-template.json b/src/aosm/azext_aosm/tests/latest/mock_vnf_OUTDATED/ubuntu-template.json new file mode 100644 index 00000000000..378927a3fe5 --- /dev/null +++ b/src/aosm/azext_aosm/tests/latest/mock_vnf_OUTDATED/ubuntu-template.json @@ -0,0 +1,118 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "bicep", + "version": "0.15.31.15270", + "templateHash": "1656082395923655778" + } + }, + "parameters": { + "location": { + "type": "string", + "defaultValue": "[resourceGroup().location]" + }, + "subnetName": { + "type": "string" + }, + "ubuntuVmName": { + "type": "string", + "defaultValue": "ubuntu-vm" + }, + "virtualNetworkId": { + "type": "string" + }, + "sshPublicKeyAdmin": { + "type": "string" + }, + "imageName": { + "type": "string" + } + }, + "variables": { + "imageResourceGroup": "[resourceGroup().name]", + "subscriptionId": "[subscription().subscriptionId]", + "vmSizeSku": "Standard_D2s_v3" + }, + "resources": [ + { + "type": "Microsoft.Network/networkInterfaces", + "apiVersion": "2021-05-01", + "name": "[format('{0}_nic', parameters('ubuntuVmName'))]", + "location": "[parameters('location')]", + "properties": { + "ipConfigurations": [ + { + "name": "ipconfig1", + "properties": { + "subnet": { + "id": "[format('{0}/subnets/{1}', parameters('virtualNetworkId'), parameters('subnetName'))]" + }, + "primary": true, + "privateIPAddressVersion": "IPv4" + } + } + ] + } + }, + { + "type": "Microsoft.Compute/virtualMachines", + "apiVersion": "2021-07-01", + "name": "[parameters('ubuntuVmName')]", + "location": "[parameters('location')]", + "properties": { + "hardwareProfile": { + "vmSize": "[variables('vmSizeSku')]" + }, + "storageProfile": { + "imageReference": { + "id": "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', variables('subscriptionId'), variables('imageResourceGroup')), 'Microsoft.Compute/images', parameters('imageName'))]" + }, + "osDisk": { + "osType": "Linux", + "name": "[format('{0}_disk', parameters('ubuntuVmName'))]", + "createOption": "FromImage", + "caching": "ReadWrite", + "writeAcceleratorEnabled": false, + "managedDisk": "[json('{\"storageAccountType\": \"Premium_LRS\"}')]", + "deleteOption": "Delete", + "diskSizeGB": 30 + } + }, + "osProfile": { + "computerName": "[parameters('ubuntuVmName')]", + "adminUsername": "azureuser", + "linuxConfiguration": { + "disablePasswordAuthentication": true, + "ssh": { + "publicKeys": [ + { + "path": "/home/azureuser/.ssh/authorized_keys", + "keyData": "[parameters('sshPublicKeyAdmin')]" + } + ] + }, + "provisionVMAgent": true, + "patchSettings": { + "patchMode": "ImageDefault", + "assessmentMode": "ImageDefault" + } + }, + "secrets": [], + "allowExtensionOperations": true + }, + "networkProfile": { + "networkInterfaces": [ + { + "id": "[resourceId('Microsoft.Network/networkInterfaces', format('{0}_nic', parameters('ubuntuVmName')))]" + } + ] + } + }, + "dependsOn": [ + "[resourceId('Microsoft.Network/networkInterfaces', format('{0}_nic', parameters('ubuntuVmName')))]" + ] + } + ] +} \ No newline at end of file diff --git a/src/aosm/azext_aosm/tests/latest/unit_test/test_artifact_builder.py b/src/aosm/azext_aosm/tests/latest/unit_test/test_artifact_builder.py index 9b0ec161e02..7be432b63b6 100644 --- a/src/aosm/azext_aosm/tests/latest/unit_test/test_artifact_builder.py +++ b/src/aosm/azext_aosm/tests/latest/unit_test/test_artifact_builder.py @@ -1,40 +1,41 @@ -# # -------------------------------------------------------------------------------------------- -# # Copyright (c) Microsoft Corporation. All rights reserved. -# # Licensed under the MIT License. See License.txt in the project root for license information. -# # -------------------------------------------------------------------------------------------- - -# from pathlib import Path -# from unittest import TestCase -# from unittest.mock import MagicMock, patch - -# from azext_aosm.definition_folder.builder.artifact_builder import ArtifactDefinitionElementBuilder - - -# class TestArtifactDefinitionElementBuilder(TestCase): -# """Test the artifact definition element builder.""" - -# @patch("pathlib.Path.write_text") -# @patch("pathlib.Path.mkdir") -# def test_write(self, mock_mkdir, mock_write_text): -# """Test writing the definition element to disk.""" - -# # Create some mocks to act as artifacts. -# artifact_1 = MagicMock() -# artifact_1.to_dict.return_value = {"abc": "def"} -# artifact_2 = MagicMock() -# artifact_2.to_dict.return_value = {"ghi": "jkl"} - -# # Create a Artifact definition element builder. -# artifact_definition_element_builder = ArtifactDefinitionElementBuilder( -# Path("/some/folder"), -# [artifact_1, artifact_2] -# ) - -# # Write the definition element to disk. -# artifact_definition_element_builder.write() - -# # Check that the definition element was written to disk. -# mock_mkdir.assert_called_once() -# artifact_1.to_dict.assert_called_once() -# artifact_2.to_dict.assert_called_once() -# mock_write_text.assert_called_once_with('[{"abc": "def"}, {"ghi": "jkl"}]') +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +from pathlib import Path +from unittest import TestCase +from unittest.mock import MagicMock, patch + +from azext_aosm.definition_folder.builder.artifact_builder import ArtifactDefinitionElementBuilder + + +class TestArtifactDefinitionElementBuilder(TestCase): + """Test the artifact definition element builder.""" + + @patch("pathlib.Path.write_text") + @patch("pathlib.Path.mkdir") + def test_write(self, mock_mkdir, mock_write_text): + """Test writing the definition element to disk.""" + + # Create some mocks to act as artifacts. + artifact_1 = MagicMock() + artifact_1.to_dict.return_value = {"abc": "def"} + artifact_2 = MagicMock() + artifact_2.to_dict.return_value = {"ghi": "jkl"} + + # Create a Artifact definition element builder. + artifact_definition_element_builder = ArtifactDefinitionElementBuilder( + Path("/some/folder"), + [artifact_1, artifact_2] + ) + + # Write the definition element to disk. + artifact_definition_element_builder.write() + + # Check that the definition element was written to disk. + mock_mkdir.assert_called_once() + artifact_1.to_dict.assert_called_once() + artifact_2.to_dict.assert_called_once() + mock_write_text.assert_called_once_with( + '[\n {\n "abc": "def"\n },\n {\n "ghi": "jkl"\n }\n]') diff --git a/src/aosm/azext_aosm/tests/latest/unit_test/test_core_vnf_handler.py b/src/aosm/azext_aosm/tests/latest/unit_test/test_core_vnf_handler.py new file mode 100644 index 00000000000..06ce64c0991 --- /dev/null +++ b/src/aosm/azext_aosm/tests/latest/unit_test/test_core_vnf_handler.py @@ -0,0 +1,105 @@ +# from unittest import TestCase +# from unittest.mock import patch, MagicMock, Mock +# from pathlib import Path +# from typing import List +# from azext_aosm.cli_handlers.onboarding_core_vnf_handler import OnboardingCoreVNFCLIHandler +# from azext_aosm.definition_folder.builder.artifact_builder import ArtifactDefinitionElementBuilder +# from azext_aosm.definition_folder.builder.bicep_builder import BicepDefinitionElementBuilder +# from azext_aosm.definition_folder.builder.local_file_builder import LocalFileBuilder +# from azext_aosm.common.constants import VNF_OUTPUT_FOLDER_FILENAME, ARTIFACT_LIST_FILENAME +# from azext_aosm.build_processors.arm_processor import AzureCoreArmBuildProcessor +# from azext_aosm.build_processors.vhd_processor import VHDProcessor +# from azext_aosm.vendored_sdks.models import ( +# AzureCoreVhdImageDeployMappingRuleProfile, AzureCoreNetworkFunctionVhdApplication, +# AzureCoreVhdImageArtifactProfile, VhdImageArtifactProfile, VhdImageMappingRuleProfile, ApplicationEnablement +# ) + +# class VNFCoreBuildTest(TestCase): + +# def setUp(self): +# self.vnf_handler = OnboardingCoreVNFCLIHandler() + +# # def test_valid_nexus_config_provided(): +# # # give it nexus specific config +# # pass + +# # def test_invalid_nexus_config_provided(): +# # # give it nexus specific config with an error +# # pass + +# # def test_core_config_provided(): +# # # give it core specific config +# # pass + +# # # def test_build_base_bicep(self): +# # # with patch("pathlib.Path.write_text") as mock_write_text: +# # # self.nexus_vnf_cli_handler.build_base_bicep() +# # # mock_write_text.assert_called() + +# def test_build_artifact_list_type(self): +# """ Testing build artifact list for Nexus VNFs + +# Test if path is as expected, and if list returned is correct type +# """ +# self.vnf_handler.processors = MagicMock() +# artifact_list = self.vnf_handler.build_artifact_list() +# self.assertEqual(artifact_list.path, Path(VNF_OUTPUT_FOLDER_FILENAME, ARTIFACT_LIST_FILENAME)) +# self.assertIsInstance(artifact_list, ArtifactDefinitionElementBuilder) + +# def test_build_resource_bicep_type(self): +# """Testing build resource bicep for Nexus VNFs + +# We only need to test the type of the bicep and the supporting files, +# and that they have the correct names. +# This is because the complicated logic is tested in the processors tests. + +# """ +# # we are testing the rest of the logic in the processors? (are we? we should) +# # TODO: fix this mocking, it works for deploymentParameters but not for actual processors (duh?) +# # mocked_input = MagicMock() +# # arm_processor = AzureCoreArmProcessor(mocked_input) +# arm_input = MagicMock() +# vhd_input = MagicMock() +# arm_processor = AzureCoreArmBuildProcessor("arm_test", arm_input) +# vhd_processor = VHDProcessor("test_vhd", vhd_input) +# arm_processor = MagicMock(spec=AzureCoreArmBuildProcessor) +# vhd_processor = MagicMock(spec=VHDProcessor) +# # arm_processor = Mock(spec=AzureCoreArmBuildProcessor) +# # vhd_processor = Mock(spec=VHDProcessor) +# # assert isinstance(arm_processor, AzureCoreArmBuildProcessor) +# # # TODO: one of these returns local file build +# # vhd_processor.generate_nf_application.return_value = AzureCoreNetworkFunctionVhdApplication(name="test",depends_on_profile=None, +# # artifact_profile=AzureCoreVhdImageArtifactProfile( +# # artifact_store=None, +# # vhd_artifact_profile=VhdImageArtifactProfile( +# # vhd_name="vhd_name", +# # vhd_version="1-0-0", +# # ),), deploy_parameters_mapping_rule_profile=AzureCoreVhdImageDeployMappingRuleProfile( +# # application_enablement=ApplicationEnablement.ENABLED, +# # vhd_image_mapping_rule_profile=VhdImageMappingRuleProfile(user_configuration=None) +# # ) +# # ) +# arm_processor.generate_params_schema.return_value = {} +# # arm_processor.generate_parameters_file.return_value = LocalFileBuilder("", {}) +# vhd_processor.generate_params_schema.return_value = {} +# # We want to test a specific private method so disable the pylint warning +# # pylint: disable=protected-access +# vhd_processor._generate_mapping_rule_profile.return_value = AzureCoreVhdImageDeployMappingRuleProfile(application_enablement=None, vhd_image_mapping_rule_profile=None) +# self.vnf_handler.processors = [arm_processor, vhd_processor] +# resource_bicep = self.vnf_handler.build_resource_bicep() +# print(resource_bicep.supporting_files[1].path) +# # TODO: check that the nexus one contains deploymentParameters, imageParameters and at least one templateParams? +# # special assert? +# self.assertIsInstance(resource_bicep.supporting_files, List[LocalFileBuilder]) +# self.assertIsInstance(resource_bicep, BicepDefinitionElementBuilder) + +# # def test_build_all_parameters_json(): +# # def test_build_artifact_manifest(self): +# # # self.vnf_handler._generate_type_specific_artifact_manifest +# # self.vnf_handler.processors = MagicMock() +# # manifest_bicep = self.vnf_handler.build_manifest_bicep() +# # # We want to test a specific private method so disable the pylint warning +# # # pylint: disable=protected-access +# # (arm_list, sa_list) = self.vnf_handler._generate_type_specific_artifact_manifest(self.vnf_handler.processors[0]) +# # self.assertEqual() + diff --git a/src/aosm/azext_aosm/tests/latest/unit_test/test_nexus_vnf_handler.py b/src/aosm/azext_aosm/tests/latest/unit_test/test_nexus_vnf_handler.py index 6ef55bad11b..a4d74cf196d 100644 --- a/src/aosm/azext_aosm/tests/latest/unit_test/test_nexus_vnf_handler.py +++ b/src/aosm/azext_aosm/tests/latest/unit_test/test_nexus_vnf_handler.py @@ -1,27 +1,61 @@ # from unittest import TestCase -# from unittest.mock import patch +# from unittest.mock import patch, MagicMock +# from pathlib import Path +# from typing import List # from azext_aosm.cli_handlers.onboarding_nexus_vnf_handler import OnboardingNexusVNFCLIHandler +# from azext_aosm.definition_folder.builder.artifact_builder import ArtifactDefinitionElementBuilder +# from azext_aosm.definition_folder.builder.bicep_builder import BicepDefinitionElementBuilder +# from azext_aosm.definition_folder.builder.local_file_builder import LocalFileBuilder +# from azext_aosm.common.constants import VNF_OUTPUT_FOLDER_FILENAME, ARTIFACT_LIST_FILENAME + # class VNFNexusBuildTest(TestCase): - + # def setUp(self): # self.nexus_handler = OnboardingNexusVNFCLIHandler() - -# def test_valid_nexus_config_provided(): -# # give it nexus specific config -# pass - -# def test_invalid_nexus_config_provided(): -# # give it nexus specific config with an error -# pass - -# def test_core_config_provided(): -# # give it core specific config -# pass - -# # def test_build_base_bicep(self): -# # with patch("pathlib.Path.write_text") as mock_write_text: -# # self.nexus_vnf_cli_handler.build_base_bicep() -# # mock_write_text.assert_called() - - \ No newline at end of file + +# # def test_valid_nexus_config_provided(): +# # # give it nexus specific config +# # pass + +# # def test_invalid_nexus_config_provided(): +# # # give it nexus specific config with an error +# # pass + +# # def test_core_config_provided(): +# # # give it core specific config +# # pass + +# # # def test_build_base_bicep(self): +# # # with patch("pathlib.Path.write_text") as mock_write_text: +# # # self.nexus_vnf_cli_handler.build_base_bicep() +# # # mock_write_text.assert_called() + +# def test_build_artifact_list_type(self): +# """ Testing build artifact list for Nexus VNFs + +# Test if path is as expected, and if list returned is correct type +# """ +# self.nexus_handler.processors = MagicMock() +# artifact_list = self.nexus_handler.build_artifact_list() +# self.assertEqual(artifact_list.path, Path(VNF_OUTPUT_FOLDER_FILENAME, ARTIFACT_LIST_FILENAME)) +# self.assertIsInstance(artifact_list, ArtifactDefinitionElementBuilder) + +# def test_build_resource_bicep_type(self): +# """Testing build resource bicep for Nexus VNFs + +# We only need to test the type of the bicep and the supporting files, +# and that they have the correct names. +# This is because the complicated logic is tested in the processors tests. + +# """ +# # we are testing the rest of the logic in the processors? (are we? we should) +# # TODO: fix this mocking, it works for deploymentParameters but not for actual processors (duh?) +# self.nexus_handler.processors = MagicMock() +# resource_bicep = self.nexus_handler.build_resource_bicep() +# print(resource_bicep.supporting_files[0].path) +# # TODO: check that the nexus one contains deploymentParameters, imageParameters and at least one templateParams? +# self.assertIsInstance(resource_bicep.supporting_files, List[LocalFileBuilder]) +# self.assertIsInstance(resource_bicep, BicepDefinitionElementBuilder) + +# # def test_build_all_parameters_json(): diff --git a/src/aosm/azext_aosm/tests/latest/unit_test/test_nsd_cli_handler.py b/src/aosm/azext_aosm/tests/latest/unit_test/test_nsd_cli_handler.py index 3a905fd18ba..87cc3f9c8e2 100644 --- a/src/aosm/azext_aosm/tests/latest/unit_test/test_nsd_cli_handler.py +++ b/src/aosm/azext_aosm/tests/latest/unit_test/test_nsd_cli_handler.py @@ -5,7 +5,7 @@ # from azure.cli.core.azclierror import ( # UnclassifiedUserFault, # ) -# TODO: Fix tests with correct mocking for input() +# # TODO: Fix tests with correct mocking for input() # @patch("pathlib.Path.exists") # class TestNsdCliHandler(TestCase): diff --git a/src/aosm/azext_aosm/tests/latest/unit_test/test_processors/test_nexus_arm_processor.py b/src/aosm/azext_aosm/tests/latest/unit_test/test_processors/test_nexus_arm_processor.py new file mode 100644 index 00000000000..78b08111e63 --- /dev/null +++ b/src/aosm/azext_aosm/tests/latest/unit_test/test_processors/test_nexus_arm_processor.py @@ -0,0 +1,24 @@ +from unittest import TestCase +from unittest.mock import patch, MagicMock +from pathlib import Path +from typing import List +from azext_aosm.build_processors.arm_processor import NexusArmBuildProcessor +from azext_aosm.inputs.arm_template_input import ArmTemplateInput + +from azext_aosm.cli_handlers.onboarding_nexus_vnf_handler import OnboardingNexusVNFCLIHandler +from azext_aosm.definition_folder.builder.artifact_builder import ArtifactDefinitionElementBuilder +from azext_aosm.definition_folder.builder.bicep_builder import BicepDefinitionElementBuilder +from azext_aosm.definition_folder.builder.local_file_builder import LocalFileBuilder +from azext_aosm.common.constants import VNF_OUTPUT_FOLDER_FILENAME, ARTIFACT_LIST_FILENAME + + +class NexusArmProcessorTest(TestCase): + + def setUp(self): + nexus_arm_input = ArmTemplateInput( + artifact_name="test-artifact-name", + artifact_version="1.1.1", + template_path="", + default_config=None + ) + self.processor = NexusArmBuildProcessor() From cadf02eaba6cf38210629a9fed898e2124d668bc Mon Sep 17 00:00:00 2001 From: Cyclam <95434717+Cyclam@users.noreply.github.com> Date: Wed, 28 Feb 2024 12:08:39 +0000 Subject: [PATCH 37/37] Use ephemeral tempdir for generated helm package .tgz file. (#151) * Use ephemeral tempdir for generated helm package .tgz file. * Fix file_path bug if .tgz file was provided by user. --------- Co-authored-by: Andy Churchard --- src/aosm/azext_aosm/common/artifact.py | 103 ++++++++++++------------- 1 file changed, 51 insertions(+), 52 deletions(-) diff --git a/src/aosm/azext_aosm/common/artifact.py b/src/aosm/azext_aosm/common/artifact.py index c74056f696a..04782b696ac 100644 --- a/src/aosm/azext_aosm/common/artifact.py +++ b/src/aosm/azext_aosm/common/artifact.py @@ -9,6 +9,7 @@ from abc import ABC, abstractmethod from functools import lru_cache from pathlib import Path +import tempfile from time import sleep from typing import Any, MutableMapping, Optional @@ -224,67 +225,65 @@ def upload( username = manifest_credentials["username"] password = manifest_credentials["acr_token"] - # TODO: Maybe port over the "if not use_manifest_permissions" feature which means you don't need to - # install docker. Although not having to install docker feels like a marginal enhancement, as most - # people playing with containers will have docker, or won't mind installing it. Note that - # use_manifest_permissions is now in command_context.cli_options['no_subscription_permissions'] - self._check_tool_installed("docker") self._check_tool_installed("helm") - # TODO: don't just dump this in /tmp - if self.file_path.is_dir(): - helm_package_cmd = [ - str(shutil.which("helm")), - "package", - self.file_path, - "--destination", - "/tmp", - ] - self._call_subprocess_raise_output(helm_package_cmd) - - # This seems to prevent occasional helm login failures - acr_login_cmd = [ - str(shutil.which("az")), - "acr", - "login", - "--name", - target_acr_name, - "--username", - username, - "--password", - password, - ] - self._call_subprocess_raise_output(acr_login_cmd) - - try: - helm_login_cmd = [ - str(shutil.which("helm")), - "registry", + # tmpdir is only used if file_path is dir, but `with` context manager is cleaner to use, so we always + # set up the tmpdir, even if it doesn't end up being used. + with tempfile.TemporaryDirectory() as tmpdir: + if self.file_path.is_dir(): + helm_package_cmd = [ + str(shutil.which("helm")), + "package", + self.file_path, + "--destination", + tmpdir, + ] + self._call_subprocess_raise_output(helm_package_cmd) + self.file_path = Path(tmpdir, f"{self.artifact_name}-{self.artifact_version}.tgz") + + # This seems to prevent occasional helm login failures + acr_login_cmd = [ + str(shutil.which("az")), + "acr", "login", - target_acr, + "--name", + target_acr_name, "--username", username, "--password", password, ] - self._call_subprocess_raise_output(helm_login_cmd) - # TODO: fix up helm package to use non-tmp path - push_command = [ - str(shutil.which("helm")), - "push", - f"/tmp/{self.artifact_name}-{self.artifact_version}.tgz", - target_acr_with_protocol, - ] - self._call_subprocess_raise_output(push_command) - finally: - helm_logout_cmd = [ - str(shutil.which("helm")), - "registry", - "logout", - target_acr, - ] - self._call_subprocess_raise_output(helm_logout_cmd) + self._call_subprocess_raise_output(acr_login_cmd) + + try: + helm_login_cmd = [ + str(shutil.which("helm")), + "registry", + "login", + target_acr, + "--username", + username, + "--password", + password, + ] + self._call_subprocess_raise_output(helm_login_cmd) + + push_command = [ + str(shutil.which("helm")), + "push", + self.file_path, + target_acr_with_protocol, + ] + self._call_subprocess_raise_output(push_command) + finally: + helm_logout_cmd = [ + str(shutil.which("helm")), + "registry", + "logout", + target_acr, + ] + self._call_subprocess_raise_output(helm_logout_cmd) logger.info("LocalFileACRArtifact uploaded %s to %s using helm push", self.file_path, target)