From 8694270801bd0a11c5ae192337fc4d6d023cbeba Mon Sep 17 00:00:00 2001 From: Cyclam <95434717+Cyclam@users.noreply.github.com> Date: Wed, 7 Feb 2024 15:23:07 +0000 Subject: [PATCH] Linting and typing (#141) Linting and type hinting fixes. --- src/aosm/azext_aosm/__init__.py | 3 +- src/aosm/azext_aosm/_client_factory.py | 4 +- src/aosm/azext_aosm/_params.py | 8 +- .../build_processors/arm_processor.py | 37 +- .../build_processors/base_processor.py | 33 +- .../build_processors/helm_chart_processor.py | 9 +- .../build_processors/nfd_processor.py | 37 +- .../build_processors/vhd_processor.py | 36 +- .../cli_handlers/onboarding_base_handler.py | 58 +- .../cli_handlers/onboarding_cnf_handler.py | 62 +- .../onboarding_nfd_base_handler.py | 3 +- .../cli_handlers/onboarding_nsd_handler.py | 100 ++- .../cli_handlers/onboarding_vnf_handler.py | 72 +- src/aosm/azext_aosm/commands.py | 4 - src/aosm/azext_aosm/common/artifact.py | 178 ++-- src/aosm/azext_aosm/common/command_context.py | 6 +- src/aosm/azext_aosm/common/constants.py | 2 +- src/aosm/azext_aosm/common/exceptions.py | 4 + src/aosm/azext_aosm/common/utils.py | 10 +- .../configuration_models/common_input.py | 8 +- .../common_parameters_config.py | 1 + .../onboarding_base_input_config.py | 4 +- .../onboarding_cnf_input_config.py | 18 +- .../onboarding_nfd_base_input_config.py | 18 +- .../onboarding_nsd_input_config.py | 8 +- .../onboarding_vnf_input_config.py | 34 +- src/aosm/azext_aosm/custom.py | 19 +- .../builder/artifact_builder.py | 10 +- .../definition_folder/builder/base_builder.py | 4 +- .../builder/bicep_builder.py | 5 +- .../builder/definition_folder_builder.py | 9 +- .../definition_folder/builder/json_builder.py | 5 +- .../builder}/local_file_builder.py | 2 +- .../reader/artifact_definition.py | 56 +- .../reader/base_definition.py | 10 +- .../reader/bicep_definition.py | 99 +- .../reader/definition_folder.py | 33 +- .../azext_aosm/inputs/helm_chart_input.py | 44 +- src/aosm/azext_aosm/inputs/nfd_input.py | 5 +- src/aosm/azext_aosm/old/_configuration.py | 771 ---------------- src/aosm/azext_aosm/old/delete/__init__.py | 5 - src/aosm/azext_aosm/old/delete/delete.py | 345 ------- src/aosm/azext_aosm/old/deploy/__init__.py | 5 - src/aosm/azext_aosm/old/deploy/artifact.py | 645 ------------- .../old/deploy/artifact_manifest.py | 169 ---- .../azext_aosm/old/deploy/deploy_with_arm.py | 731 --------------- src/aosm/azext_aosm/old/deploy/pre_deploy.py | 444 --------- .../azext_aosm/old/generate_nfd/__init__.py | 5 - .../old/generate_nfd/cnf_nfd_generator.py | 850 ------------------ .../old/generate_nfd/nfd_generator_base.py | 25 - .../templates/cnfartifactmanifest.bicep.j2 | 39 - .../templates/cnfdefinition.bicep.j2 | 79 -- .../templates/vnfartifactmanifests.bicep | 68 -- .../templates/vnfdefinition.bicep | 103 --- .../old/generate_nfd/vnf_nfd_generator.py | 340 ------- .../azext_aosm/old/generate_nsd/__init__.py | 5 - .../azext_aosm/old/generate_nsd/nf_ret.py | 185 ---- .../old/generate_nsd/nsd_generator.py | 261 ------ .../artifact_manifest_template.bicep | 39 - .../templates/nf_template.bicep.j2 | 81 -- .../templates/nsd_template.bicep.j2 | 108 --- src/aosm/azext_aosm/old/util/__init__.py | 5 - .../azext_aosm/old/util/management_clients.py | 22 - src/aosm/azext_aosm/old/util/utils.py | 25 - src/aosm/azext_aosm/old_custom.py | 525 ----------- .../tests/latest/recording_processors.py | 5 +- .../test_aosm_cnf_publish_and_delete.py | 3 +- .../test_aosm_vnf_publish_and_delete.py | 16 +- .../latest/unit_test/test_bicep_builder.py | 10 +- .../latest/unit_test/test_helm_chart_input.py | 20 +- .../unit_test/test_helm_chart_processor.py | 7 +- src/aosm/setup.py | 9 +- 72 files changed, 676 insertions(+), 6332 deletions(-) rename src/aosm/azext_aosm/{common => definition_folder/builder}/local_file_builder.py (92%) delete mode 100644 src/aosm/azext_aosm/old/_configuration.py delete mode 100644 src/aosm/azext_aosm/old/delete/__init__.py delete mode 100644 src/aosm/azext_aosm/old/delete/delete.py delete mode 100644 src/aosm/azext_aosm/old/deploy/__init__.py delete mode 100644 src/aosm/azext_aosm/old/deploy/artifact.py delete mode 100644 src/aosm/azext_aosm/old/deploy/artifact_manifest.py delete mode 100644 src/aosm/azext_aosm/old/deploy/deploy_with_arm.py delete mode 100644 src/aosm/azext_aosm/old/deploy/pre_deploy.py delete mode 100644 src/aosm/azext_aosm/old/generate_nfd/__init__.py delete mode 100644 src/aosm/azext_aosm/old/generate_nfd/cnf_nfd_generator.py delete mode 100644 src/aosm/azext_aosm/old/generate_nfd/nfd_generator_base.py delete mode 100644 src/aosm/azext_aosm/old/generate_nfd/templates/cnfartifactmanifest.bicep.j2 delete mode 100644 src/aosm/azext_aosm/old/generate_nfd/templates/cnfdefinition.bicep.j2 delete mode 100644 src/aosm/azext_aosm/old/generate_nfd/templates/vnfartifactmanifests.bicep delete mode 100644 src/aosm/azext_aosm/old/generate_nfd/templates/vnfdefinition.bicep delete mode 100644 src/aosm/azext_aosm/old/generate_nfd/vnf_nfd_generator.py delete mode 100644 src/aosm/azext_aosm/old/generate_nsd/__init__.py delete mode 100644 src/aosm/azext_aosm/old/generate_nsd/nf_ret.py delete mode 100644 src/aosm/azext_aosm/old/generate_nsd/nsd_generator.py delete mode 100644 src/aosm/azext_aosm/old/generate_nsd/templates/artifact_manifest_template.bicep delete mode 100644 src/aosm/azext_aosm/old/generate_nsd/templates/nf_template.bicep.j2 delete mode 100644 src/aosm/azext_aosm/old/generate_nsd/templates/nsd_template.bicep.j2 delete mode 100644 src/aosm/azext_aosm/old/util/__init__.py delete mode 100644 src/aosm/azext_aosm/old/util/management_clients.py delete mode 100644 src/aosm/azext_aosm/old/util/utils.py delete mode 100644 src/aosm/azext_aosm/old_custom.py diff --git a/src/aosm/azext_aosm/__init__.py b/src/aosm/azext_aosm/__init__.py index bd53e8bd88e..4314adb3c52 100644 --- a/src/aosm/azext_aosm/__init__.py +++ b/src/aosm/azext_aosm/__init__.py @@ -16,9 +16,10 @@ def __init__(self, cli_ctx=None): super().__init__(cli_ctx=cli_ctx, custom_command_type=aosm_custom) def load_command_table(self, args): - from azext_aosm.commands import load_command_table from azure.cli.core.aaz import load_aaz_command_table + from azext_aosm.commands import load_command_table + try: from . import aaz except ImportError: diff --git a/src/aosm/azext_aosm/_client_factory.py b/src/aosm/azext_aosm/_client_factory.py index ea716a48b2b..be1c30b4b6b 100644 --- a/src/aosm/azext_aosm/_client_factory.py +++ b/src/aosm/azext_aosm/_client_factory.py @@ -14,7 +14,9 @@ def cf_aosm(cli_ctx, *_) -> HybridNetworkManagementClient: # By default, get_mgmt_service_client() sets a parameter called 'base_url' when creating # the client. For us, doing so results in a key error. Setting base_url_bound=False prevents # that from happening - return get_mgmt_service_client(cli_ctx, HybridNetworkManagementClient, base_url_bound=False) + return get_mgmt_service_client( + cli_ctx, HybridNetworkManagementClient, base_url_bound=False + ) def cf_resources(cli_ctx, subscription_id=None): diff --git a/src/aosm/azext_aosm/_params.py b/src/aosm/azext_aosm/_params.py index eb23ad7e1d4..5a445c40aac 100644 --- a/src/aosm/azext_aosm/_params.py +++ b/src/aosm/azext_aosm/_params.py @@ -7,12 +7,12 @@ from azure.cli.core import AzCommandsLoader from .common.constants import ( - CNF, - VNF, - BICEP_PUBLISH, ARTIFACT_UPLOAD, - IMAGE_UPLOAD, + BICEP_PUBLISH, + CNF, HELM_TEMPLATE, + IMAGE_UPLOAD, + VNF, ) diff --git a/src/aosm/azext_aosm/build_processors/arm_processor.py b/src/aosm/azext_aosm/build_processors/arm_processor.py index c6ad47a65db..66e9145307a 100644 --- a/src/aosm/azext_aosm/build_processors/arm_processor.py +++ b/src/aosm/azext_aosm/build_processors/arm_processor.py @@ -11,17 +11,26 @@ from azext_aosm.build_processors.base_processor import BaseInputProcessor from azext_aosm.common.artifact import BaseArtifact, LocalFileACRArtifact -from azext_aosm.common.local_file_builder import LocalFileBuilder +from azext_aosm.definition_folder.builder.local_file_builder import LocalFileBuilder from azext_aosm.inputs.arm_template_input import ArmTemplateInput from azext_aosm.vendored_sdks.models import ( - ApplicationEnablement, ArmResourceDefinitionResourceElementTemplate, + ApplicationEnablement, + ArmResourceDefinitionResourceElementTemplate, ArmResourceDefinitionResourceElementTemplateDetails, - ArmTemplateArtifactProfile, ArmTemplateMappingRuleProfile, + ArmTemplateArtifactProfile, + ArmTemplateMappingRuleProfile, AzureCoreArmTemplateArtifactProfile, - AzureCoreArmTemplateDeployMappingRuleProfile, AzureCoreArtifactType, - AzureCoreNetworkFunctionArmTemplateApplication, DependsOnProfile, - ManifestArtifactFormat, NetworkFunctionApplication, NSDArtifactProfile, - ReferencedResource, ResourceElementTemplate, TemplateType) + AzureCoreArmTemplateDeployMappingRuleProfile, + AzureCoreArtifactType, + AzureCoreNetworkFunctionArmTemplateApplication, + DependsOnProfile, + ManifestArtifactFormat, + NetworkFunctionApplication, + NSDArtifactProfile, + ReferencedResource, + ResourceElementTemplate, + TemplateType, +) logger = get_logger(__name__) @@ -56,7 +65,9 @@ def get_artifact_manifest_list(self) -> List[ManifestArtifactFormat]: :return: A list of artifacts for the artifact manifest. :rtype: List[ManifestArtifactFormat] """ - logger.debug("Getting artifact manifest list for ARM template input %s.", self.name) + logger.debug( + "Getting artifact manifest list for ARM template input %s.", self.name + ) return [ ManifestArtifactFormat( artifact_name=self.input_artifact.artifact_name, @@ -122,8 +133,9 @@ def generate_resource_element_template(self) -> ResourceElementTemplate: return ArmResourceDefinitionResourceElementTemplateDetails( name=self.name, - depends_on_profile=DependsOnProfile(install_depends_on=[], - uninstall_depends_on=[], update_depends_on=[]), + depends_on_profile=DependsOnProfile( + install_depends_on=[], uninstall_depends_on=[], update_depends_on=[] + ), configuration=ArmResourceDefinitionResourceElementTemplate( template_type=TemplateType.ARM_TEMPLATE.value, parameter_values=json.dumps(parameter_values), @@ -142,8 +154,9 @@ def generate_nfvi_specific_nf_application( ) -> AzureCoreNetworkFunctionArmTemplateApplication: return AzureCoreNetworkFunctionArmTemplateApplication( name=self.name, - depends_on_profile=DependsOnProfile(install_depends_on=[], - uninstall_depends_on=[], update_depends_on=[]), + depends_on_profile=DependsOnProfile( + install_depends_on=[], uninstall_depends_on=[], update_depends_on=[] + ), artifact_type=AzureCoreArtifactType.ARM_TEMPLATE, artifact_profile=self.generate_artifact_profile(), deploy_parameters_mapping_rule_profile=self._generate_mapping_rule_profile(), diff --git a/src/aosm/azext_aosm/build_processors/base_processor.py b/src/aosm/azext_aosm/build_processors/base_processor.py index 4e48c69ca7b..02076b56e1a 100644 --- a/src/aosm/azext_aosm/build_processors/base_processor.py +++ b/src/aosm/azext_aosm/build_processors/base_processor.py @@ -10,12 +10,15 @@ from knack.log import get_logger from azext_aosm.common.artifact import BaseArtifact -from azext_aosm.common.local_file_builder import LocalFileBuilder -from azext_aosm.inputs.base_input import BaseInput -from azext_aosm.vendored_sdks.models import (ManifestArtifactFormat, - NetworkFunctionApplication, - ResourceElementTemplate) from azext_aosm.common.constants import CGS_NAME +from azext_aosm.definition_folder.builder.local_file_builder import LocalFileBuilder +from azext_aosm.inputs.base_input import BaseInput +from azext_aosm.vendored_sdks.models import ( + ManifestArtifactFormat, + NetworkFunctionApplication, + ResourceElementTemplate, +) + logger = get_logger(__name__) @@ -183,10 +186,16 @@ def generate_values_mappings( # Loop through each property in the schema. for subschema_name, subschema in schema["properties"].items(): # If the property is not in the values, and is required, add it to the values. - if "required" in schema and subschema_name not in values and subschema_name in schema["required"]: + if ( + "required" in schema + and subschema_name not in values + and subschema_name in schema["required"] + ): print(f"Adding {subschema_name} to values") if subschema["type"] == "object": - values[subschema_name] = self.generate_values_mappings(subschema, {}, is_ret) + values[subschema_name] = self.generate_values_mappings( + subschema, {}, is_ret + ) else: values[subschema_name] = ( f"{{configurationparameters('{CGS_NAME}').{self.name}.{subschema_name}}}" @@ -198,8 +207,14 @@ def generate_values_mappings( if subschema_name in values and subschema["type"] == "object": # Python evaluates {} as False, so we need to explicitly set to {} default_subschema_values = values[subschema_name] or {} - values[subschema_name] = self.generate_values_mappings(subschema, default_subschema_values, is_ret) + values[subschema_name] = self.generate_values_mappings( + subschema, default_subschema_values, is_ret + ) - logger.debug("Output of generate_values_mappings for %s: %s", self.name, json.dumps(values, indent=4)) + logger.debug( + "Output of generate_values_mappings for %s: %s", + self.name, + json.dumps(values, indent=4), + ) return values diff --git a/src/aosm/azext_aosm/build_processors/helm_chart_processor.py b/src/aosm/azext_aosm/build_processors/helm_chart_processor.py index c976c7b0edc..014ac950553 100644 --- a/src/aosm/azext_aosm/build_processors/helm_chart_processor.py +++ b/src/aosm/azext_aosm/build_processors/helm_chart_processor.py @@ -15,7 +15,7 @@ LocalFileACRArtifact, RemoteACRArtifact, ) -from azext_aosm.common.local_file_builder import LocalFileBuilder +from azext_aosm.definition_folder.builder.local_file_builder import LocalFileBuilder from azext_aosm.inputs.helm_chart_input import HelmChartInput from azext_aosm.vendored_sdks.models import ( ApplicationEnablement, @@ -68,7 +68,9 @@ def get_artifact_manifest_list(self) -> List[ManifestArtifactFormat]: :return: A list of artifacts for the artifact manifest. :rtype: List[ManifestArtifactFormat] """ - logger.debug("Getting artifact manifest list for Helm chart input %s.", self.name) + logger.debug( + "Getting artifact manifest list for Helm chart input %s.", self.name + ) artifact_manifest_list = [] artifact_manifest_list.append( ManifestArtifactFormat( @@ -200,7 +202,8 @@ def _find_chart_images(self) -> Set[Tuple[str, str]]: return images - def _find_image_lines(self, chart: HelmChartInput, image_lines: Set[str]) -> None: + @staticmethod + def _find_image_lines(chart: HelmChartInput, image_lines: Set[str]) -> None: """ Finds the lines containing image references in the given Helm chart and its dependencies. diff --git a/src/aosm/azext_aosm/build_processors/nfd_processor.py b/src/aosm/azext_aosm/build_processors/nfd_processor.py index 96205407c6c..23f9311dcbc 100644 --- a/src/aosm/azext_aosm/build_processors/nfd_processor.py +++ b/src/aosm/azext_aosm/build_processors/nfd_processor.py @@ -5,22 +5,30 @@ import json from pathlib import Path -from typing import List, Tuple, Any, Dict +from typing import Any, Dict, List, Tuple + from knack.log import get_logger from azext_aosm.build_processors.base_processor import BaseInputProcessor from azext_aosm.common.artifact import BaseArtifact, LocalFileACRArtifact -from azext_aosm.common.local_file_builder import LocalFileBuilder +from azext_aosm.common.constants import NSD_OUTPUT_FOLDER_FILENAME +from azext_aosm.definition_folder.builder.local_file_builder import LocalFileBuilder from azext_aosm.inputs.nfd_input import NFDInput from azext_aosm.vendored_sdks.models import ( - ArmResourceDefinitionResourceElementTemplate, ArtifactType, - DependsOnProfile, ManifestArtifactFormat, NetworkFunctionApplication) -from azext_aosm.vendored_sdks.models import \ - NetworkFunctionDefinitionResourceElementTemplateDetails as \ - NFDResourceElementTemplate -from azext_aosm.vendored_sdks.models import (NSDArtifactProfile, - ReferencedResource, TemplateType) -from azext_aosm.common.constants import NSD_OUTPUT_FOLDER_FILENAME + ArmResourceDefinitionResourceElementTemplate, + ArtifactType, + DependsOnProfile, + ManifestArtifactFormat, + NetworkFunctionApplication, +) +from azext_aosm.vendored_sdks.models import ( + NetworkFunctionDefinitionResourceElementTemplateDetails as NFDResourceElementTemplate, +) +from azext_aosm.vendored_sdks.models import ( + NSDArtifactProfile, + ReferencedResource, + TemplateType, +) logger = get_logger(__name__) @@ -79,7 +87,9 @@ def get_artifact_details( artifact_name=self.input_artifact.artifact_name, artifact_type=ArtifactType.OCI_ARTIFACT.value, artifact_version=self.input_artifact.artifact_version, - file_path=self.input_artifact.arm_template_output_path.relative_to(Path(NSD_OUTPUT_FOLDER_FILENAME)), + file_path=self.input_artifact.arm_template_output_path.relative_to( + Path(NSD_OUTPUT_FOLDER_FILENAME) + ), ) # Create a local file builder for the ARM template @@ -125,8 +135,9 @@ def generate_resource_element_template(self) -> NFDResourceElementTemplate: return NFDResourceElementTemplate( name=self.name, configuration=configuration, - depends_on_profile=DependsOnProfile(install_depends_on=[], - uninstall_depends_on=[], update_depends_on=[]), + depends_on_profile=DependsOnProfile( + install_depends_on=[], uninstall_depends_on=[], update_depends_on=[] + ), ) def _generate_schema( diff --git a/src/aosm/azext_aosm/build_processors/vhd_processor.py b/src/aosm/azext_aosm/build_processors/vhd_processor.py index dde6212df8a..6e636416488 100644 --- a/src/aosm/azext_aosm/build_processors/vhd_processor.py +++ b/src/aosm/azext_aosm/build_processors/vhd_processor.py @@ -9,17 +9,26 @@ from knack.log import get_logger from azext_aosm.build_processors.base_processor import BaseInputProcessor -from azext_aosm.common.artifact import (BaseArtifact, - BlobStorageAccountArtifact, - LocalFileStorageAccountArtifact) -from azext_aosm.common.local_file_builder import LocalFileBuilder +from azext_aosm.common.artifact import ( + BaseArtifact, + BlobStorageAccountArtifact, + LocalFileStorageAccountArtifact, +) +from azext_aosm.definition_folder.builder.local_file_builder import LocalFileBuilder from azext_aosm.inputs.vhd_file_input import VHDFileInput from azext_aosm.vendored_sdks.models import ( - ApplicationEnablement, ArtifactType, - AzureCoreNetworkFunctionVhdApplication, AzureCoreVhdImageArtifactProfile, - AzureCoreVhdImageDeployMappingRuleProfile, DependsOnProfile, - ManifestArtifactFormat, ReferencedResource, ResourceElementTemplate, - VhdImageArtifactProfile, VhdImageMappingRuleProfile) + ApplicationEnablement, + ArtifactType, + AzureCoreNetworkFunctionVhdApplication, + AzureCoreVhdImageArtifactProfile, + AzureCoreVhdImageDeployMappingRuleProfile, + DependsOnProfile, + ManifestArtifactFormat, + ReferencedResource, + ResourceElementTemplate, + VhdImageArtifactProfile, + VhdImageMappingRuleProfile, +) logger = get_logger(__name__) @@ -85,7 +94,9 @@ def get_artifact_details( ) artifacts.append( BlobStorageAccountArtifact( - artifact_manifest=artifact_manifest, + artifact_name=self.input_artifact.artifact_name, + artifact_type=ArtifactType.VHD_IMAGE_FILE.value, + artifact_version=self.input_artifact.artifact_version, blob_sas_uri=self.input_artifact.blob_sas_uri, ) ) @@ -108,8 +119,9 @@ def generate_nf_application(self) -> AzureCoreNetworkFunctionVhdApplication: return AzureCoreNetworkFunctionVhdApplication( name=self.name, - depends_on_profile=DependsOnProfile(install_depends_on=[], - uninstall_depends_on=[], update_depends_on=[]), + depends_on_profile=DependsOnProfile( + install_depends_on=[], uninstall_depends_on=[], update_depends_on=[] + ), artifact_profile=self._generate_artifact_profile(), deploy_parameters_mapping_rule_profile=self._generate_mapping_rule_profile(), ) diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_base_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_base_handler.py index 330c2df8c6c..3e9fcb3f9e1 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_base_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_base_handler.py @@ -8,25 +8,26 @@ from abc import ABC, abstractmethod from dataclasses import fields, is_dataclass from pathlib import Path -from typing import Optional +from typing import Optional, Union + from azure.cli.core.azclierror import UnclassifiedUserFault from jinja2 import StrictUndefined, Template from knack.log import get_logger +from azext_aosm.common.command_context import CommandContext +from azext_aosm.common.constants import DEPLOYMENT_PARAMETERS_FILENAME +from azext_aosm.configuration_models.common_parameters_config import ( + BaseCommonParametersConfig, +) from azext_aosm.configuration_models.onboarding_base_input_config import ( OnboardingBaseInputConfig, ) from azext_aosm.definition_folder.builder.definition_folder_builder import ( DefinitionFolderBuilder, ) +from azext_aosm.definition_folder.builder.local_file_builder import LocalFileBuilder from azext_aosm.definition_folder.reader.definition_folder import DefinitionFolder -from azext_aosm.common.command_context import CommandContext -from azext_aosm.configuration_models.common_parameters_config import ( - BaseCommonParametersConfig, -) -from azext_aosm.common.local_file_builder import LocalFileBuilder from azext_aosm.vendored_sdks import HybridNetworkManagementClient -from azext_aosm.common.constants import DEPLOYMENT_PARAMETERS_FILENAME logger = get_logger(__name__) @@ -38,13 +39,15 @@ def __init__( self, provided_input_path: Optional[Path] = None, aosm_client: Optional[HybridNetworkManagementClient] = None, - skip: str = None, + skip: Optional[str] = None, ): """Initialize the CLI handler.""" self.aosm_client = aosm_client self.skip = skip # If config file provided (for build, publish and delete) if provided_input_path: + # Explicitly define types + self.config: Union[OnboardingBaseInputConfig, BaseCommonParametersConfig] provided_input_path = Path(provided_input_path) # If config file is the input.jsonc for build command if provided_input_path.suffix == ".jsonc": @@ -105,6 +108,7 @@ def publish(self, command_context: CommandContext): definition_folder = DefinitionFolder( command_context.cli_options["definition_folder"] ) + assert isinstance(self.config, BaseCommonParametersConfig) definition_folder.deploy(config=self.config, command_context=command_context) def delete(self, command_context: CommandContext): @@ -116,8 +120,12 @@ def delete(self, command_context: CommandContext): # TODO: Implement def pre_validate_build(self): - """Perform all validations that need to be done before running the build command.""" - pass + """ + Perform all validations that need to be done before running the build command. + + This method must be overwritten by subclasses to be of use, but is not abstract as it's + allowed to not perform any pre-validation, in which case this base method just does nothing. + """ @abstractmethod def build_base_bicep(self): @@ -150,18 +158,19 @@ def _get_processor_list(self): raise NotImplementedError @abstractmethod - def _get_input_config(self, input_config: dict = None) -> OnboardingBaseInputConfig: + def _get_input_config( + self, input_config: Optional[dict] = None + ) -> OnboardingBaseInputConfig: """Get the configuration for the command.""" raise NotImplementedError @abstractmethod - def _get_params_config( - self, config_file: Path = None - ) -> BaseCommonParametersConfig: + def _get_params_config(self, config_file: Path) -> BaseCommonParametersConfig: """Get the parameters config for publish/delete.""" raise NotImplementedError - def _read_input_config_from_file(self, input_json_path: Path) -> dict: + @staticmethod + def _read_input_config_from_file(input_json_path: Path) -> dict: """Reads the input JSONC file, removes comments. Returns config as dictionary. @@ -172,7 +181,8 @@ def _read_input_config_from_file(self, input_json_path: Path) -> dict: return config_dict - def _render_base_bicep_contents(self, template_path): + @staticmethod + def _render_base_bicep_contents(template_path): """Write the base bicep file from given template.""" with open(template_path, "r", encoding="UTF-8") as f: template: Template = Template( @@ -183,7 +193,8 @@ def _render_base_bicep_contents(self, template_path): bicep_contents: str = template.render() return bicep_contents - def _render_definition_bicep_contents(self, template_path: Path, params): + @staticmethod + def _render_definition_bicep_contents(template_path: Path, params): """Write the definition bicep file from given template.""" with open(template_path, "r", encoding="UTF-8") as f: template: Template = Template( @@ -194,11 +205,11 @@ def _render_definition_bicep_contents(self, template_path: Path, params): bicep_contents: str = template.render(params) return bicep_contents + @staticmethod def _render_manifest_bicep_contents( - self, template_path: Path, acr_artifact_list: list, - sa_artifact_list: list = None, + sa_artifact_list: Optional[list] = None, ): """Write the manifest bicep file from given template. @@ -215,7 +226,8 @@ def _render_manifest_bicep_contents( ) return bicep_contents - def _get_template_path(self, definition_type: str, template_name: str) -> Path: + @staticmethod + def _get_template_path(definition_type: str, template_name: str) -> Path: """Get the path to a template.""" return ( Path(__file__).parent.parent @@ -329,7 +341,8 @@ def _write_config_to_file(self, output_path: Path): print(f"Empty configuration has been written to {output_path.name}") logger.info("Empty configuration has been written to %s", output_path.name) - def _check_for_overwrite(self, output_path: Path): + @staticmethod + def _check_for_overwrite(output_path: Path): """Check that the input file exists.""" if output_path.exists(): carry_on = input( @@ -354,7 +367,8 @@ def _render_deployment_params_schema( ), ) - def _build_deploy_params_schema(self, schema_properties): + @staticmethod + def _build_deploy_params_schema(schema_properties): """Build the schema for deployParameters.json.""" schema_contents = { "$schema": "https://json-schema.org/draft-07/schema#", diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_cnf_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_cnf_handler.py index de6c46bebc6..793e1ecc2c2 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_cnf_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_cnf_handler.py @@ -7,47 +7,48 @@ import json import warnings from pathlib import Path -import ruamel.yaml -from ruamel.yaml.error import ReusedAnchorWarning - -from jinja2 import Template +from typing import Optional +import ruamel.yaml from azure.cli.core.azclierror import ValidationError +from jinja2 import Template from knack.log import get_logger +from ruamel.yaml.error import ReusedAnchorWarning from azext_aosm.build_processors.helm_chart_processor import HelmChartProcessor -from azext_aosm.inputs.helm_chart_input import HelmChartInput -from azext_aosm.common.local_file_builder import LocalFileBuilder -from azext_aosm.configuration_models.onboarding_cnf_input_config import ( - OnboardingCNFInputConfig, -) -from azext_aosm.configuration_models.common_parameters_config import ( - CNFCommonParametersConfig, -) -from azext_aosm.definition_folder.builder.artifact_builder import ( - ArtifactDefinitionElementBuilder, -) -from azext_aosm.definition_folder.builder.bicep_builder import ( - BicepDefinitionElementBuilder, -) -from azext_aosm.definition_folder.builder.json_builder import ( - JSONDefinitionElementBuilder, -) from azext_aosm.common.constants import ( ARTIFACT_LIST_FILENAME, BASE_FOLDER_NAME, CNF_BASE_TEMPLATE_FILENAME, - CNF_TEMPLATE_FOLDER_NAME, CNF_DEFINITION_TEMPLATE_FILENAME, CNF_HELM_VALIDATION_ERRORS_TEMPLATE_FILENAME, CNF_INPUT_FILENAME, CNF_MANIFEST_TEMPLATE_FILENAME, CNF_OUTPUT_FOLDER_FILENAME, + CNF_TEMPLATE_FOLDER_NAME, + DEPLOYMENT_PARAMETERS_FILENAME, HELM_TEMPLATE, MANIFEST_FOLDER_NAME, NF_DEFINITION_FOLDER_NAME, - DEPLOYMENT_PARAMETERS_FILENAME, ) +from azext_aosm.common.exceptions import TemplateValidationError +from azext_aosm.configuration_models.common_parameters_config import ( + CNFCommonParametersConfig, +) +from azext_aosm.configuration_models.onboarding_cnf_input_config import ( + OnboardingCNFInputConfig, +) +from azext_aosm.definition_folder.builder.artifact_builder import ( + ArtifactDefinitionElementBuilder, +) +from azext_aosm.definition_folder.builder.bicep_builder import ( + BicepDefinitionElementBuilder, +) +from azext_aosm.definition_folder.builder.json_builder import ( + JSONDefinitionElementBuilder, +) +from azext_aosm.definition_folder.builder.local_file_builder import LocalFileBuilder +from azext_aosm.inputs.helm_chart_input import HelmChartInput from .onboarding_nfd_base_handler import OnboardingNFDBaseCLIHandler @@ -69,13 +70,13 @@ def output_folder_file_name(self) -> str: """Get the output folder file name.""" return CNF_OUTPUT_FOLDER_FILENAME - def _get_input_config(self, input_config: dict = None) -> OnboardingCNFInputConfig: + def _get_input_config(self, input_config: Optional[dict] = None) -> OnboardingCNFInputConfig: """Get the configuration for the command.""" if input_config is None: input_config = {} return OnboardingCNFInputConfig(**input_config) - def _get_params_config(self, config_file: Path = None) -> CNFCommonParametersConfig: + def _get_params_config(self, config_file: Path) -> CNFCommonParametersConfig: """Get the configuration for the command.""" with open(config_file, "r", encoding="utf-8") as _file: params_dict = json.load(_file) @@ -83,9 +84,10 @@ def _get_params_config(self, config_file: Path = None) -> CNFCommonParametersCon params_dict = {} return CNFCommonParametersConfig(**params_dict) - def _get_processor_list(self) -> [HelmChartProcessor]: + def _get_processor_list(self) -> list[HelmChartProcessor]: processor_list = [] # for each helm package, instantiate helm processor + assert isinstance(self.config, OnboardingCNFInputConfig) for helm_package in self.config.helm_packages: if helm_package.default_values: if Path(helm_package.default_values).exists(): @@ -118,12 +120,12 @@ def _validate_helm_template(self): validation_errors = {} for helm_processor in self.processors: - validation_output = helm_processor.input_artifact.validate_template() - - if validation_output: + try: + helm_processor.input_artifact.validate_template() + except TemplateValidationError as error: validation_errors[ helm_processor.input_artifact.artifact_name - ] = validation_output + ] = str(error) if validation_errors: # Create an error file using a j2 template diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_nfd_base_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_nfd_base_handler.py index 1fec72db88b..d1834fc3ddc 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_nfd_base_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_nfd_base_handler.py @@ -2,8 +2,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- -from azext_aosm.cli_handlers.onboarding_base_handler import \ - OnboardingBaseCLIHandler +from azext_aosm.cli_handlers.onboarding_base_handler import OnboardingBaseCLIHandler class OnboardingNFDBaseCLIHandler(OnboardingBaseCLIHandler): diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_nsd_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_nsd_handler.py index 9866d9de315..66322e758b2 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_nsd_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_nsd_handler.py @@ -6,36 +6,49 @@ import json from pathlib import Path +from typing import Optional from knack.log import get_logger -from azext_aosm.build_processors.arm_processor import ( - AzureCoreArmBuildProcessor) +from azext_aosm.build_processors.arm_processor import AzureCoreArmBuildProcessor from azext_aosm.build_processors.nfd_processor import NFDProcessor -from azext_aosm.cli_handlers.onboarding_nfd_base_handler import \ - OnboardingBaseCLIHandler +from azext_aosm.cli_handlers.onboarding_nfd_base_handler import OnboardingBaseCLIHandler from azext_aosm.common.constants import ( # NSD_DEFINITION_TEMPLATE_FILENAME, - ARTIFACT_LIST_FILENAME, BASE_FOLDER_NAME, MANIFEST_FOLDER_NAME, - NSD_BASE_TEMPLATE_FILENAME, NSD_TEMPLATE_FOLDER_NAME, NSD_INPUT_FILENAME, - NSD_MANIFEST_TEMPLATE_FILENAME, NSD_OUTPUT_FOLDER_FILENAME, NSD_DEFINITION_TEMPLATE_FILENAME, - CGS_FILENAME, NSD_DEFINITION_FOLDER_NAME, DEPLOYMENT_PARAMETERS_FILENAME, - TEMPLATE_PARAMETERS_FILENAME, CGS_NAME) -from azext_aosm.common.local_file_builder import LocalFileBuilder -from azext_aosm.configuration_models.common_parameters_config import \ - NSDCommonParametersConfig -from azext_aosm.configuration_models.onboarding_nsd_input_config import \ - OnboardingNSDInputConfig -from azext_aosm.definition_folder.builder.artifact_builder import \ - ArtifactDefinitionElementBuilder -from azext_aosm.definition_folder.builder.bicep_builder import \ - BicepDefinitionElementBuilder -from azext_aosm.definition_folder.builder.json_builder import \ - JSONDefinitionElementBuilder + ARTIFACT_LIST_FILENAME, + BASE_FOLDER_NAME, + CGS_FILENAME, + CGS_NAME, + DEPLOYMENT_PARAMETERS_FILENAME, + MANIFEST_FOLDER_NAME, + NSD_BASE_TEMPLATE_FILENAME, + NSD_DEFINITION_FOLDER_NAME, + NSD_DEFINITION_TEMPLATE_FILENAME, + NSD_INPUT_FILENAME, + NSD_MANIFEST_TEMPLATE_FILENAME, + NSD_OUTPUT_FOLDER_FILENAME, + NSD_TEMPLATE_FOLDER_NAME, + TEMPLATE_PARAMETERS_FILENAME, +) +from azext_aosm.configuration_models.common_parameters_config import ( + NSDCommonParametersConfig, +) +from azext_aosm.configuration_models.onboarding_nsd_input_config import ( + OnboardingNSDInputConfig, +) +from azext_aosm.definition_folder.builder.artifact_builder import ( + ArtifactDefinitionElementBuilder, +) +from azext_aosm.definition_folder.builder.bicep_builder import ( + BicepDefinitionElementBuilder, +) +from azext_aosm.definition_folder.builder.json_builder import ( + JSONDefinitionElementBuilder, +) +from azext_aosm.definition_folder.builder.local_file_builder import LocalFileBuilder from azext_aosm.inputs.arm_template_input import ArmTemplateInput from azext_aosm.inputs.nfd_input import NFDInput -from azext_aosm.vendored_sdks.models import ( - ManifestArtifactFormat, NetworkFunctionDefinitionVersion, - NetworkFunctionDefinitionVersionPropertiesFormat) +from azext_aosm.vendored_sdks import HybridNetworkManagementClient +from azext_aosm.vendored_sdks.models import NetworkFunctionDefinitionVersion logger = get_logger(__name__) @@ -56,17 +69,16 @@ def output_folder_file_name(self) -> str: @property def nfvi_site_name(self) -> str: """Return the name of the NFVI used for the NSDV.""" + assert isinstance(self.config, OnboardingNSDInputConfig) return f"{self.config.nsd_name}_NFVI" - def _get_input_config(self, input_config: dict = None) -> OnboardingNSDInputConfig: + def _get_input_config(self, input_config: Optional[dict] = None) -> OnboardingNSDInputConfig: """Get the configuration for the command.""" if input_config is None: input_config = {} return OnboardingNSDInputConfig(**input_config) - def _get_params_config( - self, config_file: dict = None - ) -> NSDCommonParametersConfig: + def _get_params_config(self, config_file: Path) -> NSDCommonParametersConfig: """Get the configuration for the command.""" with open(config_file, "r", encoding="utf-8") as _file: params_dict = json.load(_file) @@ -83,16 +95,21 @@ def _get_processor_list(self): artifact_name=resource_element.properties.artifact_name, artifact_version=resource_element.properties.version, default_config=None, - template_path=Path(resource_element.properties.file_path).absolute(), + template_path=Path( + resource_element.properties.file_path + ).absolute(), ) # TODO: generalise for nexus in nexus ready stories processor_list.append( AzureCoreArmBuildProcessor(arm_input.artifact_name, arm_input) ) elif resource_element.resource_element_type == "NF": - # TODO: change artifact name and version to the nfd name and version or justify why it was this in the first place + # TODO: change artifact name and version to the nfd name and version or justify why it was this + # in the first place # AC4 note: I couldn't find a reference in the old code, but this - # does ring a bell. Was it so the artifact manifest didn't get broken with changes to NF versions? I.e., you could make an NF version change in CGV, and the artifact manifest, which is immutable, would still be valid? + # does ring a bell. Was it so the artifact manifest didn't get broken with changes to NF versions? + # I.e., you could make an NF version change in CGV, and the artifact manifest, which is immutable, + # would still be valid? # I am concerned that if we have multiple NFs we will have clashing artifact names. # I'm not changing the behaviour right now as it's too high risk, but we should look again here. nfdv_object = self._get_nfdv(resource_element.properties) @@ -116,7 +133,9 @@ def _get_processor_list(self): processor_list.append(nfd_processor) else: # TODO: raise more specific error - raise ValueError(f"Invalid resource element type: {resource_element.resource_element_type}") + raise ValueError( + f"Invalid resource element type: {resource_element.resource_element_type}" + ) return processor_list def build_base_bicep(self): @@ -188,9 +207,7 @@ def build_resource_bicep(self): ret_list.append(nf_ret) # Adding supporting file: config mappings - deploy_values = ( - nf_ret.configuration.parameter_values - ) + deploy_values = nf_ret.configuration.parameter_values mapping_file = LocalFileBuilder( Path( NSD_OUTPUT_FOLDER_FILENAME, @@ -222,9 +239,7 @@ def build_resource_bicep(self): "template_parameters_file": TEMPLATE_PARAMETERS_FILENAME, } - bicep_contents = self._render_definition_bicep_contents( - template_path, params - ) + bicep_contents = self._render_definition_bicep_contents(template_path, params) # Generate the nsd bicep file bicep_file = BicepDefinitionElementBuilder( Path(NSD_OUTPUT_FOLDER_FILENAME, NSD_DEFINITION_FOLDER_NAME), bicep_contents @@ -251,23 +266,21 @@ def build_all_parameters_json(self): "acrManifestName": self.config.acr_artifact_store_name + "-manifest", "nsDesignGroup": self.config.nsd_name, "nsDesignVersion": self.config.nsd_version, - "nfviSiteName": self.nfvi_site_name + "nfviSiteName": self.nfvi_site_name, } base_file = JSONDefinitionElementBuilder( Path(NSD_OUTPUT_FOLDER_FILENAME), json.dumps(params_content, indent=4) ) return base_file - def _render_config_group_schema_contents(self, complete_schema, nf_names): - - required = [nf for nf in nf_names] - + @staticmethod + def _render_config_group_schema_contents(complete_schema, nf_names): params_content = { "$schema": "https://json-schema.org/draft-07/schema#", "title": f"{CGS_NAME}", "type": "object", "properties": complete_schema, - "required": required, + "required": nf_names, } return LocalFileBuilder( Path( @@ -283,6 +296,7 @@ def _get_nfdv(self, nf_properties) -> NetworkFunctionDefinitionVersion: print( f"Reading existing NFDV resource object {nf_properties.version} from group {nf_properties.name}" ) + assert isinstance(self.aosm_client, HybridNetworkManagementClient) nfdv_object = self.aosm_client.network_function_definition_versions.get( resource_group_name=nf_properties.publisher_resource_group, publisher_name=nf_properties.publisher, diff --git a/src/aosm/azext_aosm/cli_handlers/onboarding_vnf_handler.py b/src/aosm/azext_aosm/cli_handlers/onboarding_vnf_handler.py index f122af0a85f..9dfd04aed68 100644 --- a/src/aosm/azext_aosm/cli_handlers/onboarding_vnf_handler.py +++ b/src/aosm/azext_aosm/cli_handlers/onboarding_vnf_handler.py @@ -4,32 +4,36 @@ # -------------------------------------------------------------------------------------------- import json from pathlib import Path -from typing import Dict, Any +from typing import Any, Dict, Optional + from knack.log import get_logger from azext_aosm.build_processors.arm_processor import ( - AzureCoreArmBuildProcessor, BaseArmBuildProcessor) + AzureCoreArmBuildProcessor, + BaseArmBuildProcessor, +) from azext_aosm.build_processors.vhd_processor import VHDProcessor -from azext_aosm.common.constants import (ARTIFACT_LIST_FILENAME, - BASE_FOLDER_NAME, - MANIFEST_FOLDER_NAME, - NF_DEFINITION_FOLDER_NAME, - VNF_BASE_TEMPLATE_FILENAME, - VNF_TEMPLATE_FOLDER_NAME, - VNF_DEFINITION_TEMPLATE_FILENAME, - VNF_INPUT_FILENAME, - VNF_MANIFEST_TEMPLATE_FILENAME, - VNF_OUTPUT_FOLDER_FILENAME, - DEPLOYMENT_PARAMETERS_FILENAME, - VHD_PARAMETERS_FILENAME, - TEMPLATE_PARAMETERS_FILENAME) -from azext_aosm.common.local_file_builder import LocalFileBuilder -from azext_aosm.configuration_models.onboarding_vnf_input_config import ( - OnboardingVNFInputConfig, +from azext_aosm.common.constants import ( + ARTIFACT_LIST_FILENAME, + BASE_FOLDER_NAME, + DEPLOYMENT_PARAMETERS_FILENAME, + MANIFEST_FOLDER_NAME, + NF_DEFINITION_FOLDER_NAME, + TEMPLATE_PARAMETERS_FILENAME, + VHD_PARAMETERS_FILENAME, + VNF_BASE_TEMPLATE_FILENAME, + VNF_DEFINITION_TEMPLATE_FILENAME, + VNF_INPUT_FILENAME, + VNF_MANIFEST_TEMPLATE_FILENAME, + VNF_OUTPUT_FOLDER_FILENAME, + VNF_TEMPLATE_FOLDER_NAME, ) from azext_aosm.configuration_models.common_parameters_config import ( VNFCommonParametersConfig, ) +from azext_aosm.configuration_models.onboarding_vnf_input_config import ( + OnboardingVNFInputConfig, +) from azext_aosm.definition_folder.builder.artifact_builder import ( ArtifactDefinitionElementBuilder, ) @@ -39,6 +43,7 @@ from azext_aosm.definition_folder.builder.json_builder import ( JSONDefinitionElementBuilder, ) +from azext_aosm.definition_folder.builder.local_file_builder import LocalFileBuilder from azext_aosm.inputs.arm_template_input import ArmTemplateInput from azext_aosm.inputs.vhd_file_input import VHDFileInput @@ -60,15 +65,15 @@ def output_folder_file_name(self) -> str: """Get the output folder file name.""" return VNF_OUTPUT_FOLDER_FILENAME - def _get_input_config(self, input_config: Dict[str, Any] = None) -> OnboardingVNFInputConfig: + def _get_input_config( + self, input_config: Optional[Dict[str, Any]] = None + ) -> OnboardingVNFInputConfig: """Get the configuration for the command.""" if input_config is None: input_config = {} return OnboardingVNFInputConfig(**input_config) - def _get_params_config( - self, config_file: dict = None - ) -> VNFCommonParametersConfig: + def _get_params_config(self, config_file: Path) -> VNFCommonParametersConfig: """Get the configuration for the command.""" with open(config_file, "r", encoding="utf-8") as _file: params_dict = json.load(_file) @@ -195,13 +200,13 @@ def build_resource_bicep(self): # For each arm template, generate nf application if isinstance(processor, BaseArmBuildProcessor): - acr_nf_application_list.append(nf_application) # Generate local file for template_parameters + add to supporting files list params = ( - nf_application.deploy_parameters_mapping_rule_profile.template_mapping_rule_profile.template_parameters - ) + nf_application.deploy_parameters_mapping_rule_profile.template_mapping_rule_profile + ).template_parameters # Funky formatting is to stop black from reformatting to too long line + template_name = TEMPLATE_PARAMETERS_FILENAME logger.info( "Created templatateParameters as supporting file for nfDefinition bicep" @@ -212,8 +217,8 @@ def build_resource_bicep(self): sa_nf_application_list.append(nf_application) # Generate local file for vhd_parameters params = ( - nf_application.deploy_parameters_mapping_rule_profile.vhd_image_mapping_rule_profile.user_configuration - ) + nf_application.deploy_parameters_mapping_rule_profile.vhd_image_mapping_rule_profile + ).user_configuration # Funky formatting is to stop black from reformatting to too long line template_name = VHD_PARAMETERS_FILENAME else: raise TypeError(f"Type: {type(processor)} is not valid") @@ -238,11 +243,9 @@ def build_resource_bicep(self): "sa_nf_application": sa_nf_application_list[0], "deployment_parameters_file": DEPLOYMENT_PARAMETERS_FILENAME, "vhd_parameters_file": VHD_PARAMETERS_FILENAME, - "template_parameters_file": TEMPLATE_PARAMETERS_FILENAME + "template_parameters_file": TEMPLATE_PARAMETERS_FILENAME, } - bicep_contents = self._render_definition_bicep_contents( - template_path, params - ) + bicep_contents = self._render_definition_bicep_contents(template_path, params) # Create a bicep element # + add its supporting files (deploymentParameters, vhdParameters and templateParameters) @@ -268,17 +271,18 @@ def build_all_parameters_json(self): "publisherResourceGroupName": self.config.publisher_resource_group_name, "acrArtifactStoreName": self.config.acr_artifact_store_name, "saArtifactStoreName": self.config.blob_artifact_store_name, - "acrManifestName": self.config.acr_artifact_store_name + "-manifest", + "acrManifestName": self.config.acr_artifact_store_name + "-manifest", "saManifestName": self.config.blob_artifact_store_name + "-manifest", "nfDefinitionGroup": self.config.nf_name, - "nfDefinitionVersion": self.config.version + "nfDefinitionVersion": self.config.version, } base_file = JSONDefinitionElementBuilder( Path(VNF_OUTPUT_FOLDER_FILENAME), json.dumps(params_content, indent=4) ) return base_file - def _get_default_config(self, vhd): + @staticmethod + def _get_default_config(vhd): default_config = {} if vhd.image_disk_size_GB: default_config.update({"image_disk_size_GB": vhd.image_disk_size_GB}) diff --git a/src/aosm/azext_aosm/commands.py b/src/aosm/azext_aosm/commands.py index 11aeccf95d1..e0d3a4fee8e 100644 --- a/src/aosm/azext_aosm/commands.py +++ b/src/aosm/azext_aosm/commands.py @@ -5,10 +5,6 @@ from azure.cli.core import AzCommandsLoader -from ._client_factory import cf_aosm # pylint: disable=import-error - -# from azext_aosm._client_factory import cf_aosm - def load_command_table(self: AzCommandsLoader, _): with self.command_group("aosm nfd") as g: diff --git a/src/aosm/azext_aosm/common/artifact.py b/src/aosm/azext_aosm/common/artifact.py index d9d78eed6ae..60d41529cd1 100644 --- a/src/aosm/azext_aosm/common/artifact.py +++ b/src/aosm/azext_aosm/common/artifact.py @@ -2,28 +2,31 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- -from abc import ABC, abstractmethod -from functools import lru_cache import json import math -from pathlib import Path import shutil import subprocess +from abc import ABC, abstractmethod +from functools import lru_cache +from pathlib import Path from time import sleep -from typing import Dict, Optional +from typing import Any, MutableMapping, Optional -from azext_aosm.vendored_sdks.azure_storagev2.blob.v2022_11_02 import BlobClient, BlobType -from azext_aosm.vendored_sdks.models import ManifestArtifactFormat -from azext_aosm.vendored_sdks import HybridNetworkManagementClient -from azext_aosm.common.command_context import CommandContext -from azext_aosm.common.utils import convert_bicep_to_arm -from azext_aosm.configuration_models.common_parameters_config import BaseCommonParametersConfig, VNFCommonParametersConfig -from azext_aosm.vendored_sdks import HybridNetworkManagementClient -from knack.util import CLIError from knack.log import get_logger +from knack.util import CLIError from oras.client import OrasClient - +from azext_aosm.common.command_context import CommandContext +from azext_aosm.common.utils import convert_bicep_to_arm +from azext_aosm.configuration_models.common_parameters_config import ( + BaseCommonParametersConfig, + VNFCommonParametersConfig, +) +from azext_aosm.vendored_sdks import HybridNetworkManagementClient +from azext_aosm.vendored_sdks.azure_storagev2.blob.v2022_11_02 import ( + BlobClient, + BlobType, +) logger = get_logger(__name__) @@ -31,6 +34,7 @@ # TODO: Split these out into separate files, probably in a new artifacts module class BaseArtifact(ABC): """Abstract base class for artifacts.""" + def __init__(self, artifact_name: str, artifact_type: str, artifact_version: str): self.artifact_name = artifact_name self.artifact_type = artifact_type @@ -39,24 +43,24 @@ def __init__(self, artifact_name: str, artifact_type: str, artifact_version: str def to_dict(self) -> dict: """Convert an instance to a dict.""" output_dict = {"type": ARTIFACT_CLASS_TO_TYPE[type(self)]} - output_dict.update( - {k: vars(self)[k] for k in vars(self)} - ) + output_dict.update({k: vars(self)[k] for k in vars(self)}) return output_dict @abstractmethod - def upload(self, config: BaseCommonParametersConfig, command_context: CommandContext): + def upload( + self, config: BaseCommonParametersConfig, command_context: CommandContext + ): """Upload the artifact.""" - pass class BaseACRArtifact(BaseArtifact): """Abstract base class for ACR artifacts.""" @abstractmethod - def upload(self, config: BaseCommonParametersConfig, command_context: CommandContext): + def upload( + self, config: BaseCommonParametersConfig, command_context: CommandContext + ): """Upload the artifact.""" - pass @staticmethod def _check_tool_installed(tool_name: str) -> None: @@ -107,10 +111,13 @@ def _call_subprocess_raise_output(cmd: list) -> None: # Raise the error without the original exception, which may contain secrets. raise CLIError(all_output) from None + @staticmethod @lru_cache(maxsize=32) - def _manifest_credentials(self, config: BaseCommonParametersConfig, aosm_client: HybridNetworkManagementClient) -> Dict: + def _manifest_credentials( + config: BaseCommonParametersConfig, + aosm_client: HybridNetworkManagementClient, + ) -> MutableMapping[str, Any]: """Gets the details for uploading the artifacts in the manifest.""" - return aosm_client.artifact_manifests.list_credential( resource_group_name=config.publisherResourceGroupName, publisher_name=config.publisherName, @@ -119,7 +126,7 @@ def _manifest_credentials(self, config: BaseCommonParametersConfig, aosm_client: ).as_dict() @staticmethod - def _get_oras_client(manifest_credentials: Dict) -> OrasClient: + def _get_oras_client(manifest_credentials: MutableMapping[str, Any]) -> OrasClient: client = OrasClient(hostname=manifest_credentials["acr_server_url"]) client.login( username=manifest_credentials["username"], @@ -149,7 +156,9 @@ def __init__(self, artifact_name, artifact_type, artifact_version, file_path: Pa super().__init__(artifact_name, artifact_type, artifact_version) self.file_path = file_path - def upload(self, config: BaseCommonParametersConfig, command_context: CommandContext): + def upload( + self, config: BaseCommonParametersConfig, command_context: CommandContext + ): """Upload the artifact.""" logger.debug("LocalFileACRArtifact config: %s", config) @@ -158,15 +167,9 @@ def upload(self, config: BaseCommonParametersConfig, command_context: CommandCon # For NSDs, we provide paths relative to the artifacts folder, resolve them to absolute paths if not self.file_path.is_absolute(): output_folder_path = command_context.cli_options["definition_folder"] - resolved_file_path = output_folder_path.resolve() - upload_file_path = resolved_file_path / self.file_path - print("nfp", output_folder_path) - print("rfp", resolved_file_path) - print("ufp", upload_file_path) - self.file_path = upload_file_path - - # self.file_path = Path(self.file_path).resolve() - print("fp", self.file_path) + resolved_path = output_folder_path.resolve() + absolute_file_path = resolved_path / self.file_path + self.file_path = absolute_file_path if self.file_path.suffix == ".bicep": # Uploading the nf_template as part of the NSD will use this code path @@ -177,12 +180,12 @@ def upload(self, config: BaseCommonParametersConfig, command_context: CommandCon json.dump(arm_template, self.file_path.open("w")) logger.debug("Converted bicep file to ARM as: %s", self.file_path) - manifest_credentials = self._manifest_credentials(config=config, aosm_client=command_context.aosm_client) + manifest_credentials = self._manifest_credentials( + config=config, aosm_client=command_context.aosm_client + ) oras_client = self._get_oras_client(manifest_credentials=manifest_credentials) target_acr = self._get_acr(oras_client) - target = ( - f"{target_acr}/{self.artifact_name}:{self.artifact_version}" - ) + target = f"{target_acr}/{self.artifact_name}:{self.artifact_version}" logger.debug("Uploading %s to %s", self.file_path, target) retries = 0 while True: @@ -214,13 +217,21 @@ def upload(self, config: BaseCommonParametersConfig, command_context: CommandCon class RemoteACRArtifact(BaseACRArtifact): """Class for ACR artifacts from a remote ACR image.""" + def __init__( - self, artifact_name, artifact_type, artifact_version, source_registry: str, source_registry_namespace: str + self, + artifact_name, + artifact_type, + artifact_version, + source_registry: str, + source_registry_namespace: str, ): super().__init__(artifact_name, artifact_type, artifact_version) self.source_registry = source_registry self.source_registry_namespace = source_registry_namespace - self.namespace_with_slash = f"{source_registry_namespace}/" if source_registry_namespace else "" + self.namespace_with_slash = ( + f"{source_registry_namespace}/" if source_registry_namespace else "" + ) def _pull_image_to_local_registry( self, @@ -294,8 +305,11 @@ def _push_image_from_local_registry( :type target_password: str """ logger.debug("RemoteACRArtifact config: %s", config) - manifest_credentials = self._manifest_credentials(config=config, aosm_client=command_context.aosm_client) - # TODO (WIBNI): All oras_client is used for (I think) is to get the target_acr. Is there a simpler way to do this? + manifest_credentials = self._manifest_credentials( + config=config, aosm_client=command_context.aosm_client + ) + # TODO (WIBNI): All oras_client is used for (I think) is to get the target_acr. + # Is there a simpler way to do this? oras_client = self._get_oras_client(manifest_credentials=manifest_credentials) target_acr = self._get_acr(oras_client) target_username = manifest_credentials["username"] @@ -312,7 +326,9 @@ def _push_image_from_local_registry( ] self._call_subprocess_raise_output(tag_image_cmd) - logger.info("Logging into artifact store registry %s", oras_client.remote.hostname) + logger.info( + "Logging into artifact store registry %s", oras_client.remote.hostname + ) # ACR login seems to work intermittently, so we retry on failure retries = 0 while True: @@ -338,9 +354,7 @@ def _push_image_from_local_registry( sleep(3) continue logger.error( - ("Failed to login to %s as %s."), - target_acr, - target_username + ("Failed to login to %s as %s."), target_acr, target_username ) logger.debug(error, exc_info=True) raise error @@ -395,8 +409,11 @@ def _copy_image( samples/nginx:stable """ logger.debug("RemoteACRArtifact (copy_image) config: %s", config) - manifest_credentials = self._manifest_credentials(config=config, aosm_client=command_context.aosm_client) - # TODO (WIBNI): All oras_client is used for (I think) is to get the target_acr. Is there a simpler way to do this? + manifest_credentials = self._manifest_credentials( + config=config, aosm_client=command_context.aosm_client + ) + # TODO (WIBNI): All oras_client is used for (I think) is to get the target_acr. + # Is there a simpler way to do this? oras_client = self._get_oras_client(manifest_credentials=manifest_credentials) target_acr = self._get_acr(oras_client) try: @@ -489,10 +506,12 @@ def _copy_image( error, ) - def upload(self, config: BaseCommonParametersConfig, command_context: CommandContext): + def upload( + self, config: BaseCommonParametersConfig, command_context: CommandContext + ): """Upload the artifact.""" - if command_context.cli_options['no_subscription_permissions']: + if command_context.cli_options["no_subscription_permissions"]: print( f"Using docker pull and push to copy image artifact: {self.artifact_name}" ) @@ -503,9 +522,7 @@ def upload(self, config: BaseCommonParametersConfig, command_context: CommandCon f":{self.artifact_version}" ) self._pull_image_to_local_registry( - source_registry_login_server=self._clean_name( - self.source_registry - ), + source_registry_login_server=self._clean_name(self.source_registry), source_image=image_name, ) self._push_image_from_local_registry( @@ -530,11 +547,14 @@ class BaseStorageAccountArtifact(BaseArtifact): """Abstract base class for storage account artifacts.""" @abstractmethod - def upload(self, config: VNFCommonParametersConfig, command_context: CommandContext): + def upload( + self, config: BaseCommonParametersConfig, command_context: CommandContext + ): """Upload the artifact.""" - pass - def _get_blob_client(self, config: VNFCommonParametersConfig, command_context: CommandContext) -> BlobClient: + def _get_blob_client( + self, config: VNFCommonParametersConfig, command_context: CommandContext + ) -> BlobClient: container_basename = self.artifact_name.replace("-", "") container_name = f"{container_basename}-{self.artifact_version}" # For AOSM to work VHD blobs must have the suffix .vhd @@ -545,12 +565,14 @@ def _get_blob_client(self, config: VNFCommonParametersConfig, command_context: C logger.debug("container name: %s, blob name: %s", container_name, blob_name) - manifest_credentials = command_context.aosm_client.artifact_manifests.list_credential( - resource_group_name=config.publisherResourceGroupName, - publisher_name=config.publisherName, - artifact_store_name=config.saArtifactStoreName, - artifact_manifest_name=config.saManifestName, - ).as_dict() + manifest_credentials = ( + command_context.aosm_client.artifact_manifests.list_credential( + resource_group_name=config.publisherResourceGroupName, + publisher_name=config.publisherName, + artifact_store_name=config.saArtifactStoreName, + artifact_manifest_name=config.saManifestName, + ).as_dict() + ) for container_credential in manifest_credentials["container_credentials"]: if container_credential["container_name"] == container_name: @@ -567,14 +589,20 @@ class LocalFileStorageAccountArtifact(BaseStorageAccountArtifact): """Class for storage account artifacts from a local file.""" def __init__(self, artifact_name, artifact_type, artifact_version, file_path: Path): - super().__init__(artifact_name, artifact_type, artifact_version) - self.file_path = str(file_path) # TODO: Jordan cast this to str here, `str(file_path)`, check output file isn't broken, and/or is it used as a Path elsewhere? + self.file_path = str(file_path) - def upload(self, config: VNFCommonParametersConfig, command_context: CommandContext): + def upload( + self, config: BaseCommonParametersConfig, command_context: CommandContext + ): """Upload the artifact.""" + # Liskov substitution dictates we must accept BaseCommonParametersConfig, but we should + # never be calling upload on this class unless we've got VNFCommonParametersConfig + assert isinstance(config, VNFCommonParametersConfig) logger.debug("LocalFileStorageAccountArtifact config: %s", config) - blob_client = self._get_blob_client(config=config, command_context=command_context) + blob_client = self._get_blob_client( + config=config, command_context=command_context + ) logger.info("Uploading local file '%s' to blob store", self.file_path) with open(self.file_path, "rb") as artifact: blob_client.upload_blob( @@ -614,20 +642,26 @@ class BlobStorageAccountArtifact(BaseStorageAccountArtifact): # TODO (Rename): Rename class, e.g. RemoteBlobStorageAccountArtifact """Class for storage account artifacts from a remote blob.""" - blob_sas_uri: str - - def __init__(self, artifact_manifest: ManifestArtifactFormat, blob_sas_uri: str): - super().__init__(artifact_manifest) + def __init__( + self, artifact_name, artifact_type, artifact_version, blob_sas_uri: str + ): + super().__init__(artifact_name, artifact_type, artifact_version) self.blob_sas_uri = blob_sas_uri - def upload(self, config: VNFCommonParametersConfig, command_context: CommandContext): + def upload( + self, config: BaseCommonParametersConfig, command_context: CommandContext + ): """Upload the artifact.""" - + # Liskov substitution dictates we must accept BaseCommonParametersConfig, but we should + # never be calling upload on this class unless we've got VNFCommonParametersConfig + assert isinstance(config, VNFCommonParametersConfig) logger.info("Copy from SAS URL to blob store") source_blob = BlobClient.from_blob_url(self.blob_sas_uri) if source_blob.exists(): - target_blob = self._get_blob_client(config) + target_blob = self._get_blob_client( + config=config, command_context=command_context + ) logger.debug(source_blob.url) target_blob.start_copy_from_url(source_blob.url) logger.info( diff --git a/src/aosm/azext_aosm/common/command_context.py b/src/aosm/azext_aosm/common/command_context.py index a969cfb1a15..7c3d11c92c4 100644 --- a/src/aosm/azext_aosm/common/command_context.py +++ b/src/aosm/azext_aosm/common/command_context.py @@ -1,5 +1,4 @@ -from dataclasses import dataclass -from typing import Dict, Optional +from dataclasses import dataclass, field from azure.cli.core import AzCli from azure.cli.core.commands.client_factory import get_mgmt_service_client @@ -11,9 +10,8 @@ @dataclass class CommandContext: - cli_ctx: AzCli - cli_options: Optional[Dict] = None + cli_options: dict = field(default_factory=dict) def __post_init__(self): self.aosm_client: HybridNetworkManagementClient = get_mgmt_service_client( diff --git a/src/aosm/azext_aosm/common/constants.py b/src/aosm/azext_aosm/common/constants.py index f2ec18e1b5a..f9e998527dc 100644 --- a/src/aosm/azext_aosm/common/constants.py +++ b/src/aosm/azext_aosm/common/constants.py @@ -46,7 +46,7 @@ class ManifestsExist(str, Enum): MANIFEST_FOLDER_NAME = "artifactManifest" NF_DEFINITION_FOLDER_NAME = "nfDefinition" ALL_PARAMETERS_FILE_NAME = "all_deploy.parameters.json" -CGS_FILENAME = "config_group_schema.json" +CGS_FILENAME = "config-group-schema.json" CGS_NAME = "ConfigGroupSchema" DEPLOYMENT_PARAMETERS_FILENAME = "deploymentParameters.json" TEMPLATE_PARAMETERS_FILENAME = "templateParameters.json" diff --git a/src/aosm/azext_aosm/common/exceptions.py b/src/aosm/azext_aosm/common/exceptions.py index f26bed907db..35680186fe6 100644 --- a/src/aosm/azext_aosm/common/exceptions.py +++ b/src/aosm/azext_aosm/common/exceptions.py @@ -19,3 +19,7 @@ class SchemaGetOrGenerateError(Exception): class DefaultValuesNotFoundError(UserFault): """Raised when the default values file cannot be found""" + + +class TemplateValidationError(Exception): + """Raised when template validation fails""" diff --git a/src/aosm/azext_aosm/common/utils.py b/src/aosm/azext_aosm/common/utils.py index 6e6fc7fe15c..6fef0db0d60 100644 --- a/src/aosm/azext_aosm/common/utils.py +++ b/src/aosm/azext_aosm/common/utils.py @@ -3,18 +3,18 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- -import os -import tarfile -from pathlib import Path -from typing import Any, Dict, Iterable import json +import os import shutil import subprocess +import tarfile import tempfile +from pathlib import Path -from azext_aosm.common.exceptions import InvalidFileTypeError, MissingDependency from knack.log import get_logger +from azext_aosm.common.exceptions import InvalidFileTypeError, MissingDependency + logger = get_logger(__name__) diff --git a/src/aosm/azext_aosm/configuration_models/common_input.py b/src/aosm/azext_aosm/configuration_models/common_input.py index e7888b43c8e..b01175ac384 100644 --- a/src/aosm/azext_aosm/configuration_models/common_input.py +++ b/src/aosm/azext_aosm/configuration_models/common_input.py @@ -14,16 +14,18 @@ class ArmTemplatePropertiesConfig: """ARM template configuration.""" artifact_name: str = field( - default="", metadata={"comment": "Name of the artifact. Used as internal reference only."} + default="", + metadata={"comment": "Name of the artifact. Used as internal reference only."}, ) version: str = field( - default="", metadata={"comment": "Version of the artifact in 1.1.1 format."} + default="", metadata={"comment": "Version of the artifact in 1.1.1 format (three integers separated by dots)."} ) file_path: str = field( default="", metadata={ "comment": ( - "File path (absolute or relative to this configuration file) of the artifact you wish to upload from your local disk.\n" + "File path (absolute or relative to this configuration file) of the artifact you wish to upload from " + "your local disk.\n" "Use Linux slash (/) file separator even if running on Windows." ) }, diff --git a/src/aosm/azext_aosm/configuration_models/common_parameters_config.py b/src/aosm/azext_aosm/configuration_models/common_parameters_config.py index 12479490171..226a144f064 100644 --- a/src/aosm/azext_aosm/configuration_models/common_parameters_config.py +++ b/src/aosm/azext_aosm/configuration_models/common_parameters_config.py @@ -47,6 +47,7 @@ class CNFCommonParametersConfig(NFDCommonParametersConfig): @dataclass(frozen=True) class NSDCommonParametersConfig(BaseCommonParametersConfig): """Common parameters configuration for NSDs.""" + nsDesignGroup: str nsDesignVersion: str nfviSiteName: str diff --git a/src/aosm/azext_aosm/configuration_models/onboarding_base_input_config.py b/src/aosm/azext_aosm/configuration_models/onboarding_base_input_config.py index 08157bd80d3..c9a89e629e0 100644 --- a/src/aosm/azext_aosm/configuration_models/onboarding_base_input_config.py +++ b/src/aosm/azext_aosm/configuration_models/onboarding_base_input_config.py @@ -17,7 +17,9 @@ class OnboardingBaseInputConfig(ABC): location: str = field( default="", - metadata={"comment": "Azure location to use when creating resources e.g uksouth"}, + metadata={ + "comment": "Azure location to use when creating resources e.g uksouth" + }, ) publisher_name: str = field( default="", diff --git a/src/aosm/azext_aosm/configuration_models/onboarding_cnf_input_config.py b/src/aosm/azext_aosm/configuration_models/onboarding_cnf_input_config.py index dbaeb17703b..630279127a1 100644 --- a/src/aosm/azext_aosm/configuration_models/onboarding_cnf_input_config.py +++ b/src/aosm/azext_aosm/configuration_models/onboarding_cnf_input_config.py @@ -27,13 +27,13 @@ class ImageSourceConfig: ) }, ) - source_registry_namespace: str | None = field( + source_registry_namespace: str = field( default="", metadata={ "comment": ( "Optional. Namespace of the repository of the source acr registry from which to pull.\n" "For example if your repository is samples/prod/nginx then set this to samples/prod.\n" - "Leave blank if the image is in the root namespace.\n" + "Leave as empty string if the image is in the root namespace.\n" "See https://learn.microsoft.com/en-us/azure/container-registry/" "container-registry-best-practices#repository-namespaces for further details." ) @@ -55,8 +55,10 @@ class HelmPackageConfig: default="", metadata={ "comment": ( - "The file path to the helm chart on the local disk, relative to the directory from which the command is run.\n" - "Accepts .tgz, .tar or .tar.gz, or an unpacked directory. Use Linux slash (/) file separator even if running on Windows." + "The file path to the helm chart on the local disk, relative to the directory from which the " + "command is run.\n" + "Accepts .tgz, .tar or .tar.gz, or an unpacked directory. Use Linux slash (/) file separator " + "even if running on Windows." ) }, ) @@ -64,8 +66,8 @@ class HelmPackageConfig: default="", metadata={ "comment": ( - "The file path (absolute or relative to this configuration file) of YAML values file on the local disk which " - "will be used instead of the values.yaml file present in the helm chart.\n" + "The file path (absolute or relative to this configuration file) of YAML values file on the " + "local disk which will be used instead of the values.yaml file present in the helm chart.\n" "Accepts .yaml or .yml. Use Linux slash (/) file separator even if running on Windows." ) }, @@ -95,7 +97,9 @@ class OnboardingCNFInputConfig(OnboardingNFDBaseInputConfig): # TODO: Add better comment for images as not a list images: ImageSourceConfig = field( default_factory=ImageSourceConfig, - metadata={"comment": "Source of container images to be included in the CNF. Currently only one source is supported."}, + metadata={ + "comment": "Source of container images to be included in the CNF. Currently only one source is supported." + }, ) helm_packages: List[HelmPackageConfig] = field( default_factory=lambda: [HelmPackageConfig()], diff --git a/src/aosm/azext_aosm/configuration_models/onboarding_nfd_base_input_config.py b/src/aosm/azext_aosm/configuration_models/onboarding_nfd_base_input_config.py index d29827be131..9129946aa0c 100644 --- a/src/aosm/azext_aosm/configuration_models/onboarding_nfd_base_input_config.py +++ b/src/aosm/azext_aosm/configuration_models/onboarding_nfd_base_input_config.py @@ -7,18 +7,23 @@ from azure.cli.core.azclierror import ValidationError -from azext_aosm.configuration_models.onboarding_base_input_config import \ - OnboardingBaseInputConfig +from azext_aosm.configuration_models.onboarding_base_input_config import ( + OnboardingBaseInputConfig, +) @dataclass class OnboardingNFDBaseInputConfig(OnboardingBaseInputConfig): """Common input configuration for onboarding NFDs.""" - nf_name: str = field(default="", metadata={"comment": "Name of the network function."}) + nf_name: str = field( + default="", metadata={"comment": "Name of the network function."} + ) version: str = field( default="", - metadata={"comment": "Version of the network function definition in 1.1.1 format."}, + metadata={ + "comment": "Version of the network function definition in 1.1.1 format (three integers separated by dots)." + }, ) def validate(self): @@ -30,6 +35,5 @@ def validate(self): raise ValidationError("version must be set") if "-" in self.version or "." not in self.version: raise ValidationError( - "Config validation error. Version should be in" - " format 1.1.1" - ) + "Config validation error. Version must be in format 1.1.1 (three integers separated by dots)." + ) diff --git a/src/aosm/azext_aosm/configuration_models/onboarding_nsd_input_config.py b/src/aosm/azext_aosm/configuration_models/onboarding_nsd_input_config.py index 6c3151f42a4..e51334ebbd3 100644 --- a/src/aosm/azext_aosm/configuration_models/onboarding_nsd_input_config.py +++ b/src/aosm/azext_aosm/configuration_models/onboarding_nsd_input_config.py @@ -9,10 +9,10 @@ from azure.cli.core.azclierror import ValidationError -from azext_aosm.configuration_models.common_input import \ - ArmTemplatePropertiesConfig -from azext_aosm.configuration_models.onboarding_base_input_config import \ - OnboardingBaseInputConfig +from azext_aosm.configuration_models.common_input import ArmTemplatePropertiesConfig +from azext_aosm.configuration_models.onboarding_base_input_config import ( + OnboardingBaseInputConfig, +) @dataclass diff --git a/src/aosm/azext_aosm/configuration_models/onboarding_vnf_input_config.py b/src/aosm/azext_aosm/configuration_models/onboarding_vnf_input_config.py index 6ca39b390a1..6490a30614b 100644 --- a/src/aosm/azext_aosm/configuration_models/onboarding_vnf_input_config.py +++ b/src/aosm/azext_aosm/configuration_models/onboarding_vnf_input_config.py @@ -9,10 +9,10 @@ from azure.cli.core.azclierror import ValidationError -from azext_aosm.configuration_models.common_input import \ - ArmTemplatePropertiesConfig -from azext_aosm.configuration_models.onboarding_nfd_base_input_config import \ - OnboardingNFDBaseInputConfig +from azext_aosm.configuration_models.common_input import ArmTemplatePropertiesConfig +from azext_aosm.configuration_models.onboarding_nfd_base_input_config import ( + OnboardingNFDBaseInputConfig, +) @dataclass @@ -20,17 +20,24 @@ class VhdImageConfig: """Configuration for a VHD image.""" artifact_name: str = field( - default="", metadata={"comment": "Optional. Name of the artifact. Name will be generated if not supplied."} + default="", + metadata={ + "comment": "Optional. Name of the artifact. Name will be generated if not supplied." + }, ) version: str = field( - default="", metadata={"comment": "Version of the artifact in A-B-C format. Note the '-' (dash) not '.' (dot)."} + default="", + metadata={ + "comment": "Version of the artifact in A-B-C format. Note the '-' (dash) not '.' (dot)." + }, ) file_path: str = field( default="", metadata={ "comment": ( "Supply either file_path or blob_sas_url, not both.\n" - "File path (absolute or relative to this configuration file) of the artifact you wish to upload from your local disk.\n" + "File path (absolute or relative to this configuration file) of the artifact you wish to upload from " + "your local disk.\n" "Leave as empty string if not required. Use Linux slash (/) file separator even if running on Windows." ) }, @@ -39,9 +46,9 @@ class VhdImageConfig: default="", metadata={ "comment": ( - "Supply either file_path or blob_sas_url, not both.\nSAS URL of the blob artifact you wish to copy to your Artifact Store.\n" - "Leave as empty string if not required." - "Use Linux slash (/) file separator even if running on Windows." + "Supply either file_path or blob_sas_url, not both.\nSAS URL of the blob artifact you wish to copy to " + "your Artifact Store.\n" + "Leave as empty string if not required. Use Linux slash (/) file separator even if running on Windows." ) }, ) @@ -118,7 +125,10 @@ class OnboardingVNFInputConfig(OnboardingNFDBaseInputConfig): # TODO: Add better comments arm_templates: List[ArmTemplatePropertiesConfig] = field( default_factory=lambda: [ArmTemplatePropertiesConfig()], - metadata={"comment": "ARM template configuration. The ARM templates given here would deploy a VM if run. They will be used to generate the VNF."}, + metadata={ + "comment": "ARM template configuration. The ARM templates given here would deploy a VM if run. They will " + "be used to generate the VNF." + }, ) vhd: VhdImageConfig = field( @@ -158,4 +168,4 @@ def validate(self): raise ValidationError("You must include at least one arm template") for arm_template in self.arm_templates: arm_template.validate() - self.vhd.validate() \ No newline at end of file + self.vhd.validate() diff --git a/src/aosm/azext_aosm/custom.py b/src/aosm/azext_aosm/custom.py index 85fb9e6eeb9..3865e773ada 100644 --- a/src/aosm/azext_aosm/custom.py +++ b/src/aosm/azext_aosm/custom.py @@ -5,17 +5,22 @@ from __future__ import annotations from pathlib import Path +from typing import Optional + +from azure.cli.core.azclierror import UnrecognizedArgumentError +from azure.cli.core.commands import AzCliCommand + from azext_aosm.cli_handlers.onboarding_cnf_handler import OnboardingCNFCLIHandler -from azext_aosm.cli_handlers.onboarding_vnf_handler import OnboardingVNFCLIHandler from azext_aosm.cli_handlers.onboarding_nsd_handler import OnboardingNSDCLIHandler +from azext_aosm.cli_handlers.onboarding_vnf_handler import OnboardingVNFCLIHandler from azext_aosm.common.command_context import CommandContext from azext_aosm.common.constants import ALL_PARAMETERS_FILE_NAME, CNF, VNF -from azure.cli.core.commands import AzCliCommand -from azure.cli.core.azclierror import UnrecognizedArgumentError def onboard_nfd_generate_config(definition_type: str, output_file: str | None): """Generate config file for onboarding NFs.""" + # Declare types explicitly + handler: OnboardingCNFCLIHandler | OnboardingVNFCLIHandler if definition_type == CNF: handler = OnboardingCNFCLIHandler() handler.generate_config(output_file) @@ -26,8 +31,12 @@ def onboard_nfd_generate_config(definition_type: str, output_file: str | None): raise UnrecognizedArgumentError("Invalid definition type") -def onboard_nfd_build(definition_type: str, config_file: Path, skip: str = None): +def onboard_nfd_build( + definition_type: str, config_file: Path, skip: Optional[str] = None +): """Build the NF definition.""" + # Declare types explicitly + handler: OnboardingCNFCLIHandler | OnboardingVNFCLIHandler if definition_type == CNF: handler = OnboardingCNFCLIHandler(Path(config_file), skip=skip) handler.build() @@ -52,6 +61,8 @@ def onboard_nfd_publish( "definition_folder": Path(build_output_folder), }, ) + # Declare types explicitly + handler: OnboardingCNFCLIHandler | OnboardingVNFCLIHandler if definition_type == CNF: handler = OnboardingCNFCLIHandler( Path(build_output_folder, ALL_PARAMETERS_FILE_NAME) diff --git a/src/aosm/azext_aosm/definition_folder/builder/artifact_builder.py b/src/aosm/azext_aosm/definition_folder/builder/artifact_builder.py index 723e6df4fcd..0495e2a49ec 100644 --- a/src/aosm/azext_aosm/definition_folder/builder/artifact_builder.py +++ b/src/aosm/azext_aosm/definition_folder/builder/artifact_builder.py @@ -7,9 +7,10 @@ from pathlib import Path from typing import List -from azext_aosm.common.artifact import BaseArtifact from knack.log import get_logger +from azext_aosm.common.artifact import BaseArtifact + from .base_builder import BaseDefinitionElementBuilder logger = get_logger(__name__) @@ -33,9 +34,12 @@ def write(self): """Write the definition element to disk.""" self.path.mkdir(exist_ok=True) artifacts_list = [] - # TODO: handle converting path to string that doesn't couple this code to the artifact. Probably should be in to_dict method. + # TODO: Handle converting path to string that doesn't couple this code to the artifact. + # Probably should be in to_dict method. for artifact in self.artifacts: - logger.debug("Writing artifact %s as: %s", artifact.artifact_name, artifact.to_dict()) + logger.debug( + "Writing artifact %s as: %s", artifact.artifact_name, artifact.to_dict() + ) if hasattr(artifact, "file_path") and artifact.file_path is not None: artifact.file_path = str(artifact.file_path) artifacts_list.append(artifact.to_dict()) diff --git a/src/aosm/azext_aosm/definition_folder/builder/base_builder.py b/src/aosm/azext_aosm/definition_folder/builder/base_builder.py index a3e71ee6736..49041573ac5 100644 --- a/src/aosm/azext_aosm/definition_folder/builder/base_builder.py +++ b/src/aosm/azext_aosm/definition_folder/builder/base_builder.py @@ -7,14 +7,14 @@ from pathlib import Path from typing import List -from azext_aosm.common.local_file_builder import LocalFileBuilder +from azext_aosm.definition_folder.builder.local_file_builder import LocalFileBuilder class BaseDefinitionElementBuilder(ABC): """Base element definition builder.""" path: Path - supporting_files: "list[LocalFileBuilder]" + supporting_files: List[LocalFileBuilder] only_delete_on_clean: bool def __init__(self, path: Path, only_delete_on_clean: bool = False): diff --git a/src/aosm/azext_aosm/definition_folder/builder/bicep_builder.py b/src/aosm/azext_aosm/definition_folder/builder/bicep_builder.py index c49a5670e66..19b125a6414 100644 --- a/src/aosm/azext_aosm/definition_folder/builder/bicep_builder.py +++ b/src/aosm/azext_aosm/definition_folder/builder/bicep_builder.py @@ -5,8 +5,9 @@ from pathlib import Path -from azext_aosm.definition_folder.builder.base_builder import \ - BaseDefinitionElementBuilder +from azext_aosm.definition_folder.builder.base_builder import ( + BaseDefinitionElementBuilder, +) class BicepDefinitionElementBuilder(BaseDefinitionElementBuilder): diff --git a/src/aosm/azext_aosm/definition_folder/builder/definition_folder_builder.py b/src/aosm/azext_aosm/definition_folder/builder/definition_folder_builder.py index ae500c79fd3..f2e1b3182b3 100644 --- a/src/aosm/azext_aosm/definition_folder/builder/definition_folder_builder.py +++ b/src/aosm/azext_aosm/definition_folder/builder/definition_folder_builder.py @@ -4,9 +4,12 @@ # -------------------------------------------------------------------------------------------- import json -from pathlib import Path import shutil +from pathlib import Path +from typing import List + from azure.cli.core.azclierror import UnclassifiedUserFault + from azext_aosm.definition_folder.builder.base_builder import ( BaseDefinitionElementBuilder, ) @@ -17,14 +20,12 @@ JSONDefinitionElementBuilder, ) -from typing import List - class DefinitionFolderBuilder: """Builds and writes out a definition folder for an NFD or NSD.""" path: Path - elements: "list[BaseDefinitionElementBuilder]" + elements: List[BaseDefinitionElementBuilder] def __init__(self, path: Path): self.path = path diff --git a/src/aosm/azext_aosm/definition_folder/builder/json_builder.py b/src/aosm/azext_aosm/definition_folder/builder/json_builder.py index 147352dbb6f..32617905cb4 100644 --- a/src/aosm/azext_aosm/definition_folder/builder/json_builder.py +++ b/src/aosm/azext_aosm/definition_folder/builder/json_builder.py @@ -1,8 +1,9 @@ from pathlib import Path from azext_aosm.common.constants import ALL_PARAMETERS_FILE_NAME -from azext_aosm.definition_folder.builder.base_builder import \ - BaseDefinitionElementBuilder +from azext_aosm.definition_folder.builder.base_builder import ( + BaseDefinitionElementBuilder, +) class JSONDefinitionElementBuilder(BaseDefinitionElementBuilder): diff --git a/src/aosm/azext_aosm/common/local_file_builder.py b/src/aosm/azext_aosm/definition_folder/builder/local_file_builder.py similarity index 92% rename from src/aosm/azext_aosm/common/local_file_builder.py rename to src/aosm/azext_aosm/definition_folder/builder/local_file_builder.py index 3472c605998..bb9c0cad736 100644 --- a/src/aosm/azext_aosm/common/local_file_builder.py +++ b/src/aosm/azext_aosm/definition_folder/builder/local_file_builder.py @@ -3,10 +3,10 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- - from pathlib import Path +# pylint: disable=too-few-public-methods class LocalFileBuilder: """Writes locally generated files to disk.""" diff --git a/src/aosm/azext_aosm/definition_folder/reader/artifact_definition.py b/src/aosm/azext_aosm/definition_folder/reader/artifact_definition.py index 8e0cc3858eb..37004859ffc 100644 --- a/src/aosm/azext_aosm/definition_folder/reader/artifact_definition.py +++ b/src/aosm/azext_aosm/definition_folder/reader/artifact_definition.py @@ -3,21 +3,18 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- -import json import inspect +import json from pathlib import Path -from typing import List -from azext_aosm.common.artifact import ARTIFACT_TYPE_TO_CLASS, BaseArtifact +from knack.log import get_logger +from azext_aosm.common.artifact import ARTIFACT_TYPE_TO_CLASS, BaseArtifact +from azext_aosm.common.command_context import CommandContext from azext_aosm.configuration_models.common_parameters_config import ( BaseCommonParametersConfig, ) -from azext_aosm.common.command_context import CommandContext -from knack.log import get_logger - -from azext_aosm.definition_folder.reader.base_definition import \ - BaseDefinitionElement +from azext_aosm.definition_folder.reader.base_definition import BaseDefinitionElement logger = get_logger(__name__) @@ -29,40 +26,55 @@ def __init__(self, path: Path, only_delete_on_clean: bool): super().__init__(path, only_delete_on_clean) logger.debug("ArtifactDefinitionElement path: %s", path) artifact_list = json.loads((path / "artifacts.json").read_text()) - self.artifacts = [self.create_artifact_object(artifact) for artifact in artifact_list] + self.artifacts = [ + self.create_artifact_object(artifact) for artifact in artifact_list + ] # TODO: add what types are expected, and check they are those types # For filepaths, we must convert to paths again - def create_artifact_object(self, artifact: dict) -> BaseArtifact: + @staticmethod + def create_artifact_object(artifact: dict) -> BaseArtifact: """ Use reflection (via the inspect module) to identify the artifact class's required fields and create an instance of the class using the supplied artifact dict. """ if "type" not in artifact or artifact["type"] not in ARTIFACT_TYPE_TO_CLASS: - raise ValueError("Artifact type is missing or invalid for artifact {artifact}") + raise ValueError( + "Artifact type is missing or invalid for artifact {artifact}" + ) # Use reflection to get the required fields for the artifact class class_sig = inspect.signature(ARTIFACT_TYPE_TO_CLASS[artifact["type"]].__init__) - class_args = [arg for arg, _ in class_sig.parameters.items() if arg != 'self'] + class_args = [arg for arg, _ in class_sig.parameters.items() if arg != "self"] logger.debug("Artifact configuration from definition folder: %s", artifact) - logger.debug("class_args found for artifact type %s: %s", artifact["type"], class_args) + logger.debug( + "class_args found for artifact type %s: %s", artifact["type"], class_args + ) # Filter the artifact dict to only include the required fields, erroring if any are missing try: filtered_dict = {arg: artifact[arg] for arg in class_args} except KeyError as e: - raise ValueError(f"Artifact is missing required field {e}.\n" - f"Required fields are: {class_args}.\n" - f"Artifact is: {artifact}.\n" - "This is unexpected and most likely comes from manual editing " - "of the definition folder.") + raise ValueError( + f"Artifact is missing required field {e}.\n" + f"Required fields are: {class_args}.\n" + f"Artifact is: {artifact}.\n" + "This is unexpected and most likely comes from manual editing " + "of the definition folder." + ) return ARTIFACT_TYPE_TO_CLASS[artifact["type"]](**filtered_dict) - def deploy(self, config: BaseCommonParametersConfig, command_context: CommandContext): + def deploy( + self, config: BaseCommonParametersConfig, command_context: CommandContext + ): """Deploy the element.""" for artifact in self.artifacts: - logger.info("Deploying artifact %s of type %s", artifact.artifact_name, type(artifact)) + logger.info( + "Deploying artifact %s of type %s", + artifact.artifact_name, + type(artifact), + ) artifact.upload(config=config, command_context=command_context) - def delete(self): + def delete(self, config: BaseCommonParametersConfig, command_context: CommandContext): """Delete the element.""" # TODO: Implement? - pass + raise NotImplementedError diff --git a/src/aosm/azext_aosm/definition_folder/reader/base_definition.py b/src/aosm/azext_aosm/definition_folder/reader/base_definition.py index a671c5d774f..c6c49de6f10 100644 --- a/src/aosm/azext_aosm/definition_folder/reader/base_definition.py +++ b/src/aosm/azext_aosm/definition_folder/reader/base_definition.py @@ -6,10 +6,10 @@ from abc import ABC, abstractmethod from pathlib import Path +from azext_aosm.common.command_context import CommandContext from azext_aosm.configuration_models.common_parameters_config import ( BaseCommonParametersConfig, ) -from azext_aosm.common.command_context import CommandContext class BaseDefinitionElement(ABC): @@ -20,11 +20,15 @@ def __init__(self, path: Path, only_delete_on_clean: bool): self.only_delete_on_clean = only_delete_on_clean @abstractmethod - def deploy(self, config: BaseCommonParametersConfig, command_context: CommandContext): + def deploy( + self, config: BaseCommonParametersConfig, command_context: CommandContext + ): """Deploy the element.""" return NotImplementedError @abstractmethod - def delete(self, config: BaseCommonParametersConfig, command_context: CommandContext): + def delete( + self, config: BaseCommonParametersConfig, command_context: CommandContext + ): """Delete the element.""" return NotImplementedError diff --git a/src/aosm/azext_aosm/definition_folder/reader/bicep_definition.py b/src/aosm/azext_aosm/definition_folder/reader/bicep_definition.py index 67119faff33..0efe76f4d22 100644 --- a/src/aosm/azext_aosm/definition_folder/reader/bicep_definition.py +++ b/src/aosm/azext_aosm/definition_folder/reader/bicep_definition.py @@ -2,25 +2,26 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- -from dataclasses import asdict import time +from dataclasses import asdict from typing import Any, Dict +from azure.cli.core import AzCli +from azure.cli.core.azclierror import AzCLIError from azure.cli.core.commands import LongRunningOperation +from azure.core import exceptions as azure_exceptions from azure.mgmt.resource import ResourceManagementClient from azure.mgmt.resource.resources.models import DeploymentExtended +from knack.log import get_logger from azext_aosm.common.command_context import CommandContext -from azext_aosm.common.utils import convert_bicep_to_arm -from azext_aosm.configuration_models.common_parameters_config import \ - BaseCommonParametersConfig, VNFCommonParametersConfig -from azext_aosm.definition_folder.reader.base_definition import \ - BaseDefinitionElement from azext_aosm.common.constants import ManifestsExist -from azure.cli.core import AzCli -from azure.cli.core.azclierror import AzCLIError -from azure.core import exceptions as azure_exceptions -from knack.log import get_logger +from azext_aosm.common.utils import convert_bicep_to_arm +from azext_aosm.configuration_models.common_parameters_config import ( + BaseCommonParametersConfig, + VNFCommonParametersConfig, +) +from azext_aosm.definition_folder.reader.base_definition import BaseDefinitionElement logger = get_logger(__name__) @@ -28,8 +29,13 @@ class BicepDefinitionElement(BaseDefinitionElement): """Bicep definition""" + @staticmethod def _validate_and_deploy_arm_template( - self, cli_ctx: AzCli, template: Any, parameters: Dict[Any, Any], resource_group: str, resource_client: ResourceManagementClient + cli_ctx: AzCli, + template: Any, + parameters: Dict[Any, Any], + resource_group: str, + resource_client: ResourceManagementClient, ) -> Any: """ Validate and deploy an individual ARM template. @@ -53,18 +59,16 @@ def _validate_and_deploy_arm_template( validation_res = None for validation_attempt in range(2): try: - validation = ( - resource_client.deployments.begin_validate( - resource_group_name=resource_group, - deployment_name=deployment_name, - parameters={ - "properties": { - "mode": "Incremental", - "template": template, - "parameters": parameters, - } - }, - ) + validation = resource_client.deployments.begin_validate( + resource_group_name=resource_group, + deployment_name=deployment_name, + parameters={ + "properties": { + "mode": "Incremental", + "template": template, + "parameters": parameters, + } + }, ) validation_res = LongRunningOperation( cli_ctx, "Validating ARM template..." @@ -121,11 +125,11 @@ def _artifact_manifests_exist( """ try: command_context.aosm_client.artifact_manifests.get( - resource_group_name=config.publisherResourceGroupName, - publisher_name=config.publisherName, - artifact_store_name=config.acrArtifactStoreName, - artifact_manifest_name=config.acrManifestName, - ) + resource_group_name=config.publisherResourceGroupName, + publisher_name=config.publisherName, + artifact_store_name=config.acrArtifactStoreName, + artifact_manifest_name=config.acrManifestName, + ) acr_manifest_exists = True except azure_exceptions.ResourceNotFoundError: acr_manifest_exists = False @@ -147,10 +151,12 @@ def _artifact_manifests_exist( if acr_manifest_exists: return ManifestsExist.ALL - else: - return ManifestsExist.NONE - def deploy(self, config: BaseCommonParametersConfig, command_context: CommandContext): + return ManifestsExist.NONE + + def deploy( + self, config: BaseCommonParametersConfig, command_context: CommandContext + ): """Deploy the element.""" # TODO: Deploying base takes about 4 minutes, even if everything is already deployed. # We should have a check to see if it's already deployed and skip it if so. @@ -164,8 +170,10 @@ def deploy(self, config: BaseCommonParametersConfig, command_context: CommandCon # breaking this into a separate class (like we do for artifacts) is probably the right # thing to do. if self.path.name == "artifactManifest": - manifests_exist = self._artifact_manifests_exist(config=config, command_context=command_context) - if manifests_exist == ManifestsExist.ALL: + manifests_exist = self._artifact_manifests_exist( + config=config, command_context=command_context + ) + if manifests_exist == ManifestsExist.ALL: # pylint: disable=no-else-return # The manifest(s) already exist so nothing else to do for this template logger.info("Artifact manifest(s) already exist; skipping deployment.") return @@ -183,17 +191,28 @@ def deploy(self, config: BaseCommonParametersConfig, command_context: CommandCon # If none of the manifests exist, we can just go ahead and deploy the template # as normal. - logger.info("Converting bicep to ARM for '%s' template. This can take a few seconds.", self.path.name) + logger.info( + "Converting bicep to ARM for '%s' template. This can take a few seconds.", + self.path.name, + ) arm_json = convert_bicep_to_arm(self.path / "deploy.bicep") - logger.info("Deploying ARM template for %s" % self.path.name) + logger.info("Deploying ARM template for %s", self.path.name) # TODO: handle creating the resource group if it doesn't exist # Create the deploy parameters with only the parameters needed by this template parameters_in_template = arm_json["parameters"] - parameters = {k: {"value": v} for (k, v) in asdict(config).items() if k in parameters_in_template} + parameters = { + k: {"value": v} + for (k, v) in asdict(config).items() + if k in parameters_in_template + } logger.debug("All parameters provided by user: %s", config) - logger.debug("Parameters required by %s in built ARM template:%s ", self.path.name, parameters_in_template) + logger.debug( + "Parameters required by %s in built ARM template:%s ", + self.path.name, + parameters_in_template, + ) logger.debug("Filtered parameters: %s", parameters) self._validate_and_deploy_arm_template( @@ -204,7 +223,9 @@ def deploy(self, config: BaseCommonParametersConfig, command_context: CommandCon resource_client=command_context.resources_client, ) - def delete(self, config: BaseCommonParametersConfig, command_context: CommandContext): + def delete( + self, config: BaseCommonParametersConfig, command_context: CommandContext + ): """Delete the element.""" # TODO: Implement. - pass \ No newline at end of file + raise NotImplementedError diff --git a/src/aosm/azext_aosm/definition_folder/reader/definition_folder.py b/src/aosm/azext_aosm/definition_folder/reader/definition_folder.py index eb8f128befc..7dfc61c70dc 100644 --- a/src/aosm/azext_aosm/definition_folder/reader/definition_folder.py +++ b/src/aosm/azext_aosm/definition_folder/reader/definition_folder.py @@ -4,26 +4,26 @@ # -------------------------------------------------------------------------------------------- import json from pathlib import Path +from typing import Any, Dict, List + +from knack.log import get_logger +from azext_aosm.common.command_context import CommandContext from azext_aosm.configuration_models.common_parameters_config import ( BaseCommonParametersConfig, ) -from azext_aosm.definition_folder.reader.base_definition import BaseDefinitionElement -from azext_aosm.definition_folder.reader.bicep_definition import BicepDefinitionElement from azext_aosm.definition_folder.reader.artifact_definition import ( ArtifactDefinitionElement, ) -from azure.mgmt.resource import ResourceManagementClient -from azext_aosm.common.command_context import CommandContext -from knack.log import get_logger - -from typing import Any, Dict, List +from azext_aosm.definition_folder.reader.base_definition import BaseDefinitionElement +from azext_aosm.definition_folder.reader.bicep_definition import BicepDefinitionElement logger = get_logger(__name__) class DefinitionFolder: """Represents a definition folder for an NFD or NSD.""" + def __init__(self, path: Path): self.path = path try: @@ -71,14 +71,25 @@ def _parse_index_file(self, file_content: str) -> List[Dict[str, Any]]: ) return parsed_elements - def deploy(self, config: BaseCommonParametersConfig, command_context: CommandContext): + def deploy( + self, config: BaseCommonParametersConfig, command_context: CommandContext + ): """Deploy the resources defined in the folder.""" for element in self.elements: - logger.debug("Deploying definition element %s of type %s", element.path, type(element)) + logger.debug( + "Deploying definition element %s of type %s", + element.path, + type(element), + ) element.deploy(config=config, command_context=command_context) - def delete(self, resource_client: ResourceManagementClient, clean: bool = False): + def delete( + self, + config: BaseCommonParametersConfig, + command_context: CommandContext, + clean: bool = False, + ): """Delete the definition folder.""" for element in reversed(self.elements): if clean or not element.only_delete_on_clean: - element.delete(resource_client) + element.delete(config=config, command_context=command_context) diff --git a/src/aosm/azext_aosm/inputs/helm_chart_input.py b/src/aosm/azext_aosm/inputs/helm_chart_input.py index e91171e90ed..8dda26e4465 100644 --- a/src/aosm/azext_aosm/inputs/helm_chart_input.py +++ b/src/aosm/azext_aosm/inputs/helm_chart_input.py @@ -5,26 +5,27 @@ import copy import json import shutil +import subprocess import tempfile +import warnings from dataclasses import dataclass +from os import PathLike from pathlib import Path -import subprocess -from typing import Any, Dict, List, Optional, Tuple -import warnings - -import ruamel.yaml -from ruamel.yaml.error import ReusedAnchorWarning +from typing import Any, Dict, List, Optional, Tuple, Union import genson +import ruamel.yaml import yaml from knack.log import get_logger +from ruamel.yaml.error import ReusedAnchorWarning from azext_aosm.common.exceptions import ( DefaultValuesNotFoundError, MissingChartDependencyError, SchemaGetOrGenerateError, + TemplateValidationError, ) -from azext_aosm.common.utils import extract_tarfile, check_tool_installed +from azext_aosm.common.utils import check_tool_installed, extract_tarfile from azext_aosm.inputs.base_input import BaseInput logger = get_logger(__name__) @@ -98,7 +99,7 @@ def __init__( self._chart_dir = extract_tarfile(chart_path, self._temp_dir_path) self._validate() self.metadata = self._get_metadata() - self.helm_template = None + self.helm_template: Optional[str] = None self.default_config_path = default_config_path @staticmethod @@ -156,7 +157,7 @@ def validate_template(self) -> None: check_tool_installed("helm") if self.default_config_path: - cmd = [ + cmd: List[Union[str, PathLike]] = [ "helm", "template", self.artifact_name, @@ -182,7 +183,6 @@ def validate_template(self) -> None: self.artifact_name, helm_template_output, ) - return "" except subprocess.CalledProcessError as error: # Return the error message without raising an error. # The errors are going to be collected into a file by the caller of this function. @@ -196,20 +196,20 @@ def validate_template(self) -> None: error_message = error_message.replace( "\nUse --debug flag to render out invalid YAML", "" ) - return error_message + raise TemplateValidationError(error_message) def validate_values(self) -> None: """ - Confirm that the values.yaml file exists in the Helm chart directory - or that the default values are provided. + Confirm that the default values are provided or that a values.yaml file exists in the + Helm chart directory. :raises DefaultValuesNotFoundError: If the values.yaml and default values do not exist. """ logger.debug("Getting default values for Helm chart %s", self.artifact_name) - default_config = self.default_config or self._read_values_yaml() - - if not default_config: + try: + self.default_config or self._read_values_yaml() + except FileNotFoundError: logger.error("No values found for Helm chart '%s'", self.chart_path) raise DefaultValuesNotFoundError( "ERROR: No default values found for the Helm chart" @@ -281,7 +281,9 @@ def get_dependencies(self) -> List["HelmChartInput"]: :rtype: List[HelmChartInput] :raises MissingChartDependencyError: If a dependency chart is missing. """ - logger.debug("Getting dependency charts for Helm chart input, '%s'", self.artifact_name) + logger.debug( + "Getting dependency charts for Helm chart input, '%s'", self.artifact_name + ) # All dependency charts should be located in the charts directory. dependency_chart_dir = Path(self._chart_dir, "charts") @@ -422,5 +424,13 @@ def _read_values_yaml(self) -> Dict[str, Any]: content = yaml_processor.load(f) return content + logger.error( + "values.yaml|yml file not found in Helm chart '%s'", self.chart_path + ) + raise FileNotFoundError( + f"ERROR: The Helm chart '{self.chart_path}' does not contain" + "a values.yaml file." + ) + def __del__(self): shutil.rmtree(self._temp_dir_path) diff --git a/src/aosm/azext_aosm/inputs/nfd_input.py b/src/aosm/azext_aosm/inputs/nfd_input.py index 242c5029ebe..1d9fbd0ddeb 100644 --- a/src/aosm/azext_aosm/inputs/nfd_input.py +++ b/src/aosm/azext_aosm/inputs/nfd_input.py @@ -52,7 +52,10 @@ def get_defaults(self) -> Dict[str, Any]: :rtype: Dict[str, Any] """ if self.network_function_definition.id: - logger.debug("network_function_definition.id for NFD input: %s", self.network_function_definition.id) + logger.debug( + "network_function_definition.id for NFD input: %s", + self.network_function_definition.id, + ) split_id = self.network_function_definition.id.split("/") publisher_name: str = split_id[8] nfdg_name: str = split_id[10] diff --git a/src/aosm/azext_aosm/old/_configuration.py b/src/aosm/azext_aosm/old/_configuration.py deleted file mode 100644 index 3b7a408beb1..00000000000 --- a/src/aosm/azext_aosm/old/_configuration.py +++ /dev/null @@ -1,771 +0,0 @@ -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- -"""Configuration class for input config file parsing,""" -import abc -import logging -import json -import os -from dataclasses import dataclass, field, asdict -from pathlib import Path -from typing import Any, Dict, List, Optional, Union - -from azure.cli.core.azclierror import ( - InvalidArgumentValueError, - ValidationError, -) -from azext_aosm.util.constants import ( - CNF, - NF_DEFINITION_OUTPUT_BICEP_PREFIX, - NF_DEFINITION_JSON_FILENAME, - NSD, - NSD_OUTPUT_BICEP_PREFIX, - VNF, -) - -logger = logging.getLogger(__name__) - - -@dataclass -class ArtifactConfig: - # artifact.py checks for the presence of the default descriptions, change - # there if you change the descriptions. - artifact_name: str = "" - version: Optional[str] = "" - file_path: Optional[str] = None - - @classmethod - def helptext(cls) -> "ArtifactConfig": - """ - Build an object where each value is helptext for that field. - """ - return ArtifactConfig( - artifact_name="Optional. Name of the artifact.", - version="Version of the artifact in A.B.C format.", - file_path=( - "File path of the artifact you wish to upload from your local disk. " - "Relative paths are relative to the configuration file. " - "On Windows escape any backslash with another backslash." - ), - - ) - - def validate(self): - """ - Validate the configuration. - """ - if not self.version: - raise ValidationError("version must be set.") - if not self.file_path: - raise ValidationError("file_path must be set.") - - -@dataclass -class VhdArtifactConfig(ArtifactConfig): - # If you add a new property to this class, consider updating EXTRA_VHD_PARAMETERS in - # constants.py - see comment there for details. - blob_sas_url: Optional[str] = None - image_disk_size_GB: Optional[Union[str, int]] = None - image_hyper_v_generation: Optional[str] = None - image_api_version: Optional[str] = None - - def __post_init__(self): - """ - Convert parameters to the correct types. - """ - if ( - isinstance(self.image_disk_size_GB, str) - and self.image_disk_size_GB.isdigit() - ): - self.image_disk_size_GB = int(self.image_disk_size_GB) - - @classmethod - def helptext(cls) -> "VhdArtifactConfig": - """ - Build an object where each value is helptext for that field. - """ - - artifact_config = ArtifactConfig.helptext() - artifact_config.file_path = ( - "Optional. File path of the artifact you wish to upload from your local disk. " - "Delete if not required. Relative paths are relative to the configuration file." - "On Windows escape any backslash with another backslash." - ) - artifact_config.version = ( - "Version of the artifact in A-B-C format." - ) - return VhdArtifactConfig( - blob_sas_url=( - "Optional. SAS URL of the blob artifact you wish to copy to your Artifact" - " Store. Delete if not required." - ), - image_disk_size_GB=( - "Optional. Specifies the size of empty data disks in gigabytes. " - "This value cannot be larger than 1023 GB. Delete if not required." - ), - image_hyper_v_generation=( - "Optional. Specifies the HyperVGenerationType of the VirtualMachine " - "created from the image. Valid values are V1 and V2. V1 is the default if " - "not specified. Delete if not required." - ), - image_api_version=( - "Optional. The ARM API version used to create the " - "Microsoft.Compute/images resource. Delete if not required." - ), - **asdict(artifact_config), - ) - - def validate(self): - """ - Validate the configuration. - """ - if not self.version: - raise ValidationError("version must be set for vhd.") - if self.blob_sas_url and self.file_path: - raise ValidationError("Only one of file_path or blob_sas_url may be set for vhd.") - if not (self.blob_sas_url or self.file_path): - raise ValidationError("One of file_path or sas_blob_url must be set for vhd.") - - -@dataclass -class Configuration(abc.ABC): - config_file: Optional[str] = None - publisher_name: str = "" - publisher_resource_group_name: str = "" - acr_artifact_store_name: str = "" - location: str = "" - - def __post_init__(self): - """ - Set defaults for resource group and ACR as the publisher name tagged with -rg or -acr - """ - if self.publisher_name: - if not self.publisher_resource_group_name: - self.publisher_resource_group_name = f"{self.publisher_name}-rg" - if not self.acr_artifact_store_name: - self.acr_artifact_store_name = f"{self.publisher_name}-acr" - - @classmethod - def helptext(cls): - """ - Build an object where each value is helptext for that field. - """ - return Configuration( - publisher_name=( - "Name of the Publisher resource you want your definition published to. " - "Will be created if it does not exist." - ), - publisher_resource_group_name=( - "Optional. Resource group for the Publisher resource. " - "Will be created if it does not exist (with a default name if none is supplied)." - ), - acr_artifact_store_name=( - "Optional. Name of the ACR Artifact Store resource. " - "Will be created if it does not exist (with a default name if none is supplied)." - ), - location="Azure location to use when creating resources.", - ) - - def validate(self): - """ - Validate the configuration. - """ - if not self.location: - raise ValidationError("Location must be set") - if not self.publisher_name: - raise ValidationError("Publisher name must be set") - if not self.publisher_resource_group_name: - raise ValidationError("Publisher resource group name must be set") - if not self.acr_artifact_store_name: - raise ValidationError("ACR Artifact Store name must be set") - - def path_from_cli_dir(self, path: str) -> str: - """ - Convert path from config file to path from current directory. - - We assume that the path supplied in the config file is relative to the - configuration file. That isn't the same as the path relative to where ever the - CLI is being run from. This function fixes that up. - - :param path: The path relative to the config file. - """ - assert self.config_file - - # If no path has been supplied we shouldn't try to update it. - if path == "": - return "" - - # If it is an absolute path then we don't need to monkey around with it. - if os.path.isabs(path): - return path - - config_file_dir = Path(self.config_file).parent - - updated_path = str(config_file_dir / path) - - logger.debug("Updated path: %s", updated_path) - - return updated_path - - @property - def output_directory_for_build(self) -> Path: - """Base class method to ensure subclasses implement this function.""" - raise NotImplementedError("Subclass must define property") - - @property - def acr_manifest_names(self) -> List[str]: - """The list of ACR manifest names.""" - raise NotImplementedError("Subclass must define property") - - -@dataclass -class NFConfiguration(Configuration): - """Network Function configuration.""" - - nf_name: str = "" - version: str = "" - - @classmethod - def helptext(cls) -> "NFConfiguration": - """ - Build an object where each value is helptext for that field. - """ - return NFConfiguration( - nf_name="Name of NF definition", - version="Version of the NF definition in A.B.C format.", - **asdict(Configuration.helptext()), - ) - - def validate(self): - """ - Validate the configuration. - """ - super().validate() - if not self.nf_name: - raise ValidationError("nf_name must be set") - if not self.version: - raise ValidationError("version must be set") - - @property - def nfdg_name(self) -> str: - """Return the NFD Group name from the NFD name.""" - return f"{self.nf_name}-nfdg" - - @property - def acr_manifest_names(self) -> List[str]: - """ - Return the ACR manifest name from the NFD name. - - This is returned in a list for consistency with the NSConfiguration, where there - can be multiple ACR manifests. - """ - sanitized_nf_name = self.nf_name.lower().replace("_", "-") - return [f"{sanitized_nf_name}-acr-manifest-{self.version.replace('.', '-')}"] - - -@dataclass -class VNFConfiguration(NFConfiguration): - blob_artifact_store_name: str = "" - image_name_parameter: str = "" - arm_template: Union[Dict[str, str], ArtifactConfig] = field(default_factory=ArtifactConfig) - vhd: Union[Dict[str, str], VhdArtifactConfig] = field(default_factory=VhdArtifactConfig) - - @classmethod - def helptext(cls) -> "VNFConfiguration": - """ - Build an object where each value is helptext for that field. - """ - return VNFConfiguration( - blob_artifact_store_name=( - "Optional. Name of the storage account Artifact Store resource. Will be created if it " - "does not exist (with a default name if none is supplied)." - ), - image_name_parameter=( - "The parameter name in the VM ARM template which specifies the name of the " - "image to use for the VM." - ), - arm_template=ArtifactConfig.helptext(), - vhd=VhdArtifactConfig.helptext(), - **asdict(NFConfiguration.helptext()), - ) - - def __post_init__(self): - """ - Cope with deserializing subclasses from dicts to ArtifactConfig. - - Used when creating VNFConfiguration object from a loaded json config file. - """ - super().__post_init__() - if self.publisher_name and not self.blob_artifact_store_name: - self.blob_artifact_store_name = f"{self.publisher_name}-sa" - - if isinstance(self.arm_template, dict): - if self.arm_template.get("file_path"): - self.arm_template["file_path"] = self.path_from_cli_dir( - self.arm_template["file_path"] - ) - self.arm_template = ArtifactConfig(**self.arm_template) - - if isinstance(self.vhd, dict): - if self.vhd.get("file_path"): - self.vhd["file_path"] = self.path_from_cli_dir(self.vhd["file_path"]) - self.vhd = VhdArtifactConfig(**self.vhd) - - def validate(self) -> None: - """ - Validate the configuration passed in. - - :raises ValidationError for any invalid config - """ - super().validate() - - assert isinstance(self.vhd, VhdArtifactConfig) - assert isinstance(self.arm_template, ArtifactConfig) - self.vhd.validate() - self.arm_template.validate() - - assert self.vhd.version - assert self.arm_template.version - - if "." in self.vhd.version or "-" not in self.vhd.version: - raise ValidationError( - "Config validation error. VHD artifact version should be in format" - " A-B-C" - ) - if "." not in self.arm_template.version or "-" in self.arm_template.version: - raise ValidationError( - "Config validation error. ARM template artifact version should be in" - " format A.B.C" - ) - - @property - def sa_manifest_name(self) -> str: - """Return the Storage account manifest name from the NFD name.""" - sanitized_nf_name = self.nf_name.lower().replace("_", "-") - return f"{sanitized_nf_name}-sa-manifest-{self.version.replace('.', '-')}" - - @property - def output_directory_for_build(self) -> Path: - """Return the local folder for generating the bicep template to.""" - assert isinstance(self.arm_template, ArtifactConfig) - assert self.arm_template.file_path - arm_template_name = Path(self.arm_template.file_path).stem - return Path(f"{NF_DEFINITION_OUTPUT_BICEP_PREFIX}{arm_template_name}") - - -@dataclass -class HelmPackageConfig: - name: str = "" - path_to_chart: str = "" - path_to_mappings: str = "" - depends_on: List[str] = field(default_factory=lambda: []) - - @classmethod - def helptext(cls): - """ - Build an object where each value is helptext for that field. - """ - return HelmPackageConfig( - name="Name of the Helm package", - path_to_chart=( - "File path of Helm Chart on local disk. Accepts .tgz, .tar or .tar.gz." - " Use Linux slash (/) file separator even if running on Windows." - ), - path_to_mappings=( - "File path of value mappings on local disk where chosen values are replaced " - "with deploymentParameter placeholders. Accepts .yaml or .yml. If left as a " - "blank string, a value mappings file will be generated with every value " - "mapped to a deployment parameter. Use a blank string and --interactive on " - "the build command to interactively choose which values to map." - ), - depends_on=( - "Names of the Helm packages this package depends on. " - "Leave as an empty array if no dependencies" - ), - ) - - def validate(self): - """ - Validate the configuration. - """ - if not self.name: - raise ValidationError("name must be set") - if not self.path_to_chart: - raise ValidationError("path_to_chart must be set") - - -@dataclass -class CNFImageConfig: - """CNF Image config settings.""" - - source_registry: str = "" - source_registry_namespace: str = "" - source_local_docker_image: str = "" - - def __post_init__(self): - """ - Ensure that all config is lower case. - - ACR names can be uppercase but the login server is always lower case and docker - and az acr import commands require lower case. Might as well do the namespace - and docker image too although much less likely that the user has accidentally - pasted these with upper case. - """ - self.source_registry = self.source_registry.lower() - self.source_registry_namespace = self.source_registry_namespace.lower() - self.source_local_docker_image = self.source_local_docker_image.lower() - - @classmethod - def helptext(cls) -> "CNFImageConfig": - """ - Build an object where each value is helptext for that field. - """ - return CNFImageConfig( - source_registry=( - "Optional. Login server of the source acr registry from which to pull the " - "image(s). For example sourceacr.azurecr.io. Leave blank if you have set " - "source_local_docker_image." - ), - source_registry_namespace=( - "Optional. Namespace of the repository of the source acr registry from which " - "to pull. For example if your repository is samples/prod/nginx then set this to" - " samples/prod . Leave blank if the image is in the root namespace or you have " - "set source_local_docker_image." - "See https://learn.microsoft.com/en-us/azure/container-registry/" - "container-registry-best-practices#repository-namespaces for further details." - ), - source_local_docker_image=( - "Optional. Image name of the source docker image from local machine. For " - "limited use case where the CNF only requires a single docker image and exists " - "in the local docker repository. Set to blank of not required." - ), - ) - - def validate(self): - """ - Validate the configuration. - """ - if self.source_registry_namespace and not self.source_registry: - raise ValidationError( - "Config validation error. The image source registry namespace should " - "only be configured if a source registry is configured." - ) - - if self.source_registry and self.source_local_docker_image: - raise ValidationError( - "Only one of source_registry and source_local_docker_image can be set." - ) - - if not (self.source_registry or self.source_local_docker_image): - raise ValidationError( - "One of source_registry or source_local_docker_image must be set." - ) - - -@dataclass -class CNFConfiguration(NFConfiguration): - images: Union[Dict[str, str], CNFImageConfig] = field(default_factory=CNFImageConfig) - helm_packages: List[Union[Dict[str, Any], HelmPackageConfig]] = field( - default_factory=lambda: [] - ) - - def __post_init__(self): - """ - Cope with deserializing subclasses from dicts to HelmPackageConfig. - - Used when creating CNFConfiguration object from a loaded json config file. - """ - super().__post_init__() - for package_index, package in enumerate(self.helm_packages): - if isinstance(package, dict): - package["path_to_chart"] = self.path_from_cli_dir( - package["path_to_chart"] - ) - package["path_to_mappings"] = self.path_from_cli_dir( - package["path_to_mappings"] - ) - self.helm_packages[package_index] = HelmPackageConfig(**dict(package)) - if isinstance(self.images, dict): - self.images = CNFImageConfig(**self.images) - - @classmethod - def helptext(cls) -> "CNFConfiguration": - """ - Build an object where each value is helptext for that field. - """ - return CNFConfiguration( - images=CNFImageConfig.helptext(), - helm_packages=[HelmPackageConfig.helptext()], - **asdict(NFConfiguration.helptext()), - ) - - @property - def output_directory_for_build(self) -> Path: - """Return the directory the build command will writes its output to.""" - return Path(f"{NF_DEFINITION_OUTPUT_BICEP_PREFIX}{self.nf_name}") - - def validate(self): - """ - Validate the CNF config. - - :raises ValidationError: If source registry ID doesn't match the regex - """ - assert isinstance(self.images, CNFImageConfig) - super().validate() - - self.images.validate() - - for helm_package in self.helm_packages: - assert isinstance(helm_package, HelmPackageConfig) - helm_package.validate() - - -@dataclass -class NFDRETConfiguration: # pylint: disable=too-many-instance-attributes - """The configuration required for an NFDV that you want to include in an NSDV.""" - - publisher: str = "" - publisher_resource_group: str = "" - name: str = "" - version: str = "" - publisher_offering_location: str = "" - type: str = "" - multiple_instances: Union[str, bool] = False - - def __post_init__(self): - """ - Convert parameters to the correct types. - """ - # Cope with multiple_instances being supplied as a string, rather than a bool. - if isinstance(self.multiple_instances, str): - if self.multiple_instances.lower() == "true": - self.multiple_instances = True - elif self.multiple_instances.lower() == "false": - self.multiple_instances = False - - @classmethod - def helptext(cls) -> "NFDRETConfiguration": - """ - Build an object where each value is helptext for that field. - """ - return NFDRETConfiguration( - publisher="The name of the existing Network Function Definition Group to deploy using this NSD", - publisher_resource_group="The resource group that the publisher is hosted in.", - name="The name of the existing Network Function Definition Group to deploy using this NSD", - version=( - "The version of the existing Network Function Definition to base this NSD on. " - "This NSD will be able to deploy any NFDV with deployment parameters compatible " - "with this version." - ), - publisher_offering_location="The region that the NFDV is published to.", - type="Type of Network Function. Valid values are 'cnf' or 'vnf'", - multiple_instances=( - "Set to true or false. Whether the NSD should allow arbitrary numbers of this " - "type of NF. If set to false only a single instance will be allowed. Only " - "supported on VNFs, must be set to false on CNFs." - ), - ) - - def validate(self) -> None: - """ - Validate the configuration passed in. - - :raises ValidationError for any invalid config - """ - if not self.name: - raise ValidationError("Network function definition name must be set") - - if not self.publisher: - raise ValidationError(f"Publisher name must be set for {self.name}") - - if not self.publisher_resource_group: - raise ValidationError( - f"Publisher resource group name must be set for {self.name}" - ) - - if not self.version: - raise ValidationError( - f"Network function definition version must be set for {self.name}" - ) - - if not self.publisher_offering_location: - raise ValidationError( - f"Network function definition offering location must be set, for {self.name}" - ) - - if self.type not in [CNF, VNF]: - raise ValueError( - f"Network Function Type must be cnf or vnf for {self.name}" - ) - - if not isinstance(self.multiple_instances, bool): - raise ValueError( - f"multiple_instances must be a boolean for for {self.name}" - ) - - # There is currently a NFM bug that means that multiple copies of the same NF - # cannot be deployed to the same custom location: - # https://portal.microsofticm.com/imp/v3/incidents/details/405078667/home - if self.type == CNF and self.multiple_instances: - raise ValueError("Multiple instances is not supported on CNFs.") - - @property - def build_output_folder_name(self) -> Path: - """Return the local folder for generating the bicep template to.""" - current_working_directory = os.getcwd() - return Path(current_working_directory, NSD_OUTPUT_BICEP_PREFIX) - - @property - def arm_template(self) -> ArtifactConfig: - """Return the parameters of the ARM template for this RET to be uploaded as part of - the NSDV.""" - artifact = ArtifactConfig() - artifact.artifact_name = f"{self.name.lower()}_nf_artifact" - - # We want the ARM template version to match the NSD version, but we don't have - # that information here. - artifact.version = None - artifact.file_path = os.path.join( - self.build_output_folder_name, NF_DEFINITION_JSON_FILENAME - ) - return artifact - - @property - def nf_bicep_filename(self) -> str: - """Return the name of the bicep template for deploying the NFs.""" - return f"{self.name}_nf.bicep" - - @property - def resource_element_name(self) -> str: - """Return the name of the resource element.""" - artifact_name = self.arm_template.artifact_name - return f"{artifact_name}_resource_element" - - def acr_manifest_name(self, nsd_version: str) -> str: - """Return the ACR manifest name from the NFD name.""" - return ( - f"{self.name.lower().replace('_', '-')}" - f"-nf-acr-manifest-{nsd_version.replace('.', '-')}" - ) - - -@dataclass -class NSConfiguration(Configuration): - network_functions: List[Union[NFDRETConfiguration, Dict[str, Any]]] = field( - default_factory=lambda: [] - ) - nsd_name: str = "" - nsd_version: str = "" - nsdv_description: str = "" - - def __post_init__(self): - """Covert things to the correct format.""" - super().__post_init__() - if self.network_functions and isinstance(self.network_functions[0], dict): - nf_ret_list = [ - NFDRETConfiguration(**config) for config in self.network_functions - ] - self.network_functions = nf_ret_list - - @classmethod - def helptext(cls) -> "NSConfiguration": - """ - Build a NSConfiguration object where each value is helptext for that field. - """ - nsd_helptext = NSConfiguration( - network_functions=[asdict(NFDRETConfiguration.helptext())], - nsd_name=( - "Network Service Design (NSD) name. This is the collection of Network Service" - " Design Versions. Will be created if it does not exist." - ), - nsd_version=( - "Version of the NSD to be created. This should be in the format A.B.C" - ), - nsdv_description="Description of the NSDV.", - **asdict(Configuration.helptext()), - ) - - return nsd_helptext - - def validate(self): - """ - Validate the configuration passed in. - - :raises ValueError for any invalid config - """ - super().validate() - if not self.network_functions: - raise ValueError(("At least one network function must be included.")) - - for configuration in self.network_functions: - configuration.validate() - if not self.nsd_name: - raise ValueError("nsd_name must be set") - if not self.nsd_version: - raise ValueError("nsd_version must be set") - - @property - def output_directory_for_build(self) -> Path: - """Return the local folder for generating the bicep template to.""" - current_working_directory = os.getcwd() - return Path(current_working_directory, NSD_OUTPUT_BICEP_PREFIX) - - @property - def nfvi_site_name(self) -> str: - """Return the name of the NFVI used for the NSDV.""" - return f"{self.nsd_name}_NFVI" - - @property - def cg_schema_name(self) -> str: - """Return the name of the Configuration Schema used for the NSDV.""" - return f"{self.nsd_name.replace('-', '_')}_ConfigGroupSchema" - - @property - def acr_manifest_names(self) -> List[str]: - """The list of ACR manifest names for all the NF ARM templates.""" - acr_manifest_names = [] - for nf in self.network_functions: - assert isinstance(nf, NFDRETConfiguration) - acr_manifest_names.append(nf.acr_manifest_name(self.nsd_version)) - - logger.debug("ACR manifest names: %s", acr_manifest_names) - return acr_manifest_names - - -def get_configuration(configuration_type: str, config_file: str) -> Configuration: - """ - Return the correct configuration object based on the type. - - :param configuration_type: The type of configuration to return - :param config_file: The path to the config file - :return: The configuration object - """ - try: - with open(config_file, "r", encoding="utf-8") as f: - config_as_dict = json.loads(f.read()) - except json.decoder.JSONDecodeError as e: - raise InvalidArgumentValueError( - f"Config file {config_file} is not valid JSON: {e}" - ) from e - - config: Configuration - try: - if configuration_type == VNF: - config = VNFConfiguration(config_file=config_file, **config_as_dict) - elif configuration_type == CNF: - config = CNFConfiguration(config_file=config_file, **config_as_dict) - elif configuration_type == NSD: - config = NSConfiguration(config_file=config_file, **config_as_dict) - else: - raise InvalidArgumentValueError( - "Definition type not recognized, options are: vnf, cnf or nsd" - ) - except TypeError as typeerr: - raise InvalidArgumentValueError( - f"Config file {config_file} is not valid: {typeerr}" - ) from typeerr - - config.validate() - - return config diff --git a/src/aosm/azext_aosm/old/delete/__init__.py b/src/aosm/azext_aosm/old/delete/__init__.py deleted file mode 100644 index 99c0f28cd71..00000000000 --- a/src/aosm/azext_aosm/old/delete/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -# ----------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# ----------------------------------------------------------------------------- diff --git a/src/aosm/azext_aosm/old/delete/delete.py b/src/aosm/azext_aosm/old/delete/delete.py deleted file mode 100644 index 561ce51e04e..00000000000 --- a/src/aosm/azext_aosm/old/delete/delete.py +++ /dev/null @@ -1,345 +0,0 @@ -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- -"""Contains class for deploying generated definitions using the Python SDK.""" -from time import sleep -from typing import Optional - -from azure.cli.core.commands import LongRunningOperation -from azure.core.exceptions import ResourceExistsError -from azext_aosm._configuration import ( - Configuration, - NFConfiguration, - NSConfiguration, - VNFConfiguration, -) -from azext_aosm.util.management_clients import ApiClients -from azext_aosm.util.utils import input_ack -from knack.log import get_logger - -logger = get_logger(__name__) - - -class ResourceDeleter: - def __init__( - self, - api_clients: ApiClients, - config: Configuration, - cli_ctx: Optional[object] = None, - ) -> None: - """ - Initializes a new instance of the Deployer class. - - :param aosm_client: The client to use for managing AOSM resources. - :type aosm_client: HybridNetworkManagementClient - :param resource_client: The client to use for managing Azure resources. - :type resource_client: ResourceManagementClient - """ - logger.debug("Create ARM/Bicep Deployer") - self.api_clients = api_clients - self.config = config - self.cli_ctx = cli_ctx - - def delete_nfd(self, clean: bool = False, force: bool = False) -> None: - """ - Delete the NFDV and manifests. If they don't exist it still reports them as deleted. - - :param clean: Delete the NFDG, artifact stores and publisher too. Defaults to False. - Use with care. - """ - assert isinstance(self.config, NFConfiguration) - - if not force: - if clean: - print( - "Are you sure you want to delete all resources associated with NFD" - f" {self.config.nf_name} including the artifact stores and publisher" - f" {self.config.publisher_name}?" - ) - logger.warning( - "This command will fail if other NFD versions exist in the NFD group." - ) - logger.warning( - "Only do this if you are SURE you are not sharing the publisher and" - " artifact stores with other NFDs" - ) - print("There is no undo. Type the publisher name to confirm.") - if not input_ack(self.config.publisher_name.lower(), "Confirm delete:"): - print("Not proceeding with delete") - return - else: - print( - "Are you sure you want to delete the NFD Version" - f" {self.config.version} and associated manifests from group" - f" {self.config.nfdg_name} and publisher {self.config.publisher_name}?" - ) - print("There is no undo. Type 'delete' to confirm") - if not input_ack("delete", "Confirm delete:"): - print("Not proceeding with delete") - return - - self.delete_nfdv() - - if isinstance(self.config, VNFConfiguration): - self.delete_artifact_manifest("sa") - self.delete_artifact_manifest("acr") - - if clean: - logger.info("Delete called for all resources.") - self.delete_nfdg() - self.delete_artifact_store("acr") - if isinstance(self.config, VNFConfiguration): - self.delete_artifact_store("sa") - self.delete_publisher() - - def delete_nsd(self, clean: bool = False, force: bool = False) -> None: - """ - Delete the NSDV and manifests. - - If they don't exist it still reports them as deleted. - """ - assert isinstance(self.config, NSConfiguration) - - if not force: - print( - "Are you sure you want to delete the NSD Version" - f" {self.config.nsd_version}, the associated manifests" - f" {self.config.acr_manifest_names} and configuration group schema" - f" {self.config.cg_schema_name}?" - ) - if clean: - print( - f"Because of the --clean flag, the NSD {self.config.nsd_name} will also be deleted." - ) - print("There is no undo. Type 'delete' to confirm") - if not input_ack("delete", "Confirm delete:"): - print("Not proceeding with delete") - return - - self.delete_nsdv() - self.delete_artifact_manifest("acr") - self.delete_config_group_schema() - if clean: - self.delete_nsdg() - - def delete_nfdv(self): - assert isinstance(self.config, NFConfiguration) - message = ( - f"Delete NFDV {self.config.version} from group {self.config.nfdg_name} and" - f" publisher {self.config.publisher_name}" - ) - logger.debug(message) - print(message) - try: - poller = self.api_clients.aosm_client.network_function_definition_versions.begin_delete( - resource_group_name=self.config.publisher_resource_group_name, - publisher_name=self.config.publisher_name, - network_function_definition_group_name=self.config.nfdg_name, - network_function_definition_version_name=self.config.version, - ) - LongRunningOperation(self.cli_ctx, "Deleting NFDV...")(poller) - logger.info("Deleted NFDV.") - except Exception: - logger.error( - "Failed to delete NFDV %s from group %s", - self.config.version, - self.config.nfdg_name, - ) - raise - - def delete_nsdv(self): - assert isinstance(self.config, NSConfiguration) - message = ( - f"Delete NSDV {self.config.nsd_version} from group" - f" {self.config.nsd_name} and publisher {self.config.publisher_name}" - ) - logger.debug(message) - print(message) - try: - poller = self.api_clients.aosm_client.network_service_design_versions.begin_delete( - resource_group_name=self.config.publisher_resource_group_name, - publisher_name=self.config.publisher_name, - network_service_design_group_name=self.config.nsd_name, - network_service_design_version_name=self.config.nsd_version, - ) - LongRunningOperation(self.cli_ctx, "Deleting NSDV...")(poller) - logger.info("Deleted NSDV.") - except Exception: - logger.error( - "Failed to delete NSDV %s from group %s", - self.config.nsd_version, - self.config.nsd_name, - ) - raise - - def delete_artifact_manifest(self, store_type: str) -> None: - """ - _summary_ - - :param store_type: "sa" or "acr" - :raises CLIInternalError: If called with any other store type :raises - Exception if delete throws an exception - """ - if store_type == "sa": - assert isinstance(self.config, VNFConfiguration) - store_name = self.config.blob_artifact_store_name - manifest_names = [self.config.sa_manifest_name] - elif store_type == "acr": - store_name = self.config.acr_artifact_store_name - manifest_names = self.config.acr_manifest_names - else: - from azure.cli.core.azclierror import CLIInternalError - - raise CLIInternalError( - "Delete artifact manifest called for invalid store type. Valid types" - " are sa and acr." - ) - - for manifest_name in manifest_names: - message = f"Delete Artifact manifest {manifest_name} from artifact store {store_name}" - logger.debug(message) - print(message) - try: - poller = self.api_clients.aosm_client.artifact_manifests.begin_delete( - resource_group_name=self.config.publisher_resource_group_name, - publisher_name=self.config.publisher_name, - artifact_store_name=store_name, - artifact_manifest_name=manifest_name, - ) - LongRunningOperation(self.cli_ctx, "Deleting Artifact manifest...")( - poller - ) # noqa: E501 - logger.info("Deleted Artifact Manifest") - except Exception: - logger.error( - "Failed to delete Artifact manifest %s from artifact store %s", - manifest_name, - store_name, - ) - raise - - def delete_nsdg(self) -> None: - """Delete the NSD.""" - assert isinstance(self.config, NSConfiguration) - message = f"Delete NSD {self.config.nsd_name}" - logger.debug(message) - print(message) - try: - poller = ( - self.api_clients.aosm_client.network_service_design_groups.begin_delete( - resource_group_name=self.config.publisher_resource_group_name, - publisher_name=self.config.publisher_name, - network_service_design_group_name=self.config.nsd_name, - ) - ) - LongRunningOperation(self.cli_ctx, "Deleting NSD...")(poller) - logger.info("Deleted NSD") - except Exception: - logger.error("Failed to delete NSD.") - raise - - def delete_nfdg(self) -> None: - """Delete the NFDG.""" - assert isinstance(self.config, NFConfiguration) - message = f"Delete NFD Group {self.config.nfdg_name}" - logger.debug(message) - print(message) - try: - poller = self.api_clients.aosm_client.network_function_definition_groups.begin_delete( - resource_group_name=self.config.publisher_resource_group_name, - publisher_name=self.config.publisher_name, - network_function_definition_group_name=self.config.nfdg_name, - ) - LongRunningOperation(self.cli_ctx, "Deleting NFD Group...")(poller) - logger.info("Deleted NFD Group") - except Exception: - logger.error("Failed to delete NFDG.") - raise - - def delete_artifact_store(self, store_type: str) -> None: - """Delete an artifact store - :param store_type: "sa" or "acr" - :raises CLIInternalError: If called with any other store type - :raises Exception if delete throws an exception.""" - if store_type == "sa": - assert isinstance(self.config, VNFConfiguration) - store_name = self.config.blob_artifact_store_name - elif store_type == "acr": - store_name = self.config.acr_artifact_store_name - else: - from azure.cli.core.azclierror import CLIInternalError - - raise CLIInternalError( - "Delete artifact store called for invalid store type. Valid types are" - " sa and acr." - ) - message = f"Delete Artifact store {store_name}" - logger.debug(message) - print(message) - try: - poller = self.api_clients.aosm_client.artifact_stores.begin_delete( - resource_group_name=self.config.publisher_resource_group_name, - publisher_name=self.config.publisher_name, - artifact_store_name=store_name, - ) - LongRunningOperation(self.cli_ctx, "Deleting Artifact store...")(poller) - logger.info("Deleted Artifact Store") - except Exception: - logger.error("Failed to delete Artifact store %s", store_name) - raise - - def delete_publisher(self) -> None: - """ - Delete the publisher. - - Warning - dangerous - """ - message = f"Delete Publisher {self.config.publisher_name}" - logger.debug(message) - print(message) - # Occasionally nested resources that have just been deleted (e.g. artifact store) will - # still appear to exist, raising ResourceExistsError. We handle this by retrying up to - # 6 times, with a 30 second wait between each. - for attempt in range(6): - try: - poller = self.api_clients.aosm_client.publishers.begin_delete( - resource_group_name=self.config.publisher_resource_group_name, - publisher_name=self.config.publisher_name, - ) - LongRunningOperation(self.cli_ctx, "Deleting Publisher...")(poller) - logger.info("Deleted Publisher") - break - except ResourceExistsError: - if attempt == 5: - logger.error("Failed to delete publisher") - raise - logger.debug( - "ResourceExistsError: This may be nested resource is not finished deleting. Wait and retry." - ) - sleep(30) - except Exception: - logger.error("Failed to delete publisher") - raise - - def delete_config_group_schema(self) -> None: - """Delete the Configuration Group Schema.""" - assert isinstance(self.config, NSConfiguration) - message = f"Delete Configuration Group Schema {self.config.cg_schema_name}" - logger.debug(message) - print(message) - try: - poller = ( - self.api_clients.aosm_client.configuration_group_schemas.begin_delete( - resource_group_name=self.config.publisher_resource_group_name, - publisher_name=self.config.publisher_name, - configuration_group_schema_name=self.config.cg_schema_name, - ) - ) - LongRunningOperation( - self.cli_ctx, "Deleting Configuration Group Schema..." - )(poller) - logger.info("Deleted Configuration Group Schema") - except Exception: - logger.error("Failed to delete the Configuration Group Schema") - raise diff --git a/src/aosm/azext_aosm/old/deploy/__init__.py b/src/aosm/azext_aosm/old/deploy/__init__.py deleted file mode 100644 index 99c0f28cd71..00000000000 --- a/src/aosm/azext_aosm/old/deploy/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -# ----------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# ----------------------------------------------------------------------------- diff --git a/src/aosm/azext_aosm/old/deploy/artifact.py b/src/aosm/azext_aosm/old/deploy/artifact.py deleted file mode 100644 index ce8f2daa58f..00000000000 --- a/src/aosm/azext_aosm/old/deploy/artifact.py +++ /dev/null @@ -1,645 +0,0 @@ -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- - -# pylint: disable=unidiomatic-typecheck -"""A module to handle interacting with artifacts.""" -import json -import math -import shutil -import subprocess -from dataclasses import dataclass -from typing import Any, Optional, Union - -from knack.log import get_logger -from knack.util import CLIError -from oras.client import OrasClient - -from azext_aosm._configuration import ( - ArtifactConfig, - CNFImageConfig, - HelmPackageConfig, - VhdArtifactConfig, -) -from azext_aosm.vendored_sdks.azure_storagev2.blob.v2022_11_02 import ( - BlobClient, BlobType) - -logger = get_logger(__name__) - - -@dataclass -class Artifact: - """Artifact class.""" - - artifact_name: str - artifact_type: str - artifact_version: str - artifact_client: Union[BlobClient, OrasClient] - manifest_credentials: Any - - def upload( - self, - artifact_config: Union[ArtifactConfig, HelmPackageConfig], - use_manifest_permissions: bool = False, - ) -> None: - """ - Upload artifact. - - :param artifact_config: configuration for the artifact being uploaded - """ - if isinstance(self.artifact_client, OrasClient): - if isinstance(artifact_config, HelmPackageConfig): - self._upload_helm_to_acr(artifact_config, use_manifest_permissions) - elif isinstance(artifact_config, ArtifactConfig): - self._upload_arm_to_acr(artifact_config) - elif isinstance(artifact_config, CNFImageConfig): - self._upload_or_copy_image_to_acr( - artifact_config, use_manifest_permissions - ) - else: - raise ValueError(f"Unsupported artifact type: {type(artifact_config)}.") - else: - assert isinstance(artifact_config, ArtifactConfig) - self._upload_to_storage_account(artifact_config) - - def _upload_arm_to_acr(self, artifact_config: ArtifactConfig) -> None: - """ - Upload ARM artifact to ACR. - - :param artifact_config: configuration for the artifact being uploaded - """ - assert isinstance(self.artifact_client, OrasClient) - - if artifact_config.file_path: - if not self.artifact_client.remote.hostname: - raise ValueError( - "Cannot upload ARM template as OrasClient has no remote hostname." - " Please check your ACR config." - ) - target = ( - f"{self.artifact_client.remote.hostname.replace('https://', '')}" - f"/{self.artifact_name}:{self.artifact_version}" - ) - logger.debug("Uploading %s to %s", artifact_config.file_path, target) - self.artifact_client.push(files=[artifact_config.file_path], target=target) - else: - raise NotImplementedError( - "Copying artifacts is not implemented for ACR artifacts stores." - ) - - @staticmethod - def _call_subprocess_raise_output(cmd: list) -> None: - """ - Call a subprocess and raise a CLIError with the output if it fails. - - :param cmd: command to run, in list format - :raise CLIError: if the subprocess fails - """ - log_cmd = cmd.copy() - if "--password" in log_cmd: - # Do not log out passwords. - log_cmd[log_cmd.index("--password") + 1] = "[REDACTED]" - - try: - called_process = subprocess.run( - cmd, encoding="utf-8", capture_output=True, text=True, check=True - ) - logger.debug( - "Output from %s: %s. Error: %s", - log_cmd, - called_process.stdout, - called_process.stderr, - ) - except subprocess.CalledProcessError as error: - logger.debug("Failed to run %s with %s", log_cmd, error) - - all_output: str = ( - f"Command: {'' ''.join(log_cmd)}\n" - f"Output: {error.stdout}\n" - f"Error output: {error.stderr}\n" - f"Return code: {error.returncode}" - ) - logger.debug("All the output %s", all_output) - - # Raise the error without the original exception, which may contain secrets. - raise CLIError(all_output) from None - - def _upload_helm_to_acr( - self, artifact_config: HelmPackageConfig, use_manifest_permissions: bool - ) -> None: - """ - Upload artifact to ACR. This does and az acr login and then a helm push. - - Requires helm to be installed. - - :param artifact_config: configuration for the artifact being uploaded - :param use_manifest_permissions: whether to use the manifest credentials for the - upload. If False, the CLI user credentials will be used, which does not - require Docker to be installed. If True, the manifest creds will be used, - which requires Docker. - """ - self._check_tool_installed("helm") - assert isinstance(self.artifact_client, OrasClient) - chart_path = artifact_config.path_to_chart - if not self.artifact_client.remote.hostname: - raise ValueError( - "Cannot upload artifact. Oras client has no remote hostname." - ) - registry = self._get_acr() - target_registry = f"oci://{registry}" - registry_name = registry.replace(".azurecr.io", "") - - username = self.manifest_credentials["username"] - password = self.manifest_credentials["acr_token"] - - if not use_manifest_permissions: - # Note that this uses the user running the CLI's AZ login credentials, not - # the manifest credentials retrieved from the ACR. This allows users with - # enough permissions to avoid having to install docker. It logs in to the - # registry by retrieving an access token, which allows use of this command - # in environments without docker. - # It is governed by the no-subscription-permissions CLI argument which - # default to False. - logger.debug("Using CLI user credentials to log into %s", registry_name) - acr_login_with_token_cmd = [ - str(shutil.which("az")), - "acr", - "login", - "--name", - registry_name, - "--expose-token", - "--output", - "tsv", - "--query", - "accessToken", - ] - username = "00000000-0000-0000-0000-000000000000" - try: - password = subprocess.check_output( - acr_login_with_token_cmd, encoding="utf-8", text=True - ).strip() - except subprocess.CalledProcessError as error: - unauthorized = ( - error.stderr - and (" 401" in error.stderr or "unauthorized" in error.stderr) - ) or ( - error.stdout - and (" 401" in error.stdout or "unauthorized" in error.stdout) - ) - - if unauthorized: - # As we shell out the the subprocess, I think checking for these - # strings is the best check we can do for permission failures. - raise CLIError( - " Failed to login to Artifact Store ACR.\n" - " It looks like you do not have permissions. You need to have" - " the AcrPush role over the" - " whole subscription in order to be able to upload to the new" - " Artifact store.\n\nIf you do not have them then you can" - " re-run the command using the --no-subscription-permissions" - " flag to use manifest credentials scoped" - " only to the store. This requires Docker to be installed" - " locally." - ) from error - else: - # This seems to prevent occasional helm login failures - self._check_tool_installed("docker") - acr_login_cmd = [ - str(shutil.which("az")), - "acr", - "login", - "--name", - registry_name, - "--username", - username, - "--password", - password, - ] - self._call_subprocess_raise_output(acr_login_cmd) - try: - logger.debug("Uploading %s to %s", chart_path, target_registry) - helm_login_cmd = [ - str(shutil.which("helm")), - "registry", - "login", - registry, - "--username", - username, - "--password", - password, - ] - self._call_subprocess_raise_output(helm_login_cmd) - - # helm push "$chart_path" "$target_registry" - push_command = [ - str(shutil.which("helm")), - "push", - chart_path, - target_registry, - ] - self._call_subprocess_raise_output(push_command) - finally: - helm_logout_cmd = [ - str(shutil.which("helm")), - "registry", - "logout", - registry, - ] - self._call_subprocess_raise_output(helm_logout_cmd) - - @staticmethod - def _convert_to_readable_size(size_in_bytes: Optional[int]) -> str: - """Converts a size in bytes to a human readable size.""" - if size_in_bytes is None: - return "Unknown bytes" - size_name = ("B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB") - index = int(math.floor(math.log(size_in_bytes, 1024))) - power = math.pow(1024, index) - readable_size = round(size_in_bytes / power, 2) - return f"{readable_size} {size_name[index]}" - - def _vhd_upload_progress_callback( - self, current_bytes: int, total_bytes: Optional[int] - ) -> None: - """Callback function for VHD upload progress.""" - current_readable = self._convert_to_readable_size(current_bytes) - total_readable = self._convert_to_readable_size(total_bytes) - message = f"Uploaded {current_readable} of {total_readable} bytes" - logger.info(message) - print(message) - - def _upload_to_storage_account(self, artifact_config: ArtifactConfig) -> None: - """ - Upload artifact to storage account. - - :param artifact_config: configuration for the artifact being uploaded - """ - assert isinstance(self.artifact_client, BlobClient) - assert isinstance(artifact_config, ArtifactConfig) - - # If the file path is given, upload the artifact, else, copy it from an existing blob. - if artifact_config.file_path: - logger.info("Upload to blob store") - with open(artifact_config.file_path, "rb") as artifact: - self.artifact_client.upload_blob( - data=artifact, - overwrite=True, - blob_type=BlobType.PAGEBLOB, - progress_hook=self._vhd_upload_progress_callback, - ) - logger.info( - "Successfully uploaded %s to %s", - artifact_config.file_path, - self.artifact_client.account_name, - ) - else: - # Config Validation will raise error if not true - assert isinstance(artifact_config, VhdArtifactConfig) - assert artifact_config.blob_sas_url - logger.info("Copy from SAS URL to blob store") - source_blob = BlobClient.from_blob_url(artifact_config.blob_sas_url) - - if source_blob.exists(): - logger.debug(source_blob.url) - self.artifact_client.start_copy_from_url(source_blob.url) - logger.info( - "Successfully copied %s from %s to %s", - source_blob.blob_name, - source_blob.account_name, - self.artifact_client.account_name, - ) - else: - raise RuntimeError( - f"{source_blob.blob_name} does not exist in" - f" {source_blob.account_name}." - ) - - def _get_acr(self) -> str: - """ - Get the name of the ACR. - - :return: The name of the ACR - """ - assert hasattr(self.artifact_client, "remote") - if not self.artifact_client.remote.hostname: - raise ValueError( - "Cannot upload artifact. Oras client has no remote hostname." - ) - return self._clean_name(self.artifact_client.remote.hostname) - - def _get_acr_target_image( - self, - include_hostname: bool = True, - ) -> str: - """Format the acr url, artifact name and version into a target image string.""" - if include_hostname: - return f"{self._get_acr()}/{self.artifact_name}:{self.artifact_version}" - - return f"{self.artifact_name}:{self.artifact_version}" - - @staticmethod - def _check_tool_installed(tool_name: str) -> None: - """ - Check whether a tool such as docker or helm is installed. - - :param tool_name: name of the tool to check, e.g. docker - """ - if shutil.which(tool_name) is None: - raise CLIError(f"You must install {tool_name} to use this command.") - - def _upload_or_copy_image_to_acr( - self, artifact_config: CNFImageConfig, use_manifest_permissions: bool - ) -> None: - # Check whether the source registry has a namespace in the repository path - source_registry_namespace: str = "" - if artifact_config.source_registry_namespace: - source_registry_namespace = f"{artifact_config.source_registry_namespace}/" - - if artifact_config.source_local_docker_image: - # The user has provided a local docker image to use as the source - # for the images in the artifact manifest - self._check_tool_installed("docker") - print( - f"Using local docker image as source for image artifact upload for image artifact: {self.artifact_name}" - ) - self._push_image_from_local_registry( - local_docker_image=artifact_config.source_local_docker_image, - target_username=self.manifest_credentials["username"], - target_password=self.manifest_credentials["acr_token"], - ) - elif use_manifest_permissions: - self._check_tool_installed("docker") - print( - f"Using docker pull and push to copy image artifact: {self.artifact_name}" - ) - image_name = ( - f"{self._clean_name(artifact_config.source_registry)}/" - f"{source_registry_namespace}{self.artifact_name}" - f":{self.artifact_version}" - ) - self._pull_image_to_local_registry( - source_registry_login_server=self._clean_name( - artifact_config.source_registry - ), - source_image=image_name, - ) - self._push_image_from_local_registry( - local_docker_image=image_name, - target_username=self.manifest_credentials["username"], - target_password=self.manifest_credentials["acr_token"], - ) - else: - print(f"Using az acr import to copy image artifact: {self.artifact_name}") - self._copy_image( - source_registry_login_server=artifact_config.source_registry, - source_image=( - f"{source_registry_namespace}{self.artifact_name}" - f":{self.artifact_version}" - ), - ) - - def _push_image_from_local_registry( - self, - local_docker_image: str, - target_username: str, - target_password: str, - ): - """ - Push image to target registry using docker push. Requires docker. - - :param local_docker_image: name and tag of the source image on local registry - e.g. uploadacr.azurecr.io/samples/nginx:stable - :type local_docker_image: str - :param target_username: The username to use for the az acr login attempt - :type target_username: str - :param target_password: The password to use for the az acr login attempt - :type target_password: str - """ - assert hasattr(self.artifact_client, "remote") - target_acr = self._get_acr() - try: - target = self._get_acr_target_image() - print("Tagging source image") - - tag_image_cmd = [ - str(shutil.which("docker")), - "tag", - local_docker_image, - target, - ] - self._call_subprocess_raise_output(tag_image_cmd) - message = ( - "Logging into artifact store registry " - f"{self.artifact_client.remote.hostname}" - ) - - print(message) - logger.info(message) - acr_target_login_cmd = [ - str(shutil.which("az")), - "acr", - "login", - "--name", - target_acr, - "--username", - target_username, - "--password", - target_password, - ] - self._call_subprocess_raise_output(acr_target_login_cmd) - - print("Pushing target image using docker push") - push_target_image_cmd = [ - str(shutil.which("docker")), - "push", - target, - ] - self._call_subprocess_raise_output(push_target_image_cmd) - except CLIError as error: - logger.error( - ("Failed to tag and push %s to %s."), - local_docker_image, - target_acr, - ) - logger.debug(error, exc_info=True) - raise error - finally: - docker_logout_cmd = [ - str(shutil.which("docker")), - "logout", - target_acr, - ] - self._call_subprocess_raise_output(docker_logout_cmd) - - def _pull_image_to_local_registry( - self, - source_registry_login_server: str, - source_image: str, - ) -> None: - """ - Pull image to local registry using docker pull. Requires docker. - - Uses the CLI user's context to log in to the source registry. - - :param: source_registry_login_server: e.g. uploadacr.azurecr.io - :param: source_image: source docker image name e.g. - uploadacr.azurecr.io/samples/nginx:stable - """ - try: - # Login to the source registry with the CLI user credentials. This requires - # docker to be installed. - message = f"Logging into source registry {source_registry_login_server}" - print(message) - logger.info(message) - acr_source_login_cmd = [ - str(shutil.which("az")), - "acr", - "login", - "--name", - source_registry_login_server, - ] - self._call_subprocess_raise_output(acr_source_login_cmd) - message = f"Pulling source image {source_image}" - print(message) - logger.info(message) - pull_source_image_cmd = [ - str(shutil.which("docker")), - "pull", - source_image, - ] - self._call_subprocess_raise_output(pull_source_image_cmd) - except CLIError as error: - logger.error( - ( - "Failed to pull %s. Check if this image exists in the" - " source registry %s." - ), - source_image, - source_registry_login_server, - ) - logger.debug(error, exc_info=True) - raise error - finally: - docker_logout_cmd = [ - str(shutil.which("docker")), - "logout", - source_registry_login_server, - ] - self._call_subprocess_raise_output(docker_logout_cmd) - - @staticmethod - def _clean_name(registry_name: str) -> str: - """Remove https:// from the registry name.""" - return registry_name.replace("https://", "") - - def _copy_image( - self, - source_registry_login_server: str, - source_image: str, - ): - """ - Copy image from one ACR to another. - - Use az acr import to do the import image. Previously we used the python - sdk ContainerRegistryManagementClient.registries.begin_import_image - but this requires the source resource group name, which is more faff - at configuration time. - - Neither az acr import or begin_import_image support using the username - and acr_token retrieved from the manifest credentials, so this uses the - CLI users context to access both the source registry and the target - Artifact Store registry, which requires either Contributor role or a - custom role that allows the importImage action over the whole subscription. - - :param source_registry: source registry login server e.g. https://uploadacr.azurecr.io - :param source_image: source image including namespace and tags e.g. - samples/nginx:stable - """ - target_acr = self._get_acr() - try: - print("Copying artifact from source registry") - # In order to use az acr import cross subscription, we need to use a token - # to authenticate to the source registry. This is documented as the way to - # us az acr import cross-tenant, not cross-sub, but it also works - # cross-subscription, and meant we didn't have to make a breaking change to - # the format of input.json. Our usage here won't work cross-tenant since - # we're attempting to get the token (source) with the same context as that - # in which we are creating the ACR (i.e. the target tenant) - get_token_cmd = [str(shutil.which("az")), "account", "get-access-token"] - # Dont use _call_subprocess_raise_output here as we don't want to log the - # output - called_process = subprocess.run( # noqa: S603 - get_token_cmd, - encoding="utf-8", - capture_output=True, - text=True, - check=True, - ) - access_token_json = json.loads(called_process.stdout) - access_token = access_token_json["accessToken"] - except subprocess.CalledProcessError as get_token_err: - # This error is thrown from the az account get-access-token command - # If it errored we can log the output as it doesn't contain the token - logger.debug(get_token_err, exc_info=True) - raise CLIError( # pylint: disable=raise-missing-from - "Failed to import image: could not get an access token from your" - " Azure account. Try logging in again with `az login` and then re-run" - " the command. If it fails again, please raise an issue and try" - " repeating the command using the --no-subscription-permissions" - " flag to pull the image to your local machine and then" - " push it to the Artifact Store using manifest credentials scoped" - " only to the store. This requires Docker to be installed" - " locally." - ) - - try: - source = f"{self._clean_name(source_registry_login_server)}/{source_image}" - acr_import_image_cmd = [ - str(shutil.which("az")), - "acr", - "import", - "--name", - target_acr, - "--source", - source, - "--image", - self._get_acr_target_image(include_hostname=False), - "--password", - access_token, - ] - self._call_subprocess_raise_output(acr_import_image_cmd) - except CLIError as error: - logger.debug(error, exc_info=True) - if (" 401" in str(error)) or ("Unauthorized" in str(error)): - # As we shell out the the subprocess, I think checking for these strings - # is the best check we can do for permission failures. - raise CLIError( - " Failed to import image.\nIt looks like either the source_registry" - " in your config file does not exist or the image doesn't exist or" - " you do not have" - " permissions to import images. You need to have Reader/AcrPull" - f" from {source_registry_login_server}, and Contributor role +" - " AcrPush role, or a custom" - " role that allows the importImage action and AcrPush over the" - " whole subscription in order to be able to import to the new" - " Artifact store.\n\nIf you do not have the latter then you" - " can re-run the command using the --no-subscription-permissions" - " flag to pull the image to your local machine and then" - " push it to the Artifact Store using manifest credentials scoped" - " only to the store. This requires Docker to be installed" - " locally." - ) from error - - # The most likely failure is that the image already exists in the artifact - # store, so don't fail at this stage, log the error. - logger.error( - ( - "Failed to import %s to %s. Check if this image exists in the" - " source registry or is already present in the target registry.\n" - "%s" - ), - source_image, - target_acr, - error, - ) diff --git a/src/aosm/azext_aosm/old/deploy/artifact_manifest.py b/src/aosm/azext_aosm/old/deploy/artifact_manifest.py deleted file mode 100644 index ba0dc51a70c..00000000000 --- a/src/aosm/azext_aosm/old/deploy/artifact_manifest.py +++ /dev/null @@ -1,169 +0,0 @@ -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- -"""A module to handle interacting with artifact manifests.""" -from functools import cached_property, lru_cache -from typing import Any, List, Union - -from azure.cli.core.azclierror import AzCLIError -from knack.log import get_logger -from oras.client import OrasClient - -from azext_aosm._configuration import Configuration -from azext_aosm.deploy.artifact import Artifact -from azext_aosm.util.management_clients import ApiClients -from azext_aosm.vendored_sdks.models import ( - ArtifactManifest, - ArtifactType, - CredentialType, - ManifestArtifactFormat, -) -from azext_aosm.vendored_sdks.azure_storagev2.blob.v2022_11_02 import BlobClient - -logger = get_logger(__name__) - - -class ArtifactManifestOperator: - """ArtifactManifest class.""" - - # pylint: disable=too-few-public-methods - def __init__( - self, - config: Configuration, - api_clients: ApiClients, - store_name: str, - manifest_name: str, - ) -> None: - """Init.""" - self.manifest_name = manifest_name - self.api_clients = api_clients - self.config = config - self.store_name = store_name - self.artifacts = self._get_artifact_list() - - @cached_property - def _manifest_credentials(self) -> Any: - """Gets the details for uploading the artifacts in the manifest.""" - - return self.api_clients.aosm_client.artifact_manifests.list_credential( - resource_group_name=self.config.publisher_resource_group_name, - publisher_name=self.config.publisher_name, - artifact_store_name=self.store_name, - artifact_manifest_name=self.manifest_name, - ).as_dict() - - @lru_cache(maxsize=32) # noqa: B019 - def _oras_client(self, acr_url: str) -> OrasClient: - """ - Returns an OrasClient object for uploading to the artifact store ACR. - - :param arc_url: URL of the ACR backing the artifact manifest - """ - client = OrasClient(hostname=acr_url) - client.login( - username=self._manifest_credentials["username"], - password=self._manifest_credentials["acr_token"], - ) - return client - - def _get_artifact_list(self) -> List[Artifact]: - """Get the list of Artifacts in the Artifact Manifest.""" - artifacts = [] - - manifest: ArtifactManifest = ( - self.api_clients.aosm_client.artifact_manifests.get( - resource_group_name=self.config.publisher_resource_group_name, - publisher_name=self.config.publisher_name, - artifact_store_name=self.store_name, - artifact_manifest_name=self.manifest_name, - ) - ) - - # Instatiate an Artifact object for each artifact in the manifest. - if manifest.properties.artifacts: - for artifact in manifest.properties.artifacts: - if not ( - artifact.artifact_name - and artifact.artifact_type - and artifact.artifact_version - ): - raise AzCLIError( - "Cannot upload artifact. Artifact returned from " - "manifest query is missing required information." - f"{artifact}" - ) - - artifacts.append( - Artifact( - artifact_name=artifact.artifact_name, - artifact_type=artifact.artifact_type, - artifact_version=artifact.artifact_version, - artifact_client=self._get_artifact_client(artifact), - manifest_credentials=self._manifest_credentials, - ) - ) - - return artifacts - - def _get_artifact_client( - self, artifact: ManifestArtifactFormat - ) -> Union[BlobClient, OrasClient]: - """ - Get the artifact client required for uploading the artifact. - - :param artifact - a ManifestArtifactFormat with the artifact info. - """ - # Appease mypy - an error will be raised before this if these are blank - assert artifact.artifact_name - assert artifact.artifact_type - assert artifact.artifact_version - if ( - self._manifest_credentials["credential_type"] - == CredentialType.AZURE_STORAGE_ACCOUNT_TOKEN - ): - # Check we have the required artifact types for this credential. Indicates - # a coding error if we hit this but worth checking. - if not ( - artifact.artifact_type - in (ArtifactType.IMAGE_FILE, ArtifactType.VHD_IMAGE_FILE) - ): - raise AzCLIError( - f"Cannot upload artifact {artifact.artifact_name}." - " Artifact manifest credentials of type " - f"{CredentialType.AZURE_STORAGE_ACCOUNT_TOKEN} are not expected " - f"for Artifacts of type {artifact.artifact_type}" - ) - - container_basename = artifact.artifact_name.replace("-", "") - container_name = f"{container_basename}-{artifact.artifact_version}" - - # For AOSM to work VHD blobs must have the suffix .vhd - if artifact.artifact_name.endswith("-vhd"): - blob_name = f"{artifact.artifact_name[:-4].replace('-', '')}-{artifact.artifact_version}.vhd" - else: - blob_name = container_name - - logger.debug("container name: %s, blob name: %s", container_name, blob_name) - - blob_url = self._get_blob_url(container_name, blob_name) - return BlobClient.from_blob_url(blob_url) - return self._oras_client(self._manifest_credentials["acr_server_url"]) - - def _get_blob_url(self, container_name: str, blob_name: str) -> str: - """ - Get the URL for the blob to be uploaded to the storage account artifact store. - - :param container_name: name of the container - :param blob_name: the name that the blob will get uploaded with - """ - for container_credential in self._manifest_credentials["container_credentials"]: - if container_credential["container_name"] == container_name: - sas_uri = str(container_credential["container_sas_uri"]) - sas_uri_prefix, sas_uri_token = sas_uri.split("?", maxsplit=1) - - blob_url = f"{sas_uri_prefix}/{blob_name}?{sas_uri_token}" - logger.debug("Blob URL: %s", blob_url) - - return blob_url - raise KeyError(f"Manifest does not include a credential for {container_name}.") diff --git a/src/aosm/azext_aosm/old/deploy/deploy_with_arm.py b/src/aosm/azext_aosm/old/deploy/deploy_with_arm.py deleted file mode 100644 index 73da03a965a..00000000000 --- a/src/aosm/azext_aosm/old/deploy/deploy_with_arm.py +++ /dev/null @@ -1,731 +0,0 @@ -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- -"""Contains class for deploying generated definitions using ARM.""" -import json -import os -import shutil -import subprocess # noqa -import tempfile -import time -from typing import Any, Dict, Optional - -from azure.cli.core.azclierror import ValidationError -from azure.cli.core.commands import LongRunningOperation -from azure.mgmt.resource.resources.models import DeploymentExtended -from knack.log import get_logger -from knack.util import CLIError - -from azext_aosm._configuration import ( - ArtifactConfig, - CNFConfiguration, - Configuration, - NFConfiguration, - NFDRETConfiguration, - NSConfiguration, - VNFConfiguration, -) -from azext_aosm.deploy.artifact import Artifact -from azext_aosm.deploy.artifact_manifest import ArtifactManifestOperator -from azext_aosm.deploy.pre_deploy import PreDeployerViaSDK -from azext_aosm.util.constants import ( - ARTIFACT_UPLOAD, - BICEP_PUBLISH, - CNF, - CNF_DEFINITION_BICEP_TEMPLATE_FILENAME, - CNF_MANIFEST_BICEP_TEMPLATE_FILENAME, - IMAGE_UPLOAD, - NSD, - NSD_ARTIFACT_MANIFEST_BICEP_FILENAME, - NSD_BICEP_FILENAME, - VNF, - VNF_DEFINITION_BICEP_TEMPLATE_FILENAME, - VNF_MANIFEST_BICEP_TEMPLATE_FILENAME, - DeployableResourceTypes, - SkipSteps, -) -from azext_aosm.util.management_clients import ApiClients - -logger = get_logger(__name__) - - -class DeployerViaArm: # pylint: disable=too-many-instance-attributes - """ - A class to deploy Artifact Manifests, NFDs and NSDs from bicep templates using ARM. - - Uses the SDK to pre-deploy less complex resources and then ARM to deploy the bicep - templates. - """ - - def __init__( - self, - api_clients: ApiClients, - resource_type: DeployableResourceTypes, - config: Configuration, - bicep_path: Optional[str] = None, - parameters_json_file: Optional[str] = None, - manifest_bicep_path: Optional[str] = None, - manifest_params_file: Optional[str] = None, - skip: Optional[SkipSteps] = None, - cli_ctx: Optional[object] = None, - use_manifest_permissions: bool = False, - ): - """ - :param api_clients: ApiClients object for AOSM and ResourceManagement - :param config: The configuration for this NF - :param bicep_path: The path to the bicep template of the nfdv - :param parameters_json_file: path to an override file of set parameters for the nfdv - :param manifest_bicep_path: The path to the bicep template of the manifest - :param manifest_params_file: path to an override file of set parameters for - the manifest - :param skip: options to skip, either publish bicep or upload artifacts - :param cli_ctx: The CLI context. Used with CNFs and all LongRunningOperations - :param use_manifest_permissions: - CNF definition_type publish only - ignored for VNF or NSD. Causes the image - artifact copy from a source ACR to be done via docker pull and push, - rather than `az acr import`. This is slower but does not require - Contributor (or importImage action) permissions on the publisher - subscription. Also uses manifest permissions for helm chart upload. - Requires Docker to be installed locally. - """ - self.api_clients = api_clients - self.resource_type = resource_type - self.config = config - self.bicep_path = bicep_path - self.parameters_json_file = parameters_json_file - self.manifest_bicep_path = manifest_bicep_path - self.manifest_params_file = manifest_params_file - self.skip = skip - self.cli_ctx = cli_ctx - self.pre_deployer = PreDeployerViaSDK( - self.api_clients, self.config, self.cli_ctx - ) - self.use_manifest_permissions = use_manifest_permissions - - def deploy_nfd_from_bicep(self) -> None: - """ - Deploy the bicep template defining the NFD. - - Also ensure that all required predeploy resources are deployed. - """ - assert isinstance(self.config, NFConfiguration) - if self.skip == BICEP_PUBLISH: - print("Skipping bicep manifest publish") - else: - # 1) Deploy Artifact manifest bicep - # Create or check required resources - deploy_manifest_template = not self.nfd_predeploy() - if deploy_manifest_template: - self.deploy_manifest_template() - else: - print( - f"Artifact manifests exist for NFD {self.config.nf_name} " - f"version {self.config.version}" - ) - - if self.skip == ARTIFACT_UPLOAD: - print("Skipping artifact upload") - else: - # 2) Upload artifacts - must be done before nfd deployment - if self.resource_type == VNF: - self._vnfd_artifact_upload() - if self.resource_type == CNF: - self._cnfd_artifact_upload() - - if self.skip == BICEP_PUBLISH: - print("Skipping bicep nfd publish") - print("Done") - return - - # 3) Deploy NFD bicep - if not self.bicep_path: - # User has not passed in a bicep template, so we are deploying the default - # one produced from building the NFDV using this CLI - if self.resource_type == VNF: - file_name = VNF_DEFINITION_BICEP_TEMPLATE_FILENAME - if self.resource_type == CNF: - file_name = CNF_DEFINITION_BICEP_TEMPLATE_FILENAME - bicep_path = os.path.join(self.config.output_directory_for_build, file_name) - message = ( - f"Deploy bicep template for NFD {self.config.nf_name} version" - f" {self.config.version} into" - f" {self.config.publisher_resource_group_name} under publisher" - f" {self.config.publisher_name}" - ) - print(message) - logger.info(message) - logger.debug( - "Parameters used for NF definition bicep deployment: %s", - self.parameters, - ) - self.deploy_bicep_template(bicep_path, self.parameters) - print(f"Deployed NFD {self.config.nf_name} version {self.config.version}.") - - def _vnfd_artifact_upload(self) -> None: - """Uploads the VHD and ARM template artifacts.""" - assert isinstance(self.config, VNFConfiguration) - storage_account_manifest = ArtifactManifestOperator( - self.config, - self.api_clients, - self.config.blob_artifact_store_name, - self.config.sa_manifest_name, - ) - acr_manifest = ArtifactManifestOperator( - self.config, - self.api_clients, - self.config.acr_artifact_store_name, - self.config.acr_manifest_names[0], - ) - - vhd_artifact = storage_account_manifest.artifacts[0] - arm_template_artifact = acr_manifest.artifacts[0] - - vhd_config = self.config.vhd - arm_template_config = self.config.arm_template - - assert isinstance(vhd_config, ArtifactConfig) - assert isinstance(arm_template_config, ArtifactConfig) - - if self.skip == IMAGE_UPLOAD: - print("Skipping VHD artifact upload") - else: - print("Uploading VHD artifact") - vhd_artifact.upload(vhd_config) - - print("Uploading ARM template artifact") - arm_template_artifact.upload(arm_template_config) - - def _cnfd_artifact_upload(self) -> None: - """Uploads the Helm chart and any additional images.""" - assert isinstance(self.config, CNFConfiguration) - acr_properties = self.api_clients.aosm_client.artifact_stores.get( - resource_group_name=self.config.publisher_resource_group_name, - publisher_name=self.config.publisher_name, - artifact_store_name=self.config.acr_artifact_store_name, - ) - if not acr_properties.properties.storage_resource_id: - raise CLIError( - f"Artifact store {self.config.acr_artifact_store_name} " - "has no storage resource id linked" - ) - - # The artifacts from the manifest which has been deployed by bicep - acr_manifest = ArtifactManifestOperator( - self.config, - self.api_clients, - self.config.acr_artifact_store_name, - self.config.acr_manifest_names[0], - ) - - # Create a new dictionary of artifacts from the manifest, keyed by artifact name - artifact_dictionary = {} - - for artifact in acr_manifest.artifacts: - artifact_dictionary[artifact.artifact_name] = artifact - - for helm_package in self.config.helm_packages: - # Go through the helm packages in the config that the user has provided - helm_package_name = helm_package.name # type: ignore - - if helm_package_name not in artifact_dictionary: - # Helm package in the config file but not in the artifact manifest - raise CLIError( - f"Artifact {helm_package_name} not found in the artifact manifest" - ) - # Get the artifact object that came from the manifest - manifest_artifact = artifact_dictionary[helm_package_name] - - print(f"Uploading Helm package: {helm_package_name}") - - # The artifact object will use the correct client (ORAS) to upload the - # artifact - manifest_artifact.upload(helm_package, self.use_manifest_permissions) # type: ignore - - print(f"Finished uploading Helm package: {helm_package_name}") - - # Remove this helm package artifact from the dictionary. - artifact_dictionary.pop(helm_package_name) - - # All the remaining artifacts are not in the helm_packages list. We assume that - # they are images that need to be copied from another ACR or uploaded from a - # local image. - if self.skip == IMAGE_UPLOAD: - print("Skipping upload of images") - return - - # This is the first time we have easy access to the number of images to upload - # so we validate the config file here. - if ( - len(artifact_dictionary.values()) > 1 - and self.config.images.source_local_docker_image # type: ignore - ): - raise ValidationError( - "Multiple image artifacts found to upload and a local docker image" - " was specified in the config file. source_local_docker_image is only " - "supported if there is a single image artifact to upload." - ) - for artifact in artifact_dictionary.values(): - assert isinstance(artifact, Artifact) - artifact.upload(self.config.images, self.use_manifest_permissions) # type: ignore - - def nfd_predeploy(self) -> bool: - """ - All the predeploy steps for a NFD. Create publisher, artifact stores and NFDG. - - Return True if artifact manifest already exists, False otherwise - """ - logger.debug("Ensure all required resources exist") - self.pre_deployer.ensure_config_resource_group_exists() - self.pre_deployer.ensure_config_publisher_exists() - self.pre_deployer.ensure_acr_artifact_store_exists() - if self.resource_type == VNF: - self.pre_deployer.ensure_sa_artifact_store_exists() - self.pre_deployer.ensure_config_nfdg_exists() - return self.pre_deployer.do_config_artifact_manifests_exist() - - @property - def parameters(self) -> Dict[str, Any]: - if self.parameters_json_file: - message = f"Use parameters from file {self.parameters_json_file}" - logger.info(message) - print(message) - with open(self.parameters_json_file, "r", encoding="utf-8") as f: - parameters_json = json.loads(f.read()) - parameters = parameters_json["parameters"] - else: - # User has not passed in parameters file, so we use the parameters - # required from config for the default bicep template produced from - # building the NFDV using this CLI - logger.debug("Create parameters for default template.") - parameters = self.construct_parameters() - - return parameters - - def construct_parameters(self) -> Dict[str, Any]: - """ - Create the parmeters dictionary for vnfdefinitions.bicep. - - VNF specific. - """ - if self.resource_type == VNF: - assert isinstance(self.config, VNFConfiguration) - assert isinstance(self.config.vhd, ArtifactConfig) - assert isinstance(self.config.arm_template, ArtifactConfig) - return { - "location": {"value": self.config.location}, - "publisherName": {"value": self.config.publisher_name}, - "acrArtifactStoreName": {"value": self.config.acr_artifact_store_name}, - "saArtifactStoreName": {"value": self.config.blob_artifact_store_name}, - "nfName": {"value": self.config.nf_name}, - "nfDefinitionGroup": {"value": self.config.nfdg_name}, - "nfDefinitionVersion": {"value": self.config.version}, - "vhdVersion": {"value": self.config.vhd.version}, - "armTemplateVersion": {"value": self.config.arm_template.version}, - } - if self.resource_type == CNF: - assert isinstance(self.config, CNFConfiguration) - return { - "location": {"value": self.config.location}, - "publisherName": {"value": self.config.publisher_name}, - "acrArtifactStoreName": {"value": self.config.acr_artifact_store_name}, - "nfDefinitionGroup": {"value": self.config.nfdg_name}, - "nfDefinitionVersion": {"value": self.config.version}, - } - if self.resource_type == NSD: - assert isinstance(self.config, NSConfiguration) - return { - "location": {"value": self.config.location}, - "publisherName": {"value": self.config.publisher_name}, - "acrArtifactStoreName": {"value": self.config.acr_artifact_store_name}, - "nsDesignGroup": {"value": self.config.nsd_name}, - "nsDesignVersion": {"value": self.config.nsd_version}, - "nfviSiteName": {"value": self.config.nfvi_site_name}, - } - raise TypeError( - "Unexpected config type. Expected [VNFConfiguration|CNFConfiguration|NSConfiguration]," - f" received {type(self.config)}" - ) - - def construct_manifest_parameters(self) -> Dict[str, Any]: - """Create the parmeters dictionary for VNF, CNF or NSD.""" - if self.resource_type == VNF: - assert isinstance(self.config, VNFConfiguration) - assert isinstance(self.config.vhd, ArtifactConfig) - assert isinstance(self.config.arm_template, ArtifactConfig) - return { - "location": {"value": self.config.location}, - "publisherName": {"value": self.config.publisher_name}, - "acrArtifactStoreName": {"value": self.config.acr_artifact_store_name}, - "saArtifactStoreName": {"value": self.config.blob_artifact_store_name}, - "acrManifestName": {"value": self.config.acr_manifest_names[0]}, - "saManifestName": {"value": self.config.sa_manifest_name}, - "nfName": {"value": self.config.nf_name}, - "vhdVersion": {"value": self.config.vhd.version}, - "armTemplateVersion": {"value": self.config.arm_template.version}, - } - if self.resource_type == CNF: - assert isinstance(self.config, CNFConfiguration) - return { - "location": {"value": self.config.location}, - "publisherName": {"value": self.config.publisher_name}, - "acrArtifactStoreName": {"value": self.config.acr_artifact_store_name}, - "acrManifestName": {"value": self.config.acr_manifest_names[0]}, - } - if self.resource_type == NSD: - assert isinstance(self.config, NSConfiguration) - - arm_template_names = [] - - for nf in self.config.network_functions: - assert isinstance(nf, NFDRETConfiguration) - arm_template_names.append(nf.arm_template.artifact_name) - - # Set the artifact version to be the same as the NSD version, so that they - # don't get over written when a new NSD is published. - return { - "location": {"value": self.config.location}, - "publisherName": {"value": self.config.publisher_name}, - "acrArtifactStoreName": {"value": self.config.acr_artifact_store_name}, - "acrManifestNames": {"value": self.config.acr_manifest_names}, - "armTemplateNames": {"value": arm_template_names}, - "armTemplateVersion": {"value": self.config.nsd_version}, - } - raise ValueError("Unknown configuration type") - - def deploy_nsd_from_bicep(self) -> None: - """ - Deploy the bicep template defining the VNFD. - - Also ensure that all required predeploy resources are deployed. - """ - assert isinstance(self.config, NSConfiguration) - if not self.skip == BICEP_PUBLISH: - # 1) Deploy Artifact manifest bicep - if not self.bicep_path: - # User has not passed in a bicep template, so we are deploying the default - # one produced from building the NSDV using this CLI - bicep_path = os.path.join( - self.config.output_directory_for_build, - NSD_BICEP_FILENAME, - ) - - logger.debug(self.parameters) - - # Create or check required resources - deploy_manifest_template = not self.nsd_predeploy() - - if deploy_manifest_template: - self.deploy_manifest_template() - else: - logger.debug( - "Artifact manifests %s already exist", - self.config.acr_manifest_names, - ) - print("Artifact manifests already exist") - - if self.skip == ARTIFACT_UPLOAD: - print("Skipping artifact upload") - else: - # 2) Upload artifacts - must be done before nsd deployment - for manifest, nf in zip( - self.config.acr_manifest_names, self.config.network_functions - ): - assert isinstance(nf, NFDRETConfiguration) - acr_manifest = ArtifactManifestOperator( - self.config, - self.api_clients, - self.config.acr_artifact_store_name, - manifest, - ) - - # Convert the NF bicep to ARM - arm_template_artifact_json = self.convert_bicep_to_arm( - os.path.join( - self.config.output_directory_for_build, nf.nf_bicep_filename - ) - ) - - arm_template_artifact = acr_manifest.artifacts[0] - - # appease mypy - assert ( - nf.arm_template.file_path - ), "Config missing ARM template file path" - with open(nf.arm_template.file_path, "w", encoding="utf-8") as file: - file.write(json.dumps(arm_template_artifact_json, indent=4)) - - print(f"Uploading ARM template artifact: {nf.arm_template.file_path}") - arm_template_artifact.upload(nf.arm_template) - - if self.skip == BICEP_PUBLISH: - print("Skipping bicep nsd publish") - print("Done") - return - - # 3) Deploy NSD bicep - if not self.bicep_path: - # User has not passed in a bicep template, so we are deploying the default - # one produced from building the NSDV using this CLI - bicep_path = os.path.join( - self.config.output_directory_for_build, - NSD_BICEP_FILENAME, - ) - message = ( - f"Deploy bicep template for NSDV {self.config.nsd_version} " - f"into {self.config.publisher_resource_group_name} under publisher " - f"{self.config.publisher_name}" - ) - print(message) - logger.info(message) - self.deploy_bicep_template(bicep_path, self.parameters) - print( - f"Deployed NSD {self.config.nsd_name} " - f"version {self.config.nsd_version}." - ) - - def deploy_manifest_template(self) -> None: - """Deploy the bicep template defining the manifest.""" - print("Deploy bicep template for Artifact manifests") - logger.debug("Deploy manifest bicep") - - if not self.manifest_bicep_path: - file_name: str = "" - if self.resource_type == NSD: - file_name = NSD_ARTIFACT_MANIFEST_BICEP_FILENAME - if self.resource_type == VNF: - file_name = VNF_MANIFEST_BICEP_TEMPLATE_FILENAME - if self.resource_type == CNF: - file_name = CNF_MANIFEST_BICEP_TEMPLATE_FILENAME - - manifest_bicep_path = os.path.join( - str(self.config.output_directory_for_build), - file_name, - ) - if not self.manifest_params_file: - manifest_params = self.construct_manifest_parameters() - else: - logger.info("Use provided manifest parameters") - with open(self.manifest_params_file, "r", encoding="utf-8") as f: - manifest_json = json.loads(f.read()) - manifest_params = manifest_json["parameters"] - self.deploy_bicep_template(manifest_bicep_path, manifest_params) - - def nsd_predeploy(self) -> bool: - """ - All the predeploy steps for a NSD. Check if the RG, publisher, ACR, NSD and - artifact manifest exist. - - Return True if artifact manifest already exists, False otherwise - """ - logger.debug("Ensure all required resources exist") - self.pre_deployer.ensure_config_resource_group_exists() - self.pre_deployer.ensure_config_publisher_exists() - self.pre_deployer.ensure_acr_artifact_store_exists() - self.pre_deployer.ensure_config_nsd_exists() - return self.pre_deployer.do_config_artifact_manifests_exist() - - def deploy_bicep_template( - self, bicep_template_path: str, parameters: Dict[Any, Any] - ) -> Any: - """ - Deploy a bicep template. - - :param bicep_template_path: Path to the bicep template - :param parameters: Parameters for the bicep template - :return Any output that the template produces - """ - logger.info("Deploy %s", bicep_template_path) - logger.debug("Parameters: %s", parameters) - arm_template_json = self.convert_bicep_to_arm(bicep_template_path) - - return self.validate_and_deploy_arm_template( - arm_template_json, parameters, self.config.publisher_resource_group_name - ) - - def resource_exists(self, resource_name: str) -> bool: - """ - Determine if a resource with the given name exists. - - :param resource_name: The name of the resource to check. - """ - logger.debug("Check if %s exists", resource_name) - resources = self.api_clients.resource_client.resources.list_by_resource_group( - resource_group_name=self.config.publisher_resource_group_name - ) - - resource_exists = False - - for resource in resources: - if resource.name == resource_name: - resource_exists = True - break - - return resource_exists - - def validate_and_deploy_arm_template( - self, template: Any, parameters: Dict[Any, Any], resource_group: str - ) -> Any: - """ - Validate and deploy an individual ARM template. - - This ARM template will be created in the resource group passed in. - - :param template: The JSON contents of the template to deploy - :param parameters: The JSON contents of the parameters file - :param resource_group: The name of the resource group that has been deployed - - :return: Output dictionary from the bicep template. - :raise RuntimeError if validation or deploy fails - """ - # Get current time from the time module and remove all digits after the decimal - # point - current_time = str(time.time()).split(".", maxsplit=1)[0] - - # Add a timestamp to the deployment name to ensure it is unique - deployment_name = f"AOSM_CLI_deployment_{current_time}" - - # Validation is automatically re-attempted in live runs, but not in test - # playback, causing them to fail. This explicitly re-attempts validation to - # ensure the tests pass - validation_res = None - for validation_attempt in range(2): - try: - validation = ( - self.api_clients.resource_client.deployments.begin_validate( - resource_group_name=resource_group, - deployment_name=deployment_name, - parameters={ - "properties": { - "mode": "Incremental", - "template": template, - "parameters": parameters, - } - }, - ) - ) - validation_res = LongRunningOperation( - self.cli_ctx, "Validating ARM template..." - )(validation) - break - except Exception: # pylint: disable=broad-except - if validation_attempt == 1: - raise - - if not validation_res: - # Don't expect to hit this but it appeases mypy - raise RuntimeError(f"Validation of template {template} failed.") - - logger.debug("Validation Result %s", validation_res) - if validation_res.error: - # Validation failed so don't even try to deploy - logger.error( - ( - "Template for resource group %s has failed validation. The message" - " was: %s. See logs for additional details." - ), - resource_group, - validation_res.error.message, - ) - logger.debug( - ( - "Template for resource group %s failed validation." - " Full error details: %s" - ), - resource_group, - validation_res.error, - ) - raise RuntimeError("Azure template validation failed.") - - # Validation succeeded so proceed with deployment - logger.debug("Successfully validated resources for %s", resource_group) - - poller = self.api_clients.resource_client.deployments.begin_create_or_update( - resource_group_name=resource_group, - deployment_name=deployment_name, - parameters={ - "properties": { - "mode": "Incremental", - "template": template, - "parameters": parameters, - } - }, - ) - logger.debug(poller) - - # Wait for the deployment to complete and get the outputs - deployment: DeploymentExtended = LongRunningOperation( - self.cli_ctx, "Deploying ARM template" - )(poller) - logger.debug("Finished deploying") - - if deployment.properties is not None: - depl_props = deployment.properties - else: - raise RuntimeError("The deployment has no properties.\nAborting") - logger.debug("Deployed: %s %s %s", deployment.name, deployment.id, depl_props) - - if depl_props.provisioning_state != "Succeeded": - logger.debug("Failed to provision: %s", depl_props) - raise RuntimeError( - "Deploy of template to resource group" - f" {resource_group} proceeded but the provisioning" - f" state returned is {depl_props.provisioning_state}." - "\nAborting" - ) - logger.debug( - "Provisioning state of deployment %s : %s", - resource_group, - depl_props.provisioning_state, - ) - - return depl_props.outputs - - @staticmethod - def convert_bicep_to_arm(bicep_template_path: str) -> Any: - """ - Convert a bicep template into an ARM template. - - :param bicep_template_path: The path to the bicep template to be converted - :return: Output dictionary from the bicep template. - """ - logger.debug("Converting %s to ARM template", bicep_template_path) - - with tempfile.TemporaryDirectory() as tmpdir: - bicep_filename = os.path.basename(bicep_template_path) - arm_template_name = bicep_filename.replace(".bicep", ".json") - - try: - bicep_output = subprocess.run( # noqa - [ - str(shutil.which("az")), - "bicep", - "build", - "--file", - bicep_template_path, - "--outfile", - os.path.join(tmpdir, arm_template_name), - ], - check=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - logger.debug("az bicep output: %s", str(bicep_output)) - except subprocess.CalledProcessError as err: - logger.error( - ( - "ARM template compilation failed! See logs for full " - "output. The failing command was %s" - ), - err.cmd, - ) - logger.debug("bicep build stdout: %s", err.stdout) - logger.debug("bicep build stderr: %s", err.stderr) - raise - - with open( - os.path.join(tmpdir, arm_template_name), "r", encoding="utf-8" - ) as template_file: - arm_json = json.loads(template_file.read()) - - return arm_json diff --git a/src/aosm/azext_aosm/old/deploy/pre_deploy.py b/src/aosm/azext_aosm/old/deploy/pre_deploy.py deleted file mode 100644 index b34a4929cc9..00000000000 --- a/src/aosm/azext_aosm/old/deploy/pre_deploy.py +++ /dev/null @@ -1,444 +0,0 @@ -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- -"""Contains class for deploying resources required by NFDs/NSDs via the SDK.""" - -from typing import Optional - -from azure.cli.core.azclierror import AzCLIError -from azure.cli.core.commands import LongRunningOperation -from azure.core import exceptions as azure_exceptions -from azure.mgmt.resource.resources.models import ResourceGroup -from knack.log import get_logger - -from azext_aosm._configuration import ( - Configuration, - VNFConfiguration, -) -from azext_aosm.util.management_clients import ApiClients -from azext_aosm.vendored_sdks.models import ( - ArtifactStore, - ArtifactStorePropertiesFormat, - ArtifactStoreType, - NetworkFunctionDefinitionGroup, - NetworkServiceDesignGroup, - ProvisioningState, - Publisher, - PublisherPropertiesFormat, - ManagedServiceIdentity -) - -logger = get_logger(__name__) - - -class PreDeployerViaSDK: - """ - A class for checking or publishing resources required by NFDs/NSDs. - - Uses the SDK to deploy rather than ARM, as the objects it deploys are not complex. - """ - - def __init__( - self, - api_clients: ApiClients, - config: Configuration, - cli_ctx: Optional[object] = None, - ) -> None: - """ - Initializes a new instance of the Deployer class. - - :param api_clients: ApiClients object for AOSM and ResourceManagement - :param config: The configuration for this NF - :param cli_ctx: The CLI context. Used with all LongRunningOperation calls. - """ - - self.api_clients = api_clients - self.config = config - self.cli_ctx = cli_ctx - - def ensure_resource_group_exists(self, resource_group_name: str) -> None: - """ - Checks whether a particular resource group exists on the subscription, and - attempts to create it if not. - - :param resource_group_name: The name of the resource group - """ - if not self.api_clients.resource_client.resource_groups.check_existence( - resource_group_name - ): - logger.info("RG %s not found. Create it.", resource_group_name) - print(f"Creating resource group {resource_group_name}.") - rg_params: ResourceGroup = ResourceGroup(location=self.config.location) - self.api_clients.resource_client.resource_groups.create_or_update( - resource_group_name, rg_params - ) - else: - print(f"Resource group {resource_group_name} exists.") - self.api_clients.resource_client.resource_groups.get(resource_group_name) - - def ensure_config_resource_group_exists(self) -> None: - """ - Ensures that the resource group exists. - - Finds the parameters from self.config - """ - self.ensure_resource_group_exists(self.config.publisher_resource_group_name) - - def ensure_publisher_exists( - self, resource_group_name: str, publisher_name: str, location: str - ) -> None: - """ - Ensures that the publisher exists in the resource group. - - :param resource_group_name: The name of the resource group. - :type resource_group_name: str - :param publisher_name: The name of the publisher. - :type publisher_name: str - :param location: The location of the publisher. - :type location: str - """ - - try: - publisher = self.api_clients.aosm_client.publishers.get( - resource_group_name, publisher_name - ) - print( - f"Publisher {publisher.name} exists in resource group" - f" {resource_group_name}" - ) - except azure_exceptions.ResourceNotFoundError: - # Create the publisher with default SAMI and private scope - logger.info("Creating publisher %s if it does not exist", publisher_name) - print( - f"Creating publisher {publisher_name} in resource group" - f" {resource_group_name}" - ) - publisher_properties = PublisherPropertiesFormat(scope="Private") - publisher_sami = ManagedServiceIdentity(type="SystemAssigned") - poller = self.api_clients.aosm_client.publishers.begin_create_or_update( - resource_group_name=resource_group_name, - publisher_name=publisher_name, - parameters=Publisher(location=location, properties=publisher_properties, identity=publisher_sami), - ) - LongRunningOperation(self.cli_ctx, "Creating publisher...")(poller) - - def ensure_config_publisher_exists(self) -> None: - """ - Ensures that the publisher exists in the resource group. - - Finds the parameters from self.config - """ - self.ensure_publisher_exists( - resource_group_name=self.config.publisher_resource_group_name, - publisher_name=self.config.publisher_name, - location=self.config.location, - ) - - def ensure_artifact_store_exists( - self, - resource_group_name: str, - publisher_name: str, - artifact_store_name: str, - artifact_store_type: ArtifactStoreType, - location: str, - ) -> None: - """ - Ensures that the artifact store exists in the resource group. - - :param resource_group_name: The name of the resource group. - :type resource_group_name: str - :param publisher_name: The name of the publisher. - :type publisher_name: str - :param artifact_store_name: The name of the artifact store. - :type artifact_store_name: str - :param artifact_store_type: The type of the artifact store. - :type artifact_store_type: ArtifactStoreType - :param location: The location of the artifact store. - :type location: str - """ - logger.info( - "Creating artifact store %s if it does not exist", - artifact_store_name, - ) - try: - self.api_clients.aosm_client.artifact_stores.get( - resource_group_name=resource_group_name, - publisher_name=publisher_name, - artifact_store_name=artifact_store_name, - ) - print( - f"Artifact store {artifact_store_name} exists in resource group" - f" {resource_group_name}" - ) - except azure_exceptions.ResourceNotFoundError as ex: - print( - f"Create Artifact Store {artifact_store_name} of type" - f" {artifact_store_type}" - ) - artifact_store_properties = ArtifactStorePropertiesFormat(store_type=artifact_store_type) - poller = ( - self.api_clients.aosm_client.artifact_stores.begin_create_or_update( - resource_group_name=resource_group_name, - publisher_name=publisher_name, - artifact_store_name=artifact_store_name, - parameters=ArtifactStore( - location=location, - properties=artifact_store_properties, - ), - ) - ) - # LongRunningOperation waits for provisioning state Succeeded before - # carrying on - artifactStore: ArtifactStore = LongRunningOperation( - self.cli_ctx, "Creating Artifact Store..." - )(poller) - - if artifactStore.properties.provisioning_state != ProvisioningState.SUCCEEDED: - logger.debug("Failed to provision artifact store: %s", artifactStore.name) - raise RuntimeError( - "Creation of artifact store proceeded, but the provisioning" - f" state returned is {artifactStore.properties.provisioning_state}. " - "\nAborting" - ) from ex - logger.debug( - "Provisioning state of %s: %s", - artifact_store_name, - artifactStore.properties.provisioning_state, - ) - - def ensure_acr_artifact_store_exists(self) -> None: - """ - Ensures that the ACR Artifact store exists. - - Finds the parameters from self.config - """ - self.ensure_artifact_store_exists( - self.config.publisher_resource_group_name, - self.config.publisher_name, - self.config.acr_artifact_store_name, - ArtifactStoreType.AZURE_CONTAINER_REGISTRY, # type: ignore - self.config.location, - ) - - def ensure_sa_artifact_store_exists(self) -> None: - """ - Ensures that the Storage Account Artifact store for VNF exists. - - Finds the parameters from self.config - """ - if not isinstance(self.config, VNFConfiguration): - # This is a coding error but worth checking. - raise AzCLIError( - "Cannot check that the storage account artifact store exists as " - "the configuration file doesn't map to VNFConfiguration" - ) - - self.ensure_artifact_store_exists( - self.config.publisher_resource_group_name, - self.config.publisher_name, - self.config.blob_artifact_store_name, - ArtifactStoreType.AZURE_STORAGE_ACCOUNT, # type: ignore - self.config.location, - ) - - def ensure_nfdg_exists( - self, - resource_group_name: str, - publisher_name: str, - nfdg_name: str, - location: str, - ): - """ - Ensures that the network function definition group exists in the resource group. - - :param resource_group_name: The name of the resource group. - :type resource_group_name: str - :param publisher_name: The name of the publisher. - :type publisher_name: str - :param nfdg_name: The name of the network function definition group. - :type nfdg_name: str - :param location: The location of the network function definition group. - :type location: str - """ - - logger.info( - "Creating network function definition group %s if it does not exist", - nfdg_name, - ) - - try: - self.api_clients.aosm_client.network_function_definition_groups.get( - resource_group_name=resource_group_name, - publisher_name=publisher_name, - network_function_definition_group_name=nfdg_name, - ) - print( - f"Network function definition group {nfdg_name} exists in resource" - f" group {resource_group_name}" - ) - except azure_exceptions.ResourceNotFoundError as ex: - print(f"Create Network Function Definition Group {nfdg_name}") - poller = self.api_clients.aosm_client.network_function_definition_groups.begin_create_or_update( - resource_group_name=resource_group_name, - publisher_name=publisher_name, - network_function_definition_group_name=nfdg_name, - parameters=NetworkFunctionDefinitionGroup(location=location), - ) - - # Asking for result waits for provisioning state Succeeded before carrying - # on - nfdg: NetworkFunctionDefinitionGroup = LongRunningOperation( - self.cli_ctx, "Creating Network Function Definition Group..." - )(poller) - - if nfdg.properties.provisioning_state != ProvisioningState.SUCCEEDED: - logger.debug( - "Failed to provision Network Function Definition Group: %s", - nfdg.name, - ) - raise RuntimeError( - "Creation of Network Function Definition Group proceeded, but the" - f" provisioning state returned is {nfdg.properties.provisioning_state}." - " \nAborting" - ) from ex - logger.debug( - "Provisioning state of %s: %s", nfdg_name, nfdg.properties.provisioning_state - ) - - def ensure_config_nfdg_exists( - self, - ): - """ - Ensures that the Network Function Definition Group exists. - - Finds the parameters from self.config - """ - self.ensure_nfdg_exists( - self.config.publisher_resource_group_name, - self.config.publisher_name, - self.config.nfdg_name, - self.config.location, - ) - - def ensure_config_nsd_exists( - self, - ): - """ - Ensures that the Network Service Design exists. - - Finds the parameters from self.config - """ - self.ensure_nsd_exists( - self.config.publisher_resource_group_name, - self.config.publisher_name, - self.config.nsd_name, - self.config.location, - ) - - def does_artifact_manifest_exist( - self, rg_name: str, publisher_name: str, store_name: str, manifest_name: str - ) -> bool: - try: - self.api_clients.aosm_client.artifact_manifests.get( - resource_group_name=rg_name, - publisher_name=publisher_name, - artifact_store_name=store_name, - artifact_manifest_name=manifest_name, - ) - logger.debug("Artifact manifest %s exists", manifest_name) - return True - except azure_exceptions.ResourceNotFoundError: - logger.debug("Artifact manifest %s does not exist", manifest_name) - return False - - def do_config_artifact_manifests_exist( - self, - ) -> bool: - """Returns True if all required manifests exist, False otherwise.""" - all_acr_mannys_exist = True - any_acr_mannys_exist: bool = not self.config.acr_manifest_names - - for manifest in self.config.acr_manifest_names: - acr_manny_exists: bool = self.does_artifact_manifest_exist( - rg_name=self.config.publisher_resource_group_name, - publisher_name=self.config.publisher_name, - store_name=self.config.acr_artifact_store_name, - manifest_name=manifest, - ) - all_acr_mannys_exist &= acr_manny_exists - any_acr_mannys_exist |= acr_manny_exists - - if isinstance(self.config, VNFConfiguration): - sa_manny_exists: bool = self.does_artifact_manifest_exist( - rg_name=self.config.publisher_resource_group_name, - publisher_name=self.config.publisher_name, - store_name=self.config.blob_artifact_store_name, - manifest_name=self.config.sa_manifest_name, - ) - if all_acr_mannys_exist and sa_manny_exists: - return True - if any_acr_mannys_exist or sa_manny_exists: - raise AzCLIError( - "Only a subset of artifact manifest exists. Cannot proceed. Please delete" - " the NFDV or NSDV as appropriate using the `az aosm nfd delete` or " - "`az aosm nsd delete` command." - ) - return False - - return all_acr_mannys_exist - - def ensure_nsd_exists( - self, - resource_group_name: str, - publisher_name: str, - nsd_name: str, - location: str, - ): - """ - Ensures that the network service design group exists in the resource group. - - :param resource_group_name: The name of the resource group. - :type resource_group_name: str - :param publisher_name: The name of the publisher. - :type publisher_name: str - :param nsd_name: The name of the network service design group. - :type nsd_name: str - :param location: The location of the network service design group. - :type location: str - """ - print( - f"Creating Network Service Design {nsd_name} if it does not exist", - ) - logger.info( - "Creating Network Service Design %s if it does not exist", - nsd_name, - ) - poller = self.api_clients.aosm_client.network_service_design_groups.begin_create_or_update( - resource_group_name=resource_group_name, - publisher_name=publisher_name, - network_service_design_group_name=nsd_name, - parameters=NetworkServiceDesignGroup(location=location), - ) - LongRunningOperation(self.cli_ctx, "Creating Network Service Design...")(poller) - - def resource_exists_by_name(self, rg_name: str, resource_name: str) -> bool: - """ - Determine if a resource with the given name exists. No checking is done as - to the type. - - :param resource_name: The name of the resource to check. - """ - logger.debug("Check if %s exists", resource_name) - resources = self.api_clients.resource_client.resources.list_by_resource_group( - resource_group_name=rg_name - ) - - resource_exists = False - - for resource in resources: - if resource.name == resource_name: - resource_exists = True - break - - return resource_exists diff --git a/src/aosm/azext_aosm/old/generate_nfd/__init__.py b/src/aosm/azext_aosm/old/generate_nfd/__init__.py deleted file mode 100644 index 99c0f28cd71..00000000000 --- a/src/aosm/azext_aosm/old/generate_nfd/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -# ----------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# ----------------------------------------------------------------------------- diff --git a/src/aosm/azext_aosm/old/generate_nfd/cnf_nfd_generator.py b/src/aosm/azext_aosm/old/generate_nfd/cnf_nfd_generator.py deleted file mode 100644 index e3027debd0b..00000000000 --- a/src/aosm/azext_aosm/old/generate_nfd/cnf_nfd_generator.py +++ /dev/null @@ -1,850 +0,0 @@ -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- -"""Contains a class for generating CNF NFDs and associated resources.""" -import json -import re -import shutil -import tarfile -import tempfile -from dataclasses import dataclass -from pathlib import Path -from typing import Any, Dict, Iterator, List, Optional, Tuple - -import yaml -from azure.cli.core.azclierror import FileOperationError, InvalidTemplateError -from jinja2 import StrictUndefined, Template -from knack.log import get_logger - -from azext_aosm._configuration import CNFConfiguration, HelmPackageConfig -from azext_aosm.generate_nfd.nfd_generator_base import NFDGenerator -from azext_aosm.util.constants import ( - CNF_DEFINITION_BICEP_TEMPLATE_FILENAME, - CNF_DEFINITION_JINJA2_SOURCE_TEMPLATE_FILENAME, - CNF_MANIFEST_BICEP_TEMPLATE_FILENAME, - CNF_MANIFEST_JINJA2_SOURCE_TEMPLATE_FILENAME, - CNF_VALUES_SCHEMA_FILENAME, - CONFIG_MAPPINGS_DIR_NAME, - DEPLOYMENT_PARAMETER_MAPPING_REGEX, - DEPLOYMENT_PARAMETERS_FILENAME, - GENERATED_VALUES_MAPPINGS_DIR_NAME, - IMAGE_NAME_AND_VERSION_REGEX, - IMAGE_PATH_REGEX, - IMAGE_PULL_SECRETS_START_STRING, - IMAGE_START_STRING, - SCHEMA_PREFIX, - SCHEMAS_DIR_NAME, -) -from azext_aosm.util.utils import input_ack - -logger = get_logger(__name__) - - -@dataclass -class Artifact: - """Information about an artifact.""" - - name: str - version: str - - -@dataclass -class NFApplicationConfiguration: # pylint: disable=too-many-instance-attributes - name: str - chartName: str - chartVersion: str - releaseName: str - dependsOnProfile: List[str] - registryValuesPaths: List[str] - imagePullSecretsValuesPaths: List[str] - valueMappingsFile: str - - def __post_init__(self): - """Format the fields based on the NFDV validation rules.""" - self._format_name() - self._format_release_name() - - def _format_name(self): - """ - Format the name field. - - The name should start with a alphabetic character, have alphanumeric characters - or '-' in-between and end with alphanumerc character, and be less than 64 - characters long. See NfdVersionValidationHelper.cs in pez codebase - """ - # Replace any non (alphanumeric or '-') characters with '-' - self.name = re.sub("[^0-9a-zA-Z-]+", "-", self.name) - # Strip leading or trailing - - self.name = self.name.strip("-") - self.name = self.name[:64] - - if not self.name: - raise InvalidTemplateError( - "The name field of the NF application configuration for helm package " - f"{self.chartName} is empty after removing invalid characters. " - "Valid characters are alphanumeric and '-'. Please fix this in the name" - " field for the helm package in your input config file." - ) - - def _format_release_name(self): - """ - Format release name. - - It must consist of lower case alphanumeric characters, '-' or '.', and must - start and end with an alphanumeric character See - AzureArcKubernetesRuleBuilderExtensions.cs and - AzureArcKubernetesNfValidationMessage.cs in pez codebase - """ - self.releaseName = self.releaseName.lower() - # Replace any non (alphanumeric or '-' or '.') characters with '-' - self.releaseName = re.sub("[^0-9a-z-.]+", "-", self.releaseName) - # Strip leading - or . - self.releaseName = self.releaseName.strip("-") - self.releaseName = self.releaseName.strip(".") - if not self.releaseName: - raise InvalidTemplateError( - "The releaseName field of the NF application configuration for helm " - f"chart {self.chartName} is empty after formatting and removing invalid" - "characters. Valid characters are alphanumeric, -.' and '-' and the " - "releaseName must start and end with an alphanumeric character. The " - "value of this field is taken from Chart.yaml within the helm package. " - "Please fix up the helm package. Before removing invalid characters" - f", the releaseName was {self.chartName}." - ) - - -@dataclass -class ImageInfo: - parameter: List[str] - name: str - version: str - - -class CnfNfdGenerator(NFDGenerator): # pylint: disable=too-many-instance-attributes - """ - CNF NFD Generator. - - This takes a config file, and outputs: - - A bicep file for the NFDV - - Parameters files that are used by the NFDV bicep file, these are the - deployParameters and the mapping profiles of those deploy parameters - - A bicep file for the Artifact manifests - """ - - def __init__(self, config: CNFConfiguration, interactive: bool = False): - """ - Create a new CNF NFD Generator. - - Interactive parameter is only used if the user wants to generate the values - mapping file from the values.yaml in the helm package, and also requires the - mapping file in config to be blank. - """ - self.config = config - self.nfd_jinja2_template_path = ( - Path(__file__).parent - / "templates" - / CNF_DEFINITION_JINJA2_SOURCE_TEMPLATE_FILENAME - ) - self.manifest_jinja2_template_path = ( - Path(__file__).parent - / "templates" - / CNF_MANIFEST_JINJA2_SOURCE_TEMPLATE_FILENAME - ) - self.output_directory: Path = self.config.output_directory_for_build - self._cnfd_bicep_path = ( - self.output_directory / CNF_DEFINITION_BICEP_TEMPLATE_FILENAME - ) - self._tmp_dir: Optional[Path] = None - - self.artifacts: List[Artifact] = [] - self.nf_application_configurations: List[NFApplicationConfiguration] = [] - self.deployment_parameter_schema: Dict[str, Any] = SCHEMA_PREFIX - self.interactive = interactive - - def generate_nfd(self) -> None: - """Generate a CNF NFD which comprises a group, an Artifact Manifest and an NFDV.""" - - # Create temporary directory. - with tempfile.TemporaryDirectory() as tmpdirname: - self._tmp_dir = Path(tmpdirname) - try: - for helm_package in self.config.helm_packages: - # Unpack the chart into the tmp directory - assert isinstance(helm_package, HelmPackageConfig) - - self._extract_chart(Path(helm_package.path_to_chart)) - - # TODO: Validate charts - - # Create a chart mapping schema if none has been passed in. - if not helm_package.path_to_mappings: - self._generate_chart_value_mappings(helm_package) - - # Get schema for each chart - # (extract mappings and relevant parts of the schema) - # + Add that schema to the big schema. - self.deployment_parameter_schema["properties"].update( - self._get_chart_mapping_schema(helm_package) - ) - - # Get all image line matches for files in the chart. - # Do this here so we don't have to do it multiple times. - image_line_matches = self._find_image_parameter_from_chart( - helm_package - ) - - # Creates a flattened list of image registry paths to prevent set error - image_registry_paths: List[str] = [] - for image_info in image_line_matches: - image_registry_paths += image_info.parameter - - # Generate the NF application configuration for the chart - # passed to jinja2 renderer to render bicep template - self.nf_application_configurations.append( - self._generate_nf_application_config( - helm_package, - image_registry_paths, - self._find_image_pull_secrets_parameter_from_chart( - helm_package - ), - ) - ) - # Workout the list of artifacts for the chart and - # update the list for the NFD with any unique artifacts. - chart_artifacts = self._get_artifact_list( - helm_package, image_line_matches - ) - self.artifacts += [ - a for a in chart_artifacts if a not in self.artifacts - ] - self._write_nfd_bicep_file() - self._write_schema_to_file() - self._write_manifest_bicep_contents() - self._copy_to_output_directory() - print( - f"Generated NFD bicep template created in {self.output_directory}" - ) - print( - "Please review these templates. When you are happy with them run " - "`az aosm nfd publish` with the same arguments." - ) - except InvalidTemplateError as e: - raise e - - @property - def nfd_bicep_path(self) -> Optional[Path]: - """Returns the path to the bicep file for the NFD if it has been created.""" - if self._cnfd_bicep_path.exists(): - return self._cnfd_bicep_path - return None - - def _extract_chart(self, path: Path) -> None: - """ - Extract the chart into the tmp directory. - - :param path: The path to helm package - """ - assert self._tmp_dir - - logger.debug("Extracting helm package %s", path) - - file_extension = path.suffix - if file_extension in (".gz", ".tgz"): - with tarfile.open(path, "r:gz") as tar: - tar.extractall(path=self._tmp_dir) - - elif file_extension == ".tar": - with tarfile.open(path, "r:") as tar: - tar.extractall(path=self._tmp_dir) - - else: - raise InvalidTemplateError( - f"ERROR: The helm package '{path}' is not a .tgz, .tar or .tar.gz file." - " Please fix this and run the command again." - ) - - def _generate_chart_value_mappings(self, helm_package: HelmPackageConfig) -> None: - """ - Optional function to create a chart value mappings file with every value being a deployParameter. - - Expected use when a helm chart is very simple and user wants every value to be a - deployment parameter. - """ - assert self._tmp_dir - logger.debug( - "Creating chart value mappings file for %s", helm_package.path_to_chart - ) - print(f"Creating chart value mappings file for {helm_package.path_to_chart}.") - - # Get all the values files in the chart - top_level_values_yaml = self._read_top_level_values_yaml(helm_package) - - mapping_to_write = self._replace_values_with_deploy_params( - top_level_values_yaml, None - ) - - # Write the mapping to a file - mapping_directory: Path = self._tmp_dir / GENERATED_VALUES_MAPPINGS_DIR_NAME - mapping_directory.mkdir(exist_ok=True) - mapping_filepath = ( - mapping_directory / f"{helm_package.name}-generated-mapping.yaml" - ) - with open(mapping_filepath, "w", encoding="UTF-8") as mapping_file: - yaml.dump(mapping_to_write, mapping_file) - - # Update the config that points to the mapping file - helm_package.path_to_mappings = str(mapping_filepath) - - def _read_top_level_values_yaml( - self, helm_package: HelmPackageConfig - ) -> Dict[str, Any]: - """ - Return a dictionary of the values.yaml|yml read from the root of the helm package. - - :param helm_package: The helm package to look in - :type helm_package: HelmPackageConfig - :raises FileOperationError: if no values.yaml|yml found - :return: A dictionary of the yaml read from the file - :rtype: Dict[str, Any] - """ - assert self._tmp_dir - for file in Path(self._tmp_dir / helm_package.name).iterdir(): - if file.name in ("values.yaml", "values.yml"): - with file.open(encoding="UTF-8") as values_file: - values_yaml = yaml.safe_load(values_file) - return values_yaml - - raise FileOperationError( - "Cannot find top level values.yaml/.yml file in Helm package." - ) - - def _write_manifest_bicep_contents(self) -> None: - """Write the bicep file for the Artifact Manifest to the temp directory.""" - assert self._tmp_dir - - with open(self.manifest_jinja2_template_path, "r", encoding="UTF-8") as f: - template: Template = Template( - f.read(), - undefined=StrictUndefined, - ) - - bicep_contents: str = template.render( - artifacts=self.artifacts, - ) - - path = self._tmp_dir / CNF_MANIFEST_BICEP_TEMPLATE_FILENAME - with open(path, "w", encoding="utf-8") as f: - f.write(bicep_contents) - - logger.info("Created artifact manifest bicep template: %s", path) - - def _write_nfd_bicep_file(self) -> None: - """Write the bicep file for the NFD to the temp directory.""" - assert self._tmp_dir - with open(self.nfd_jinja2_template_path, "r", encoding="UTF-8") as f: - template: Template = Template( - f.read(), - undefined=StrictUndefined, - ) - - bicep_contents: str = template.render( - nf_application_configurations=self.nf_application_configurations, - ) - - path = self._tmp_dir / CNF_DEFINITION_BICEP_TEMPLATE_FILENAME - with open(path, "w", encoding="utf-8") as f: - f.write(bicep_contents) - - logger.info("Created NFD bicep template: %s", path) - - def _write_schema_to_file(self) -> None: - """Write the schema to file deploymentParameters.json to the temp directory.""" - logger.debug("Create deploymentParameters.json") - assert self._tmp_dir - - full_schema = self._tmp_dir / DEPLOYMENT_PARAMETERS_FILENAME - with open(full_schema, "w", encoding="UTF-8") as f: - json.dump(self.deployment_parameter_schema, f, indent=4) - - logger.debug("%s created", full_schema) - - def _copy_to_output_directory(self) -> None: - """ - Copy files from the temp directory to the output directory. - - Files are the config mappings, schema and bicep templates (artifact manifest and - NFDV). - """ - assert self._tmp_dir - - logger.info("Create NFD bicep %s", self.output_directory) - - Path(self.output_directory / SCHEMAS_DIR_NAME).mkdir( - parents=True, exist_ok=True - ) - - # Copy the nfd and the manifest bicep files to the output directory - shutil.copy( - self._tmp_dir / CNF_DEFINITION_BICEP_TEMPLATE_FILENAME, - self.output_directory, - ) - shutil.copy( - self._tmp_dir / CNF_MANIFEST_BICEP_TEMPLATE_FILENAME, self.output_directory - ) - - # Copy any generated values mappings YAML files to the corresponding directory in - # the output directory so that the user can edit them and re-run the build if - # required - if Path(self._tmp_dir / GENERATED_VALUES_MAPPINGS_DIR_NAME).exists(): - shutil.copytree( - self._tmp_dir / GENERATED_VALUES_MAPPINGS_DIR_NAME, - self.output_directory / GENERATED_VALUES_MAPPINGS_DIR_NAME, - ) - - # Copy the JSON config mappings and deploymentParameters schema that are used - # for the NFD to the output directory - shutil.copytree( - self._tmp_dir / CONFIG_MAPPINGS_DIR_NAME, - self.output_directory / CONFIG_MAPPINGS_DIR_NAME, - dirs_exist_ok=True, - ) - shutil.copy( - self._tmp_dir / DEPLOYMENT_PARAMETERS_FILENAME, - self.output_directory / SCHEMAS_DIR_NAME / DEPLOYMENT_PARAMETERS_FILENAME, - ) - - logger.info("Copied files to %s", self.output_directory) - - def _generate_nf_application_config( - self, - helm_package: HelmPackageConfig, - image_registry_path: List[str], - image_pull_secret_line_matches: List[str], - ) -> NFApplicationConfiguration: - """Generate NF application config.""" - (name, version) = self._get_chart_name_and_version(helm_package) - - registry_values_paths = set(image_registry_path) - image_pull_secrets_values_paths = set(image_pull_secret_line_matches) - - return NFApplicationConfiguration( - name=helm_package.name, - chartName=name, - chartVersion=version, - releaseName=name, - dependsOnProfile=helm_package.depends_on, - registryValuesPaths=list(registry_values_paths), - imagePullSecretsValuesPaths=list(image_pull_secrets_values_paths), - valueMappingsFile=self._jsonify_value_mappings(helm_package), - ) - - @staticmethod - def _find_yaml_files(directory: Path) -> Iterator[Path]: - """ - Find all yaml files recursively in given directory. - - :param directory: The directory to search. - """ - yield from directory.glob("**/*.yaml") - yield from directory.glob("**/*.yml") - - def _find_image_parameter_from_chart( - self, helm_package_config: HelmPackageConfig - ) -> List[ImageInfo]: - """ - Find pattern matches in Helm chart for the names of the image parameters. - - :param helm_package: The helm package config. - - Returns list of tuples containing the list of image - paths and the name and version of the image. e.g. (Values.foo.bar.repoPath, foo, - 1.2.3) - """ - assert self._tmp_dir - chart_dir = self._tmp_dir / helm_package_config.name - matches = [] - path = [] - - for file in self._find_yaml_files(chart_dir): - with open(file, "r", encoding="UTF-8") as f: - logger.debug("Searching for %s in %s", IMAGE_START_STRING, file) - for line in f: - if IMAGE_START_STRING in line: - logger.debug("Found %s in %s", IMAGE_START_STRING, line) - path = re.findall(IMAGE_PATH_REGEX, line) - - # If "image:", search for chart name and version - name_and_version = re.search(IMAGE_NAME_AND_VERSION_REGEX, line) - logger.debug( - "Regex match for name and version is %s", - name_and_version, - ) - - if name_and_version and len(name_and_version.groups()) == 2: - logger.debug( - "Found image name and version %s %s", - name_and_version.group("name"), - name_and_version.group("version"), - ) - matches.append( - ImageInfo( - path, - name_and_version.group("name"), - name_and_version.group("version"), - ) - ) - else: - logger.debug("No image name and version found") - return matches - - def _find_image_pull_secrets_parameter_from_chart( - self, helm_package_config: HelmPackageConfig - ) -> List[str]: - """ - Find pattern matches in Helm chart for the ImagePullSecrets parameter. - - :param helm_package: The helm package config. - - Returns list of lists containing image pull - secrets paths, e.g. Values.foo.bar.imagePullSecret - """ - assert self._tmp_dir - chart_dir = self._tmp_dir / helm_package_config.name - matches = [] - path = [] - - for file in self._find_yaml_files(chart_dir): - with open(file, "r", encoding="UTF-8") as f: - logger.debug( - "Searching for %s in %s", IMAGE_PULL_SECRETS_START_STRING, file - ) - for line in f: - if IMAGE_PULL_SECRETS_START_STRING in line: - logger.debug( - "Found %s in %s", IMAGE_PULL_SECRETS_START_STRING, line - ) - path = re.findall(IMAGE_PATH_REGEX, line) - matches += path - return matches - - def _get_artifact_list( - self, - helm_package: HelmPackageConfig, - image_line_matches: List[ImageInfo], - ) -> List[Artifact]: - """ - Get the list of artifacts for the chart. - - :param helm_package: The helm package config. - :param image_line_matches: The list of image line matches. - """ - artifact_list = [] - (name, version) = self._get_chart_name_and_version(helm_package) - helm_artifact = Artifact(name, version) - - artifact_list.append(helm_artifact) - for image_info in image_line_matches: - artifact_list.append(Artifact(image_info.name, image_info.version)) - - return artifact_list - - def _get_chart_mapping_schema( - self, helm_package: HelmPackageConfig - ) -> Dict[Any, Any]: - """ - Get the schema for the non default values (those with {deploymentParameter...}). - Based on the user provided values schema. - - param helm_package: The helm package config. - """ - assert self._tmp_dir - logger.debug("Get chart mapping schema for %s", helm_package.name) - - mappings_path = helm_package.path_to_mappings - values_schema = self._tmp_dir / helm_package.name / CNF_VALUES_SCHEMA_FILENAME - if not Path(mappings_path).exists(): - raise InvalidTemplateError( - f"ERROR: The helm package '{helm_package.name}' does not have a valid values" - " mappings file. The file at '{helm_package.path_to_mappings}' does not exist." - "\nPlease fix this and run the command again." - ) - if not values_schema.exists(): - raise InvalidTemplateError( - f"ERROR: The helm package '{helm_package.name}' is missing {CNF_VALUES_SCHEMA_FILENAME}." - "\nPlease fix this and run the command again." - ) - - with open(mappings_path, "r", encoding="utf-8") as stream: - values_data = yaml.load(stream, Loader=yaml.SafeLoader) - - with open(values_schema, "r", encoding="utf-8") as f: - schema_data = json.load(f) - - try: - deploy_params_dict = self.traverse_dict( - values_data, DEPLOYMENT_PARAMETER_MAPPING_REGEX - ) - logger.debug("Deploy params dict is %s", deploy_params_dict) - new_schema = self.search_schema(deploy_params_dict, schema_data) - except KeyError as e: - raise InvalidTemplateError( - "ERROR: There is a problem with your schema or values for the helm" - f" package '{helm_package.name}'." - "\nPlease fix this and run the command again." - ) from e - - logger.debug("Generated chart mapping schema for %s", helm_package.name) - return new_schema - - @staticmethod - def traverse_dict( - dict_to_search: Dict[Any, Any], target_regex: str - ) -> Dict[str, List[str]]: - """ - Traverse the dictionary provided and return a dictionary of all the values that match the target regex, - with the key being the deploy parameter and the value being the path (as a list) to the value. - e.g. {"foo": ["global", "foo", "bar"]} - - :param d: The dictionary to traverse. - :param target: The regex to search for. - """ - - # pylint: disable=too-many-nested-blocks - @dataclass - class DictNode: - # The dictionary under this node - sub_dict: Dict[Any, Any] - - # The path to this node under the main dictionary - position_path: List[str] - - # Initialize the stack with the dictionary and an empty path - stack: List[DictNode] = [DictNode(dict_to_search, [])] - result = {} # Initialize empty dictionary to store the results - while stack: # While there are still items in the stack - # Pop the last item from the stack and unpack it into node (the dictionary) and path - node = stack.pop() - - # For each key-value pair in the popped item - for key, value in node.sub_dict.items(): - # If the value is a dictionary - if isinstance(value, dict): - # Add the dictionary to the stack with the path - stack.append(DictNode(value, node.position_path + [key])) - - # If the value is a string + matches target regex - elif isinstance(value, str): - # Take the match i.e, foo from {deployParameter.foo} - match = re.search(target_regex, value) - - # Add it to the result dictionary with its path as the value - if match: - result[match.group(1)] = node.position_path + [key] - - elif isinstance(value, list): - logger.debug("Found a list %s", value) - for item in value: - logger.debug("Found an item %s", item) - - if isinstance(item, str): - match = re.search(target_regex, item) - - if match: - result[match.group(1)] = node.position_path + [key] - - elif isinstance(item, dict): - stack.append(DictNode(item, node.position_path + [key])) - - elif isinstance(item, list): - # We should fix this but for now just log a warning and - # carry on - logger.warning( - "Values mapping file contains a list of lists " - "at path %s, which this tool cannot parse. " - "Please check the output configMappings and schemas " - "files and check that they are as required.", - node.position_path + [key], - ) - return result - - @staticmethod - def search_schema( - deployParams_paths: Dict[str, List[str]], full_schema - ) -> Dict[str, Dict[str, str]]: - """ - Search through the provided schema for the types of the deployment parameters. - This assumes that the type of the key will be the type of the deployment parameter. - e.g. if foo: {deployParameter.bar} and foo is type string, then bar is type string. - - Returns a dictionary of the deployment parameters in the format: - {"foo": {"type": "string"}, "bar": {"type": "string"}} - - param deployParams_paths: a dictionary of all the deploy parameters to search for, - with the key being the deploy parameter and the value being the - path to the value. - e.g. {"foo": ["global", "foo", "bar"]} - param full_schema: The schema to search through. - """ - new_schema = {} - no_schema_list = [] - for deploy_param, path_list in deployParams_paths.items(): - logger.debug( - "Searching for %s in schema at path %s", deploy_param, path_list - ) - node = full_schema - for path in path_list: - if "properties" in node.keys(): - logger.debug( - "Searching properties for %s in schema at path %s", - deploy_param, - path, - ) - node = node["properties"][path] - else: - logger.debug("No schema node found for %s", deploy_param) - no_schema_list.append(deploy_param) - new_schema.update({deploy_param: {"type": "string"}}) - if deploy_param not in new_schema: - param_type = node.get("type", None) - if param_type == "array": - # If the type is an array, we need to get the type of the items. - # (This currently only supports a single type, not a list of types. - # If a list is provided, we default to string.) - array_item_schema = node.get("items", {}) - if isinstance(array_item_schema, dict): - param_type = array_item_schema.get("type", None) - else: - logger.debug("Array item schema is not a dict (probably a list)") - param_type = None - if not param_type: - logger.debug("No type found for %s", deploy_param) - no_schema_list.append(deploy_param) - param_type = "string" - new_schema.update({deploy_param: {"type": param_type}}) - if no_schema_list: - logger.warning( - "No schema or type found for deployment parameter(s): %s", no_schema_list - ) - logger.warning( - "We default these parameters to type string. " - "Please edit schemas/%s in the output before publishing " - "if this is wrong", - DEPLOYMENT_PARAMETERS_FILENAME, - ) - return new_schema - - def _replace_values_with_deploy_params( - self, - values_yaml_dict, - param_prefix: Optional[str] = None, - ) -> Dict[Any, Any]: - """ - Given the yaml dictionary read from values.yaml, replace all the values with {deploymentParameter.keyname}. - - Thus creating a values mapping file if the user has not provided one in config. - """ - logger.debug("Replacing values with deploy parameters") - final_values_mapping_dict: Dict[Any, Any] = {} - for k, v in values_yaml_dict.items(): # pylint: disable=too-many-nested-blocks - # if value is a string and contains deployParameters. - logger.debug("Processing key %s", k) - param_name = k if param_prefix is None else f"{param_prefix}_{k}" - if isinstance(v, dict): - final_values_mapping_dict[k] = self._replace_values_with_deploy_params( - v, param_name - ) - elif isinstance(v, list): - final_values_mapping_dict[k] = [] - for index, item in enumerate(v): - param_name = ( - f"{param_prefix}_{k}_{index}" - if param_prefix - else f"{k}_{index}" - ) - if isinstance(item, dict): - final_values_mapping_dict[k].append( - self._replace_values_with_deploy_params(item, param_name) - ) - elif isinstance(item, (str, int, bool)) or not item: - if self.interactive: - if not input_ack( - "y", f"Expose parameter {param_name}? y/n " - ): - logger.debug("Excluding parameter %s", param_name) - final_values_mapping_dict[k].append(item) - continue - replacement_value = f"{{deployParameters.{param_name}}}" - final_values_mapping_dict[k].append(replacement_value) - else: - raise ValueError( - f"Found an unexpected type {type(item)} of key {k} in " - "values.yaml, cannot generate values mapping file." - ) - elif isinstance(v, (str, int, bool)) or not v: - # Replace the parameter with {deploymentParameter.keyname} - # If v is blank we don't know what type it is. Assuming it is an - # empty string (but do this after checking for dict and list) - if self.interactive: - # Interactive mode. Prompt user to include or exclude parameters - # This requires the enter key after the y/n input which isn't ideal - if not input_ack("y", f"Expose parameter {param_name}? y/n "): - logger.debug("Excluding parameter %s", param_name) - final_values_mapping_dict.update({k: v}) - continue - replacement_value = f"{{deployParameters.{param_name}}}" - - # add the schema for k (from the big schema) to the (smaller) schema - final_values_mapping_dict.update({k: replacement_value}) - else: - raise ValueError( - f"Found an unexpected type {type(v)} of key {k} in values.yaml, " - "cannot generate values mapping file." - ) - - return final_values_mapping_dict - - def _get_chart_name_and_version( - self, helm_package: HelmPackageConfig - ) -> Tuple[str, str]: - """Get the name and version of the chart.""" - assert self._tmp_dir - chart_path = self._tmp_dir / helm_package.name / "Chart.yaml" - - if not chart_path.exists(): - raise InvalidTemplateError( - f"There is no Chart.yaml file in the helm package '{helm_package.name}'. " - "\nPlease fix this and run the command again." - ) - - with open(chart_path, "r", encoding="utf-8") as f: - data = yaml.load(f, Loader=yaml.FullLoader) - if "name" in data and "version" in data: - chart_name = data["name"] - chart_version = data["version"] - else: - raise FileOperationError( - "A name or version is missing from Chart.yaml in the helm package" - f" '{helm_package.name}'." - "\nPlease fix this and run the command again." - ) - - return (chart_name, chart_version) - - def _jsonify_value_mappings(self, helm_package: HelmPackageConfig) -> str: - """Yaml->JSON values mapping file, then return the filename.""" - assert self._tmp_dir - mappings_yaml_file = helm_package.path_to_mappings - mappings_dir = self._tmp_dir / CONFIG_MAPPINGS_DIR_NAME - mappings_output_file = mappings_dir / f"{helm_package.name}-mappings.json" - - mappings_dir.mkdir(exist_ok=True) - - with open(mappings_yaml_file, "r", encoding="utf-8") as f: - data = yaml.load(f, Loader=yaml.FullLoader) - - with open(mappings_output_file, "w", encoding="utf-8") as file: - json.dump(data, file, indent=4) - - logger.debug("Generated parameter mappings for %s", helm_package.name) - return f"{helm_package.name}-mappings.json" diff --git a/src/aosm/azext_aosm/old/generate_nfd/nfd_generator_base.py b/src/aosm/azext_aosm/old/generate_nfd/nfd_generator_base.py deleted file mode 100644 index 4141665dfdf..00000000000 --- a/src/aosm/azext_aosm/old/generate_nfd/nfd_generator_base.py +++ /dev/null @@ -1,25 +0,0 @@ -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- -"""Contains a base class for generating NFDs.""" -from abc import ABC, abstractmethod -from pathlib import Path -from typing import Optional - -from knack.log import get_logger - -logger = get_logger(__name__) - - -class NFDGenerator(ABC): - """A class for generating an NFD from a config file.""" - - @abstractmethod - def generate_nfd(self) -> None: - ... - - @property - @abstractmethod - def nfd_bicep_path(self) -> Optional[Path]: - ... diff --git a/src/aosm/azext_aosm/old/generate_nfd/templates/cnfartifactmanifest.bicep.j2 b/src/aosm/azext_aosm/old/generate_nfd/templates/cnfartifactmanifest.bicep.j2 deleted file mode 100644 index 0f0eeb99767..00000000000 --- a/src/aosm/azext_aosm/old/generate_nfd/templates/cnfartifactmanifest.bicep.j2 +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright (c) Microsoft Corporation. - -// This file creates an Artifact Manifest for a CNF -param location string -@description('Name of an existing publisher, expected to be in the resource group where you deploy the template') -param publisherName string -@description('Name of an existing ACR-backed Artifact Store, deployed under the publisher.') -param acrArtifactStoreName string -@description('Name of the manifest to deploy for the ACR-backed Artifact Store') -param acrManifestName string - -// Created by the az aosm definition publish command before the template is deployed -resource publisher 'Microsoft.HybridNetwork/publishers@2023-09-01' existing = { - name: publisherName - scope: resourceGroup() -} - -// Created by the az aosm definition publish command before the template is deployed -resource acrArtifactStore 'Microsoft.HybridNetwork/publishers/artifactStores@2023-09-01' existing = { - parent: publisher - name: acrArtifactStoreName -} - -resource acrArtifactManifest 'Microsoft.Hybridnetwork/publishers/artifactStores/artifactManifests@2023-09-01' = { - parent: acrArtifactStore - name: acrManifestName - location: location - properties: { - artifacts: [ - {%- for artifact in artifacts %} - { - artifactName: '{{ artifact.name }}' - artifactType: 'OCIArtifact' - artifactVersion: '{{ artifact.version }}' - } - {%- endfor %} - ] - } -} diff --git a/src/aosm/azext_aosm/old/generate_nfd/templates/cnfdefinition.bicep.j2 b/src/aosm/azext_aosm/old/generate_nfd/templates/cnfdefinition.bicep.j2 deleted file mode 100644 index 4eeadfe6338..00000000000 --- a/src/aosm/azext_aosm/old/generate_nfd/templates/cnfdefinition.bicep.j2 +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright (c) Microsoft Corporation. - -// This file creates an NF definition for a CNF -param location string -@description('Name of an existing publisher, expected to be in the resource group where you deploy the template') -param publisherName string -@description('Name of an existing ACR-backed Artifact Store, deployed under the publisher.') -param acrArtifactStoreName string -@description('Name of an existing Network Function Definition Group') -param nfDefinitionGroup string -@description('The version of the NFDV you want to deploy, in format A.B.C') -param nfDefinitionVersion string - -// Created by the az aosm definition publish command before the template is deployed -resource publisher 'Microsoft.HybridNetwork/publishers@2023-09-01' existing = { - name: publisherName - scope: resourceGroup() -} - -// Created by the az aosm definition publish command before the template is deployed -resource acrArtifactStore 'Microsoft.HybridNetwork/publishers/artifactStores@2023-09-01' existing = { - parent: publisher - name: acrArtifactStoreName -} - -// Created by the az aosm definition publish command before the template is deployed -resource nfdg 'Microsoft.Hybridnetwork/publishers/networkfunctiondefinitiongroups@2023-09-01' existing = { - parent: publisher - name: nfDefinitionGroup -} - -resource nfdv 'Microsoft.Hybridnetwork/publishers/networkfunctiondefinitiongroups/networkfunctiondefinitionversions@2023-09-01' = { - parent: nfdg - name: nfDefinitionVersion - location: location - properties: { - // versionState should be changed to 'Active' once it is finalized. - versionState: 'Preview' - {#- Note that all paths in bicep must be specified using the forward slash #} - {#- (/) character even if running on Windows. #} - {#- See https://learn.microsoft.com/en-us/azure/azure-resource-manager/bicep/modules#local-file #} - deployParameters: string(loadJsonContent('schemas/deploymentParameters.json')) - networkFunctionType: 'ContainerizedNetworkFunction' - networkFunctionTemplate: { - nfviType: 'AzureArcKubernetes' - networkFunctionApplications: [ - {%- for configuration in nf_application_configurations %} - { - artifactType: 'HelmPackage' - name: '{{ configuration.name }}' - dependsOnProfile: { - installDependsOn: {{ configuration.dependsOnProfile }} - } - artifactProfile: { - artifactStore: { - id: acrArtifactStore.id - } - helmArtifactProfile: { - helmPackageName: '{{ configuration.chartName }}' - helmPackageVersionRange: '{{ configuration.chartVersion }}' - registryValuesPaths: {{ configuration.registryValuesPaths }} - imagePullSecretsValuesPaths: {{ configuration.imagePullSecretsValuesPaths }} - } - } - deployParametersMappingRuleProfile: { - applicationEnablement: 'Enabled' - helmMappingRuleProfile: { - releaseNamespace: '{{ configuration.releaseName }}' - releaseName: '{{ configuration.releaseName }}' - helmPackageVersion: '{{ configuration.chartVersion }}' - values: string(loadJsonContent('configMappings/{{ configuration.valueMappingsFile }}')) - } - } - } - {%- endfor %} - ] - } - } -} diff --git a/src/aosm/azext_aosm/old/generate_nfd/templates/vnfartifactmanifests.bicep b/src/aosm/azext_aosm/old/generate_nfd/templates/vnfartifactmanifests.bicep deleted file mode 100644 index 109cca9c766..00000000000 --- a/src/aosm/azext_aosm/old/generate_nfd/templates/vnfartifactmanifests.bicep +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright (c) Microsoft Corporation. - -// This file creates an NF definition for a VNF -param location string -@description('Name of an existing publisher, expected to be in the resource group where you deploy the template') -param publisherName string -@description('Name of an existing ACR-backed Artifact Store, deployed under the publisher.') -param acrArtifactStoreName string -@description('Name of an existing Storage Account-backed Artifact Store, deployed under the publisher.') -param saArtifactStoreName string -@description('Name of the manifest to deploy for the ACR-backed Artifact Store') -param acrManifestName string -@description('Name of the manifest to deploy for the Storage Account-backed Artifact Store') -param saManifestName string -@description('Name of Network Function. Used predominantly as a prefix for other variable names') -param nfName string -@description('The version that you want to name the NFM VHD artifact, in format A-B-C. e.g. 6-13-0') -param vhdVersion string -@description('The name under which to store the ARM template') -param armTemplateVersion string - -// Created by the az aosm definition publish command before the template is deployed -resource publisher 'Microsoft.HybridNetwork/publishers@2023-09-01' existing = { - name: publisherName - scope: resourceGroup() -} - -// Created by the az aosm definition publish command before the template is deployed -resource acrArtifactStore 'Microsoft.HybridNetwork/publishers/artifactStores@2023-09-01' existing = { - parent: publisher - name: acrArtifactStoreName -} - -// Created by the az aosm definition publish command before the template is deployed -resource saArtifactStore 'Microsoft.HybridNetwork/publishers/artifactStores@2023-09-01' existing = { - parent: publisher - name: saArtifactStoreName -} - -resource saArtifactManifest 'Microsoft.Hybridnetwork/publishers/artifactStores/artifactManifests@2023-09-01' = { - parent: saArtifactStore - name: saManifestName - location: location - properties: { - artifacts: [ - { - artifactName: '${nfName}-vhd' - artifactType: 'VhdImageFile' - artifactVersion: vhdVersion - } - ] - } -} - -resource acrArtifactManifest 'Microsoft.Hybridnetwork/publishers/artifactStores/artifactManifests@2023-09-01' = { - parent: acrArtifactStore - name: acrManifestName - location: location - properties: { - artifacts: [ - { - artifactName: '${nfName}-arm-template' - artifactType: 'ArmTemplate' - artifactVersion: armTemplateVersion - } - ] - } -} diff --git a/src/aosm/azext_aosm/old/generate_nfd/templates/vnfdefinition.bicep b/src/aosm/azext_aosm/old/generate_nfd/templates/vnfdefinition.bicep deleted file mode 100644 index 9deaeffd182..00000000000 --- a/src/aosm/azext_aosm/old/generate_nfd/templates/vnfdefinition.bicep +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright (c) Microsoft Corporation. - -// This file creates an NF definition for a VNF -param location string -@description('Name of an existing publisher, expected to be in the resource group where you deploy the template') -param publisherName string -@description('Name of an existing ACR-backed Artifact Store, deployed under the publisher.') -param acrArtifactStoreName string -@description('Name of an existing Storage Account-backed Artifact Store, deployed under the publisher.') -param saArtifactStoreName string -@description('Name of Network Function. Used predominantly as a prefix for other variable names') -param nfName string -@description('Name of an existing Network Function Definition Group') -param nfDefinitionGroup string -@description('The version of the NFDV you want to deploy, in format A.B.C') -param nfDefinitionVersion string -@description('The version that you want to name the NFM VHD artifact, in format A-B-C. e.g. 6-13-0') -param vhdVersion string -@description('The version that you want to name the NFM template artifact, in format A.B.C. e.g. 6.13.0. If testing for development, you can use any numbers you like.') -param armTemplateVersion string - -// Created by the az aosm definition publish command before the template is deployed -resource publisher 'Microsoft.HybridNetwork/publishers@2023-09-01' existing = { - name: publisherName - scope: resourceGroup() -} - -// Created by the az aosm definition publish command before the template is deployed -resource acrArtifactStore 'Microsoft.HybridNetwork/publishers/artifactStores@2023-09-01' existing = { - parent: publisher - name: acrArtifactStoreName -} - -// Created by the az aosm definition publish command before the template is deployed -resource saArtifactStore 'Microsoft.HybridNetwork/publishers/artifactStores@2023-09-01' existing = { - parent: publisher - name: saArtifactStoreName -} - -// Created by the az aosm definition publish command before the template is deployed -resource nfdg 'Microsoft.Hybridnetwork/publishers/networkfunctiondefinitiongroups@2023-09-01' existing = { - parent: publisher - name: nfDefinitionGroup -} - -resource nfdv 'Microsoft.Hybridnetwork/publishers/networkfunctiondefinitiongroups/networkfunctiondefinitionversions@2023-09-01' = { - parent: nfdg - name: nfDefinitionVersion - location: location - properties: { - // versionState should be changed to 'Active' once it is finalized. - versionState: 'Preview' - deployParameters: string(loadJsonContent('schemas/deploymentParameters.json')) - networkFunctionType: 'VirtualNetworkFunction' - networkFunctionTemplate: { - nfviType: 'AzureCore' - networkFunctionApplications: [ - { - artifactType: 'VhdImageFile' - name: '${nfName}Image' - dependsOnProfile: null - artifactProfile: { - vhdArtifactProfile: { - vhdName: '${nfName}-vhd' - vhdVersion: vhdVersion - } - artifactStore: { - id: saArtifactStore.id - } - } - // mapping deploy param vals to vals required by this network function application object - deployParametersMappingRuleProfile: { - vhdImageMappingRuleProfile: { - userConfiguration: string(loadJsonContent('configMappings/vhdParameters.json')) - } - // ?? - applicationEnablement: 'Unknown' - } - } - { - artifactType: 'ArmTemplate' - name: nfName - dependsOnProfile: null - artifactProfile: { - templateArtifactProfile: { - templateName: '${nfName}-arm-template' - templateVersion: armTemplateVersion - } - artifactStore: { - id: acrArtifactStore.id - } - } - deployParametersMappingRuleProfile: { - templateMappingRuleProfile: { - templateParameters: string(loadJsonContent('configMappings/templateParameters.json')) - } - applicationEnablement: 'Unknown' - } - } - ] - } - } -} diff --git a/src/aosm/azext_aosm/old/generate_nfd/vnf_nfd_generator.py b/src/aosm/azext_aosm/old/generate_nfd/vnf_nfd_generator.py deleted file mode 100644 index e415817594e..00000000000 --- a/src/aosm/azext_aosm/old/generate_nfd/vnf_nfd_generator.py +++ /dev/null @@ -1,340 +0,0 @@ -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- -"""Contains a class for generating VNF NFDs and associated resources.""" - -import json -import shutil -import tempfile -from functools import cached_property -from pathlib import Path -from typing import Any, Dict, Optional - -from knack.log import get_logger - -from azext_aosm._configuration import ArtifactConfig, VNFConfiguration -from azext_aosm.generate_nfd.nfd_generator_base import NFDGenerator -from azext_aosm.util.constants import ( - CONFIG_MAPPINGS_DIR_NAME, - DEPLOYMENT_PARAMETERS_FILENAME, - EXTRA_VHD_PARAMETERS, - OPTIONAL_DEPLOYMENT_PARAMETERS_FILENAME, - OPTIONAL_DEPLOYMENT_PARAMETERS_HEADING, - SCHEMA_PREFIX, - SCHEMAS_DIR_NAME, - TEMPLATE_PARAMETERS_FILENAME, - VHD_PARAMETERS_FILENAME, - VNF_DEFINITION_BICEP_TEMPLATE_FILENAME, - VNF_MANIFEST_BICEP_TEMPLATE_FILENAME, -) -from azext_aosm.util.utils import input_ack, snake_case_to_camel_case - -logger = get_logger(__name__) - -# Different types are used in ARM templates and NFDs. The list accepted by NFDs is -# documented in the AOSM meta-schema. This will be published in the future but for now -# can be found in -# https://microsoft.sharepoint.com/:w:/t/NSODevTeam/Ec7ovdKroSRIv5tumQnWIE0BE-B2LykRcll2Qb9JwfVFMQ -ARM_TO_JSON_PARAM_TYPES: Dict[str, str] = { - "int": "integer", - "securestring": "string", - "bool": "boolean", -} - - -class VnfNfdGenerator(NFDGenerator): - # pylint: disable=too-many-instance-attributes - """ - VNF NFD Generator. - - This takes a source ARM template and a config file, and outputs: - - A bicep file for the NFDV - - Parameters files that are used by the NFDV bicep file, these are the - deployParameters and the mapping profiles of those deploy parameters - - A bicep file for the Artifact manifests - - @param order_params: whether to order the deployment and template output parameters - with those without a default first, then those with a default. - Those without a default will definitely be required to be - exposed, those with a default may not be. - @param interactive: whether to prompt the user to confirm the parameters to be - exposed. - """ - - def __init__( - self, config: VNFConfiguration, order_params: bool, interactive: bool - ): - self.config = config - - assert isinstance(self.config.arm_template, ArtifactConfig) - assert self.config.arm_template.file_path - - self.arm_template_path = Path(self.config.arm_template.file_path) - self.output_directory: Path = self.config.output_directory_for_build - - self._vnfd_bicep_path = Path( - self.output_directory, VNF_DEFINITION_BICEP_TEMPLATE_FILENAME - ) - self._manifest_bicep_path = Path( - self.output_directory, VNF_MANIFEST_BICEP_TEMPLATE_FILENAME - ) - self.order_params = order_params - self.interactive = interactive - self._tmp_dir: Optional[Path] = None - self.image_name = f"{self.config.nf_name}Image" - - def generate_nfd(self) -> None: - """ - Generate a VNF NFD which comprises an group, an Artifact Manifest and a NFDV. - - Create a bicep template for an NFD from the ARM template for the VNF. - """ - # Create temporary directory. - with tempfile.TemporaryDirectory() as tmpdirname: - self._tmp_dir = Path(tmpdirname) - - self._create_parameter_files() - self._copy_to_output_directory() - print( - f"Generated NFD bicep templates created in {self.output_directory}" - ) - print( - "Please review these templates. When you are happy with them run " - "`az aosm nfd publish` with the same arguments." - ) - - @property - def nfd_bicep_path(self) -> Optional[Path]: - """Returns the path to the bicep file for the NFD if it has been created.""" - if self._vnfd_bicep_path.exists(): - return self._vnfd_bicep_path - return None - - @property - def manifest_bicep_path(self) -> Optional[Path]: - """Returns the path to the bicep file for the NFD if it has been created.""" - if self._manifest_bicep_path.exists(): - return self._manifest_bicep_path - return None - - @cached_property - def vm_parameters(self) -> Dict[str, Any]: - """The parameters from the VM ARM template.""" - with open(self.arm_template_path, "r", encoding="utf-8") as _file: - data = json.load(_file) - if "parameters" in data: - parameters: Dict[str, Any] = data["parameters"] - else: - print( - "No parameters found in the template provided. " - "Your NFD will have no deployParameters" - ) - parameters = {} - - return parameters - - @property - def vm_parameters_ordered(self) -> Dict[str, Any]: - """The parameters from the VM ARM template, ordered as those without defaults then those with.""" - vm_parameters_no_default: Dict[str, Any] = {} - vm_parameters_with_default: Dict[str, Any] = {} - has_default_field: bool = False - has_default: bool = False - - for key in self.vm_parameters: - # Order parameters into those with and without defaults - has_default_field = "defaultValue" in self.vm_parameters[key] - has_default = ( - has_default_field - and not self.vm_parameters[key]["defaultValue"] == "" - ) - - if has_default: - vm_parameters_with_default[key] = self.vm_parameters[key] - else: - vm_parameters_no_default[key] = self.vm_parameters[key] - - return {**vm_parameters_no_default, **vm_parameters_with_default} - - def _create_parameter_files(self) -> None: - """Create the deployment, template and VHD parameter files.""" - assert self._tmp_dir - tmp_schemas_directory: Path = self._tmp_dir / SCHEMAS_DIR_NAME - tmp_schemas_directory.mkdir() - self.write_deployment_parameters(tmp_schemas_directory) - - tmp_mappings_directory: Path = self._tmp_dir / CONFIG_MAPPINGS_DIR_NAME - tmp_mappings_directory.mkdir() - self.write_template_parameters(tmp_mappings_directory) - self.write_vhd_parameters(tmp_mappings_directory) - - def write_deployment_parameters(self, directory: Path) -> None: - """ - Write out the NFD deploymentParameters.json file to `directory` - - :param directory: The directory to put this file in. - """ - logger.debug("Create deploymentParameters.json") - - nfd_parameters = {} - nfd_parameters_with_default = {} - vm_parameters_to_exclude = [] - - vm_parameters = ( - self.vm_parameters_ordered - if self.order_params - else self.vm_parameters - ) - - for key in vm_parameters: - if key == self.config.image_name_parameter: - # There is only one correct answer for the image name, so don't ask the - # user, instead it is hardcoded in config mappings. - continue - - # Order parameters into those without and then with defaults - has_default_field = "defaultValue" in self.vm_parameters[key] - has_default = ( - has_default_field - and not self.vm_parameters[key]["defaultValue"] == "" - ) - - if self.interactive and has_default: - # Interactive mode. Prompt user to include or exclude parameters - # This requires the enter key after the y/n input which isn't ideal - if not input_ack("y", f"Expose parameter {key}? y/n "): - logger.debug("Excluding parameter %s", key) - vm_parameters_to_exclude.append(key) - continue - - # Map ARM parameter types to JSON parameter types accepted by AOSM - arm_type = self.vm_parameters[key]["type"] - json_type = ARM_TO_JSON_PARAM_TYPES.get(arm_type.lower(), arm_type) - - if has_default: - nfd_parameters_with_default[key] = {"type": json_type} - - nfd_parameters[key] = {"type": json_type} - - # Now we are out of the vm_parameters loop, we can remove the excluded - # parameters so they don't get included in templateParameters.json - # Remove from both ordered and unordered dicts - for key in vm_parameters_to_exclude: - self.vm_parameters.pop(key, None) - - deployment_parameters_path = directory / DEPLOYMENT_PARAMETERS_FILENAME - - # Heading for the deployParameters schema - deploy_parameters_full: Dict[str, Any] = SCHEMA_PREFIX - deploy_parameters_full["properties"].update(nfd_parameters) - - with open(deployment_parameters_path, "w", encoding="utf-8") as _file: - _file.write(json.dumps(deploy_parameters_full, indent=4)) - - logger.debug("%s created", deployment_parameters_path) - if self.order_params: - print( - "Deployment parameters for the generated NFDV are ordered by those " - "without defaults first to make it easier to choose which to expose." - ) - - # Extra output file to help the user know which parameters are optional - if not self.interactive: - if nfd_parameters_with_default: - optional_deployment_parameters_path = ( - directory / OPTIONAL_DEPLOYMENT_PARAMETERS_FILENAME - ) - with open( - optional_deployment_parameters_path, "w", encoding="utf-8" - ) as _file: - _file.write(OPTIONAL_DEPLOYMENT_PARAMETERS_HEADING) - _file.write( - json.dumps(nfd_parameters_with_default, indent=4) - ) - print( - "Optional ARM parameters detected. Created " - f"{OPTIONAL_DEPLOYMENT_PARAMETERS_FILENAME} to help you choose which " - "to expose." - ) - - def write_template_parameters(self, directory: Path) -> None: - """ - Write out the NFD templateParameters.json file to `directory`. - - :param directory: The directory to put this file in. - """ - logger.debug("Create %s", TEMPLATE_PARAMETERS_FILENAME) - vm_parameters = ( - self.vm_parameters_ordered - if self.order_params - else self.vm_parameters - ) - - template_parameters = {} - - for key in vm_parameters: - if key == self.config.image_name_parameter: - template_parameters[key] = self.image_name - continue - - template_parameters[key] = f"{{deployParameters.{key}}}" - - template_parameters_path = directory / TEMPLATE_PARAMETERS_FILENAME - - with open(template_parameters_path, "w", encoding="utf-8") as _file: - _file.write(json.dumps(template_parameters, indent=4)) - - logger.debug("%s created", template_parameters_path) - - def write_vhd_parameters(self, directory: Path) -> None: - """ - Write out the NFD vhdParameters.json file to `directory`. - - :param directory: The directory to put this file in. - """ - vhd_config = self.config.vhd - # vhdImageMappingRuleProfile userConfiguration within the NFDV API accepts azureDeployLocation - # as the location where the image resource should be created from the VHD. The CLI does not - # expose this as it defaults to the NF deploy location, and we can't think of situations where - # it should be different. - vhd_parameters = { - "imageName": self.image_name, - **{ - snake_case_to_camel_case(key): value - for key, value in vhd_config.__dict__.items() - if key in EXTRA_VHD_PARAMETERS and value is not None - }, - } - - vhd_parameters_path = directory / VHD_PARAMETERS_FILENAME - with open(vhd_parameters_path, "w", encoding="utf-8") as _file: - _file.write(json.dumps(vhd_parameters, indent=4)) - - logger.debug("%s created", vhd_parameters_path) - - def _copy_to_output_directory(self) -> None: - """Copy the static bicep templates and generated config mappings and schema into the build output directory.""" - logger.info("Create NFD bicep %s", self.output_directory) - assert self._tmp_dir - Path(self.output_directory).mkdir(exist_ok=True) - - static_bicep_templates_dir = Path(__file__).parent / "templates" - - static_vnfd_bicep_path = ( - static_bicep_templates_dir / VNF_DEFINITION_BICEP_TEMPLATE_FILENAME - ) - shutil.copy(static_vnfd_bicep_path, self.output_directory) - - static_manifest_bicep_path = ( - static_bicep_templates_dir / VNF_MANIFEST_BICEP_TEMPLATE_FILENAME - ) - shutil.copy(static_manifest_bicep_path, self.output_directory) - # Copy everything in the temp directory to the output directory - shutil.copytree( - self._tmp_dir, - self.output_directory, - dirs_exist_ok=True, - ) - - logger.info("Copied files to %s", self.output_directory) diff --git a/src/aosm/azext_aosm/old/generate_nsd/__init__.py b/src/aosm/azext_aosm/old/generate_nsd/__init__.py deleted file mode 100644 index 99c0f28cd71..00000000000 --- a/src/aosm/azext_aosm/old/generate_nsd/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -# ----------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# ----------------------------------------------------------------------------- diff --git a/src/aosm/azext_aosm/old/generate_nsd/nf_ret.py b/src/aosm/azext_aosm/old/generate_nsd/nf_ret.py deleted file mode 100644 index 69589e69242..00000000000 --- a/src/aosm/azext_aosm/old/generate_nsd/nf_ret.py +++ /dev/null @@ -1,185 +0,0 @@ -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- -"""Handles the creation of a resource element template for a network function.""" - -import json - -from typing import Dict, Any, List, Union -from knack.log import get_logger - -from azext_aosm._configuration import NFDRETConfiguration -from azext_aosm.util.constants import CNF, VNF -from azext_aosm.util.management_clients import ApiClients -from azext_aosm.vendored_sdks.models import NetworkFunctionDefinitionVersion, NFVIType - - -logger = get_logger(__name__) - - -class NFRETGenerator: - """Represents a single network function resource element template within an NSD.""" - - def __init__( - self, api_clients: ApiClients, config: NFDRETConfiguration, cg_schema_name: str - ) -> None: - self.config = config - self.cg_schema_name = cg_schema_name - nfdv = self._get_nfdv(config, api_clients) - print( - f"Finding the deploy parameters for {self.config.name}:{self.config.version}" - ) - - if not nfdv.properties.deploy_parameters: - raise NotImplementedError( - f"NFDV {self.config.name} has no deploy parameters, cannot generate NSD." - ) - self.deploy_parameters: Dict[str, Any] = json.loads( - nfdv.properties.deploy_parameters - ) - - self.nfd_group_name = self.config.name.replace("-", "_") - self.nfdv_parameter_name = f"{self.nfd_group_name}_nfd_version" - self.config_mapping_filename = f"{self.config.name}_config_mapping.json" - - @staticmethod - def _get_nfdv( - config: NFDRETConfiguration, api_clients: ApiClients - ) -> NetworkFunctionDefinitionVersion: - """Get the existing NFDV resource object.""" - print( - f"Reading existing NFDV resource object {config.version} from group {config.name}" - ) - nfdv_object = api_clients.aosm_client.network_function_definition_versions.get( - resource_group_name=config.publisher_resource_group, - publisher_name=config.publisher, - network_function_definition_group_name=config.name, - network_function_definition_version_name=config.version, - ) - return nfdv_object - - @property - def config_mappings(self) -> Dict[str, Any]: - """ - Return the contents of the config mapping file for this RET. - - Output will look something like: - { - "deploymentParametersObject": { - "deploymentParameters": [ - "{configurationparameters('foo_ConfigGroupSchema').bar.deploymentParameters}" - ] - }, - "nginx_nfdg_nfd_version": "{configurationparameters('foo_ConfigGroupSchema').bar.bar_nfd_version}", - "managedIdentity": "{configurationparameters('foo_ConfigGroupSchema').managedIdentity}", - "customLocationId": "{configurationparameters('foo_ConfigGroupSchema').bar.customLocationId}" - } - """ - nf = self.config.name - - logger.debug("Create %s", self.config_mapping_filename) - - deployment_parameters: Union[ - str, List[str] - ] = f"{{configurationparameters('{self.cg_schema_name}').{nf}.deploymentParameters}}" - - if not self.config.multiple_instances: - assert isinstance(deployment_parameters, str) - deployment_parameters = [deployment_parameters] - - deployment_parameters_object = {"deploymentParameters": deployment_parameters} - - version_parameter = ( - f"{{configurationparameters('{self.cg_schema_name}')." - f"{nf}.{self.nfdv_parameter_name}}}" - ) - - config_mappings = { - "deploymentParametersObject": deployment_parameters_object, - self.nfdv_parameter_name: version_parameter, - "managedIdentity": f"{{configurationparameters('{self.cg_schema_name}').managedIdentity}}", - } - - if self.config.type == CNF: - config_mappings[ - "customLocationId" - ] = f"{{configurationparameters('{self.cg_schema_name}').{nf}.customLocationId}}" - - return config_mappings - - @property - def nf_bicep_substitutions(self) -> Dict[str, Any]: - """Returns the jinja2 parameters for the NF bicep template template.""" - return { - "network_function_name": self.config.name, - "publisher_name": self.config.publisher, - "publisher_resource_group": self.config.publisher_resource_group, - "network_function_definition_group_name": (self.config.name), - "network_function_definition_version_parameter": (self.nfdv_parameter_name), - "network_function_definition_offering_location": ( - self.config.publisher_offering_location - ), - # Ideally we would use the network_function_type from reading the actual - # NF, as we do for deployParameters, but the SDK currently doesn't - # support this and needs to be rebuilt to do so. - "nfvi_type": ( - NFVIType.AZURE_CORE.value # type: ignore[attr-defined] # pylint: disable=no-member - if self.config.type == VNF - else NFVIType.AZURE_ARC_KUBERNETES.value # type: ignore[attr-defined] # pylint: disable=no-member - ), - "CNF": self.config.type == CNF, - } - - @property - def config_schema_snippet(self) -> Dict[str, Any]: - """Return the CGS snippet for this NF.""" - nfdv_version_description_string = ( - f"The version of the {self.config.name} " - "NFD to use. This version must be compatible with (have the same " - "parameters exposed as) " - f"{self.config.name}." - ) - - if self.config.multiple_instances: - deploy_parameters = { - "type": "array", - "items": { - "type": "object", - "properties": self.deploy_parameters["properties"], - }, - } - else: - deploy_parameters = { - "type": "object", - "properties": self.deploy_parameters["properties"], - } - - nf_schema: Dict[str, Any] = { - "type": "object", - "properties": { - "deploymentParameters": deploy_parameters, - self.nfdv_parameter_name: { - "type": "string", - "description": nfdv_version_description_string, - }, - }, - "required": ["deploymentParameters", self.nfdv_parameter_name], - } - - if self.config.type == CNF: - custom_location_description_string = ( - "The custom location ID of the ARC-Enabled AKS Cluster to deploy the CNF " - "to. Should be of the form " - "'/subscriptions/{subscriptionId}/resourcegroups" - "/{resourceGroupName}/providers/microsoft.extendedlocation/" - "customlocations/{customLocationName}'" - ) - - nf_schema["properties"]["customLocationId"] = { - "type": "string", - "description": custom_location_description_string, - } - nf_schema["required"].append("customLocationId") - - return nf_schema diff --git a/src/aosm/azext_aosm/old/generate_nsd/nsd_generator.py b/src/aosm/azext_aosm/old/generate_nsd/nsd_generator.py deleted file mode 100644 index adee03677e6..00000000000 --- a/src/aosm/azext_aosm/old/generate_nsd/nsd_generator.py +++ /dev/null @@ -1,261 +0,0 @@ -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- -"""Contains a class for generating NSDs and associated resources.""" -import json -import os -import shutil -import tempfile -from functools import cached_property -from typing import Any, Dict - -from jinja2 import Template -from knack.log import get_logger - -from azext_aosm._configuration import NFDRETConfiguration, NSConfiguration -from azext_aosm.generate_nsd.nf_ret import NFRETGenerator -from azext_aosm.util.constants import ( - CONFIG_MAPPINGS_DIR_NAME, - NF_TEMPLATE_JINJA2_SOURCE_TEMPLATE, - NSD_ARTIFACT_MANIFEST_BICEP_FILENAME, - NSD_ARTIFACT_MANIFEST_SOURCE_TEMPLATE_FILENAME, - NSD_BICEP_FILENAME, - NSD_DEFINITION_JINJA2_SOURCE_TEMPLATE, - SCHEMAS_DIR_NAME, - TEMPLATES_DIR_NAME, -) -from azext_aosm.util.management_clients import ApiClients - -logger = get_logger(__name__) - -# Different types are used in Bicep templates and NFDs. The list accepted by NFDs is -# documented in the AOSM meta-schema. This will be published in the future but for now -# can be found in -# https://microsoft.sharepoint.com/:w:/t/NSODevTeam/Ec7ovdKroSRIv5tumQnWIE0BE-B2LykRcll2Qb9JwfVFMQ -NFV_TO_BICEP_PARAM_TYPES: Dict[str, str] = { - "integer": "int", - "boolean": "bool", -} - - -class NSDGenerator: # pylint: disable=too-few-public-methods - """ - NSD Generator. - - This takes a config file and a set of NFDV deploy_parameters and outputs: - - A bicep file for the NSDV - - Parameters files that are used by the NSDV bicep file, these are the - schemas and the mapping profiles of those schemas parameters - - A bicep file for the Artifact manifest - - A bicep and JSON file defining the Network Function that will - be deployed by the NSDV - """ - - def __init__(self, api_clients: ApiClients, config: NSConfiguration): - self.config = config - self.nsd_bicep_template_name = NSD_DEFINITION_JINJA2_SOURCE_TEMPLATE - self.nsd_bicep_output_name = NSD_BICEP_FILENAME - - self.nf_ret_generators = [] - - for nf_config in self.config.network_functions: - assert isinstance(nf_config, NFDRETConfiguration) - self.nf_ret_generators.append( - NFRETGenerator(api_clients, nf_config, self.config.cg_schema_name) - ) - - def generate_nsd(self) -> None: - """Generate a NSD templates which includes an Artifact Manifest, NFDV and NF templates.""" - logger.info("Generate NSD bicep templates") - - # Create temporary folder. - with tempfile.TemporaryDirectory() as tmpdirname: - self._write_config_group_schema_json(tmpdirname) - self._write_config_mapping_files(tmpdirname) - self._write_nsd_manifest(tmpdirname) - self._write_nf_bicep_files(tmpdirname) - self._write_nsd_bicep(tmpdirname) - - self._copy_to_output_folder(tmpdirname) - print( - "Generated NSD bicep templates created in" - f" {self.config.output_directory_for_build}" - ) - print( - "Please review these templates. When you are happy with them run " - "`az aosm nsd publish` with the same arguments." - ) - - @cached_property - def _config_group_schema_dict(self) -> Dict[str, Any]: - """ - :return: The Config Group Schema as a dictionary. - - See src/aosm/azext_aosm/tests/latest/nsd_output/*/schemas for examples of the - output from this function. - """ - managed_identity_description_string = ( - "The managed identity to use to deploy NFs within this SNS. This should " - "be of the form '/subscriptions/{subscriptionId}/resourceGroups/" - "{resourceGroupName}/providers/Microsoft.ManagedIdentity/" - "userAssignedIdentities/{identityName}. " - "If you wish to use a system assigned identity, set this to a blank string." - ) - - properties = { - nf.config.name: nf.config_schema_snippet for nf in self.nf_ret_generators - } - - properties.update( - { - "managedIdentity": { - "type": "string", - "description": managed_identity_description_string, - } - } - ) - - required = [nf.config.name for nf in self.nf_ret_generators] - required.append("managedIdentity") - - cgs_dict: Dict[str, Any] = { - "$schema": "https://json-schema.org/draft-07/schema#", - "title": self.config.cg_schema_name, - "type": "object", - "properties": properties, - "required": required, - } - - return cgs_dict - - def _write_config_group_schema_json(self, output_directory) -> None: - """Create a file containing the json schema for the CGS.""" - temp_schemas_folder_path = os.path.join(output_directory, SCHEMAS_DIR_NAME) - os.mkdir(temp_schemas_folder_path) - - logger.debug("Create %s.json", self.config.cg_schema_name) - - schema_path = os.path.join( - temp_schemas_folder_path, f"{self.config.cg_schema_name}.json" - ) - - with open(schema_path, "w", encoding="utf-8") as _file: - _file.write(json.dumps(self._config_group_schema_dict, indent=4)) - - logger.debug("%s created", schema_path) - - def _write_config_mapping_files(self, output_directory) -> None: - """Write out a config mapping file for each NF.""" - temp_mappings_folder_path = os.path.join( - output_directory, CONFIG_MAPPINGS_DIR_NAME - ) - - os.mkdir(temp_mappings_folder_path) - - for nf in self.nf_ret_generators: - config_mappings_path = os.path.join( - temp_mappings_folder_path, nf.config_mapping_filename - ) - - with open(config_mappings_path, "w", encoding="utf-8") as _file: - _file.write(json.dumps(nf.config_mappings, indent=4)) - - logger.debug("%s created", config_mappings_path) - - def _write_nf_bicep_files(self, output_directory) -> None: - """ - Write bicep files for deploying NFs. - - In the publish step these bicep files will be uploaded to the publisher storage - account as artifacts. - """ - for nf in self.nf_ret_generators: - substitutions = {"location": self.config.location} - substitutions.update(nf.nf_bicep_substitutions) - - self._generate_bicep( - NF_TEMPLATE_JINJA2_SOURCE_TEMPLATE, - os.path.join(output_directory, nf.config.nf_bicep_filename), - substitutions, - ) - - def _write_nsd_bicep(self, output_directory) -> None: - """Write out the NSD bicep file.""" - ret_names = [nf.config.resource_element_name for nf in self.nf_ret_generators] - arm_template_names = [ - nf.config.arm_template.artifact_name for nf in self.nf_ret_generators - ] - config_mapping_files = [ - nf.config_mapping_filename for nf in self.nf_ret_generators - ] - - # We want the armTemplateVersion to be the same as the NSD Version. That means - # that if we create a new NSDV then the existing artifacts won't be overwritten. - params = { - "nfvi_site_name": self.config.nfvi_site_name, - "armTemplateNames": arm_template_names, - "armTemplateVersion": self.config.nsd_version, - "cg_schema_name": self.config.cg_schema_name, - "nsdv_description": self.config.nsdv_description, - "ResourceElementName": ret_names, - "configMappingFiles": config_mapping_files, - "nf_count": len(self.nf_ret_generators), - } - - self._generate_bicep( - self.nsd_bicep_template_name, - os.path.join(output_directory, self.nsd_bicep_output_name), - params, - ) - - def _write_nsd_manifest(self, output_directory) -> None: - """Write out the NSD manifest bicep file.""" - logger.debug("Create NSD manifest") - - self._generate_bicep( - NSD_ARTIFACT_MANIFEST_SOURCE_TEMPLATE_FILENAME, - os.path.join(output_directory, NSD_ARTIFACT_MANIFEST_BICEP_FILENAME), - {}, - ) - - @staticmethod - def _generate_bicep( - template_name: str, output_file_name: str, params: Dict[Any, Any] - ) -> None: - """ - Render the bicep templates with the correct parameters and copy them into the build output folder. - - :param template_name: The name of the template to render - :param output_file_name: The name of the output file - :param params: The parameters to render the template with - """ - - code_dir = os.path.dirname(__file__) - - bicep_template_path = os.path.join(code_dir, TEMPLATES_DIR_NAME, template_name) - - with open(bicep_template_path, "r", encoding="utf-8") as file: - bicep_contents = file.read() - - bicep_template = Template(bicep_contents) - - # Render all the relevant parameters in the bicep template - rendered_template = bicep_template.render(**params) - - with open(output_file_name, "w", encoding="utf-8") as file: - file.write(rendered_template) - - def _copy_to_output_folder(self, temp_dir) -> None: - """Copy the bicep templates, config mappings and schema into the build output folder.""" - - logger.info("Create NSD bicep %s", self.config.output_directory_for_build) - os.mkdir(self.config.output_directory_for_build) - - shutil.copytree( - temp_dir, - self.config.output_directory_for_build, - dirs_exist_ok=True, - ) - - logger.info("Copied files to %s", self.config.output_directory_for_build) diff --git a/src/aosm/azext_aosm/old/generate_nsd/templates/artifact_manifest_template.bicep b/src/aosm/azext_aosm/old/generate_nsd/templates/artifact_manifest_template.bicep deleted file mode 100644 index 34ac9ca3fdb..00000000000 --- a/src/aosm/azext_aosm/old/generate_nsd/templates/artifact_manifest_template.bicep +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright (c) Microsoft Corporation. - -// This file creates an Artifact Manifest for a NSD -param location string -@description('Name of an existing publisher, expected to be in the resource group where you deploy the template') -param publisherName string -@description('Name of an existing ACR-backed Artifact Store, deployed under the publisher.') -param acrArtifactStoreName string -@description('Name of the manifest to deploy for the ACR-backed Artifact Store') -param acrManifestNames array -@description('The name under which to store the ARM template') -param armTemplateNames array -@description('The version that you want to name the NFM template artifact, in format A.B.C. e.g. 6.13.0. If testing for development, you can use any numbers you like.') -param armTemplateVersion string - -resource publisher 'Microsoft.HybridNetwork/publishers@2023-09-01' existing = { - name: publisherName - scope: resourceGroup() -} - -resource acrArtifactStore 'Microsoft.HybridNetwork/publishers/artifactStores@2023-09-01' existing = { - parent: publisher - name: acrArtifactStoreName -} - -resource acrArtifactManifests 'Microsoft.Hybridnetwork/publishers/artifactStores/artifactManifests@2023-09-01' = [for (values, i) in armTemplateNames: { - parent: acrArtifactStore - name: acrManifestNames[i] - location: location - properties: { - artifacts: [ - { - artifactName: armTemplateNames[i] - artifactType: 'ArmTemplate' - artifactVersion: armTemplateVersion - } - ] - } -}] diff --git a/src/aosm/azext_aosm/old/generate_nsd/templates/nf_template.bicep.j2 b/src/aosm/azext_aosm/old/generate_nsd/templates/nf_template.bicep.j2 deleted file mode 100644 index 8a53daa9edf..00000000000 --- a/src/aosm/azext_aosm/old/generate_nsd/templates/nf_template.bicep.j2 +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Highly Confidential Material -// -// The template that the NSD invokes to create the Network Function from a published NFDV. - -@description('Publisher where the NFD is published') -param publisherName string = '{{publisher_name}}' - -@description('Resource group where the NFD publisher exists') -param publisherResourceGroup string = '{{publisher_resource_group}}' - -@description('NFD Group name for the Network Function') -param networkFunctionDefinitionGroupName string = '{{network_function_definition_group_name}}' - -@description('NFD version') -param {{network_function_definition_version_parameter}} string - -@description('The managed identity that should be used to create the NF.') -param managedIdentity string - -{%- if CNF %} -@description('The custom location of the ARC-enabled AKS cluster to create the NF.') -param customLocationId string -{%- endif %} - -param location string = '{{location}}' - -param nfviType string = '{{nfvi_type}}' - -param resourceGroupId string = resourceGroup().id - -@secure() -param deploymentParametersObject object - -var deploymentParameters = deploymentParametersObject.deploymentParameters - -var identityObject = (managedIdentity == '') ? { - type: 'SystemAssigned' -} : { - type: 'UserAssigned' - userAssignedIdentities: { - '${managedIdentity}': {} - } -} - -resource publisher 'Microsoft.HybridNetwork/publishers@2023-09-01' existing = { - name: publisherName - scope: resourceGroup(publisherResourceGroup) -} - -resource nfdg 'Microsoft.Hybridnetwork/publishers/networkfunctiondefinitiongroups@2023-09-01' existing = { - parent: publisher - name: networkFunctionDefinitionGroupName -} - -resource nfdv 'Microsoft.Hybridnetwork/publishers/networkfunctiondefinitiongroups/networkfunctiondefinitionversions@2023-09-01' existing = { - parent: nfdg - name: {{network_function_definition_version_parameter}} - -} - -resource nf_resource 'Microsoft.HybridNetwork/networkFunctions@2023-09-01' = [for (values, i) in deploymentParameters: { - name: '{{network_function_name}}${i}' - location: location - identity: identityObject - properties: { - networkFunctionDefinitionVersionResourceReference: { - id: nfdv.id - idType: 'Open' - } - nfviType: nfviType -{%- if CNF %} - nfviId: customLocationId -{%- else %} - nfviId: resourceGroupId -{%- endif %} - allowSoftwareUpdate: true - configurationType: 'Secret' - secretDeploymentValues: string(values) - } -}] diff --git a/src/aosm/azext_aosm/old/generate_nsd/templates/nsd_template.bicep.j2 b/src/aosm/azext_aosm/old/generate_nsd/templates/nsd_template.bicep.j2 deleted file mode 100644 index 57d787c4803..00000000000 --- a/src/aosm/azext_aosm/old/generate_nsd/templates/nsd_template.bicep.j2 +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Highly Confidential Material -// -// Bicep template to create an Artifact Manifest, Config Group Schema and NSDV. -// -// Requires an existing NFDV from which the values will be populated. - -param location string -@description('Name of an existing publisher, expected to be in the resource group where you deploy the template') -param publisherName string -@description('Name of an existing ACR-backed Artifact Store, deployed under the publisher.') -param acrArtifactStoreName string -@description('Name of an existing Network Service Design Group') -param nsDesignGroup string -@description('The version of the NSDV you want to create, in format A.B.C') -param nsDesignVersion string -@description('Name of the nfvi site') -param nfviSiteName string = '{{nfvi_site_name}}' - -// The publisher resource is the top level AOSM resource under which all other designer resources -// are created. -resource publisher 'Microsoft.HybridNetwork/publishers@2023-09-01' existing = { - name: publisherName - scope: resourceGroup() -} - -// The artifact store is the resource in which all the artifacts required to deploy the NF are stored. -// The artifact store is created by the az aosm CLI before this template is deployed. -resource acrArtifactStore 'Microsoft.HybridNetwork/publishers/artifactStores@2023-09-01' existing = { - parent: publisher - name: acrArtifactStoreName -} - -// Created up-front, the NSD Group is the parent resource under which all NSD versions will be created. -resource nsdGroup 'Microsoft.Hybridnetwork/publishers/networkservicedesigngroups@2023-09-01' existing = { - parent: publisher - name: nsDesignGroup -} - -// The configuration group schema defines the configuration required to deploy the NSD. The NSD references this object in the -// `configurationgroupsSchemaReferences` and references the values in the schema in the `parameterValues`. -// The operator will create a config group values object that will satisfy this schema. -resource cgSchema 'Microsoft.Hybridnetwork/publishers/configurationGroupSchemas@2023-09-01' = { - parent: publisher - name: '{{cg_schema_name}}' - location: location - properties: { - schemaDefinition: string(loadJsonContent('schemas/{{cg_schema_name}}.json')) - } -} - -// The NSD version -resource nsdVersion 'Microsoft.Hybridnetwork/publishers/networkservicedesigngroups/networkservicedesignversions@2023-09-01' = { - parent: nsdGroup - name: nsDesignVersion - location: location - properties: { - description: '{{nsdv_description}}' - // The version state can be Preview, Active or Deprecated. - // Once in an Active state, the NSDV becomes immutable. - versionState: 'Preview' - // The `configurationgroupsSchemaReferences` field contains references to the schemas required to - // be filled out to configure this NSD. - configurationGroupSchemaReferences: { - {{cg_schema_name}}: { - id: cgSchema.id - } - } - // This details the NFVIs that should be available in the Site object created by the operator. - nfvisFromSite: { - nfvi1: { - name: nfviSiteName - type: 'AzureCore' - } - } - // This field lists the templates that will be deployed by AOSM and the config mappings - // to the values in the CG schemas. - resourceElementTemplates: [ -{%- for index in range(nf_count) %} - { - name: '{{ResourceElementName[index]}}' - // The type of resource element can be ArmResourceDefinition, ConfigurationDefinition or NetworkFunctionDefinition. - type: 'NetworkFunctionDefinition' - // The configuration object may be different for different types of resource element. - configuration: { - // This field points AOSM at the artifact in the artifact store. - artifactProfile: { - artifactStoreReference: { - id: acrArtifactStore.id - } - artifactName: '{{armTemplateNames[index]}}' - artifactVersion: '{{armTemplateVersion}}' - } - templateType: 'ArmTemplate' - // The parameter values map values from the CG schema, to values required by the template - // deployed by this resource element. - parameterValues: string(loadJsonContent('configMappings/{{configMappingFiles[index]}}')) - } - dependsOnProfile: { - installDependsOn: [] - uninstallDependsOn: [] - updateDependsOn: [] - } - } -{%- endfor %} - ] - } -} diff --git a/src/aosm/azext_aosm/old/util/__init__.py b/src/aosm/azext_aosm/old/util/__init__.py deleted file mode 100644 index 99c0f28cd71..00000000000 --- a/src/aosm/azext_aosm/old/util/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -# ----------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# ----------------------------------------------------------------------------- diff --git a/src/aosm/azext_aosm/old/util/management_clients.py b/src/aosm/azext_aosm/old/util/management_clients.py deleted file mode 100644 index 936e0d16ec7..00000000000 --- a/src/aosm/azext_aosm/old/util/management_clients.py +++ /dev/null @@ -1,22 +0,0 @@ -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- -"""Clients for the python SDK along with useful caches.""" - -from dataclasses import dataclass - -from azure.mgmt.resource import ResourceManagementClient -from knack.log import get_logger - -from azext_aosm.vendored_sdks import HybridNetworkManagementClient - -logger = get_logger(__name__) - - -@dataclass -class ApiClients: - """A class for API Clients needed throughout.""" - - aosm_client: HybridNetworkManagementClient - resource_client: ResourceManagementClient diff --git a/src/aosm/azext_aosm/old/util/utils.py b/src/aosm/azext_aosm/old/util/utils.py deleted file mode 100644 index 91d144a764c..00000000000 --- a/src/aosm/azext_aosm/old/util/utils.py +++ /dev/null @@ -1,25 +0,0 @@ -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- -"""Utility functions.""" - - -def input_ack(ack: str, request_to_user: str) -> bool: - """ - Overarching function to request, sanitise and return True if input is specified ack. - - This prints the question string and asks for user input. which is santised by - removing all whitespaces in the string, and made lowercase. True is returned if the - user input is equal to supplied acknowledgement string and False if anything else - """ - unsanitised_ans = input(request_to_user) - return str(unsanitised_ans.strip().replace(" ", "").lower()) == ack - - -def snake_case_to_camel_case(text): - """Converts snake case to camel case.""" - components = text.split("_") - return components[0] + "".join( - x[0].upper() + x[1:] for x in components[1:] - ) diff --git a/src/aosm/azext_aosm/old_custom.py b/src/aosm/azext_aosm/old_custom.py deleted file mode 100644 index 008b2210a21..00000000000 --- a/src/aosm/azext_aosm/old_custom.py +++ /dev/null @@ -1,525 +0,0 @@ -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- - -import json -import os -import shutil -from dataclasses import asdict -from pathlib import Path -from typing import Optional - -from azure.cli.core.azclierror import ( - CLIInternalError, - InvalidArgumentValueError, - UnclassifiedUserFault, -) -from azure.cli.core.commands import AzCliCommand -from azure.core import exceptions as azure_exceptions -from knack.log import get_logger - -from azext_aosm._client_factory import cf_features, cf_resources -from azext_aosm._configuration import ( - CNFConfiguration, - Configuration, - NFConfiguration, - NSConfiguration, - VNFConfiguration, - get_configuration, -) -from azext_aosm.delete.delete import ResourceDeleter -from azext_aosm.deploy.deploy_with_arm import DeployerViaArm -from azext_aosm.generate_nfd.cnf_nfd_generator import CnfNfdGenerator -from azext_aosm.generate_nfd.nfd_generator_base import NFDGenerator -from azext_aosm.generate_nfd.vnf_nfd_generator import VnfNfdGenerator -from azext_aosm.generate_nsd.nsd_generator import NSDGenerator -from azext_aosm.util.constants import ( - AOSM_FEATURE_NAMESPACE, - AOSM_REQUIRED_FEATURES, - CNF, - NSD, - VNF, - DeployableResourceTypes, - SkipSteps, -) -from azext_aosm.util.management_clients import ApiClients -from azext_aosm.vendored_sdks import HybridNetworkManagementClient - -logger = get_logger(__name__) - - -def build_definition( - definition_type: str, - config_file: str, - order_params: bool = False, - interactive: bool = False, - force: bool = False, -): - """ - Build a definition. - - :param definition_type: VNF or CNF - :param config_file: path to the file - :param order_params: VNF definition_type only - ignored for CNF. Order - deploymentParameters schema and configMappings to have the parameters without - default values at the top. - :param interactive: Whether to prompt for input when creating deploy parameters - mapping files - :param force: force the build even if the design has already been built - """ - - # Read the config from the given file - config = _get_config_from_file( - config_file=config_file, configuration_type=definition_type - ) - assert isinstance(config, NFConfiguration) - - # Generate the NFD and the artifact manifest. - _generate_nfd( - definition_type=definition_type, - config=config, - order_params=order_params, - interactive=interactive, - force=force, - ) - - -def generate_definition_config(definition_type: str, output_file: str = "input.json"): - """ - Generate an example config file for building a definition. - - :param definition_type: CNF, VNF - :param output_file: path to output config file, defaults to "input.json" - """ - config: Configuration - if definition_type == CNF: - config = CNFConfiguration.helptext() - elif definition_type == VNF: - config = VNFConfiguration.helptext() - else: - raise ValueError("definition_type must be CNF or VNF") - - _generate_config(configuration=config, output_file=output_file) - - -def _get_config_from_file(config_file: str, configuration_type: str) -> Configuration: - """ - Read input config file JSON and turn it into a Configuration object. - - :param config_file: path to the file - :param configuration_type: VNF, CNF or NSD - :returns: The Configuration object - """ - - if not os.path.exists(config_file): - raise InvalidArgumentValueError( - f"Config file {config_file} not found. Please specify a valid config file" - " path." - ) - - config = get_configuration(configuration_type, config_file) - return config - - -def _generate_nfd( - definition_type: str, - config: NFConfiguration, - order_params: bool, - interactive: bool, - force: bool = False, -): - """Generate a Network Function Definition for the given type and config.""" - nfd_generator: NFDGenerator - if definition_type == VNF: - assert isinstance(config, VNFConfiguration) - nfd_generator = VnfNfdGenerator(config, order_params, interactive) - elif definition_type == CNF: - assert isinstance(config, CNFConfiguration) - nfd_generator = CnfNfdGenerator(config, interactive) - else: - raise CLIInternalError( - "Generate NFD called for unrecognised definition_type. Only VNF and CNF" - " have been implemented." - ) - if nfd_generator.nfd_bicep_path: - if not force: - carry_on = input( - f"The {nfd_generator.nfd_bicep_path.parent} directory already exists -" - " delete it and continue? (y/n)" - ) - if carry_on != "y": - raise UnclassifiedUserFault("User aborted!") - - shutil.rmtree(nfd_generator.nfd_bicep_path.parent) - nfd_generator.generate_nfd() - - -def _check_features_enabled(cmd: AzCliCommand): - """ - Check that the required Azure features are enabled on the subscription. - - :param cmd: The AzCLICommand object for the original command that was run, we use - this to retrieve the CLI context in order to get the features client for access - to the features API. - """ - features_client = cf_features(cmd.cli_ctx) - # Check that the required features are enabled on the subscription - for feature in AOSM_REQUIRED_FEATURES: - try: - feature_result = features_client.features.get( - resource_provider_namespace=AOSM_FEATURE_NAMESPACE, - feature_name=feature, - ) - if ( - not feature_result - or not feature_result.properties.state == "Registered" - ): - # We don't want to log the name of the feature to the user as it is - # a hidden feature. We do want to log it to the debug log though. - logger.debug( - "Feature %s is not registered on the subscription.", feature - ) - raise CLIInternalError( - "Your Azure subscription has not been fully onboarded to AOSM. " - "Please see the AOSM onboarding documentation for more information." - ) - except azure_exceptions.ResourceNotFoundError as rerr: - # If the feature is not found, it is not registered, but also something has - # gone wrong with the CLI code and onboarding instructions. - logger.debug( - "Feature not found error - Azure doesn't recognise the feature %s." - "This indicates a coding error or error with the AOSM onboarding " - "instructions.", - feature, - ) - logger.debug(rerr) - raise CLIInternalError( - "CLI encountered an error checking that your " - "subscription has been onboarded to AOSM. Please raise an issue against" - " the CLI." - ) from rerr - - -def publish_definition( - cmd: AzCliCommand, - client: HybridNetworkManagementClient, - definition_type, - config_file, - definition_file: Optional[str] = None, - parameters_json_file: Optional[str] = None, - manifest_file: Optional[str] = None, - manifest_params_file: Optional[str] = None, - skip: Optional[SkipSteps] = None, - no_subscription_permissions: bool = False, -): - """ - Publish a generated definition. - - :param cmd: The AzCLICommand object for the command that was run, we use this to - find the CLI context (from which, for example, subscription id and - credentials can be found, and other clients can be generated.) - :param client: The AOSM client. This is created in _client_factory.py and passed - in by commands.py - we could alternatively just use cf_aosm as - we use cf_resources, but other extensions seem to pass a client - around like this. - :type client: HybridNetworkManagementClient - :param definition_type: VNF or CNF - :param config_file: Path to the config file for the NFDV - :param definition_file: Optional path to a bicep template to deploy, in case the - user wants to edit the built NFDV template. If omitted, the default built NFDV - template will be used. - :param parameters_json_file: Optional path to a parameters file for the bicep file, - in case the user wants to edit the built NFDV template. If omitted, parameters - from config will be turned into parameters for the bicep file - :param manifest_file: Optional path to an override bicep template to deploy - manifests - :param manifest_params_file: Optional path to an override bicep parameters file for - manifest parameters - :param skip: options to skip, either publish bicep or upload artifacts - :param no_subscription_permissions: - CNF definition_type publish only - ignored for VNF. Causes the image - artifact copy from a source ACR to be done via docker pull and push, - rather than `az acr import`. This is slower but does not require - Contributor (or importImage action) and AcrPush permissions on the publisher - subscription. It requires Docker to be installed. - """ - # Check that the required features are enabled on the subscription - _check_features_enabled(cmd) - - print("Publishing definition.") - api_clients = ApiClients( - aosm_client=client, - resource_client=cf_resources(cmd.cli_ctx), - ) - - config = _get_config_from_file( - config_file=config_file, configuration_type=definition_type - ) - - deployer = DeployerViaArm( - api_clients, - resource_type=definition_type, - config=config, - bicep_path=definition_file, - parameters_json_file=parameters_json_file, - manifest_bicep_path=manifest_file, - manifest_params_file=manifest_params_file, - skip=skip, - cli_ctx=cmd.cli_ctx, - use_manifest_permissions=no_subscription_permissions, - ) - deployer.deploy_nfd_from_bicep() - - -def delete_published_definition( - cmd: AzCliCommand, - client: HybridNetworkManagementClient, - definition_type, - config_file, - clean=False, - force=False, -): - """ - Delete a published definition. - - :param cmd: The AzCLICommand object for the command that was run, we use this to - find the CLI context (from which, for example, subscription id and - credentials can be found, and other clients can be generated.) - :param client: The AOSM client. This is created in _client_factory.py and passed - in by commands.py - we could alternatively just use cf_aosm as - we use cf_resources, but other extensions seem to pass a client - around like this. - :param definition_type: CNF or VNF - :param config_file: Path to the config file - :param clean: if True, will delete the NFDG, artifact stores and publisher too. - Defaults to False. Only works if no resources have those as a parent. Use - with care. - :param force: if True, will not prompt for confirmation before deleting the resources. - """ - # Check that the required features are enabled on the subscription - _check_features_enabled(cmd) - - config = _get_config_from_file( - config_file=config_file, configuration_type=definition_type - ) - - api_clients = ApiClients( - aosm_client=client, resource_client=cf_resources(cmd.cli_ctx) - ) - - delly = ResourceDeleter(api_clients, config, cmd.cli_ctx) - if definition_type == VNF: - delly.delete_nfd(clean=clean, force=force) - elif definition_type == CNF: - delly.delete_nfd(clean=clean, force=force) - else: - raise ValueError( - "Definition type must be either 'vnf' or 'cnf'. Definition type" - f" {definition_type} is not recognised." - ) - - -def generate_design_config(output_file: str = "input.json"): - """ - Generate an example config file for building a NSD. - - :param output_file: path to output config file, defaults to "input.json" - :type output_file: str, optional - """ - _generate_config(NSConfiguration.helptext(), output_file) - - -def _generate_config(configuration: Configuration, output_file: str = "input.json"): - """ - Generic generate config function for NFDs and NSDs. - - :param configuration: The Configuration object with helptext filled in for each of - the fields. - :param output_file: path to output config file, defaults to "input.json" - """ - # Config file is a special parameter on the configuration objects. It is the path - # to the configuration file, rather than an input parameter. It therefore shouldn't - # be included here. - config = asdict(configuration) - config.pop("config_file") - - config_as_dict = json.dumps(config, indent=4) - - if Path(output_file).exists(): - carry_on = input( - f"The file {output_file} already exists - do you want to overwrite it?" - " (y/n)" - ) - if carry_on != "y": - raise UnclassifiedUserFault("User aborted!") - - with open(output_file, "w", encoding="utf-8") as f: - f.write(config_as_dict) - if isinstance(configuration, NSConfiguration): - prtName = "design" - else: - prtName = "definition" - print(f"Empty {prtName} configuration has been written to {output_file}") - logger.info( - "Empty %s configuration has been written to %s", prtName, output_file - ) - - -def build_design( - cmd: AzCliCommand, - client: HybridNetworkManagementClient, - config_file: str, - force: bool = False, -): - """ - Build a Network Service Design. - - :param cmd: The AzCLICommand object for the command that was run, we use this to - find the CLI context (from which, for example, subscription id and - credentials can be found, and other clients can be generated.) - :param client: The AOSM client. This is created in _client_factory.py and passed - in by commands.py - we could alternatively just use cf_aosm as - we use cf_resources, but other extensions seem to pass a client - around like this. - :type client: HybridNetworkManagementClient - :param config_file: path to the file - :param force: force the build, even if the design has already been built - """ - - api_clients = ApiClients( - aosm_client=client, resource_client=cf_resources(cmd.cli_ctx) - ) - - # Read the config from the given file - config = _get_config_from_file(config_file=config_file, configuration_type=NSD) - assert isinstance(config, NSConfiguration) - config.validate() - - # Generate the NSD and the artifact manifest. - # This function should not be taking deploy parameters - _generate_nsd( - config=config, - api_clients=api_clients, - force=force, - ) - - -def delete_published_design( - cmd: AzCliCommand, - client: HybridNetworkManagementClient, - config_file, - clean=False, - force=False, -): - """ - Delete a published NSD. - - :param cmd: The AzCLICommand object for the command that was run, we use this to - find the CLI context (from which, for example, subscription id and - credentials can be found, and other clients can be generated.) - :param client: The AOSM client. This is created in _client_factory.py and passed - in by commands.py - we could alternatively just use cf_aosm as - we use cf_resources, but other extensions seem to pass a client - around like this. - :param config_file: Path to the config file - :param clean: if True, will delete the NSD, artifact stores and publisher too. - Defaults to False. Only works if no resources have those as a parent. - Use with care. - :param clean: if True, will delete the NSD on top of the other resources. - :param force: if True, will not prompt for confirmation before deleting the resources. - """ - # Check that the required features are enabled on the subscription - _check_features_enabled(cmd) - - config = _get_config_from_file(config_file=config_file, configuration_type=NSD) - - api_clients = ApiClients( - aosm_client=client, resource_client=cf_resources(cmd.cli_ctx) - ) - - destroyer = ResourceDeleter(api_clients, config, cmd.cli_ctx) - destroyer.delete_nsd(clean=clean, force=force) - - -def publish_design( - cmd: AzCliCommand, - client: HybridNetworkManagementClient, - config_file, - design_file: Optional[str] = None, - parameters_json_file: Optional[str] = None, - manifest_file: Optional[str] = None, - manifest_params_file: Optional[str] = None, - skip: Optional[SkipSteps] = None, -): - """ - Publish a generated design. - - :param cmd: The AzCLICommand object for the command that was run, we use this to - find the CLI context (from which, for example, subscription id and - credentials can be found, and other clients can be generated.) - :param client: The AOSM client. This is created in _client_factory.py and passed - in by commands.py - we could alternatively just use cf_aosm as - we use cf_resources, but other extensions seem to pass a client - around like this. - :type client: HybridNetworkManagementClient - :param config_file: Path to the config file for the NSDV - :param design_file: Optional path to an override bicep template to deploy the NSDV. - :param parameters_json_file: Optional path to a parameters file for the bicep file, - in case the user wants to edit the built NSDV template. If - omitted, parameters from config will be turned into parameters - for the bicep file - :param manifest_file: Optional path to an override bicep template to deploy - manifests - :param manifest_params_file: Optional path to an override bicep parameters - file for manifest parameters - :param skip: options to skip, either publish bicep or upload artifacts - """ - # Check that the required features are enabled on the subscription - _check_features_enabled(cmd) - - print("Publishing design.") - api_clients = ApiClients( - aosm_client=client, resource_client=cf_resources(cmd.cli_ctx) - ) - - config = _get_config_from_file(config_file=config_file, configuration_type=NSD) - assert isinstance(config, NSConfiguration) - config.validate() - - deployer = DeployerViaArm( - api_clients, - resource_type=DeployableResourceTypes.NSD, - config=config, - bicep_path=design_file, - parameters_json_file=parameters_json_file, - manifest_bicep_path=manifest_file, - manifest_params_file=manifest_params_file, - skip=skip, - cli_ctx=cmd.cli_ctx, - ) - - deployer.deploy_nsd_from_bicep() - - -def _generate_nsd( - config: NSConfiguration, api_clients: ApiClients, force: bool = False -): - """Generate a Network Service Design for the given config.""" - if config: - nsd_generator = NSDGenerator(config=config, api_clients=api_clients) - else: - raise CLIInternalError("Generate NSD called without a config file") - - if os.path.exists(config.output_directory_for_build): - if not force: - carry_on = input( - f"The folder {config.output_directory_for_build} already exists - delete it" - " and continue? (y/n)" - ) - if carry_on != "y": - raise UnclassifiedUserFault("User aborted! ") - - shutil.rmtree(config.output_directory_for_build) - - nsd_generator.generate_nsd() diff --git a/src/aosm/azext_aosm/tests/latest/recording_processors.py b/src/aosm/azext_aosm/tests/latest/recording_processors.py index ae7b36b061f..e57db0346b0 100644 --- a/src/aosm/azext_aosm/tests/latest/recording_processors.py +++ b/src/aosm/azext_aosm/tests/latest/recording_processors.py @@ -7,11 +7,12 @@ # the recordings so that we can avoid checking in secrets to the repo. # -------------------------------------------------------------------------------------------- -from azure.cli.testsdk.scenario_tests import RecordingProcessor -from azure.cli.testsdk.scenario_tests.utilities import is_text_payload import json import re +from azure.cli.testsdk.scenario_tests import RecordingProcessor +from azure.cli.testsdk.scenario_tests.utilities import is_text_payload + MOCK_TOKEN = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" MOCK_SAS_URI = "https://xxxxxxxxxxxxxxx.blob.core.windows.net" MOCK_STORAGE_ACCOUNT_SR = "&si=StorageAccountAccessPolicy&sr=xxxxxxxxxxxxxxxxxxxx" diff --git a/src/aosm/azext_aosm/tests/latest/test_aosm_cnf_publish_and_delete.py b/src/aosm/azext_aosm/tests/latest/test_aosm_cnf_publish_and_delete.py index 21935070f30..38d43d5e92c 100644 --- a/src/aosm/azext_aosm/tests/latest/test_aosm_cnf_publish_and_delete.py +++ b/src/aosm/azext_aosm/tests/latest/test_aosm_cnf_publish_and_delete.py @@ -15,9 +15,10 @@ import os from typing import Dict + from azure.cli.testsdk import LiveScenarioTest, ResourceGroupPreparer -from knack.log import get_logger from jinja2 import Template +from knack.log import get_logger logger = get_logger(__name__) diff --git a/src/aosm/azext_aosm/tests/latest/test_aosm_vnf_publish_and_delete.py b/src/aosm/azext_aosm/tests/latest/test_aosm_vnf_publish_and_delete.py index 39e099f7918..6f25a3d785b 100644 --- a/src/aosm/azext_aosm/tests/latest/test_aosm_vnf_publish_and_delete.py +++ b/src/aosm/azext_aosm/tests/latest/test_aosm_vnf_publish_and_delete.py @@ -13,10 +13,10 @@ # -------------------------------------------------------------------------------------------- import os + from azure.cli.testsdk import LiveScenarioTest, ResourceGroupPreparer -from knack.log import get_logger from jinja2 import Template - +from knack.log import get_logger logger = get_logger(__name__) @@ -24,9 +24,7 @@ NFD_INPUT_FILE_NAME = "vnf_input.json" NSD_INPUT_TEMPLATE_NAME = "vnf_nsd_input_template.json" NSD_INPUT_FILE_NAME = "nsd_input.json" -ARM_TEMPLATE_RELATIVE_PATH = ( - "scenario_test_mocks/vnf_mocks/ubuntu_template.json" -) +ARM_TEMPLATE_RELATIVE_PATH = "scenario_test_mocks/vnf_mocks/ubuntu_template.json" def update_resource_group_in_input_file( @@ -67,9 +65,7 @@ def update_resource_group_in_input_file( class VnfNsdTest(LiveScenarioTest): """This class contains the integration tests for the aosm extension for vnf definition type.""" - @ResourceGroupPreparer( - name_prefix="cli_test_vnf_nsd_", location="uaenorth" - ) + @ResourceGroupPreparer(name_prefix="cli_test_vnf_nsd_", location="uaenorth") def test_vnf_nsd_publish_and_delete(self, resource_group): """ This test creates a vnf nfd and nsd, publishes them, and then deletes them. @@ -108,9 +104,7 @@ def test_vnf_nsd_publish_and_delete(self, resource_group): finally: # If the command fails, then the test should fail. # We still need to clean up the resources, so we run the delete command. - self.cmd( - f'az aosm nsd delete -f "{nsd_input_file_path}" --clean --force' - ) + self.cmd(f'az aosm nsd delete -f "{nsd_input_file_path}" --clean --force') self.cmd( f'az aosm nfd delete --definition-type vnf -f "{nfd_input_file_path}" --clean --force' ) diff --git a/src/aosm/azext_aosm/tests/latest/unit_test/test_bicep_builder.py b/src/aosm/azext_aosm/tests/latest/unit_test/test_bicep_builder.py index 529a65dbf73..11b9c699575 100644 --- a/src/aosm/azext_aosm/tests/latest/unit_test/test_bicep_builder.py +++ b/src/aosm/azext_aosm/tests/latest/unit_test/test_bicep_builder.py @@ -7,7 +7,9 @@ from unittest import TestCase from unittest.mock import MagicMock, patch -from azext_aosm.definition_folder.builder.bicep_builder import BicepDefinitionElementBuilder +from azext_aosm.definition_folder.builder.bicep_builder import ( + BicepDefinitionElementBuilder, +) class TestBicepDefinitionElementBuilder(TestCase): @@ -20,8 +22,7 @@ def test_write(self, mock_mkdir, mock_write_text): # Create a Bicep definition element builder. bicep_definition_element_builder = BicepDefinitionElementBuilder( - Path("/some/folder"), - "some bicep content" + Path("/some/folder"), "some bicep content" ) # Write the definition element to disk. @@ -38,8 +39,7 @@ def test_write_supporting_files(self, mock_mkdir, mock_write_text): # Create a Bicep definition element builder. bicep_definition_element_builder = BicepDefinitionElementBuilder( - Path("/some/folder"), - "some bicep content" + Path("/some/folder"), "some bicep content" ) # Create some mocks to act as supporting files. diff --git a/src/aosm/azext_aosm/tests/latest/unit_test/test_helm_chart_input.py b/src/aosm/azext_aosm/tests/latest/unit_test/test_helm_chart_input.py index 98f34185b0d..b492dff9b92 100644 --- a/src/aosm/azext_aosm/tests/latest/unit_test/test_helm_chart_input.py +++ b/src/aosm/azext_aosm/tests/latest/unit_test/test_helm_chart_input.py @@ -3,15 +3,16 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- import logging -from unittest import TestCase -from pathlib import Path import os import sys +from pathlib import Path +from unittest import TestCase -from azext_aosm.inputs.helm_chart_input import ( - HelmChartInput, +from azext_aosm.common.exceptions import ( + DefaultValuesNotFoundError, + TemplateValidationError, ) -from azext_aosm.common.exceptions import DefaultValuesNotFoundError +from azext_aosm.inputs.helm_chart_input import HelmChartInput code_directory = os.path.dirname(__file__) parent_directory = os.path.abspath(os.path.join(code_directory, "..")) @@ -41,9 +42,8 @@ def test_validate_template_valid_chart(self): ), ) - output = helm_chart_input.validate_template() - - assert output == "" + # A valid template does not raise exceptions or return anything. + helm_chart_input.validate_template() def test_validate_template_invalid_chart(self): """Test validating an invalid Helm chart using helm template.""" @@ -59,9 +59,7 @@ def test_validate_template_invalid_chart(self): ), ) - output = helm_chart_input.validate_template() - - assert output != "" + self.assertRaises(TemplateValidationError, helm_chart_input.validate_template) def test_validate_values(self): """Test validating whether values exist in a helm chart.""" diff --git a/src/aosm/azext_aosm/tests/latest/unit_test/test_helm_chart_processor.py b/src/aosm/azext_aosm/tests/latest/unit_test/test_helm_chart_processor.py index 5697109e5df..83873e3a34e 100644 --- a/src/aosm/azext_aosm/tests/latest/unit_test/test_helm_chart_processor.py +++ b/src/aosm/azext_aosm/tests/latest/unit_test/test_helm_chart_processor.py @@ -1,10 +1,9 @@ import os +from pathlib import Path from unittest import TestCase from unittest.mock import Mock -from azext_aosm.build_processors.helm_chart_processor import ( - HelmChartProcessor, -) -from pathlib import Path + +from azext_aosm.build_processors.helm_chart_processor import HelmChartProcessor code_directory = os.path.dirname(__file__) parent_directory = os.path.abspath(os.path.join(code_directory, "..")) diff --git a/src/aosm/setup.py b/src/aosm/setup.py index c9bcfbae75c..36a210d0207 100644 --- a/src/aosm/setup.py +++ b/src/aosm/setup.py @@ -5,6 +5,7 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- from codecs import open + from setuptools import find_packages, setup try: @@ -32,7 +33,13 @@ "License :: OSI Approved :: MIT License", ] -DEPENDENCIES = ["oras~=0.1.19", "azure-storage-blob>=12.15.0", "jinja2>=3.1.2", "genson>=1.2.2", "ruamel.yaml>=0.17.4"] +DEPENDENCIES = [ + "oras~=0.1.19", + "azure-storage-blob>=12.15.0", + "jinja2>=3.1.2", + "genson>=1.2.2", + "ruamel.yaml>=0.17.4", +] with open("README.md", "r", encoding="utf-8") as f: README = f.read()