[RDBMS] Private DNS zone parameter added for restore command, high av…

…ailability validator (#18218)

* add private dns zone for restore and ha validator, template for github actions

* change auto registration and fix autogenerated name

* test  fixed

* add param explanation

* Fix typo

* Add functionalities

* update functionalities

* fix setup py

* delete automatic action run

* Update running condition

src/azure-cli/azure/cli/command_modules/rdbms/_flexible_server_util.py

@@ -13,13 +13,15 @@
 import string
 import yaml
 from knack.log import get_logger
+from msrestazure.tools import parse_resource_id
+from msrestazure.azure_exceptions import CloudError
+from azure.cli.core.azclierror import AuthenticationError
 from azure.core.paging import ItemPaged
 from azure.cli.core.commands.client_factory import get_subscription_id
 from azure.cli.core.commands import LongRunningOperation, _is_poller
 from azure.cli.core.azclierror import RequiredArgumentMissingError, InvalidArgumentValueError
 from azure.cli.command_modules.role.custom import create_service_principal_for_rbac
 from azure.mgmt.resource.resources.models import ResourceGroup
-from msrestazure.tools import parse_resource_id
 from ._client_factory import resource_client_factory, cf_mysql_flexible_location_capabilities, cf_postgres_flexible_location_capabilities
 
 logger = get_logger(__name__)
@@ -182,8 +184,11 @@ def get_mysql_list_skus_info(cmd, location):
 
 def _parse_list_skus(result, database_engine):
     result = _get_list_from_paged_response(result)
+    single_az = False
     if not result:
         raise InvalidArgumentValueError("No available SKUs in this location")
+    if len(result) == 1:
+        single_az = True
 
     tiers = result[0].supported_flexible_server_editions
     tiers_dict = {}
@@ -219,8 +224,8 @@ def _parse_list_skus(result, database_engine):
         tiers_dict[tier_name] = tier_dict
 
     if database_engine == 'mysql':
-        return tiers_dict, iops_dict
-    return tiers_dict
+        return tiers_dict, iops_dict, single_az
+    return tiers_dict, single_az
 
 
 def _get_available_values(sku_info, argument, tier=None):
@@ -333,9 +338,13 @@ def run_subprocess_get_output(command):
     return process
 
 
-def register_credential_secrets(cmd, server, repository):
+def register_credential_secrets(cmd, database_engine, server, repository):
+    logger.warning('Adding secret "AZURE_CREDENTIALS" to github repository')
     resource_group = parse_resource_id(server.id)["resource_group"]
-    scope = "/subscriptions/{}/resourceGroups/{}".format(get_subscription_id(cmd.cli_ctx), resource_group)
+    provider = "DBforMySQL"
+    if database_engine == "postgresql":
+        provider = "DBforPostgreSQL"
+    scope = "/subscriptions/{}/resourceGroups/{}/providers/Microsoft.{}/flexibleServers/{}".format(get_subscription_id(cmd.cli_ctx), resource_group, provider, server.name)
 
     app = create_service_principal_for_rbac(cmd, name=server.name, role='contributor', scopes=[scope])
     app['clientId'], app['clientSecret'], app['tenantId'] = app.pop('appId'), app.pop('password'), app.pop('tenant')
@@ -356,13 +365,14 @@ def register_credential_secrets(cmd, server, repository):
     os.remove(credential_file)
 
 
-def register_connection_secrets(cmd, database_engine, server, database_name, administrator_login, administrator_login_password, repository):
+def register_connection_secrets(cmd, database_engine, server, database_name, administrator_login, administrator_login_password, repository, connection_string_name):
+    logger.warning("Added secret %s to github repository", connection_string_name)
     if database_engine == 'postgresql':
         connection_string = "host={} port=5432 dbname={} user={} password={} sslmode=require".format(server.fully_qualified_domain_name, database_name, administrator_login, administrator_login_password)
-        run_subprocess('gh secret set {} --repo {} -b"{}"'.format(AZURE_POSTGRESQL_CONNECTION_STRING, repository, connection_string))
+        run_subprocess('gh secret set {} --repo {} -b"{}"'.format(connection_string_name, repository, connection_string))
     elif database_engine == 'mysql':
         connection_string = "Server={}; Port=3306; Database={}; Uid={}; Pwd={}; SslMode=Preferred;".format(server.fully_qualified_domain_name, database_name, administrator_login, administrator_login_password)
-        run_subprocess('gh secret set {} --repo {} -b"{}"'.format(AZURE_MYSQL_CONNECTION_STRING, repository, connection_string))
+        run_subprocess('gh secret set {} --repo {} -b"{}"'.format(connection_string_name, repository, connection_string))
 
 
 def fill_action_template(cmd, database_engine, server, database_name, administrator_login, administrator_login_password, file_name, action_name, repository):
@@ -373,28 +383,40 @@ def fill_action_template(cmd, database_engine, server, database_name, administra
 
     process = run_subprocess_get_output("gh secret list --repo {}".format(repository))
     github_secrets = process.stdout.read().strip().decode('UTF-8')
-    connection_string = AZURE_POSTGRESQL_CONNECTION_STRING if database_engine == 'postgresql' else AZURE_MYSQL_CONNECTION_STRING
+    # connection_string = AZURE_POSTGRESQL_CONNECTION_STRING if database_engine == 'postgresql' else AZURE_MYSQL_CONNECTION_STRING
 
     if AZURE_CREDENTIALS not in github_secrets:
-        register_credential_secrets(cmd,
-                                    server=server,
-                                    repository=repository)
-
-    if connection_string not in github_secrets:
+        try:
+            register_credential_secrets(cmd,
+                                        database_engine=database_engine,
+                                        server=server,
+                                        repository=repository)
+        except CloudError:
+            raise AuthenticationError('You do not have authorization to create a service principal to run azure service in github actions. \n'
+                                      'Please create a service principal that has access to the database server and add "AZURE_CREDENTIALS" secret to your github repository. \n'
+                                      'Follow the instruction here "aka.ms/github-actions-azure-credentials".')
+
+    connection_string_name = server.name.upper().replace("-", "_") + "_" + database_name.upper().replace("-", "_") + "_" + database_engine.upper() + "_CONNECTION_STRING"
+    if connection_string_name not in github_secrets:
         register_connection_secrets(cmd,
                                     database_engine=database_engine,
                                     server=server,
                                     database_name=database_name,
                                     administrator_login=administrator_login,
                                     administrator_login_password=administrator_login_password,
-                                    repository=repository)
+                                    repository=repository,
+                                    connection_string_name=connection_string_name)
 
     current_location = os.path.dirname(__file__)
 
     with open(current_location + "/templates/" + database_engine + "_githubaction_template.yaml", "r") as template_file:
         template = yaml.safe_load(template_file)
         template['jobs']['build']['steps'][2]['with']['server-name'] = server.fully_qualified_domain_name
-        template['jobs']['build']['steps'][2]['with']['sql-file'] = file_name
+        if database_engine == 'postgresql':
+            template['jobs']['build']['steps'][2]['with']['plsql-file'] = file_name
+        else:
+            template['jobs']['build']['steps'][2]['with']['sql-file'] = file_name
+        template['jobs']['build']['steps'][2]['with']['connection-string'] = "${{ secrets." + connection_string_name + " }}"
         with open(action_dir + action_name + '.yml', 'w', encoding='utf8') as yml_file:
             yml_file.write("on: [workflow_dispatch]\n")
             yml_file.write(yaml.dump(template))

src/azure-cli/azure/cli/command_modules/rdbms/_params.py

@@ -305,7 +305,7 @@ def _flexible_server_params(command_group):
             c.argument('high_availability', default="Disabled", options_list=['--high-availability'], help='Enable or disable high availability feature.  Default value is Disabled. High availability can only be set during flexible server create time')
             c.argument('assign_identity', options_list=['--assign-identity'],
                        help='Generate and assign an Azure Active Directory Identity for this server for use with key management services like Azure KeyVault. No need to enter extra argument.')
-            c.argument('private_dns_zone_arguments', options_list=['--private-dns-zone'], help='The name or id of new or existing private dns zone. You can use the private dns zone from same resource group, different resource group, or different subscription. If you want to use a zone from different resource group or subscription, please provide resource Id. CLI creates a new private dns zone within the same resource group if not provided by users.')
+            c.argument('private_dns_zone_arguments', options_list=['--private-dns-zone'], help='This parameter only applies for a server with private access. The name or id of new or existing private dns zone. You can use the private dns zone from same resource group, different resource group, or different subscription. If you want to use a zone from different resource group or subscription, please provide resource Id. CLI creates a new private dns zone within the same resource group if not provided by users.')
             c.argument('database_name', id_part=None, arg_type=database_name_setter_arg_type, options_list=['--database-name', '-d'], help='The name of the database to be created when provisioning the database server')
 
         with self.argument_context('{} flexible-server delete'.format(command_group)) as c:
@@ -324,6 +324,8 @@ def _flexible_server_params(command_group):
                            help='The name of the source server to restore from.')
                 c.argument('zone', options_list=['--zone'],
                            help='Availability zone into which to provision the resource.')
+                c.argument('private_dns_zone_arguments', options_list=['--private-dns-zone'],
+                           help='This parameter only applies for a server with private access. The name or id of new or existing private dns zone. You can use the private dns zone from same resource group, different resource group, or different subscription. If you want to use a zone from different resource group or subscription, please provide resource Id. CLI creates a new private dns zone within the same resource group if not provided by users.')
             elif command_group == 'mysql':
                 c.argument('source_server', options_list=['--source-server'],
                            help='The name or resource ID of the source server to restore from.')

src/azure-cli/azure/cli/command_modules/rdbms/flexible_server_custom_common.py

@@ -283,15 +283,14 @@ def github_actions_setup(cmd, client, resource_group_name, server_name, database
     if allow_push:
         logger.warning("Pushing the created action file to origin %s branch", branch)
         run_subprocess("git push origin {}".format(branch))
-        github_actions_run(action_name, branch)
     else:
         logger.warning('You did not set --allow-push parameter. Please push the prepared file %s to your remote repo and run "deploy run" command to activate the workflow.', action_path)
 
 
 def github_actions_run(action_name, branch):
 
     gitcli_check_and_login()
-    logger.warning("Created event for %s.yml in branch %s", action_name, branch)
+    logger.warning("Created an event for %s.yml in branch %s", action_name, branch)
     run_subprocess("gh workflow run {}.yml --ref {}".format(action_name, branch))
 
 

src/azure-cli/azure/cli/command_modules/rdbms/flexible_server_custom_mysql.py

@@ -39,7 +39,7 @@ def flexible_server_create(cmd, client, resource_group_name=None, server_name=No
     # validator
     if location is None:
         location = DEFAULT_LOCATION_MySQL
-    sku_info, iops_info = get_mysql_list_skus_info(cmd, location)
+    sku_info, iops_info, single_az = get_mysql_list_skus_info(cmd, location)
     mysql_arguments_validator(tier, sku_name, storage_mb, backup_retention, sku_info, version=version)
 
     db_context = DbContext(
@@ -49,6 +49,8 @@ def flexible_server_create(cmd, client, resource_group_name=None, server_name=No
     if high_availability is not None and high_availability.lower() == 'enabled':
         if tier == 'Burstable':
             raise ArgumentUsageError("High availability is not supported for Burstable tier")
+        if single_az:
+            raise ArgumentUsageError("This region is single availability zone. High availability is not supported in a single availability zone region.")
 
     # Raise error when user passes values for both parameters
     if subnet_arm_resource_id is not None and public_access is not None:
@@ -173,13 +175,12 @@ def flexible_server_update_custom_func(cmd, instance,
                                        tags=None,
                                        auto_grow=None,
                                        assign_identity=False,
-                                       ha_enabled=None,
                                        replication_role=None,
                                        maintenance_window=None,
                                        iops=None):
     # validator
     location = ''.join(instance.location.lower().split())
-    sku_info, iops_info = get_mysql_list_skus_info(cmd, location)
+    sku_info, iops_info, _ = get_mysql_list_skus_info(cmd, location)
     mysql_arguments_validator(tier, sku_name, storage_mb, backup_retention, sku_info, instance=instance)
 
     server_module_path = instance.__module__
@@ -247,7 +248,6 @@ def flexible_server_update_custom_func(cmd, instance,
                              ssl_enforcement=ssl_enforcement,
                              delegated_subnet_arguments=instance.delegated_subnet_arguments,
                              tags=tags,
-                             ha_enabled=ha_enabled,
                              replication_role=replication_role)
 
     if assign_identity:
@@ -543,8 +543,7 @@ def _determine_iops(storage_gb, iops_info, iops_input, tier, sku_name):
 def get_free_iops(storage_in_mb, iops_info, tier, sku_name):
     free_iops = MINIMUM_IOPS + (storage_in_mb // 1024) * 3
     max_supported_iops = iops_info[tier][sku_name]  # free iops cannot exceed maximum supported iops for the sku
-    logger.warning(iops_info[tier])
-    logger.warning(iops_info[tier][sku_name])
+
     return min(free_iops, max_supported_iops)
 
 

src/azure-cli/azure/cli/command_modules/rdbms/flexible_server_custom_postgres.py

@@ -29,7 +29,7 @@
 # region create without args
 # pylint: disable=too-many-locals
 # pylint: disable=too-many-statements
-# pylint: disable=raise-missing-from
+# pylint: disable=raise-missing-from, unbalanced-tuple-unpacking
 def flexible_server_create(cmd, client,
                            resource_group_name=None, server_name=None,
                            location=None, backup_retention=None,
@@ -44,7 +44,7 @@ def flexible_server_create(cmd, client,
     # validator
     if location is None:
         location = DEFAULT_LOCATION_PG
-    sku_info = get_postgres_list_skus_info(cmd, location)
+    sku_info, single_az = get_postgres_list_skus_info(cmd, location)
     pg_arguments_validator(tier, sku_name, storage_mb, sku_info, version=version)
     storage_mb *= 1024
 
@@ -55,6 +55,8 @@ def flexible_server_create(cmd, client,
     if high_availability is not None and high_availability.lower() == 'enabled':
         if tier == 'Burstable':
             raise ArgumentUsageError("High availability is not supported for Burstable tier")
+        if single_az:
+            raise ArgumentUsageError("This region is single availability zone. High availability is not supported in a single availability zone region.")
 
     # Raise error when user passes values for both parameters
     if subnet_arm_resource_id is not None and public_access is not None:
@@ -138,7 +140,8 @@ def flexible_server_create(cmd, client,
 
 def flexible_server_restore(cmd, client,
                             resource_group_name, server_name,
-                            source_server, restore_point_in_time=None, location=None, zone=None, no_wait=False):
+                            source_server, restore_point_in_time=None, location=None, zone=None, no_wait=False,
+                            private_dns_zone_arguments=None):
     provider = 'Microsoft.DBforPostgreSQL'
     validate_server_name(cf_postgres_check_resource_availability(cmd.cli_ctx, '_'), server_name, 'Microsoft.DBforPostgreSQL/flexibleServers')
 
@@ -169,7 +172,19 @@ def flexible_server_restore(cmd, client,
     try:
         source_server_object = client.get(id_parts['resource_group'], id_parts['name'])
         parameters.location = source_server_object.location
-        parameters.private_dns_zone_arguments = source_server_object.private_dns_zone_arguments
+        if source_server_object.public_network_access == 'Disabled':
+            parameters.private_dns_zone_arguments = source_server_object.private_dns_zone_arguments
+            if private_dns_zone_arguments is not None:
+                subnet_id = source_server_object.delegated_subnet_arguments.subnet_arm_resource_id
+                private_dns_zone_id = prepare_private_dns_zone(cmd,
+                                                               'PostgreSQL',
+                                                               resource_group_name,
+                                                               server_name,
+                                                               private_dns_zone=private_dns_zone_arguments,
+                                                               subnet_id=subnet_id,
+                                                               location=location)
+                parameters.private_dns_zone_arguments = postgresql_flexibleservers.models.ServerPropertiesPrivateDnsZoneArguments(private_dns_zone_arm_resource_id=private_dns_zone_id)
+
     except Exception as e:
         raise ResourceNotFoundError(e)
 
@@ -190,7 +205,7 @@ def flexible_server_update_custom_func(cmd, instance,
 
     # validator
     location = ''.join(instance.location.lower().split())
-    sku_info = get_postgres_list_skus_info(cmd, location)
+    sku_info, _ = get_postgres_list_skus_info(cmd, location)
     pg_arguments_validator(tier, sku_name, storage_mb, sku_info, instance=instance)
 
     server_module_path = instance.__module__