[HDInsight] - Support ESP, BYOK and update tags (#8294)

src/command_modules/azure-cli-hdinsight/HISTORY.rst

@@ -3,6 +3,16 @@
 Release History
 ===============
 
+0.3.0
++++++
+
+* BREAKING CHANGE: `create`, `application create`: Removed the `--virtual-network` and `--subnet-name` parameters.
+                   `create`: Change the `--storage-account` to accept name or id of storage account instead of blob endpoints.
+* `create`: added the `--vnet-name` and `--subnet-name` parameters.
+* `create`: added support for Enterprise Security Package and disk encryption
+* Added `rotate-disk-encryption-key` command
+* Added `update` command
+
 0.2.0
 +++++
 

src/command_modules/azure-cli-hdinsight/azure/cli/command_modules/hdinsight/_client_factory.py

@@ -17,6 +17,13 @@ def cf_storage(cli_ctx, *_, **__):
     return get_mgmt_service_client(cli_ctx, StorageManagementClient)
 
 
+def cf_network(cli_ctx, aux_subscriptions=None, **_):
+    from azure.cli.core.profiles import ResourceType
+    from azure.cli.core.commands.client_factory import get_mgmt_service_client
+    return get_mgmt_service_client(cli_ctx, ResourceType.MGMT_NETWORK,
+                                   aux_subscriptions=aux_subscriptions)
+
+
 def cf_hdinsight(cli_ctx, *_, **__):
     from azure.cli.core.commands.client_factory import get_mgmt_service_client
     from azure.mgmt.hdinsight import HDInsightManagementClient

src/command_modules/azure-cli-hdinsight/azure/cli/command_modules/hdinsight/_completers.py

@@ -4,37 +4,14 @@
 # --------------------------------------------------------------------------------------------
 
 from azure.cli.core.decorators import Completer
-from ._client_factory import cf_storage
+from ._client_factory import cf_network
 
 
+# pylint: disable=inconsistent-return-statements
 @Completer
-def storage_account_completion_list(cmd, prefix, namespace, **kwargs):  # pylint: disable=unused-argument
-    storage_client = cf_storage(cmd.cli_ctx)
-    rg = getattr(namespace, 'resource_group_name', None)
-    if rg:
-        storage_accounts = storage_client.storage_accounts.list_by_resource_group(rg)
-    else:
-        storage_accounts = storage_client.storage_accounts.list()
-
-    def extract_blob_endpoint(storage_account):
-        return storage_account and storage_account.primary_endpoints and storage_account.primary_endpoints.blob
-
-    def extract_host(uri):
-        import re
-        return re.search('//(.*)/', uri).groups()[0]
-
-    return [extract_host(extract_blob_endpoint(s)) for s in storage_accounts]
-
-
-@Completer
-def storage_account_key_completion_list(cmd, prefix, namespace, **kwargs):  # pylint: disable=unused-argument
-    from .util import get_key_for_storage_account
-
-    storage_endpoint = getattr(namespace, 'storage_account', None)
-    if not storage_endpoint:
-        return []
-
-    rg = getattr(namespace, 'resource_group_name', None)
-
-    key = get_key_for_storage_account(cmd, storage_endpoint, rg)
-    return [key] if key else []
+def subnet_completion_list(cmd, prefix, namespace, **kwargs):  # pylint: disable=unused-argument
+    client = cf_network(cmd.cli_ctx)
+    if namespace.resource_group_name and namespace.vnet_name:
+        rg = namespace.resource_group_name
+        vnet = namespace.vnet_name
+        return [r.name for r in client.subnets.list(resource_group_name=rg, virtual_network_name=vnet)]

src/command_modules/azure-cli-hdinsight/azure/cli/command_modules/hdinsight/_help.py

@@ -18,10 +18,27 @@
     examples:
         - name: Create a cluster with an existing storage account.
           text: |-
-              az hdinsight create -t spark -n MyCluster -g MyResourceGroup \\
-              -p {HTTP_password} \\
-              --storage-account MyStorageAccount.blob.core.windows.net \\
-              --storage-account-key {storage_account_key}
+              az hdinsight create -t spark -g MyResourceGroup -n MyCluster \\
+              -p "HttpPassword1234!" \\
+              --storage-account MyStorageAccount
+        - name: Create a cluster with Enterprise Security Package.
+          text: |-
+              az hdinsight create -t spark -g MyResourceGroup -n MyCluster \\
+              -p "HttpPassword1234!" \\
+              --storage-account MyStorageAccount \\
+              --subnet "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/MyRG/providers/Microsoft.Network/virtualNetworks/MyVnet/subnets/subnet1" \\
+              --domain "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/MyRG/providers/Microsoft.AAD/domainServices/MyDomain.onmicrosoft.com" \\
+              --assign-identity "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/MyMsiRG/providers/Microsoft.ManagedIdentity/userAssignedIdentities/MyMSI" \\
+              --cluster-admin-account MyAdminAccount@MyDomain.onmicrosoft.com
+        - name: Create a Kafka cluster with disk encryption. See https://docs.microsoft.com/en-us/azure/hdinsight/kafka/apache-kafka-byok.
+          text: |-
+             az hdinsight create -t kafka -g MyResourceGroup -n MyCluster \\
+             -p "HttpPassword1234!" --workernode-data-disks-per-node 2 \\
+             --storage-account MyStorageAccount \\
+             --encryption-key-name kafkaClusterKey \\
+             --encryption-key-version 00000000000000000000000000000000 \\
+             --encryption-vault-uri https://MyKeyVault.vault.azure.net \\
+             --assign-identity MyMSI
 """
 
 helps['hdinsight list'] = """
@@ -34,6 +51,11 @@
     short-summary: Place the CLI in a waiting state until an operation is complete.
 """
 
+helps['hdinsight rotate-disk-encryption-key'] = """
+    type: command
+    short-summary: Rotate disk encryption key of the specified HDInsight cluster.
+"""
+
 helps['hdinsight application'] = """
     type: group
     short-summary: Manage HDInsight applications.
@@ -45,14 +67,14 @@
     examples:
         - name: Create an application with a script URI.
           text: |-
-              az hdinsight application create -n MyCluster -g MyResourceGroup \\
+              az hdinsight application create -g MyResourceGroup -n MyCluster \\
               --application-name MyApplication \\
               --script-uri https://path/to/install/script.sh \\
               --script-action-name MyScriptAction \\
               --script-parameters '"-option value"'
         - name: Create an application with a script URI and specified edge node size.
           text: |-
-              az hdinsight application create -n MyCluster -g MyResourceGroup \\
+              az hdinsight application create -g MyResourceGroup -n MyCluster \\
               --application-name MyApplication \\
               --script-uri https://path/to/install/script.sh \\
               --script-action-name MyScriptAction \\

src/command_modules/azure-cli-hdinsight/azure/cli/command_modules/hdinsight/_params.py

@@ -4,24 +4,33 @@
 # --------------------------------------------------------------------------------------------
 
 from azure.cli.core.commands.parameters import get_enum_type, name_type, tags_type, get_resource_name_completion_list, \
-    get_generic_completion_list
-from ._validators import validate_component_version
+    get_generic_completion_list, get_three_state_flag
+from ._validators import (validate_component_version,
+                          validate_storage_account,
+                          validate_msi,
+                          validate_subnet,
+                          validate_domain_service)
 
 # Cluster types may be added in the future. Therefore, this list can be used for completion, but not input validation.
 known_cluster_types = ["hadoop", "interactivehive", "hbase", "kafka", "storm", "spark", "rserver", "mlservices"]
+
+# Known role (node) types.
 known_role_types = ["headnode", "workernode", "zookeepernode", "edgenode"]
 
 
 # pylint: disable=too-many-statements
 def load_arguments(self, _):
-    from ._completers import storage_account_completion_list, storage_account_key_completion_list
+    from ._completers import subnet_completion_list
     from knack.arguments import CLIArgumentType
+    from azure.mgmt.hdinsight.models import Tier, JsonWebKeyEncryptionAlgorithm
     node_size_type = CLIArgumentType(arg_group='Node',
                                      help='The size of the node. See also: https://docs.microsoft.com/en-us/azure/'
                                           'hdinsight/hdinsight-hadoop-provision-linux-clusters#configure-cluster-size')
 
     # cluster
     with self.argument_context('hdinsight') as c:
+
+        # Cluster
         c.argument('cluster_name', arg_type=name_type,
                    completer=get_resource_name_completion_list('Microsoft.HDInsight/clusters'),
                    help='The name of the cluster.')
@@ -43,18 +52,26 @@ def load_arguments(self, _):
                         'hdinsight-versions')
         c.argument('cluster_configurations', arg_group='Cluster',
                    help='Extra configurations of various components, in JSON.')
-        c.argument('cluster_tier', arg_type=get_enum_type(['standard', 'premium']), arg_group='Cluster',
-                   help='The tier of the cluster: standard or premium.')
+        c.argument('cluster_tier', arg_type=get_enum_type(Tier), arg_group='Cluster',
+                   help='The tier of the cluster')
+        c.argument('esp', arg_group='Cluster', arg_type=get_three_state_flag(),
+                   help='Specify to create cluster with Enterprise Security Package')
+
+        # HTTP
         c.argument('http_username', options_list=['--http-user', '-u'], arg_group='HTTP',
                    help='HTTP username for the cluster.  Default: admin.')
         c.argument('http_password', options_list=['--http-password', '-p'], arg_group='HTTP',
                    help='HTTP password for the cluster.')
+
+        # SSH
         c.argument('ssh_username', options_list=['--ssh-user', '-U'], arg_group='SSH',
                    help='SSH username for the cluster nodes.')
         c.argument('ssh_password', options_list=['--ssh-password', '-P'], arg_group='SSH',
                    help='SSH password for the cluster nodes. If none specified, uses the HTTP password.')
         c.argument('ssh_public_key', options_list=['--ssh-public-key', '-K'], arg_group='SSH',
                    help='SSH public key for the cluster nodes.')
+
+        # Node
         c.argument('headnode_size', arg_type=node_size_type)
         c.argument('workernode_size', arg_type=node_size_type)
         c.argument('workernode_data_disks_per_node', arg_group='Node',
@@ -66,26 +83,66 @@ def load_arguments(self, _):
                    help='The size of the data disk in GB, e.g. 1023.')
         c.argument('zookeepernode_size', arg_type=node_size_type)
         c.argument('edgenode_size', arg_type=node_size_type)
-        c.argument('workernode_count', options_list=['--size', '-s'], arg_group='Cluster',
+        c.argument('workernode_count', options_list=['--size', '-s'], arg_group='Node',
                    help='The number of worker nodes in the cluster.')
-        c.argument('storage_account', arg_group='Storage', completer=storage_account_completion_list,
-                   help='The storage account, e.g. "<name>.blob.core.windows.net".')
-        c.argument('storage_account_key', arg_group='Storage', completer=storage_account_key_completion_list,
+
+        # Storage
+        c.argument('storage_account', arg_group='Storage', validator=validate_storage_account,
+                   completer=get_resource_name_completion_list('Microsoft.Storage/storageAccounts'),
+                   help='The name or ID of the storage account.')
+        c.argument('storage_account_key', arg_group='Storage',
                    help='The storage account key. A key can be retrieved automatically '
                         'if the user has access to the storage account.')
         c.argument('storage_default_container', arg_group='Storage',
                    help='The storage container the cluster will use. '
                         'Uses the cluster name if none was specified. (WASB only)')
         c.argument('storage_default_filesystem', arg_group='Storage',
                    help='The storage filesystem the cluster will use. (DFS only)')
-        c.argument('virtual_network', arg_group='Network',
-                   help='The virtual network resource ID of an existing virtual network.')
-        c.argument('subnet_name', arg_group='Network',
-                   help='The name of the subnet in the specified virtual network.')
+
+        # Network
+        c.argument('vnet_name', arg_group='Network', validator=validate_subnet,
+                   completer=get_resource_name_completion_list('Microsoft.Network/virtualNetworks'),
+                   help='The name of a virtual network.')
+        c.argument('subnet', arg_group='Network',
+                   completer=subnet_completion_list,
+                   help='The name or ID of subnet. If name is supplied, `--vnet-name` must be supplied.')
+
+        # Script Action
         c.argument('script_action_name', arg_group='Script Action', help='The name of the script action.')
         c.argument('script_uri', arg_group='Script Action', help='The URI to the script.')
         c.argument('script_parameters', arg_group='Script Action', help='The parameters for the script.')
 
+        # Domain Service
+        c.argument('domain', arg_group='Domain Service', validator=validate_domain_service,
+                   help='The name or resource ID of the user\'s Azure Active Directory Domain Service. '
+                        'Required only when create cluster with Enterprise Security Package.')
+        c.argument('cluster_users_group_dns', arg_group='Domain Service', nargs='+',
+                   help='A space-delimited list of Distinguished Names for cluster user groups. '
+                        'Required only when create cluster with Enterprise Security Package. ')
+        c.argument('cluster_admin_password', arg_group='Domain Service',
+                   help='The domain admin password. '
+                        'Required only when create cluster with Enterprise Security Package.')
+        c.argument('cluster_admin_account', arg_group='Domain Service',
+                   help='The domain user account that will have admin privileges on the cluster. '
+                        'Required only when create cluster with Enterprise Security Package.')
+        c.argument('ldaps_urls', arg_group='Domain Service', nargs='+',
+                   help='A space-delimited list of LDAPS protocol URLs to communicate with the Active Directory. '
+                        'Required only when create cluster with Enterprise Security Package.')
+
+        # Customer Managed Key
+        c.argument('encryption_vault_uri', arg_group='Customer Managed Key',
+                   help='Base key vault URI where the customers key is located eg. https://myvault.vault.azure.net')
+        c.argument('encryption_key_name', arg_group='Customer Managed Key',
+                   help='Key name that is used for enabling disk encryption.')
+        c.argument('encryption_key_version', arg_group='Customer Managed Key',
+                   help='Key version that is used for enabling disk encryption.')
+        c.argument('encryption_algorithm', arg_type=get_enum_type(JsonWebKeyEncryptionAlgorithm),
+                   arg_group='Customer Managed Key', help='Algorithm identifier for encryption.')
+
+        # Managed Service Identity
+        c.argument('assign_identity', arg_group='Managed Service Identity', validator=validate_msi,
+                   help="The name or ID of user assigned identity.")
+
     # application
     with self.argument_context('hdinsight application') as c:
         c.argument('application_name', arg_group='Application', help='The constant value for the application name.')

src/command_modules/azure-cli-hdinsight/azure/cli/command_modules/hdinsight/_validators.py

@@ -11,3 +11,75 @@ def validate_component_version(namespace):
         if any(invalid_component_versions):
             raise ValueError('Component verions must be in the form component=version. '
                              'Invalid component version(s): {}'.format(', '.join(invalid_component_versions)))
+
+
+# Validate storage account.
+def validate_storage_account(cmd, namespace):
+    from azure.cli.core.commands.client_factory import get_subscription_id
+    from msrestazure.tools import is_valid_resource_id, resource_id
+    if namespace.storage_account:
+        if not is_valid_resource_id(namespace.storage_account):
+            namespace.storage_account = resource_id(
+                subscription=get_subscription_id(cmd.cli_ctx),
+                resource_group=namespace.resource_group_name,
+                namespace='Microsoft.Storage', type='storageAccounts',
+                name=namespace.storage_account
+            )
+
+
+# Validates if a subnet id or name have been given by the user. If subnet id is given, vnet-name should not be provided.
+def validate_subnet(cmd, namespace):
+    from msrestazure.tools import resource_id, is_valid_resource_id
+    from azure.cli.core.commands.client_factory import get_subscription_id
+    from knack.util import CLIError
+
+    subnet = namespace.subnet
+    subnet_is_id = is_valid_resource_id(subnet)
+    vnet = namespace.vnet_name
+
+    if (subnet_is_id and not vnet) or (not subnet and not vnet):
+        pass
+    elif subnet and not subnet_is_id and vnet:
+        namespace.subnet = resource_id(
+            subscription=get_subscription_id(cmd.cli_ctx),
+            resource_group=namespace.resource_group_name,
+            namespace='Microsoft.Network',
+            type='virtualNetworks',
+            name=vnet,
+            child_type_1='subnets',
+            child_name_1=subnet)
+    else:
+        raise CLIError('usage error: [--subnet ID | --subnet NAME --vnet-name NAME]')
+    delattr(namespace, 'vnet_name')
+
+
+# Validate managed identity.
+def validate_msi(cmd, namespace):
+    from azure.cli.core.commands.client_factory import get_subscription_id
+    from msrestazure.tools import is_valid_resource_id, resource_id
+
+    if namespace.assign_identity is not None:
+        if not is_valid_resource_id(namespace.assign_identity):
+            namespace.assign_identity = resource_id(
+                subscription=get_subscription_id(cmd.cli_ctx),
+                resource_group=namespace.resource_group_name,
+                namespace='Microsoft.ManagedIdentity',
+                type='userAssignedIdentities',
+                name=namespace.assign_identity
+            )
+
+
+# Validate domain service.
+def validate_domain_service(cmd, namespace):
+    from azure.cli.core.commands.client_factory import get_subscription_id
+    from msrestazure.tools import is_valid_resource_id, resource_id
+
+    if namespace.domain:
+        if not is_valid_resource_id(namespace.domain):
+            namespace.domain = resource_id(
+                subscription=get_subscription_id(cmd.cli_ctx),
+                resource_group=namespace.resource_group_name,
+                namespace='Microsoft.AAD',
+                type='domainServices',
+                name=namespace.domain
+            )

src/command_modules/azure-cli-hdinsight/azure/cli/command_modules/hdinsight/commands.py

@@ -52,6 +52,8 @@ def load_command_table(self, _):
         g.custom_command('list', 'list_clusters')
         g.wait_command('wait')
         g.command('delete', 'delete', confirmation=True, supports_no_wait=True)
+        g.custom_command('rotate-disk-encryption-key', 'rotate_hdi_cluster_key', supports_no_wait=True)
+        g.command('update', 'update', supports_no_wait=True)
 
     # usage operations
     with self.command_group('hdinsight', hdinsight_locations_sdk, client_factory=cf_hdinsight_locations) as g: