From ad077612754f9f6207c81fb707641865e173b4f0 Mon Sep 17 00:00:00 2001 From: Azure SDK for Python bot Date: Thu, 27 Sep 2018 22:33:20 +0000 Subject: [PATCH 1/2] Generated from 9dec6b08742c433d0429a8737a65fdc690863d6b Adding parameters --- .../azure/mgmt/datafactory/models/__init__.py | 3 ++ .../hd_insight_on_demand_linked_service.py | 6 +++ ...hd_insight_on_demand_linked_service_py3.py | 8 ++- .../mgmt/datafactory/models/script_action.py | 49 +++++++++++++++++++ .../datafactory/models/script_action_py3.py | 49 +++++++++++++++++++ 5 files changed, 114 insertions(+), 1 deletion(-) create mode 100644 azure-mgmt-datafactory/azure/mgmt/datafactory/models/script_action.py create mode 100644 azure-mgmt-datafactory/azure/mgmt/datafactory/models/script_action_py3.py diff --git a/azure-mgmt-datafactory/azure/mgmt/datafactory/models/__init__.py b/azure-mgmt-datafactory/azure/mgmt/datafactory/models/__init__.py index 436179448b73..e958df5cca8c 100644 --- a/azure-mgmt-datafactory/azure/mgmt/datafactory/models/__init__.py +++ b/azure-mgmt-datafactory/azure/mgmt/datafactory/models/__init__.py @@ -73,6 +73,7 @@ from .responsys_linked_service_py3 import ResponsysLinkedService from .azure_databricks_linked_service_py3 import AzureDatabricksLinkedService from .azure_data_lake_analytics_linked_service_py3 import AzureDataLakeAnalyticsLinkedService + from .script_action_py3 import ScriptAction from .hd_insight_on_demand_linked_service_py3 import HDInsightOnDemandLinkedService from .salesforce_marketing_cloud_linked_service_py3 import SalesforceMarketingCloudLinkedService from .netezza_linked_service_py3 import NetezzaLinkedService @@ -418,6 +419,7 @@ from .responsys_linked_service import ResponsysLinkedService from .azure_databricks_linked_service import AzureDatabricksLinkedService from .azure_data_lake_analytics_linked_service import AzureDataLakeAnalyticsLinkedService + from .script_action import ScriptAction from .hd_insight_on_demand_linked_service import HDInsightOnDemandLinkedService from .salesforce_marketing_cloud_linked_service import SalesforceMarketingCloudLinkedService from .netezza_linked_service import NetezzaLinkedService @@ -831,6 +833,7 @@ 'ResponsysLinkedService', 'AzureDatabricksLinkedService', 'AzureDataLakeAnalyticsLinkedService', + 'ScriptAction', 'HDInsightOnDemandLinkedService', 'SalesforceMarketingCloudLinkedService', 'NetezzaLinkedService', diff --git a/azure-mgmt-datafactory/azure/mgmt/datafactory/models/hd_insight_on_demand_linked_service.py b/azure-mgmt-datafactory/azure/mgmt/datafactory/models/hd_insight_on_demand_linked_service.py index 8069bd2d2d26..bd84aabc5012 100644 --- a/azure-mgmt-datafactory/azure/mgmt/datafactory/models/hd_insight_on_demand_linked_service.py +++ b/azure-mgmt-datafactory/azure/mgmt/datafactory/models/hd_insight_on_demand_linked_service.py @@ -132,6 +132,10 @@ class HDInsightOnDemandLinkedService(LinkedService): :param zookeeper_node_size: Specifies the size of the Zoo Keeper node for the HDInsight cluster. :type zookeeper_node_size: object + :param script_actions: Custom script actions to run on HDI ondemand + cluster once it's up. Please refer to + https://docs.microsoft.com/en-us/azure/hdinsight/hdinsight-hadoop-customize-cluster-linux?toc=%2Fen-us%2Fazure%2Fhdinsight%2Fr-server%2FTOC.json&bc=%2Fen-us%2Fazure%2Fbread%2Ftoc.json#understanding-script-actions. + :type script_actions: list[~azure.mgmt.datafactory.models.ScriptAction] """ _validation = { @@ -182,6 +186,7 @@ class HDInsightOnDemandLinkedService(LinkedService): 'head_node_size': {'key': 'typeProperties.headNodeSize', 'type': 'object'}, 'data_node_size': {'key': 'typeProperties.dataNodeSize', 'type': 'object'}, 'zookeeper_node_size': {'key': 'typeProperties.zookeeperNodeSize', 'type': 'object'}, + 'script_actions': {'key': 'typeProperties.scriptActions', 'type': '[ScriptAction]'}, } def __init__(self, **kwargs): @@ -216,4 +221,5 @@ def __init__(self, **kwargs): self.head_node_size = kwargs.get('head_node_size', None) self.data_node_size = kwargs.get('data_node_size', None) self.zookeeper_node_size = kwargs.get('zookeeper_node_size', None) + self.script_actions = kwargs.get('script_actions', None) self.type = 'HDInsightOnDemand' diff --git a/azure-mgmt-datafactory/azure/mgmt/datafactory/models/hd_insight_on_demand_linked_service_py3.py b/azure-mgmt-datafactory/azure/mgmt/datafactory/models/hd_insight_on_demand_linked_service_py3.py index 75f601f6a97b..5566a022bda2 100644 --- a/azure-mgmt-datafactory/azure/mgmt/datafactory/models/hd_insight_on_demand_linked_service_py3.py +++ b/azure-mgmt-datafactory/azure/mgmt/datafactory/models/hd_insight_on_demand_linked_service_py3.py @@ -132,6 +132,10 @@ class HDInsightOnDemandLinkedService(LinkedService): :param zookeeper_node_size: Specifies the size of the Zoo Keeper node for the HDInsight cluster. :type zookeeper_node_size: object + :param script_actions: Custom script actions to run on HDI ondemand + cluster once it's up. Please refer to + https://docs.microsoft.com/en-us/azure/hdinsight/hdinsight-hadoop-customize-cluster-linux?toc=%2Fen-us%2Fazure%2Fhdinsight%2Fr-server%2FTOC.json&bc=%2Fen-us%2Fazure%2Fbread%2Ftoc.json#understanding-script-actions. + :type script_actions: list[~azure.mgmt.datafactory.models.ScriptAction] """ _validation = { @@ -182,9 +186,10 @@ class HDInsightOnDemandLinkedService(LinkedService): 'head_node_size': {'key': 'typeProperties.headNodeSize', 'type': 'object'}, 'data_node_size': {'key': 'typeProperties.dataNodeSize', 'type': 'object'}, 'zookeeper_node_size': {'key': 'typeProperties.zookeeperNodeSize', 'type': 'object'}, + 'script_actions': {'key': 'typeProperties.scriptActions', 'type': '[ScriptAction]'}, } - def __init__(self, *, cluster_size, time_to_live, version, linked_service_name, host_subscription_id, tenant, cluster_resource_group, additional_properties=None, connect_via=None, description: str=None, parameters=None, annotations=None, service_principal_id=None, service_principal_key=None, cluster_name_prefix=None, cluster_user_name=None, cluster_password=None, cluster_ssh_user_name=None, cluster_ssh_password=None, additional_linked_service_names=None, hcatalog_linked_service_name=None, cluster_type=None, spark_version=None, core_configuration=None, h_base_configuration=None, hdfs_configuration=None, hive_configuration=None, map_reduce_configuration=None, oozie_configuration=None, storm_configuration=None, yarn_configuration=None, encrypted_credential=None, head_node_size=None, data_node_size=None, zookeeper_node_size=None, **kwargs) -> None: + def __init__(self, *, cluster_size, time_to_live, version, linked_service_name, host_subscription_id, tenant, cluster_resource_group, additional_properties=None, connect_via=None, description: str=None, parameters=None, annotations=None, service_principal_id=None, service_principal_key=None, cluster_name_prefix=None, cluster_user_name=None, cluster_password=None, cluster_ssh_user_name=None, cluster_ssh_password=None, additional_linked_service_names=None, hcatalog_linked_service_name=None, cluster_type=None, spark_version=None, core_configuration=None, h_base_configuration=None, hdfs_configuration=None, hive_configuration=None, map_reduce_configuration=None, oozie_configuration=None, storm_configuration=None, yarn_configuration=None, encrypted_credential=None, head_node_size=None, data_node_size=None, zookeeper_node_size=None, script_actions=None, **kwargs) -> None: super(HDInsightOnDemandLinkedService, self).__init__(additional_properties=additional_properties, connect_via=connect_via, description=description, parameters=parameters, annotations=annotations, **kwargs) self.cluster_size = cluster_size self.time_to_live = time_to_live @@ -216,4 +221,5 @@ def __init__(self, *, cluster_size, time_to_live, version, linked_service_name, self.head_node_size = head_node_size self.data_node_size = data_node_size self.zookeeper_node_size = zookeeper_node_size + self.script_actions = script_actions self.type = 'HDInsightOnDemand' diff --git a/azure-mgmt-datafactory/azure/mgmt/datafactory/models/script_action.py b/azure-mgmt-datafactory/azure/mgmt/datafactory/models/script_action.py new file mode 100644 index 000000000000..50bc0131a5cf --- /dev/null +++ b/azure-mgmt-datafactory/azure/mgmt/datafactory/models/script_action.py @@ -0,0 +1,49 @@ +# coding=utf-8 +# -------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# +# Code generated by Microsoft (R) AutoRest Code Generator. +# Changes may cause incorrect behavior and will be lost if the code is +# regenerated. +# -------------------------------------------------------------------------- + +from msrest.serialization import Model + + +class ScriptAction(Model): + """Custom script action to run on HDI ondemand cluster once it's up. + + All required parameters must be populated in order to send to Azure. + + :param name: Required. The user provided name of the script action. + :type name: str + :param uri: Required. The URI for the script action. + :type uri: str + :param roles: Required. The node types on which the script action should + be executed. + :type roles: object + :param parameters: The parameters for the script action. + :type parameters: str + """ + + _validation = { + 'name': {'required': True}, + 'uri': {'required': True}, + 'roles': {'required': True}, + } + + _attribute_map = { + 'name': {'key': 'name', 'type': 'str'}, + 'uri': {'key': 'uri', 'type': 'str'}, + 'roles': {'key': 'roles', 'type': 'object'}, + 'parameters': {'key': 'parameters', 'type': 'str'}, + } + + def __init__(self, **kwargs): + super(ScriptAction, self).__init__(**kwargs) + self.name = kwargs.get('name', None) + self.uri = kwargs.get('uri', None) + self.roles = kwargs.get('roles', None) + self.parameters = kwargs.get('parameters', None) diff --git a/azure-mgmt-datafactory/azure/mgmt/datafactory/models/script_action_py3.py b/azure-mgmt-datafactory/azure/mgmt/datafactory/models/script_action_py3.py new file mode 100644 index 000000000000..c0e278073219 --- /dev/null +++ b/azure-mgmt-datafactory/azure/mgmt/datafactory/models/script_action_py3.py @@ -0,0 +1,49 @@ +# coding=utf-8 +# -------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# +# Code generated by Microsoft (R) AutoRest Code Generator. +# Changes may cause incorrect behavior and will be lost if the code is +# regenerated. +# -------------------------------------------------------------------------- + +from msrest.serialization import Model + + +class ScriptAction(Model): + """Custom script action to run on HDI ondemand cluster once it's up. + + All required parameters must be populated in order to send to Azure. + + :param name: Required. The user provided name of the script action. + :type name: str + :param uri: Required. The URI for the script action. + :type uri: str + :param roles: Required. The node types on which the script action should + be executed. + :type roles: object + :param parameters: The parameters for the script action. + :type parameters: str + """ + + _validation = { + 'name': {'required': True}, + 'uri': {'required': True}, + 'roles': {'required': True}, + } + + _attribute_map = { + 'name': {'key': 'name', 'type': 'str'}, + 'uri': {'key': 'uri', 'type': 'str'}, + 'roles': {'key': 'roles', 'type': 'object'}, + 'parameters': {'key': 'parameters', 'type': 'str'}, + } + + def __init__(self, *, name: str, uri: str, roles, parameters: str=None, **kwargs) -> None: + super(ScriptAction, self).__init__(**kwargs) + self.name = name + self.uri = uri + self.roles = roles + self.parameters = parameters From 60e4d33e7857fddb9b76f7ab172c2736d855dbcc Mon Sep 17 00:00:00 2001 From: azuresdkci Date: Thu, 27 Sep 2018 22:35:48 +0000 Subject: [PATCH 2/2] Packaging update of azure-mgmt-datafactory --- azure-mgmt-datafactory/MANIFEST.in | 1 - azure-mgmt-datafactory/azure/__init__.py | 2 +- azure-mgmt-datafactory/azure/mgmt/__init__.py | 2 +- azure-mgmt-datafactory/azure_bdist_wheel.py | 54 ------------------- azure-mgmt-datafactory/setup.cfg | 1 - azure-mgmt-datafactory/setup.py | 17 +++--- 6 files changed, 11 insertions(+), 66 deletions(-) delete mode 100644 azure-mgmt-datafactory/azure_bdist_wheel.py diff --git a/azure-mgmt-datafactory/MANIFEST.in b/azure-mgmt-datafactory/MANIFEST.in index 9ecaeb15de50..bb37a2723dae 100644 --- a/azure-mgmt-datafactory/MANIFEST.in +++ b/azure-mgmt-datafactory/MANIFEST.in @@ -1,2 +1 @@ include *.rst -include azure_bdist_wheel.py \ No newline at end of file diff --git a/azure-mgmt-datafactory/azure/__init__.py b/azure-mgmt-datafactory/azure/__init__.py index de40ea7ca058..0260537a02bb 100644 --- a/azure-mgmt-datafactory/azure/__init__.py +++ b/azure-mgmt-datafactory/azure/__init__.py @@ -1 +1 @@ -__import__('pkg_resources').declare_namespace(__name__) +__path__ = __import__('pkgutil').extend_path(__path__, __name__) \ No newline at end of file diff --git a/azure-mgmt-datafactory/azure/mgmt/__init__.py b/azure-mgmt-datafactory/azure/mgmt/__init__.py index de40ea7ca058..0260537a02bb 100644 --- a/azure-mgmt-datafactory/azure/mgmt/__init__.py +++ b/azure-mgmt-datafactory/azure/mgmt/__init__.py @@ -1 +1 @@ -__import__('pkg_resources').declare_namespace(__name__) +__path__ = __import__('pkgutil').extend_path(__path__, __name__) \ No newline at end of file diff --git a/azure-mgmt-datafactory/azure_bdist_wheel.py b/azure-mgmt-datafactory/azure_bdist_wheel.py deleted file mode 100644 index 8a81d1b61775..000000000000 --- a/azure-mgmt-datafactory/azure_bdist_wheel.py +++ /dev/null @@ -1,54 +0,0 @@ -#------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -#-------------------------------------------------------------------------- - -from distutils import log as logger -import os.path - -from wheel.bdist_wheel import bdist_wheel -class azure_bdist_wheel(bdist_wheel): - """The purpose of this class is to build wheel a little differently than the sdist, - without requiring to build the wheel from the sdist (i.e. you can build the wheel - directly from source). - """ - - description = "Create an Azure wheel distribution" - - user_options = bdist_wheel.user_options + \ - [('azure-namespace-package=', None, - "Name of the deepest nspkg used")] - - def initialize_options(self): - bdist_wheel.initialize_options(self) - self.azure_namespace_package = None - - def finalize_options(self): - bdist_wheel.finalize_options(self) - if self.azure_namespace_package and not self.azure_namespace_package.endswith("-nspkg"): - raise ValueError("azure_namespace_package must finish by -nspkg") - - def run(self): - if not self.distribution.install_requires: - self.distribution.install_requires = [] - self.distribution.install_requires.append( - "{}>=2.0.0".format(self.azure_namespace_package)) - bdist_wheel.run(self) - - def write_record(self, bdist_dir, distinfo_dir): - if self.azure_namespace_package: - # Split and remove last part, assuming it's "nspkg" - subparts = self.azure_namespace_package.split('-')[0:-1] - folder_with_init = [os.path.join(*subparts[0:i+1]) for i in range(len(subparts))] - for azure_sub_package in folder_with_init: - init_file = os.path.join(bdist_dir, azure_sub_package, '__init__.py') - if os.path.isfile(init_file): - logger.info("manually remove {} while building the wheel".format(init_file)) - os.remove(init_file) - else: - raise ValueError("Unable to find {}. Are you sure of your namespace package?".format(init_file)) - bdist_wheel.write_record(self, bdist_dir, distinfo_dir) -cmdclass = { - 'bdist_wheel': azure_bdist_wheel, -} diff --git a/azure-mgmt-datafactory/setup.cfg b/azure-mgmt-datafactory/setup.cfg index 856f4164982c..3c6e79cf31da 100644 --- a/azure-mgmt-datafactory/setup.cfg +++ b/azure-mgmt-datafactory/setup.cfg @@ -1,3 +1,2 @@ [bdist_wheel] universal=1 -azure-namespace-package=azure-mgmt-nspkg \ No newline at end of file diff --git a/azure-mgmt-datafactory/setup.py b/azure-mgmt-datafactory/setup.py index 98b696f39617..8fb4b233c1b8 100644 --- a/azure-mgmt-datafactory/setup.py +++ b/azure-mgmt-datafactory/setup.py @@ -10,12 +10,6 @@ import os.path from io import open from setuptools import find_packages, setup -try: - from azure_bdist_wheel import cmdclass -except ImportError: - from distutils import log as logger - logger.warn("Wheel is not available, disabling bdist_wheel hook") - cmdclass = {} # Change the PACKAGE_NAME only to change folder and different name PACKAGE_NAME = "azure-mgmt-datafactory" @@ -76,11 +70,18 @@ 'License :: OSI Approved :: MIT License', ], zip_safe=False, - packages=find_packages(exclude=["tests"]), + packages=find_packages(exclude=[ + 'tests', + # Exclude packages that will be covered by PEP420 or nspkg + 'azure', + 'azure.mgmt', + ]), install_requires=[ 'msrest>=0.5.0', 'msrestazure>=0.4.32,<2.0.0', 'azure-common~=1.1', ], - cmdclass=cmdclass + extras_require={ + ":python_version<'3.0'": ['azure-mgmt-nspkg'], + } )