[AIRFLOW-4767] Fix errors in the documentation of Dataproc Operator (#…

…5487)

airflow/contrib/example_dags/example_gcp_dataproc_pig_operator.py

@@ -16,7 +16,6 @@
 # KIND, either express or implied. See the License for the
 # specific language governing permissions and limitations
 # under the License.
-
 """
 Example Airflow DAG for Google Dataproc PigOperator
 """
@@ -50,7 +49,7 @@
  num_workers=2
  )
 
- pig_taks = DataProcPigOperator(
+ pig_task = DataProcPigOperator(
  task_id="pig_task",
  query="define sin HiveUDF('sin');",
  region=REGION,
@@ -64,4 +63,4 @@
  region=REGION
  )
 
- create_task >> pig_taks >> delete_task
+ create_task >> pig_task >> delete_task

airflow/contrib/operators/dataproc_operator.py

@@ -696,7 +696,7 @@ class DataProcPigOperator(DataProcJobBaseOperator):
  :param query: The query or reference to the query
  file (pg or pig extension). (templated)
  :type query: str
- :param query_uri: The uri of a pig script on Cloud Storage.
+ :param query_uri: The HCFS URI of the script that contains the Pig queries.
  :type query_uri: str
  :param variables: Map of named parameters for the query. (templated)
  :type variables: dict
@@ -750,7 +750,7 @@ class DataProcHiveOperator(DataProcJobBaseOperator):
 
  :param query: The query or reference to the query file (q extension).
  :type query: str
- :param query_uri: The uri of a hive script on Cloud Storage.
+ :param query_uri: The HCFS URI of the script that contains the Hive queries.
  :type query_uri: str
  :param variables: Map of named parameters for the query.
  :type variables: dict
@@ -804,7 +804,7 @@ class DataProcSparkSqlOperator(DataProcJobBaseOperator):
 
  :param query: The query or reference to the query file (q extension). (templated)
  :type query: str
- :param query_uri: The uri of a spark sql script on Cloud Storage.
+ :param query_uri: The HCFS URI of the script that contains the SQL queries.
  :type query_uri: str
  :param variables: Map of named parameters for the query. (templated)
  :type variables: dict
@@ -854,8 +854,8 @@ class DataProcSparkOperator(DataProcJobBaseOperator):
  """
  Start a Spark Job on a Cloud DataProc cluster.
 
- :param main_jar: URI of the job jar provisioned on Cloud Storage. (use this or
-   the main_class, not both together).
+ :param main_jar: The HCFS URI of the jar file that contains the main class
+ (use this or the main_class, not both together).
  :type main_jar: str
  :param main_class: Name of the job class. (use this or the main_jar, not both
  together).
@@ -917,8 +917,8 @@ class DataProcHadoopOperator(DataProcJobBaseOperator):
  """
  Start a Hadoop Job on a Cloud DataProc cluster.
 
- :param main_jar: URI of the job jar provisioned on Cloud Storage. (use this or
-   the main_class, not both together).
+ :param main_jar: The HCFS URI of the jar file containing the main class
+ (use this or the main_class, not both together).
  :type main_jar: str
  :param main_class: Name of the job class. (use this or the main_jar, not both
  together).

tests/contrib/operators/test_dataproc_operator.py

@@ -640,6 +640,7 @@ def test_hook_correct_region():
  )
 
  dataproc_task.execute(None)
+
  mock_hook.return_value.submit.assert_called_once_with(mock.ANY, mock.ANY,
  GCP_REGION, mock.ANY)