From c17034e9babac7ab881cf5ed905245966c012680 Mon Sep 17 00:00:00 2001 From: BobDu Date: Thu, 17 Jul 2025 17:44:04 +0800 Subject: [PATCH 1/2] Fix EMR operator parameter documentation and naming - Correct parameter names (waiter_check_interval_seconds -> waiter_delay) - Improve English grammar and wording in docstrings - Standardize parameter documentation format - Fix default value descriptions to match actual code Signed-off-by: BobDu --- .../providers/amazon/aws/operators/emr.py | 20 ++++++++++--------- .../amazon/aws/example_emr_serverless.py | 4 ++-- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/providers/amazon/src/airflow/providers/amazon/aws/operators/emr.py b/providers/amazon/src/airflow/providers/amazon/aws/operators/emr.py index 047d3563fbd09..7dbe37de08ecc 100644 --- a/providers/amazon/src/airflow/providers/amazon/aws/operators/emr.py +++ b/providers/amazon/src/airflow/providers/amazon/aws/operators/emr.py @@ -970,7 +970,7 @@ class EmrServerlessCreateApplicationOperator(AwsBaseOperator[EmrServerlessHook]) :param release_label: The EMR release version associated with the application. :param job_type: The type of application you want to start, such as Spark or Hive. - :param wait_for_completion: If true, wait for the Application to start before returning. Default to True. + :param wait_for_completion: If true, wait for the Application to start before returning. Defaults to True. If set to False, ``waiter_max_attempts`` and ``waiter_delay`` will only be applied when waiting for the application to be in the ``CREATED`` state. :param client_request_token: The client idempotency token of the application to create. @@ -985,8 +985,9 @@ class EmrServerlessCreateApplicationOperator(AwsBaseOperator[EmrServerlessHook]) :param verify: Whether or not to verify SSL certificates. See: https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html :param waiter_max_attempts: Number of times the waiter should poll the application to check the state. - If not set, the waiter will use its default value. + Defaults to 25 if not set. :param waiter_delay: Number of seconds between polling the state of the application. + Defaults to 60 seconds if not set. :param deferrable: If True, the operator will wait asynchronously for application to be created. This implies waiting for completion. This mode requires aiobotocore module to be installed. (default: False, but can be overridden in config file by setting default_deferrable to True) @@ -1117,8 +1118,8 @@ class EmrServerlessStartJobOperator(AwsBaseOperator[EmrServerlessHook]): Its value must be unique for each request. :param config: Optional dictionary for arbitrary parameters to the boto API start_job_run call. :param wait_for_completion: If true, waits for the job to start before returning. Defaults to True. - If set to False, ``waiter_countdown`` and ``waiter_check_interval_seconds`` will only be applied - when waiting for the application be to in the ``STARTED`` state. + If set to False, ``waiter_max_attempts`` and ``waiter_delay`` will only be applied + when waiting for the application to be in the ``STARTED`` state. :param aws_conn_id: The Airflow connection used for AWS credentials. If this is ``None`` or empty then the default boto3 behaviour is used. If running Airflow in a distributed manner and aws_conn_id is None or @@ -1129,8 +1130,9 @@ class EmrServerlessStartJobOperator(AwsBaseOperator[EmrServerlessHook]): https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html :param name: Name for the EMR Serverless job. If not provided, a default name will be assigned. :param waiter_max_attempts: Number of times the waiter should poll the application to check the state. - If not set, the waiter will use its default value. + Defaults to 25 if not set. :param waiter_delay: Number of seconds between polling the state of the job run. + Defaults to 60 seconds if not set. :param deferrable: If True, the operator will wait asynchronously for the crawl to complete. This implies waiting for completion. This mode requires aiobotocore module to be installed. (default: False, but can be overridden in config file by setting default_deferrable to True) @@ -1439,9 +1441,9 @@ class EmrServerlessStopApplicationOperator(AwsBaseOperator[EmrServerlessHook]): If you want to wait for the jobs to finish gracefully, use :class:`airflow.providers.amazon.aws.sensors.emr.EmrServerlessJobSensor` :param waiter_max_attempts: Number of times the waiter should poll the application to check the state. - Default is 25. + Defaults to 25 if not set. :param waiter_delay: Number of seconds between polling the state of the application. - Default is 60 seconds. + Defaults to 60 seconds if not set. :param deferrable: If True, the operator will wait asynchronously for the application to stop. This implies waiting for completion. This mode requires aiobotocore module to be installed. (default: False, but can be overridden in config file by setting default_deferrable to True) @@ -1574,9 +1576,9 @@ class EmrServerlessDeleteApplicationOperator(EmrServerlessStopApplicationOperato :param verify: Whether or not to verify SSL certificates. See: https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html :param waiter_max_attempts: Number of times the waiter should poll the application to check the state. - Defaults to 25. + Defaults to 25 if not set. :param waiter_delay: Number of seconds between polling the state of the application. - Defaults to 60 seconds. + Defaults to 60 seconds if not set. :param deferrable: If True, the operator will wait asynchronously for application to be deleted. This implies waiting for completion. This mode requires aiobotocore module to be installed. (default: False, but can be overridden in config file by setting default_deferrable to True) diff --git a/providers/amazon/tests/system/amazon/aws/example_emr_serverless.py b/providers/amazon/tests/system/amazon/aws/example_emr_serverless.py index bd90e6fe0c4f6..3485234afbb11 100644 --- a/providers/amazon/tests/system/amazon/aws/example_emr_serverless.py +++ b/providers/amazon/tests/system/amazon/aws/example_emr_serverless.py @@ -134,7 +134,7 @@ force_stop=True, ) # [END howto_operator_emr_serverless_stop_application] - stop_app.waiter_check_interval_seconds = 1 + stop_app.waiter_delay = 1 # [START howto_operator_emr_serverless_delete_application] delete_app = EmrServerlessDeleteApplicationOperator( @@ -142,7 +142,7 @@ application_id=emr_serverless_app_id, ) # [END howto_operator_emr_serverless_delete_application] - delete_app.waiter_check_interval_seconds = 1 + delete_app.waiter_delay = 1 delete_app.trigger_rule = TriggerRule.ALL_DONE delete_s3_bucket = S3DeleteBucketOperator( From 2fad22038603e81b2d56f4cdd809ce254b552902 Mon Sep 17 00:00:00 2001 From: BobDu Date: Thu, 17 Jul 2025 18:35:51 +0800 Subject: [PATCH 2/2] Fix duplicate wait_for_completion parameter documentation in EmrAddStepsOperator Signed-off-by: BobDu --- .../amazon/src/airflow/providers/amazon/aws/operators/emr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/providers/amazon/src/airflow/providers/amazon/aws/operators/emr.py b/providers/amazon/src/airflow/providers/amazon/aws/operators/emr.py index 7dbe37de08ecc..4377ccd89bc70 100644 --- a/providers/amazon/src/airflow/providers/amazon/aws/operators/emr.py +++ b/providers/amazon/src/airflow/providers/amazon/aws/operators/emr.py @@ -89,9 +89,9 @@ class EmrAddStepsOperator(AwsBaseOperator[EmrHook]): :param steps: boto3 style steps or reference to a steps file (must be '.json') to be added to the jobflow. (templated) :param wait_for_completion: If True, the operator will wait for all the steps to be completed. + Defaults to False. Note: When deferrable=True, this parameter will not take effect. :param execution_role_arn: The ARN of the runtime role for a step on the cluster. :param do_xcom_push: if True, job_flow_id is pushed to XCom with key job_flow_id. - :param wait_for_completion: Whether to wait for job run completion. (default: True) :param deferrable: If True, the operator will wait asynchronously for the job to complete. This implies waiting for completion. This mode requires aiobotocore module to be installed. (default: False)