From f931bdbaa31a437dea54a7f722a405434d66fc4c Mon Sep 17 00:00:00 2001 From: Niko Oliveira Date: Thu, 26 Jun 2025 21:41:18 -0700 Subject: [PATCH] Fix GlueJobOperator deferred waiting (#52314) In #52243 the waiting was moved from custom code within the glue hook to using the aws base waiters when deferring Glue jobs. The Trigger was given inappropriate inputs which caused it to wait for zero attempts, which causes our tests to fail. This change moves to using the common parameters we use for other operators in deferrable with the same defaults as the Trigger has. Note: previously this Operator used to wait indefinitely for the job to either complete or fail. The default now waits for 75 minutes. The aws base waiter has no ability to wait indefinitely, nor do I think it should, that feels like a bug to me. So I'm considering this slight behaviour change a bug fix of a bug fix. --- .../providers/databricks/hooks/databricks.py | 54 +++++++++---------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/providers/databricks/src/airflow/providers/databricks/hooks/databricks.py b/providers/databricks/src/airflow/providers/databricks/hooks/databricks.py index 56bf1da68fd11..8ae436dba1d56 100644 --- a/providers/databricks/src/airflow/providers/databricks/hooks/databricks.py +++ b/providers/databricks/src/airflow/providers/databricks/hooks/databricks.py @@ -37,33 +37,33 @@ from airflow.exceptions import AirflowException from airflow.providers.databricks.hooks.databricks_base import BaseDatabricksHook -GET_CLUSTER_ENDPOINT = ("GET", "api/2.0/clusters/get") -RESTART_CLUSTER_ENDPOINT = ("POST", "api/2.0/clusters/restart") -START_CLUSTER_ENDPOINT = ("POST", "api/2.0/clusters/start") -TERMINATE_CLUSTER_ENDPOINT = ("POST", "api/2.0/clusters/delete") - -CREATE_ENDPOINT = ("POST", "api/2.1/jobs/create") -RESET_ENDPOINT = ("POST", "api/2.1/jobs/reset") -UPDATE_ENDPOINT = ("POST", "api/2.1/jobs/update") -RUN_NOW_ENDPOINT = ("POST", "api/2.1/jobs/run-now") -SUBMIT_RUN_ENDPOINT = ("POST", "api/2.1/jobs/runs/submit") -GET_RUN_ENDPOINT = ("GET", "api/2.1/jobs/runs/get") -CANCEL_RUN_ENDPOINT = ("POST", "api/2.1/jobs/runs/cancel") -DELETE_RUN_ENDPOINT = ("POST", "api/2.1/jobs/runs/delete") -REPAIR_RUN_ENDPOINT = ("POST", "api/2.1/jobs/runs/repair") -OUTPUT_RUNS_JOB_ENDPOINT = ("GET", "api/2.1/jobs/runs/get-output") -CANCEL_ALL_RUNS_ENDPOINT = ("POST", "api/2.1/jobs/runs/cancel-all") - -INSTALL_LIBS_ENDPOINT = ("POST", "api/2.0/libraries/install") -UNINSTALL_LIBS_ENDPOINT = ("POST", "api/2.0/libraries/uninstall") - -LIST_JOBS_ENDPOINT = ("GET", "api/2.1/jobs/list") -LIST_PIPELINES_ENDPOINT = ("GET", "api/2.0/pipelines") - -WORKSPACE_GET_STATUS_ENDPOINT = ("GET", "api/2.0/workspace/get-status") - -SPARK_VERSIONS_ENDPOINT = ("GET", "api/2.0/clusters/spark-versions") -SQL_STATEMENTS_ENDPOINT = "api/2.0/sql/statements" +GET_CLUSTER_ENDPOINT = ("GET", "2.0/clusters/get") +RESTART_CLUSTER_ENDPOINT = ("POST", "2.0/clusters/restart") +START_CLUSTER_ENDPOINT = ("POST", "2.0/clusters/start") +TERMINATE_CLUSTER_ENDPOINT = ("POST", "2.0/clusters/delete") + +CREATE_ENDPOINT = ("POST", "2.1/jobs/create") +RESET_ENDPOINT = ("POST", "2.1/jobs/reset") +UPDATE_ENDPOINT = ("POST", "2.1/jobs/update") +RUN_NOW_ENDPOINT = ("POST", "2.1/jobs/run-now") +SUBMIT_RUN_ENDPOINT = ("POST", "2.1/jobs/runs/submit") +GET_RUN_ENDPOINT = ("GET", "2.1/jobs/runs/get") +CANCEL_RUN_ENDPOINT = ("POST", "2.1/jobs/runs/cancel") +DELETE_RUN_ENDPOINT = ("POST", "2.1/jobs/runs/delete") +REPAIR_RUN_ENDPOINT = ("POST", "2.1/jobs/runs/repair") +OUTPUT_RUNS_JOB_ENDPOINT = ("GET", "2.1/jobs/runs/get-output") +CANCEL_ALL_RUNS_ENDPOINT = ("POST", "2.1/jobs/runs/cancel-all") + +INSTALL_LIBS_ENDPOINT = ("POST", "2.0/libraries/install") +UNINSTALL_LIBS_ENDPOINT = ("POST", "2.0/libraries/uninstall") + +LIST_JOBS_ENDPOINT = ("GET", "2.1/jobs/list") +LIST_PIPELINES_ENDPOINT = ("GET", "2.0/pipelines") + +WORKSPACE_GET_STATUS_ENDPOINT = ("GET", "2.0/workspace/get-status") + +SPARK_VERSIONS_ENDPOINT = ("GET", "2.0/clusters/spark-versions") +SQL_STATEMENTS_ENDPOINT = "2.0/sql/statements" class RunLifeCycleState(Enum):