Skip to content

Adding gcp_conn_id to template_fields for BigQuery and Dataproc Operators #58250

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
potiuk merged 1 commit into from
Nov 13, 2025
Merged

Adding gcp_conn_id to template_fields for BigQuery and Dataproc Operators #58250

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions providers/google/src/airflow/providers/google/cloud/operators/bigquery.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1012,6 +1012,7 @@ class BigQueryGetDataOperator(GoogleCloudBaseOperator, _BigQueryOperatorsEncrypt
"project_id",
"max_results",
"selected_fields",
"gcp_conn_id",
"impersonation_chain",
)
ui_color = BigQueryUIColors.QUERY.value
Expand Down Expand Up @@ -1253,6 +1254,7 @@ class BigQueryCreateTableOperator(GoogleCloudBaseOperator):
"table_resource",
"project_id",
"gcs_schema_object",
"gcp_conn_id",
"impersonation_chain",
)
template_fields_renderers = {"table_resource": "json"}
Expand Down Expand Up @@ -1415,6 +1417,7 @@ class BigQueryDeleteDatasetOperator(GoogleCloudBaseOperator):
template_fields: Sequence[str] = (
"dataset_id",
"project_id",
"gcp_conn_id",
"impersonation_chain",
)
ui_color = BigQueryUIColors.DATASET.value
Expand Down Expand Up @@ -1494,6 +1497,7 @@ class BigQueryCreateEmptyDatasetOperator(GoogleCloudBaseOperator):
"dataset_id",
"project_id",
"dataset_reference",
"gcp_conn_id",
"impersonation_chain",
)
template_fields_renderers = {"dataset_reference": "json"}
Expand Down Expand Up @@ -1598,6 +1602,7 @@ class BigQueryGetDatasetOperator(GoogleCloudBaseOperator):
template_fields: Sequence[str] = (
"dataset_id",
"project_id",
"gcp_conn_id",
"impersonation_chain",
)
ui_color = BigQueryUIColors.DATASET.value
Expand Down Expand Up @@ -1662,6 +1667,7 @@ class BigQueryGetDatasetTablesOperator(GoogleCloudBaseOperator):
template_fields: Sequence[str] = (
"dataset_id",
"project_id",
"gcp_conn_id",
"impersonation_chain",
)
ui_color = BigQueryUIColors.DATASET.value
Expand Down Expand Up @@ -1732,6 +1738,7 @@ class BigQueryUpdateTableOperator(GoogleCloudBaseOperator):
"dataset_id",
"table_id",
"project_id",
"gcp_conn_id",
"impersonation_chain",
)
template_fields_renderers = {"table_resource": "json"}
Expand Down Expand Up @@ -1838,6 +1845,7 @@ class BigQueryUpdateDatasetOperator(GoogleCloudBaseOperator):
template_fields: Sequence[str] = (
"dataset_id",
"project_id",
"gcp_conn_id",
"impersonation_chain",
)
template_fields_renderers = {"dataset_resource": "json"}
Expand Down Expand Up @@ -1913,6 +1921,7 @@ class BigQueryDeleteTableOperator(GoogleCloudBaseOperator):

template_fields: Sequence[str] = (
"deletion_dataset_table",
"gcp_conn_id",
"impersonation_chain",
)
ui_color = BigQueryUIColors.TABLE.value
Expand Down Expand Up @@ -2007,6 +2016,7 @@ class BigQueryUpsertTableOperator(GoogleCloudBaseOperator):
template_fields: Sequence[str] = (
"dataset_id",
"table_resource",
"gcp_conn_id",
"impersonation_chain",
"project_id",
)
Expand Down Expand Up @@ -2134,6 +2144,7 @@ class BigQueryUpdateTableSchemaOperator(GoogleCloudBaseOperator):
"dataset_id",
"table_id",
"project_id",
"gcp_conn_id",
"impersonation_chain",
)
template_fields_renderers = {"schema_fields_updates": "json"}
Expand Down Expand Up @@ -2263,6 +2274,7 @@ class BigQueryInsertJobOperator(GoogleCloudBaseOperator, _BigQueryInsertJobOpera
template_fields: Sequence[str] = (
"configuration",
"job_id",
"gcp_conn_id",
"impersonation_chain",
"project_id",
)
Expand Down
60 changes: 51 additions & 9 deletions providers/google/src/airflow/providers/google/cloud/operators/dataproc.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -628,6 +628,7 @@ class DataprocCreateClusterOperator(GoogleCloudBaseOperator):
"virtual_cluster_config",
"cluster_name",
"labels",
"gcp_conn_id",
"impersonation_chain",
)
template_fields_renderers = {"cluster_config": "json", "virtual_cluster_config": "json"}
Expand Down Expand Up @@ -951,7 +952,13 @@ class DataprocDeleteClusterOperator(GoogleCloudBaseOperator):
:param polling_interval_seconds: Time (seconds) to wait between calls to check the cluster status.
"""

template_fields: Sequence[str] = ("project_id", "region", "cluster_name", "impersonation_chain")
template_fields: Sequence[str] = (
"project_id",
"region",
"cluster_name",
"gcp_conn_id",
"impersonation_chain",
)

def __init__(
self,
Expand Down Expand Up @@ -1077,6 +1084,7 @@ class _DataprocStartStopClusterBaseOperator(GoogleCloudBaseOperator):
"region",
"project_id",
"request_id",
"gcp_conn_id",
"impersonation_chain",
)

Expand Down Expand Up @@ -1422,7 +1430,7 @@ class DataprocCreateWorkflowTemplateOperator(GoogleCloudBaseOperator):
:param metadata: Additional metadata that is provided to the method.
"""

template_fields: Sequence[str] = ("region", "template")
template_fields: Sequence[str] = ("region", "template", "gcp_conn_id")
template_fields_renderers = {"template": "json"}
operator_extra_links = (DataprocWorkflowTemplateLink(),)

Expand Down Expand Up @@ -1516,7 +1524,13 @@ class DataprocInstantiateWorkflowTemplateOperator(GoogleCloudBaseOperator):
:param cancel_on_kill: Flag which indicates whether cancel the workflow, when on_kill is called
"""

template_fields: Sequence[str] = ("template_id", "impersonation_chain", "request_id", "parameters")
template_fields: Sequence[str] = (
"template_id",
"gcp_conn_id",
"impersonation_chain",
"request_id",
"parameters",
)
template_fields_renderers = {"parameters": "json"}
operator_extra_links = (DataprocWorkflowLink(),)

Expand Down Expand Up @@ -1664,7 +1678,7 @@ class DataprocInstantiateInlineWorkflowTemplateOperator(GoogleCloudBaseOperator)
:param cancel_on_kill: Flag which indicates whether cancel the workflow, when on_kill is called
"""

template_fields: Sequence[str] = ("template", "impersonation_chain")
template_fields: Sequence[str] = ("template", "gcp_conn_id", "impersonation_chain")
template_fields_renderers = {"template": "json"}
operator_extra_links = (DataprocWorkflowLink(),)

Expand Down Expand Up @@ -1832,7 +1846,14 @@ class DataprocSubmitJobOperator(GoogleCloudBaseOperator):
:param wait_timeout: How many seconds wait for job to be ready. Used only if ``asynchronous`` is False
"""

template_fields: Sequence[str] = ("project_id", "region", "job", "impersonation_chain", "request_id")
template_fields: Sequence[str] = (
"project_id",
"region",
"job",
"gcp_conn_id",
"impersonation_chain",
"request_id",
)
template_fields_renderers = {"job": "json"}

operator_extra_links = (DataprocJobLink(),)
Expand Down Expand Up @@ -2031,6 +2052,7 @@ class DataprocUpdateClusterOperator(GoogleCloudBaseOperator):
"region",
"request_id",
"project_id",
"gcp_conn_id",
"impersonation_chain",
)
operator_extra_links = (DataprocClusterLink(),)
Expand Down Expand Up @@ -2166,6 +2188,7 @@ class DataprocDiagnoseClusterOperator(GoogleCloudBaseOperator):
"project_id",
"region",
"cluster_name",
"gcp_conn_id",
"impersonation_chain",
"tarball_gcs_dir",
"diagnosis_interval",
Expand Down Expand Up @@ -2312,6 +2335,7 @@ class DataprocCreateBatchOperator(GoogleCloudBaseOperator):
"batch",
"batch_id",
"region",
"gcp_conn_id",
"impersonation_chain",
)
operator_extra_links = (DataprocBatchLink(),)
Expand Down Expand Up @@ -2620,7 +2644,13 @@ class DataprocDeleteBatchOperator(GoogleCloudBaseOperator):
account from the list granting this role to the originating account (templated).
"""

template_fields: Sequence[str] = ("batch_id", "region", "project_id", "impersonation_chain")
template_fields: Sequence[str] = (
"batch_id",
"region",
"project_id",
"gcp_conn_id",
"impersonation_chain",
)

def __init__(
self,
Expand Down Expand Up @@ -2684,7 +2714,13 @@ class DataprocGetBatchOperator(GoogleCloudBaseOperator):
account from the list granting this role to the originating account (templated).
"""

template_fields: Sequence[str] = ("batch_id", "region", "project_id", "impersonation_chain")
template_fields: Sequence[str] = (
"batch_id",
"region",
"project_id",
"gcp_conn_id",
"impersonation_chain",
)
operator_extra_links = (DataprocBatchLink(),)

def __init__(
Expand Down Expand Up @@ -2760,7 +2796,7 @@ class DataprocListBatchesOperator(GoogleCloudBaseOperator):
:param order_by: How to order results as specified in ListBatchesRequest
"""

template_fields: Sequence[str] = ("region", "project_id", "impersonation_chain")
template_fields: Sequence[str] = ("region", "project_id", "gcp_conn_id", "impersonation_chain")
operator_extra_links = (DataprocBatchesListLink(),)

def __init__(
Expand Down Expand Up @@ -2834,7 +2870,13 @@ class DataprocCancelOperationOperator(GoogleCloudBaseOperator):
account from the list granting this role to the originating account (templated).
"""

template_fields: Sequence[str] = ("operation_name", "region", "project_id", "impersonation_chain")
template_fields: Sequence[str] = (
"operation_name",
"region",
"project_id",
"gcp_conn_id",
"impersonation_chain",
)

def __init__(
self,
Expand Down